ic_memory/runtime/backing.rs
1use ic_platform::stable_structures::{Memory, memory_manager::VirtualMemory};
2use std::{
3 cell::{Cell, RefCell},
4 rc::Rc,
5};
6
7// Physical custody, immutable geometry and exact manager metadata authority are
8// shared by the runtime and every handle, including detached ledger handles.
9pub(super) struct GrowthState<M: Memory> {
10 // Fields drop in declaration order: retire the claim while its backing's
11 // identity allocation is still alive, before another thread can reuse it.
12 pub _claim: super::custody::BackingClaim,
13 pub backing: Rc<M>,
14 pub bucket_size_pages: u16,
15 pub authority: RefCell<super::layout::ManagerAuthority>,
16 pub open_slots: [Cell<u64>; 4],
17}
18
19impl<M: Memory> GrowthState<M> {
20 pub fn verify(&self) -> Result<(), super::MemoryManagerLayoutError> {
21 self.authority.borrow().verify(self.backing.as_ref())
22 }
23}
24
25///
26/// RuntimeMemory
27///
28/// Exclusive virtual memory leased through a runtime's committed authority.
29/// Implements [`Memory`] for stable structures without exposing the backing
30/// memory or an alternate manager. The lease cannot be cloned; dropping it
31/// permits another open of the same allocation. Moving it into a collection
32/// transfers the lease until that collection releases the memory.
33/// The handle can outlive its runtime and retains the backing claim. A replacement
34/// runtime is refused until every handle releases it. Custom backing identities
35/// must follow [`super::MemoryBacking`]; raw aliased access remains prohibited.
36/// IO checks the virtual extent before upstream bucket translation. Empty IO
37/// is valid at the end of memory, but not beyond it. Reads preserve the upstream
38/// optimized support for uninitialized destinations through `Memory::read_unsafe`.
39/// Growth reserves physical capacity before assigning manager buckets. Ordinary
40/// backing refusal, arithmetic overflow and bucket exhaustion return typed errors
41/// without changing virtual extents or manager metadata. Backing implementations
42/// must obey the `Memory` contract; traps and partial writes are not transactions
43/// on native memory. Typed collections retain their own failure behavior.
44/// Size and IO trust exclusive backing custody between full metadata checks at
45/// construction, initial bootstrap, lease opening and growth. Raw changes during
46/// a live lease are prohibited; ordinary IO does not scan for them. Detected
47/// divergence permanently poisons this owner, including its existing handles.
48/// Invalid IO bounds still refuse before backing access.
49///
50
51pub struct RuntimeMemory<M: Memory> {
52 pub(super) memory: VirtualMemory<Rc<M>>,
53 pub(super) growth: Rc<GrowthState<M>>,
54 pub(super) id: u8,
55}
56
57impl<M: Memory> Drop for RuntimeMemory<M> {
58 fn drop(&mut self) {
59 let word = &self.growth.open_slots[usize::from(self.id / 64)];
60 word.set(word.get() & !(1_u64 << (self.id % 64)));
61 }
62}
63
64impl<M: Memory> RuntimeMemory<M> {
65 // The manager bounds page counts by 32768 buckets of at most u16::MAX
66 // pages, so converting its virtual extent to bytes cannot overflow u64.
67 // Check before the upstream cache: its buckets can include virtual slack,
68 // and its span/translation arithmetic is unchecked in release builds.
69 #[expect(
70 clippy::inline_always,
71 reason = "matched PocketIC measurements reduce IO instructions with bounded Wasm growth"
72 )]
73 #[inline(always)]
74 fn check_io_bounds(&self, offset: u64, count: usize) {
75 let extent = self.memory.size() * crate::constants::WASM_PAGE_SIZE_BYTES;
76 assert!(
77 offset <= extent && count as u64 <= extent - offset,
78 "virtual memory access out of bounds",
79 );
80 self.growth
81 .authority
82 .borrow()
83 .check_healthy()
84 .expect("persisted manager metadata differs from runtime authority");
85 }
86
87 /// Grow by the requested pages, returning the previous virtual page count.
88 ///
89 /// Capacity is reserved before assigning manager buckets. Ordinary refusal
90 /// preserves virtual extents and manager metadata and permits retry. The
91 /// upstream [`Memory::grow`] adapter translates errors into its required
92 /// `-1` sentinel; direct runtime callers receive [`super::RuntimeGrowError`].
93 /// A zero-page request verifies metadata and returns the current extent
94 /// without growth or writes, after checking for reentrant growth.
95 ///
96 /// # Panics
97 ///
98 /// Panics if a private growth-accounting invariant is broken or backing
99 /// memory panics.
100 pub fn grow(&self, pages: u64) -> Result<u64, super::RuntimeGrowError> {
101 use super::RuntimeGrowError;
102 let mut authority = self
103 .growth
104 .authority
105 .try_borrow_mut()
106 .map_err(|_| RuntimeGrowError::ReentrantAccess)?;
107 authority.verify(self.growth.backing.as_ref())?;
108 let old_pages = self.memory.size();
109 if pages == 0 {
110 return Ok(old_pages);
111 }
112 let new_pages = old_pages
113 .checked_add(pages)
114 .ok_or(RuntimeGrowError::ArithmeticOverflow)?;
115 let bucket_pages = u64::from(self.growth.bucket_size_pages);
116 let extra = new_pages.div_ceil(bucket_pages) - old_pages.div_ceil(bucket_pages);
117 let total = u64::from(authority.allocated_buckets())
118 .checked_add(extra)
119 .ok_or(RuntimeGrowError::ArithmeticOverflow)?;
120 if total > u64::from(super::layout::BUCKET_CAPACITY) {
121 return Err(RuntimeGrowError::BucketExhausted {
122 required_buckets: total,
123 capacity: super::layout::BUCKET_CAPACITY,
124 });
125 }
126 #[expect(
127 clippy::cast_possible_truncation,
128 reason = "admission bounds total by the u16 bucket capacity"
129 )]
130 let total_buckets = total as u16;
131 let required_pages = 1 + total * bucket_pages;
132 let physical_pages = self.growth.backing.size();
133 if required_pages > physical_pages
134 && self.growth.backing.grow(required_pages - physical_pages) < 0
135 {
136 return Err(RuntimeGrowError::BackingRefused {
137 additional_pages: required_pages - physical_pages,
138 });
139 }
140 // The pinned manager's only refusal is bucket exhaustion, already
141 // checked above while all handles share this exclusive reservation.
142 // Physical capacity is reserved before it assigns any buckets.
143 assert_eq!(
144 self.memory.grow(pages),
145 old_pages.cast_signed(),
146 "preflighted manager growth returns the previous virtual extent"
147 );
148 authority.record_growth(self.id, new_pages, total_buckets);
149 authority.verify(self.growth.backing.as_ref())?;
150 Ok(old_pages)
151 }
152}
153
154impl<M: Memory> Memory for RuntimeMemory<M> {
155 fn size(&self) -> u64 {
156 self.growth
157 .authority
158 .borrow()
159 .check_healthy()
160 .expect("persisted manager metadata differs from runtime authority");
161 self.memory.size()
162 }
163 fn grow(&self, pages: u64) -> i64 {
164 // The upstream trait fixes the sentinel contract. Direct calls use the
165 // inherent typed method; virtual extents fit in i64 by bucket capacity.
166 Self::grow(self, pages).map_or(-1, u64::cast_signed)
167 }
168 fn read(&self, offset: u64, dst: &mut [u8]) {
169 self.check_io_bounds(offset, dst.len());
170 self.memory.read(offset, dst);
171 }
172 #[expect(
173 unsafe_code,
174 reason = "delegate the upstream raw-read contract unchanged"
175 )]
176 unsafe fn read_unsafe(&self, offset: u64, dst: *mut u8, count: usize) {
177 self.check_io_bounds(offset, count);
178 // SAFETY: The caller supplies a valid destination disjoint from this
179 // memory and its backing. Forwarding preserves the pointer and count;
180 // VirtualMemory owns bucket translation and initializes the destination
181 // on success. After a panic, initialization must not be assumed.
182 unsafe { self.memory.read_unsafe(offset, dst, count) }
183 }
184 fn write(&self, offset: u64, src: &[u8]) {
185 self.check_io_bounds(offset, src.len());
186 self.memory.write(offset, src);
187 }
188}