ic_memory/runtime/
mod.rs

1mod admission;
2#[cfg(test)]
3mod admission_tests;
4mod adoption;
5#[cfg(test)]
6mod adoption_tests;
7mod allocations;
8mod backing;
9mod config;
10mod diagnostics;
11mod error;
12#[cfg(any(test, feature = "test-support"))]
13#[path = "../testing/diagnostics.rs"]
14pub mod fixture_diagnostics;
15mod layout;
16mod policy;
17
18#[cfg(all(test, not(target_arch = "wasm32")))]
19mod allocation_tests;
20#[cfg(test)]
21mod growth_tests;
22#[cfg(test)]
23#[expect(
24    unsafe_code,
25    reason = "exercise raw reads with valid uninitialized destinations"
26)]
27mod read_tests;
28#[cfg(test)]
29mod request_tests;
30#[cfg(test)]
31mod tests;
32
33pub use admission::{BootstrapAdmission, BootstrapAdmissionError};
34pub use adoption::RuntimeAdoptionError;
35
36pub use allocations::{
37    MemoryAllocation, MemoryAllocationSummary, MemoryAllocations, MemoryBindingSummary,
38};
39pub use backing::RuntimeMemory;
40pub use config::MemoryManagerConfig;
41pub use error::{
42    MemoryResolutionError, RuntimeBootstrapError, RuntimeConstructionError, RuntimeDiagnosticError,
43    RuntimeGrowError, RuntimeOpenError,
44};
45pub use layout::MemoryManagerLayoutError;
46pub use policy::GenericAllocationPolicy;
47
48use crate::{
49    LedgerCommitStore, PolicyIdentity, RuntimeBootstrapPolicy, STABLE_CELL_VALUE_OFFSET,
50    StableCellLedgerError,
51    requests::SealedDeclarationSnapshot,
52    slot::MEMORY_MANAGER_LEDGER_ID,
53    stable_cell::{EncodedLedgerRecord, decode_stable_cell_commit_store_from_memory},
54};
55use ic_platform::stable_structures::{
56    Cell, Memory,
57    memory_manager::{MemoryId, MemoryManager},
58};
59
60use std::rc::Rc;
61
62enum RuntimeLifecycle {
63    Unbootstrapped,
64    Bootstrapped {
65        allocations: Vec<crate::AllocationDeclaration>,
66        generation: u64,
67        binding: RuntimeBootstrapBinding,
68    },
69}
70
71struct RuntimeBootstrapBinding {
72    source: SealedDeclarationSnapshot,
73    policy_identity: PolicyIdentity,
74    pool: crate::MemoryAllocationPool,
75}
76
77///
78/// MemoryRuntime
79///
80/// Canonical owner of allocation bootstrap state for one backing memory.
81///
82/// The runtime owns its `MemoryManager`, allocation-ledger persistence, bootstrap
83/// lifecycle, committed allocation capability, opens, and diagnostics. Static
84/// linked-program declarations are supplied separately as one immutable
85/// [`SealedDeclarationSnapshot`].
86///
87/// Each bootstrap attempt fallibly decodes its ledger record from persisted
88/// memory. Capacity-checked writes use `Cell`; diagnostics also read persisted
89/// memory directly.
90///
91/// `M` needs only [`Memory`]. The runtime does not require the backing memory
92/// to be `Send`, `Sync`, `Clone`, or `'static`.
93///
94/// The host must supply exclusive backing custody: no other runtime, manager or
95/// raw IO may access the same backing while this runtime or any of its leases
96/// lives. Aliases hidden behind a generic [`Memory`] cannot be detected here.
97/// Drop every lease and the runtime before constructing a replacement on retained
98/// backing. Opens are exclusive within this runtime, including across warm
99/// bootstrap and authority verification.
100///
101
102pub struct MemoryRuntime<M: Memory> {
103    memory_manager: MemoryManager<Rc<M>>,
104    // Shared backing, immutable geometry and live accounting belong to growth.
105    // Handles reserve physical capacity; the manager owns bucket metadata.
106    growth: Rc<backing::GrowthState<M>>,
107    lifecycle: RuntimeLifecycle,
108}
109
110impl<M: Memory> MemoryRuntime<M> {
111    /// Construct an unbootstrapped runtime without overwriting foreign memory.
112    ///
113    /// Empty backing memory is initialized as an
114    /// [`MemoryManager`]. Nonempty memory must pass bounded
115    /// validation of the current manager header, bucket table, and extents; otherwise
116    /// construction returns a typed error before the manager can
117    /// write its header or allocation table. A pre-grown blank memory is
118    /// nonempty and is therefore rejected rather than assumed disposable.
119    ///
120    /// # Errors
121    ///
122    /// Returns [`RuntimeConstructionError::ForeignMemory`] for nonempty memory
123    /// without `MemoryManager` magic, or
124    /// [`RuntimeConstructionError::UnsupportedMemoryManagerVersion`] when the
125    /// magic is recognized but the layout version is not current. Invalid
126    /// metadata returns [`RuntimeConstructionError::Layout`]. Refused fresh
127    /// metadata growth returns [`RuntimeConstructionError::Growth`] without
128    /// writes, allowing the same backing memory to be retried. Reopening honors
129    /// the actual persisted bucket size; only fresh memory uses 128 pages.
130    pub fn new(memory: M) -> Result<Self, RuntimeConstructionError> {
131        Self::construct(memory, None)
132    }
133
134    /// Construct with an explicit immutable bucket policy. Existing memory must
135    /// match exactly; mismatches fail before manager initialization or writes.
136    ///
137    /// # Errors
138    ///
139    /// Returns the construction errors described by [`Self::new`], or
140    /// [`RuntimeConstructionError::BucketSizeMismatch`] when existing geometry
141    /// differs from `config`.
142    pub fn new_with_config(
143        memory: M,
144        config: MemoryManagerConfig,
145    ) -> Result<Self, RuntimeConstructionError> {
146        Self::construct(memory, Some(config))
147    }
148
149    fn construct(
150        memory: M,
151        requested: Option<MemoryManagerConfig>,
152    ) -> Result<Self, RuntimeConstructionError> {
153        if cfg!(target_endian = "big") {
154            return Err(MemoryManagerLayoutError::UnsupportedByteOrder.into());
155        }
156        let (bucket_size_pages, allocated_buckets) = if memory.size() == 0 {
157            // Reserve the metadata page before the dependency writes its header.
158            // This fresh allocation is owned here; caller-supplied nonempty
159            // memory still passes the layout checks below before any writes.
160            if memory.grow(1) == -1 {
161                return Err(RuntimeGrowError::BackingRefused {
162                    additional_pages: 1,
163                }
164                .into());
165            }
166            (requested.unwrap_or_default().bucket_size_pages(), 0)
167        } else {
168            let measured = layout::read(&memory)?;
169            let actual = measured.bucket_pages;
170            if let Some(config) = requested {
171                check_bucket_size(actual, config)?;
172            }
173            (actual, measured.allocated_buckets)
174        };
175        let backing = Rc::new(memory);
176        let growth = Rc::new(backing::GrowthState {
177            backing: Rc::clone(&backing),
178            bucket_size_pages,
179            allocated_buckets: std::cell::RefCell::new(allocated_buckets),
180            open_slots: [const { std::cell::Cell::new(0) }; 4],
181        });
182        Ok(Self {
183            memory_manager: MemoryManager::init_with_bucket_size(
184                Rc::clone(&backing),
185                bucket_size_pages,
186            ),
187            growth,
188            lifecycle: RuntimeLifecycle::Unbootstrapped,
189        })
190    }
191
192    /// Return the immutable bucket configuration bound to this runtime's manager.
193    #[must_use]
194    pub fn memory_manager_config(&self) -> MemoryManagerConfig {
195        // Construction has already validated the nonzero persisted setting.
196        MemoryManagerConfig::from_validated(self.growth.bucket_size_pages)
197    }
198
199    /// Return whether this runtime has published committed allocation authority.
200    #[must_use]
201    pub const fn is_bootstrapped(&self) -> bool {
202        matches!(self.lifecycle, RuntimeLifecycle::Bootstrapped { .. })
203    }
204
205    /// Bootstrap this backing memory from one immutable declaration snapshot.
206    ///
207    /// Recovery, metadata admission, logical resolution, policy evaluation,
208    /// staging, persistence and binding publication are local to this runtime.
209    /// A repeated call is idempotent only when the sealed declaration snapshot,
210    /// canonical host pool and [`RuntimeBootstrapPolicy::runtime_bootstrap_identity`] match the
211    /// successful bootstrap. A mismatch returns a typed error without
212    /// advancing the durable generation or re-evaluating policy.
213    /// Independently sealed snapshots match when their canonical contents are equal.
214    ///
215    /// # Panics
216    ///
217    /// Panics if a private runtime or encoding invariant is broken, or backing
218    /// memory or a policy callback panics.
219    pub fn bootstrap<P: RuntimeBootstrapPolicy>(
220        &mut self,
221        declarations: &SealedDeclarationSnapshot,
222        pool: &crate::MemoryAllocationPool,
223        policy: &P,
224    ) -> Result<u64, RuntimeBootstrapError<P::Error>> {
225        match &self.lifecycle {
226            RuntimeLifecycle::Unbootstrapped => self.bootstrap_unbootstrapped(
227                declarations,
228                pool,
229                policy,
230                policy.runtime_bootstrap_identity(),
231            ),
232            RuntimeLifecycle::Bootstrapped { .. } => {
233                self.verify_bootstrap(declarations, pool, policy)
234            }
235        }
236    }
237
238    /// Verify the exact host bootstrap binding through a shared runtime reference.
239    ///
240    /// Returns the established generation only when declarations, pool and the
241    /// policy identity match. Evaluates the policy identity once, but never runs
242    /// admission or allocation validation, reads backing memory, writes, grows,
243    /// or opens an allocation. An unbootstrapped runtime is refused rather than
244    /// initialized. Hosts may share one committed runtime through `Rc` without
245    /// reconstructing its binding checks or attempting mutable bootstrap again.
246    pub fn verify_bootstrap<P: RuntimeBootstrapPolicy>(
247        &self,
248        declarations: &SealedDeclarationSnapshot,
249        pool: &crate::MemoryAllocationPool,
250        policy: &P,
251    ) -> Result<u64, RuntimeBootstrapError<P::Error>> {
252        let RuntimeLifecycle::Bootstrapped {
253            binding,
254            generation,
255            ..
256        } = &self.lifecycle
257        else {
258            return Err(RuntimeBootstrapError::NotBootstrapped);
259        };
260        binding.validate(declarations, pool, &policy.runtime_bootstrap_identity())?;
261        Ok(*generation)
262    }
263
264    fn validate_pool_custody(
265        &self,
266        occupied: &[bool; crate::constants::MAX_ALLOCATIONS],
267        pool: &crate::MemoryAllocationPool,
268    ) -> Result<(), super::MemoryResolutionError> {
269        for id in 0..crate::MEMORY_MANAGER_INVALID_ID {
270            if pool.contains(id) && self.memory_size_pages(id) != 0 && !occupied[usize::from(id)] {
271                return Err(super::MemoryResolutionError::UnmanagedAllocation { id });
272            }
273        }
274        Ok(())
275    }
276
277    fn bootstrap_unbootstrapped<P: RuntimeBootstrapPolicy>(
278        &mut self,
279        declarations: &SealedDeclarationSnapshot,
280        pool: &crate::MemoryAllocationPool,
281        policy: &P,
282        policy_identity: PolicyIdentity,
283    ) -> Result<u64, RuntimeBootstrapError<P::Error>> {
284        let mut memory = self
285            .memory(MEMORY_MANAGER_LEDGER_ID)
286            .expect("governance memory is not publicly leasable");
287        let mut record = decode_stable_cell_commit_store_from_memory(&memory)?;
288        let recovered = record.recover_or_genesis()?;
289        let occupied = recovered.ledger().occupied_slots();
290        self.validate_pool_custody(&occupied, pool)?;
291        if memory.size() == 0 {
292            // Empty memory decodes to an uninitialized record. Preserve fresh
293            // cell acquisition before admission without persisting genesis.
294            memory = write_ledger_cell(memory, &LedgerCommitStore::default())?;
295        }
296        let mut admission = BootstrapAdmission::new(recovered.ledger(), declarations, pool);
297        let preparation = policy.prepare_bootstrap(&mut admission);
298        let historical = admission.complete()?;
299        preparation.map_err(RuntimeBootstrapError::AdmissionPolicy)?;
300        let resolved = declarations.resolve(&recovered, &occupied, historical, pool)?;
301        let (generation, resolved) = {
302            let (staged, resolved) = crate::ledger::stage_generation(recovered, resolved, policy)?;
303            record.commit_generation(&staged)?;
304            (staged.current_generation(), resolved)
305        };
306        drop(write_ledger_cell(memory, &record)?);
307        // Publish application bindings only after the durable Cell write succeeds.
308        let mut allocations = resolved;
309        allocations.retain(|declaration| {
310            !crate::is_ic_memory_stable_key(declaration.stable_key().as_str())
311        });
312        self.lifecycle = RuntimeLifecycle::Bootstrapped {
313            allocations,
314            generation,
315            binding: RuntimeBootstrapBinding {
316                source: declarations.clone(),
317                policy_identity,
318                pool: pool.clone(),
319            },
320        };
321        Ok(generation)
322    }
323
324    /// Borrow this runtime's persisted application bindings without IO.
325    /// These passive views cannot open memory independently of the runtime.
326    pub fn committed_allocations(
327        &self,
328    ) -> Result<&[crate::AllocationDeclaration], RuntimeOpenError> {
329        match &self.lifecycle {
330            RuntimeLifecycle::Unbootstrapped => Err(RuntimeOpenError::NotBootstrapped),
331            RuntimeLifecycle::Bootstrapped { allocations, .. } => Ok(allocations),
332        }
333    }
334
335    /// Return the in-memory persisted generation without decoding the ledger.
336    pub const fn committed_generation(&self) -> Result<u64, RuntimeOpenError> {
337        match &self.lifecycle {
338            RuntimeLifecycle::Unbootstrapped => Err(RuntimeOpenError::NotBootstrapped),
339            RuntimeLifecycle::Bootstrapped { generation, .. } => Ok(*generation),
340        }
341    }
342
343    /// Borrow the host pool bound by successful bootstrap for host projections.
344    /// This is current policy, never evidence of historical ownership or free IDs.
345    pub const fn allocation_pool(&self) -> Result<&crate::MemoryAllocationPool, RuntimeOpenError> {
346        match &self.lifecycle {
347            RuntimeLifecycle::Unbootstrapped => Err(RuntimeOpenError::NotBootstrapped),
348            RuntimeLifecycle::Bootstrapped { binding, .. } => Ok(&binding.pool),
349        }
350    }
351
352    /// Lease by durable key using this runtime's persisted current bindings.
353    ///
354    /// A second live open returns [`RuntimeOpenError::AlreadyOpen`] without IO.
355    /// Dropping the handle, or a collection that owns it, permits reopening.
356    pub fn open_memory(&self, stable_key: &str) -> Result<RuntimeMemory<M>, RuntimeOpenError> {
357        self.memory(self.memory_id(stable_key)?)
358    }
359
360    /// Resolve an application key's committed ID without opening memory,
361    /// reading history, or changing the host's policy or bucket configuration.
362    pub fn memory_id(&self, stable_key: &str) -> Result<u8, RuntimeOpenError> {
363        crate::key::validate(stable_key)?;
364        if crate::is_ic_memory_stable_key(stable_key) {
365            return Err(RuntimeOpenError::ReservedStableKey {
366                stable_key: stable_key.to_string(),
367            });
368        }
369        let declaration =
370            crate::declaration::declaration_for_key(self.committed_allocations()?, stable_key)
371                .ok_or_else(|| RuntimeOpenError::StableKeyNotCommitted(stable_key.to_string()))?;
372        Ok(declaration.slot().id())
373    }
374
375    fn memory(&self, id: u8) -> Result<RuntimeMemory<M>, RuntimeOpenError> {
376        let word = &self.growth.open_slots[usize::from(id / 64)];
377        let mask = 1_u64 << (id % 64);
378        if word.get() & mask != 0 {
379            return Err(RuntimeOpenError::AlreadyOpen { id });
380        }
381        let memory = self.memory_manager.get(MemoryId::new(id));
382        word.set(word.get() | mask);
383        Ok(RuntimeMemory {
384            memory,
385            growth: Rc::clone(&self.growth),
386            id,
387        })
388    }
389
390    // Size-only observations need no handle carrying runtime growth authority.
391    fn memory_size_pages(&self, id: u8) -> u64 {
392        self.memory_manager.get(MemoryId::new(id)).size()
393    }
394
395    fn ledger_record_from_memory(&self) -> Result<LedgerCommitStore, StableCellLedgerError> {
396        // Decoding only reads; the manager handle retains backing lifetime
397        // without carrying the growth owner needed by writable runtime handles.
398        let memory = self
399            .memory_manager
400            .get(MemoryId::new(MEMORY_MANAGER_LEDGER_ID));
401        decode_stable_cell_commit_store_from_memory(&memory)
402    }
403}
404
405impl RuntimeBootstrapBinding {
406    fn validate<P>(
407        &self,
408        declarations: &SealedDeclarationSnapshot,
409        pool: &crate::MemoryAllocationPool,
410        policy_identity: &PolicyIdentity,
411    ) -> Result<(), RuntimeBootstrapError<P>> {
412        if &self.source != declarations {
413            return Err(RuntimeBootstrapError::DeclarationSnapshotMismatch);
414        }
415        if &self.pool != pool {
416            return Err(RuntimeBootstrapError::AllocationPoolMismatch);
417        }
418        if &self.policy_identity != policy_identity {
419            return Err(RuntimeBootstrapError::PolicyIdentityMismatch {
420                established: self.policy_identity,
421                requested: *policy_identity,
422            });
423        }
424        Ok(())
425    }
426}
427
428fn write_ledger_cell<M: Memory, P>(
429    memory: RuntimeMemory<M>,
430    record: &LedgerCommitStore,
431) -> Result<RuntimeMemory<M>, RuntimeBootstrapError<P>> {
432    let encoded = EncodedLedgerRecord::new(record).map_err(|value_size| {
433        RuntimeBootstrapError::StableCellLedgerWriteTooLarge { value_size }
434    })?;
435    let value_size = encoded.len();
436    // The record limit is below u32::MAX, including the eight-byte cell header.
437    let required_bytes = STABLE_CELL_VALUE_OFFSET + value_size as u64;
438    let available_bytes = memory.size().saturating_mul(crate::WASM_PAGE_SIZE_BYTES);
439    if required_bytes > available_bytes {
440        let grow_by = (required_bytes - available_bytes).div_ceil(crate::WASM_PAGE_SIZE_BYTES);
441        memory.grow(grow_by)?;
442    }
443    Ok(Cell::new(memory, encoded).into_memory())
444}
445
446const fn check_bucket_size(
447    actual: u16,
448    requested: MemoryManagerConfig,
449) -> Result<(), RuntimeConstructionError> {
450    if actual != requested.bucket_size_pages() {
451        return Err(RuntimeConstructionError::BucketSizeMismatch {
452            persisted: actual,
453            requested: requested.bucket_size_pages(),
454        });
455    }
456    Ok(())
457}
ic_memory/runtime/mod.rs

ic_memory/runtime/
mod.rs
