ic_memory/runtime/
mod.rs

1mod admission;
2#[cfg(test)]
3mod admission_tests;
4mod adoption;
5#[cfg(test)]
6mod adoption_tests;
7mod allocations;
8mod backing;
9mod config;
10mod diagnostics;
11mod error;
12mod layout;
13mod policy;
14
15#[cfg(test)]
16mod allocation_tests;
17#[cfg(test)]
18mod growth_tests;
19#[cfg(test)]
20#[expect(
21    unsafe_code,
22    reason = "exercise raw reads with valid uninitialized destinations"
23)]
24mod read_tests;
25#[cfg(test)]
26mod request_tests;
27#[cfg(test)]
28mod tests;
29
30pub use admission::{BootstrapAdmission, BootstrapAdmissionError};
31pub use adoption::RuntimeAdoptionError;
32
33pub use allocations::{
34    MemoryAllocation, MemoryAllocationSummary, MemoryAllocations, MemoryBindingSummary,
35};
36pub use backing::RuntimeMemory;
37pub use config::MemoryManagerConfig;
38pub use error::{
39    MemoryResolutionError, RuntimeBootstrapError, RuntimeConstructionError, RuntimeDiagnosticError,
40    RuntimeGrowError, RuntimeOpenError,
41};
42pub use layout::MemoryManagerLayoutError;
43pub use policy::GenericAllocationPolicy;
44
45use crate::{
46    LedgerCommitStore, PolicyIdentity, RuntimeBootstrapPolicy, STABLE_CELL_VALUE_OFFSET,
47    StableCellLedgerError,
48    requests::SealedDeclarationSnapshot,
49    slot::MEMORY_MANAGER_LEDGER_ID,
50    stable_cell::{EncodedLedgerRecord, decode_stable_cell_commit_store_from_memory},
51};
52use ic_stable_structures::{
53    Cell, Memory,
54    memory_manager::{MemoryId, MemoryManager},
55};
56
57use std::rc::Rc;
58
59enum RuntimeLifecycle {
60    Unbootstrapped,
61    Bootstrapped {
62        allocations: Vec<crate::AllocationDeclaration>,
63        generation: u64,
64        binding: RuntimeBootstrapBinding,
65    },
66}
67
68struct RuntimeBootstrapBinding {
69    source: SealedDeclarationSnapshot,
70    policy_identity: PolicyIdentity,
71    pool: crate::MemoryAllocationPool,
72}
73
74///
75/// MemoryRuntime
76///
77/// Canonical owner of allocation bootstrap state for one backing memory.
78///
79/// The runtime owns its `MemoryManager`, allocation-ledger persistence, bootstrap
80/// lifecycle, committed allocation capability, opens, and diagnostics. Static
81/// linked-program declarations are supplied separately as one immutable
82/// [`SealedDeclarationSnapshot`].
83///
84/// Each bootstrap attempt fallibly decodes its ledger record from persisted
85/// memory. Capacity-checked writes use `Cell`; diagnostics also read persisted
86/// memory directly.
87///
88/// `M` needs only [`Memory`]. The runtime does not require the backing memory
89/// to be `Send`, `Sync`, `Clone`, or `'static`.
90///
91/// The host must supply exclusive backing custody: no other runtime, manager or
92/// raw IO may access the same backing while this runtime or any of its leases
93/// lives. Aliases hidden behind a generic [`Memory`] cannot be detected here.
94/// Drop every lease and the runtime before constructing a replacement on retained
95/// backing. Opens are exclusive within this runtime, including across warm
96/// bootstrap and authority verification.
97///
98
99pub struct MemoryRuntime<M: Memory> {
100    memory_manager: MemoryManager<Rc<M>>,
101    // Shared backing, immutable geometry and live accounting belong to growth.
102    // Handles reserve physical capacity; the manager owns bucket metadata.
103    growth: Rc<backing::GrowthState<M>>,
104    lifecycle: RuntimeLifecycle,
105}
106
107impl<M: Memory> MemoryRuntime<M> {
108    /// Construct an unbootstrapped runtime without overwriting foreign memory.
109    ///
110    /// Empty backing memory is initialized as an
111    /// `ic_stable_structures::MemoryManager`. Nonempty memory must pass bounded
112    /// validation of the current manager header, bucket table, and extents; otherwise
113    /// construction returns a typed error before the manager can
114    /// write its header or allocation table. A pre-grown blank memory is
115    /// nonempty and is therefore rejected rather than assumed disposable.
116    ///
117    /// # Errors
118    ///
119    /// Returns [`RuntimeConstructionError::ForeignMemory`] for nonempty memory
120    /// without `MemoryManager` magic, or
121    /// [`RuntimeConstructionError::UnsupportedMemoryManagerVersion`] when the
122    /// magic is recognized but the layout version is not current. Invalid
123    /// metadata returns [`RuntimeConstructionError::Layout`]. Refused fresh
124    /// metadata growth returns [`RuntimeConstructionError::Growth`] without
125    /// writes, allowing the same backing memory to be retried. Reopening honors
126    /// the actual persisted bucket size; only fresh memory uses 128 pages.
127    pub fn new(memory: M) -> Result<Self, RuntimeConstructionError> {
128        Self::construct(memory, None)
129    }
130
131    /// Construct with an explicit immutable bucket policy. Existing memory must
132    /// match exactly; mismatches fail before manager initialization or writes.
133    ///
134    /// # Errors
135    ///
136    /// Returns the construction errors described by [`Self::new`], or
137    /// [`RuntimeConstructionError::BucketSizeMismatch`] when existing geometry
138    /// differs from `config`.
139    pub fn new_with_config(
140        memory: M,
141        config: MemoryManagerConfig,
142    ) -> Result<Self, RuntimeConstructionError> {
143        Self::construct(memory, Some(config))
144    }
145
146    fn construct(
147        memory: M,
148        requested: Option<MemoryManagerConfig>,
149    ) -> Result<Self, RuntimeConstructionError> {
150        if cfg!(target_endian = "big") {
151            return Err(MemoryManagerLayoutError::UnsupportedByteOrder.into());
152        }
153        let (bucket_size_pages, allocated_buckets) = if memory.size() == 0 {
154            // Reserve the metadata page before the dependency writes its header.
155            // This fresh allocation is owned here; caller-supplied nonempty
156            // memory still passes the layout checks below before any writes.
157            if memory.grow(1) == -1 {
158                return Err(RuntimeGrowError::BackingRefused {
159                    additional_pages: 1,
160                }
161                .into());
162            }
163            (requested.unwrap_or_default().bucket_size_pages(), 0)
164        } else {
165            let measured = layout::read(&memory)?;
166            let actual = measured.bucket_pages;
167            if let Some(config) = requested {
168                check_bucket_size(actual, config)?;
169            }
170            (actual, measured.allocated_buckets)
171        };
172        let backing = Rc::new(memory);
173        let growth = Rc::new(backing::GrowthState {
174            backing: Rc::clone(&backing),
175            bucket_size_pages,
176            allocated_buckets: std::cell::RefCell::new(allocated_buckets),
177            open_slots: [const { std::cell::Cell::new(0) }; 4],
178        });
179        Ok(Self {
180            memory_manager: MemoryManager::init_with_bucket_size(
181                Rc::clone(&backing),
182                bucket_size_pages,
183            ),
184            growth,
185            lifecycle: RuntimeLifecycle::Unbootstrapped,
186        })
187    }
188
189    /// Return the immutable bucket configuration bound to this runtime's manager.
190    #[must_use]
191    pub fn memory_manager_config(&self) -> MemoryManagerConfig {
192        // Construction has already validated the nonzero persisted setting.
193        MemoryManagerConfig::from_validated(self.growth.bucket_size_pages)
194    }
195
196    /// Return whether this runtime has published committed allocation authority.
197    #[must_use]
198    pub const fn is_bootstrapped(&self) -> bool {
199        matches!(self.lifecycle, RuntimeLifecycle::Bootstrapped { .. })
200    }
201
202    /// Bootstrap this backing memory from one immutable declaration snapshot.
203    ///
204    /// Recovery, metadata admission, logical resolution, policy evaluation,
205    /// staging, persistence and binding publication are local to this runtime.
206    /// A repeated call is idempotent only when the sealed declaration snapshot,
207    /// canonical host pool and [`RuntimeBootstrapPolicy::runtime_bootstrap_identity`] match the
208    /// successful bootstrap. A mismatch returns a typed error without
209    /// advancing the durable generation or re-evaluating policy.
210    /// Independently sealed snapshots match when their canonical contents are equal.
211    ///
212    /// # Panics
213    ///
214    /// Panics if a private runtime or encoding invariant is broken, or backing
215    /// memory or a policy callback panics.
216    pub fn bootstrap<P: RuntimeBootstrapPolicy>(
217        &mut self,
218        declarations: &SealedDeclarationSnapshot,
219        pool: &crate::MemoryAllocationPool,
220        policy: &P,
221    ) -> Result<u64, RuntimeBootstrapError<P::Error>> {
222        let policy_identity = policy.runtime_bootstrap_identity();
223        match &self.lifecycle {
224            RuntimeLifecycle::Unbootstrapped => {
225                self.bootstrap_unbootstrapped(declarations, pool, policy, policy_identity)?;
226            }
227            RuntimeLifecycle::Bootstrapped { binding, .. } => {
228                binding.validate(declarations, pool, &policy_identity)?;
229            }
230        }
231        match &self.lifecycle {
232            RuntimeLifecycle::Bootstrapped { generation, .. } => Ok(*generation),
233            RuntimeLifecycle::Unbootstrapped => {
234                unreachable!("successful bootstrap publishes committed allocations")
235            }
236        }
237    }
238
239    fn validate_pool_custody(
240        &self,
241        occupied: &[bool; crate::constants::MAX_ALLOCATIONS],
242        pool: &crate::MemoryAllocationPool,
243    ) -> Result<(), super::MemoryResolutionError> {
244        for id in 0..crate::MEMORY_MANAGER_INVALID_ID {
245            if pool.contains(id) && self.memory_size_pages(id) != 0 && !occupied[usize::from(id)] {
246                return Err(super::MemoryResolutionError::UnmanagedAllocation { id });
247            }
248        }
249        Ok(())
250    }
251
252    fn bootstrap_unbootstrapped<P: RuntimeBootstrapPolicy>(
253        &mut self,
254        declarations: &SealedDeclarationSnapshot,
255        pool: &crate::MemoryAllocationPool,
256        policy: &P,
257        policy_identity: PolicyIdentity,
258    ) -> Result<(), RuntimeBootstrapError<P::Error>> {
259        let mut memory = self
260            .memory(MEMORY_MANAGER_LEDGER_ID)
261            .expect("governance memory is not publicly leasable");
262        let mut record = decode_stable_cell_commit_store_from_memory(&memory)?;
263        let recovered = record.recover_or_genesis()?;
264        let occupied = recovered.ledger().occupied_slots();
265        self.validate_pool_custody(&occupied, pool)?;
266        if memory.size() == 0 {
267            // Empty memory decodes to an uninitialized record. Preserve fresh
268            // cell acquisition before admission without persisting genesis.
269            memory = write_ledger_cell(memory, &LedgerCommitStore::default())?;
270        }
271        let mut admission = BootstrapAdmission::new(recovered.ledger(), declarations, pool);
272        let preparation = policy.prepare_bootstrap(&mut admission);
273        let historical = admission.complete()?;
274        preparation.map_err(RuntimeBootstrapError::AdmissionPolicy)?;
275        let resolved = declarations.resolve(&recovered, &occupied, historical, pool)?;
276        let (generation, resolved) = {
277            let (staged, resolved) = crate::ledger::stage_generation(recovered, resolved, policy)?;
278            record.commit_generation(&staged)?;
279            (staged.current_generation(), resolved)
280        };
281        drop(write_ledger_cell(memory, &record)?);
282        // Publish application bindings only after the durable Cell write succeeds.
283        let mut allocations = resolved;
284        allocations.retain(|declaration| {
285            !crate::is_ic_memory_stable_key(declaration.stable_key().as_str())
286        });
287        self.lifecycle = RuntimeLifecycle::Bootstrapped {
288            allocations,
289            generation,
290            binding: RuntimeBootstrapBinding {
291                source: declarations.clone(),
292                policy_identity,
293                pool: pool.clone(),
294            },
295        };
296        Ok(())
297    }
298
299    /// Borrow this runtime's persisted application bindings without IO.
300    /// These passive views cannot open memory independently of the runtime.
301    pub fn committed_allocations(
302        &self,
303    ) -> Result<&[crate::AllocationDeclaration], RuntimeOpenError> {
304        match &self.lifecycle {
305            RuntimeLifecycle::Unbootstrapped => Err(RuntimeOpenError::NotBootstrapped),
306            RuntimeLifecycle::Bootstrapped { allocations, .. } => Ok(allocations),
307        }
308    }
309
310    /// Return the in-memory persisted generation without decoding the ledger.
311    pub const fn committed_generation(&self) -> Result<u64, RuntimeOpenError> {
312        match &self.lifecycle {
313            RuntimeLifecycle::Unbootstrapped => Err(RuntimeOpenError::NotBootstrapped),
314            RuntimeLifecycle::Bootstrapped { generation, .. } => Ok(*generation),
315        }
316    }
317
318    /// Borrow the host pool bound by successful bootstrap for host projections.
319    /// This is current policy, never evidence of historical ownership or free IDs.
320    pub const fn allocation_pool(&self) -> Result<&crate::MemoryAllocationPool, RuntimeOpenError> {
321        match &self.lifecycle {
322            RuntimeLifecycle::Unbootstrapped => Err(RuntimeOpenError::NotBootstrapped),
323            RuntimeLifecycle::Bootstrapped { binding, .. } => Ok(&binding.pool),
324        }
325    }
326
327    /// Lease by durable key using this runtime's persisted current bindings.
328    ///
329    /// A second live open returns [`RuntimeOpenError::AlreadyOpen`] without IO.
330    /// Dropping the handle, or a collection that owns it, permits reopening.
331    pub fn open_memory(&self, stable_key: &str) -> Result<RuntimeMemory<M>, RuntimeOpenError> {
332        self.memory(self.memory_id(stable_key)?)
333    }
334
335    /// Resolve an application key's committed ID without opening memory,
336    /// reading history, or changing the host's policy or bucket configuration.
337    pub fn memory_id(&self, stable_key: &str) -> Result<u8, RuntimeOpenError> {
338        crate::key::validate(stable_key)?;
339        if crate::is_ic_memory_stable_key(stable_key) {
340            return Err(RuntimeOpenError::ReservedStableKey {
341                stable_key: stable_key.to_string(),
342            });
343        }
344        let declaration =
345            crate::declaration::declaration_for_key(self.committed_allocations()?, stable_key)
346                .ok_or_else(|| RuntimeOpenError::StableKeyNotCommitted(stable_key.to_string()))?;
347        Ok(declaration.slot().id())
348    }
349
350    fn memory(&self, id: u8) -> Result<RuntimeMemory<M>, RuntimeOpenError> {
351        let word = &self.growth.open_slots[usize::from(id / 64)];
352        let mask = 1_u64 << (id % 64);
353        if word.get() & mask != 0 {
354            return Err(RuntimeOpenError::AlreadyOpen { id });
355        }
356        let memory = self.memory_manager.get(MemoryId::new(id));
357        word.set(word.get() | mask);
358        Ok(RuntimeMemory {
359            memory,
360            growth: Rc::clone(&self.growth),
361            id,
362        })
363    }
364
365    // Size-only observations need no handle carrying runtime growth authority.
366    fn memory_size_pages(&self, id: u8) -> u64 {
367        self.memory_manager.get(MemoryId::new(id)).size()
368    }
369
370    fn ledger_record_from_memory(&self) -> Result<LedgerCommitStore, StableCellLedgerError> {
371        // Decoding only reads; the manager handle retains backing lifetime
372        // without carrying the growth owner needed by writable runtime handles.
373        let memory = self
374            .memory_manager
375            .get(MemoryId::new(MEMORY_MANAGER_LEDGER_ID));
376        decode_stable_cell_commit_store_from_memory(&memory)
377    }
378}
379
380impl RuntimeBootstrapBinding {
381    fn validate<P>(
382        &self,
383        declarations: &SealedDeclarationSnapshot,
384        pool: &crate::MemoryAllocationPool,
385        policy_identity: &PolicyIdentity,
386    ) -> Result<(), RuntimeBootstrapError<P>> {
387        if &self.source != declarations {
388            return Err(RuntimeBootstrapError::DeclarationSnapshotMismatch);
389        }
390        if &self.pool != pool {
391            return Err(RuntimeBootstrapError::AllocationPoolMismatch);
392        }
393        if &self.policy_identity != policy_identity {
394            return Err(RuntimeBootstrapError::PolicyIdentityMismatch {
395                established: self.policy_identity,
396                requested: *policy_identity,
397            });
398        }
399        Ok(())
400    }
401}
402
403fn write_ledger_cell<M: Memory, P>(
404    memory: RuntimeMemory<M>,
405    record: &LedgerCommitStore,
406) -> Result<RuntimeMemory<M>, RuntimeBootstrapError<P>> {
407    let encoded = EncodedLedgerRecord::new(record).map_err(|value_size| {
408        RuntimeBootstrapError::StableCellLedgerWriteTooLarge { value_size }
409    })?;
410    let value_size = encoded.len();
411    // The record limit is below u32::MAX, including the eight-byte cell header.
412    let required_bytes = STABLE_CELL_VALUE_OFFSET + value_size as u64;
413    let available_bytes = memory.size().saturating_mul(crate::WASM_PAGE_SIZE_BYTES);
414    if required_bytes > available_bytes {
415        let grow_by = (required_bytes - available_bytes).div_ceil(crate::WASM_PAGE_SIZE_BYTES);
416        memory.grow(grow_by)?;
417    }
418    Ok(Cell::new(memory, encoded).into_memory())
419}
420
421const fn check_bucket_size(
422    actual: u16,
423    requested: MemoryManagerConfig,
424) -> Result<(), RuntimeConstructionError> {
425    if actual != requested.bucket_size_pages() {
426        return Err(RuntimeConstructionError::BucketSizeMismatch {
427            persisted: actual,
428            requested: requested.bucket_size_pages(),
429        });
430    }
431    Ok(())
432}
ic_memory/runtime/mod.rs

ic_memory/runtime/
mod.rs
