ic_memory/runtime/
mod.rs

1mod admission;
2#[cfg(test)]
3mod admission_tests;
4mod adoption;
5#[cfg(test)]
6mod adoption_tests;
7mod allocations;
8mod backing;
9pub mod bootstrap;
10mod config;
11mod diagnostics;
12mod error;
13mod layout;
14mod policy;
15
16#[cfg(test)]
17mod allocation_tests;
18#[cfg(test)]
19mod growth_tests;
20#[cfg(test)]
21#[expect(
22    unsafe_code,
23    reason = "exercise raw reads with valid uninitialized destinations"
24)]
25mod read_tests;
26#[cfg(test)]
27mod request_tests;
28#[cfg(test)]
29mod tests;
30
31pub use admission::{BootstrapAdmission, BootstrapAdmissionError};
32pub use adoption::RuntimeAdoptionError;
33
34pub use allocations::{
35    AllocationBinding, MemoryAllocation, MemoryAllocationSummary, MemoryAllocations,
36    MemoryBindingSummary,
37};
38pub use backing::RuntimeMemory;
39pub use config::MemoryManagerConfig;
40pub use error::{
41    MemoryResolutionError, RuntimeBootstrapError, RuntimeConstructionError, RuntimeDiagnosticError,
42    RuntimeGrowError, RuntimeOpenError,
43};
44pub use layout::MemoryManagerLayoutError;
45pub use policy::GenericAllocationPolicy;
46
47use crate::{
48    CommittedAllocations, LedgerCommitStore, PolicyIdentity, RuntimeBootstrapPolicy,
49    STABLE_CELL_VALUE_OFFSET, StableCellLedgerError,
50    requests::SealedDeclarationSnapshot,
51    slot::MEMORY_MANAGER_LEDGER_ID,
52    stable_cell::{EncodedLedgerRecord, decode_stable_cell_commit_store_from_memory},
53};
54use ic_stable_structures::{
55    Cell, Memory,
56    memory_manager::{MemoryId, MemoryManager},
57};
58
59use std::rc::Rc;
60
61enum RuntimeLifecycle {
62    Unbootstrapped,
63    Bootstrapped {
64        committed_allocations: CommittedAllocations,
65        binding: RuntimeBootstrapBinding,
66    },
67}
68
69struct RuntimeBootstrapBinding {
70    source: SealedDeclarationSnapshot,
71    policy_identity: PolicyIdentity,
72    pool: crate::MemoryAllocationPool,
73}
74
75///
76/// MemoryRuntime
77///
78/// Canonical owner of allocation bootstrap state for one backing memory.
79///
80/// The runtime owns its `MemoryManager`, allocation-ledger persistence, bootstrap
81/// lifecycle, committed allocation capability, opens, and diagnostics. Static
82/// linked-program declarations are supplied separately as one immutable
83/// [`SealedDeclarationSnapshot`].
84///
85/// Each bootstrap attempt fallibly decodes its ledger record from persisted
86/// memory. Capacity-checked writes use `Cell`; diagnostics also read persisted
87/// memory directly.
88///
89/// `M` needs only [`Memory`]. The runtime does not require the backing memory
90/// to be `Send`, `Sync`, `Clone`, or `'static`.
91///
92
93pub struct MemoryRuntime<M: Memory> {
94    memory_manager: MemoryManager<Rc<M>>,
95    // Shared backing, immutable geometry and live accounting belong to growth.
96    // Handles reserve physical capacity; the manager owns bucket metadata.
97    growth: Rc<backing::GrowthState<M>>,
98    lifecycle: RuntimeLifecycle,
99}
100
101impl<M: Memory> MemoryRuntime<M> {
102    /// Construct an unbootstrapped runtime without overwriting foreign memory.
103    ///
104    /// Empty backing memory is initialized as an
105    /// `ic_stable_structures::MemoryManager`. Nonempty memory must pass bounded
106    /// validation of the current manager header, bucket table, and extents; otherwise
107    /// construction returns a typed error before the manager can
108    /// write its header or allocation table. A pre-grown blank memory is
109    /// nonempty and is therefore rejected rather than assumed disposable.
110    ///
111    /// # Errors
112    ///
113    /// Returns [`RuntimeConstructionError::ForeignMemory`] for nonempty memory
114    /// without `MemoryManager` magic, or
115    /// [`RuntimeConstructionError::UnsupportedMemoryManagerVersion`] when the
116    /// magic is recognized but the layout version is not current. Invalid
117    /// metadata returns [`RuntimeConstructionError::Layout`]. Refused fresh
118    /// metadata growth returns [`RuntimeConstructionError::Growth`] without
119    /// writes, allowing the same backing memory to be retried. Reopening honors
120    /// the actual persisted bucket size; only fresh memory uses 128 pages.
121    pub fn new(memory: M) -> Result<Self, RuntimeConstructionError> {
122        Self::construct(memory, None)
123    }
124
125    /// Construct with an explicit immutable bucket policy. Existing memory must
126    /// match exactly; mismatches fail before manager initialization or writes.
127    ///
128    /// # Errors
129    ///
130    /// Returns the construction errors described by [`Self::new`], or
131    /// [`RuntimeConstructionError::BucketSizeMismatch`] when existing geometry
132    /// differs from `config`.
133    pub fn new_with_config(
134        memory: M,
135        config: MemoryManagerConfig,
136    ) -> Result<Self, RuntimeConstructionError> {
137        Self::construct(memory, Some(config))
138    }
139
140    fn construct(
141        memory: M,
142        requested: Option<MemoryManagerConfig>,
143    ) -> Result<Self, RuntimeConstructionError> {
144        if cfg!(target_endian = "big") {
145            return Err(MemoryManagerLayoutError::UnsupportedByteOrder.into());
146        }
147        let (bucket_size_pages, allocated_buckets) = if memory.size() == 0 {
148            // Reserve the metadata page before the dependency writes its header.
149            // This fresh allocation is owned here; caller-supplied nonempty
150            // memory still passes the layout checks below before any writes.
151            if memory.grow(1) == -1 {
152                return Err(RuntimeGrowError::BackingRefused {
153                    additional_pages: 1,
154                }
155                .into());
156            }
157            (requested.unwrap_or_default().bucket_size_pages(), 0)
158        } else {
159            let measured = layout::read(&memory)?;
160            let actual = measured.bucket_pages;
161            if let Some(config) = requested {
162                check_bucket_size(actual, config)?;
163            }
164            (actual, measured.allocated_buckets)
165        };
166        let backing = Rc::new(memory);
167        let growth = Rc::new(backing::GrowthState {
168            backing: Rc::clone(&backing),
169            bucket_size_pages,
170            allocated_buckets: std::cell::RefCell::new(allocated_buckets),
171        });
172        Ok(Self {
173            memory_manager: MemoryManager::init_with_bucket_size(
174                Rc::clone(&backing),
175                bucket_size_pages,
176            ),
177            growth,
178            lifecycle: RuntimeLifecycle::Unbootstrapped,
179        })
180    }
181
182    /// Return the immutable bucket configuration bound to this runtime's manager.
183    #[must_use]
184    pub fn memory_manager_config(&self) -> MemoryManagerConfig {
185        // Construction has already validated the nonzero persisted setting.
186        MemoryManagerConfig::from_validated(self.growth.bucket_size_pages)
187    }
188
189    /// Return whether this runtime has published committed allocation authority.
190    #[must_use]
191    pub const fn is_bootstrapped(&self) -> bool {
192        matches!(self.lifecycle, RuntimeLifecycle::Bootstrapped { .. })
193    }
194
195    /// Bootstrap this backing memory from one immutable declaration snapshot.
196    ///
197    /// Recovery, metadata admission, logical resolution, policy evaluation,
198    /// staging, persistence and capability publication are local to this runtime.
199    /// A repeated call is idempotent only when the sealed declaration snapshot,
200    /// canonical host pool and [`RuntimeBootstrapPolicy::runtime_bootstrap_identity`] match the
201    /// successful bootstrap. A mismatch returns a typed error without
202    /// advancing the durable generation or re-evaluating policy.
203    /// Independently sealed snapshots match when their canonical contents are equal.
204    ///
205    /// # Panics
206    ///
207    /// Panics if a private runtime or encoding invariant is broken, or backing
208    /// memory or a policy callback panics.
209    pub fn bootstrap<P: RuntimeBootstrapPolicy>(
210        &mut self,
211        declarations: &SealedDeclarationSnapshot,
212        pool: &crate::MemoryAllocationPool,
213        policy: &P,
214    ) -> Result<&CommittedAllocations, RuntimeBootstrapError<P::Error>> {
215        let policy_identity = policy.runtime_bootstrap_identity()?;
216        match &self.lifecycle {
217            RuntimeLifecycle::Unbootstrapped => {
218                self.bootstrap_unbootstrapped(declarations, pool, policy, policy_identity)?;
219            }
220            RuntimeLifecycle::Bootstrapped { binding, .. } => {
221                binding.validate(declarations, pool, &policy_identity)?;
222            }
223        }
224        match &self.lifecycle {
225            RuntimeLifecycle::Bootstrapped {
226                committed_allocations,
227                ..
228            } => Ok(committed_allocations),
229            RuntimeLifecycle::Unbootstrapped => {
230                unreachable!("successful bootstrap publishes committed allocations")
231            }
232        }
233    }
234
235    fn validate_pool_custody(
236        &self,
237        occupied: &[bool; crate::constants::MAX_ALLOCATIONS],
238        pool: &crate::MemoryAllocationPool,
239    ) -> Result<(), super::MemoryResolutionError> {
240        for id in 0..crate::MEMORY_MANAGER_INVALID_ID {
241            if pool.contains(id) && self.memory_size_pages(id) != 0 && !occupied[usize::from(id)] {
242                return Err(super::MemoryResolutionError::UnmanagedAllocation { id });
243            }
244        }
245        Ok(())
246    }
247
248    fn bootstrap_unbootstrapped<P: RuntimeBootstrapPolicy>(
249        &mut self,
250        declarations: &SealedDeclarationSnapshot,
251        pool: &crate::MemoryAllocationPool,
252        policy: &P,
253        policy_identity: PolicyIdentity,
254    ) -> Result<(), RuntimeBootstrapError<P::Error>> {
255        let memory = self.memory(MEMORY_MANAGER_LEDGER_ID);
256        let mut record = decode_stable_cell_commit_store_from_memory(&memory)?;
257        let recovered = record.recover_or_genesis()?;
258        let occupied = recovered.ledger().occupied_slots();
259        self.validate_pool_custody(&occupied, pool)?;
260        if memory.size() == 0 {
261            // Empty memory decodes to an uninitialized record. Preserve fresh
262            // cell acquisition before admission without persisting genesis.
263            write_ledger_cell(memory.clone(), &LedgerCommitStore::default())?;
264        }
265        let mut admission = BootstrapAdmission::new(recovered.ledger(), declarations, pool);
266        let preparation = policy.prepare_bootstrap(&mut admission);
267        let historical = admission.complete()?;
268        preparation.map_err(RuntimeBootstrapError::AdmissionPolicy)?;
269        let resolved = declarations.resolve(recovered.ledger(), &occupied, historical, pool)?;
270        let commit =
271            crate::runtime::bootstrap::prepare_commit(&mut record, recovered, resolved, policy)?;
272        write_ledger_cell(memory, &record)?;
273        let committed = commit.confirm_persisted();
274        self.lifecycle = RuntimeLifecycle::Bootstrapped {
275            committed_allocations: committed,
276            binding: RuntimeBootstrapBinding {
277                source: declarations.clone(),
278                policy_identity,
279                pool: pool.clone(),
280            },
281        };
282        Ok(())
283    }
284
285    /// Borrow this runtime's committed allocation-open capability.
286    pub const fn committed_allocations(&self) -> Result<&CommittedAllocations, RuntimeOpenError> {
287        match &self.lifecycle {
288            RuntimeLifecycle::Unbootstrapped => Err(RuntimeOpenError::NotBootstrapped),
289            RuntimeLifecycle::Bootstrapped {
290                committed_allocations,
291                ..
292            } => Ok(committed_allocations),
293        }
294    }
295
296    /// Open by durable key using only this runtime's persisted current capability.
297    pub fn open_memory(&self, stable_key: &str) -> Result<RuntimeMemory<M>, RuntimeOpenError> {
298        Ok(self.memory(self.memory_id(stable_key)?))
299    }
300
301    /// Resolve an application key's committed ID without opening memory,
302    /// reading history, or changing the host's policy or bucket configuration.
303    pub fn memory_id(&self, stable_key: &str) -> Result<u8, RuntimeOpenError> {
304        crate::key::validate(stable_key)?;
305        if crate::is_ic_memory_stable_key(stable_key) {
306            return Err(RuntimeOpenError::ReservedStableKey {
307                stable_key: stable_key.to_string(),
308            });
309        }
310        let declaration = crate::capability::declaration_for_key(
311            self.committed_allocations()?.declarations(),
312            stable_key,
313        )
314        .ok_or_else(|| RuntimeOpenError::StableKeyNotCommitted(stable_key.to_string()))?;
315        Ok(declaration.slot().id())
316    }
317
318    fn memory(&self, id: u8) -> RuntimeMemory<M> {
319        RuntimeMemory {
320            memory: self.memory_manager.get(MemoryId::new(id)),
321            growth: Rc::clone(&self.growth),
322        }
323    }
324
325    // Size-only observations need no handle carrying runtime growth authority.
326    fn memory_size_pages(&self, id: u8) -> u64 {
327        self.memory_manager.get(MemoryId::new(id)).size()
328    }
329
330    fn ledger_record_from_memory(&self) -> Result<LedgerCommitStore, StableCellLedgerError> {
331        // Decoding only reads; the manager handle retains backing lifetime
332        // without carrying the growth owner needed by writable runtime handles.
333        let memory = self
334            .memory_manager
335            .get(MemoryId::new(MEMORY_MANAGER_LEDGER_ID));
336        decode_stable_cell_commit_store_from_memory(&memory)
337    }
338}
339
340impl RuntimeBootstrapBinding {
341    fn validate<P>(
342        &self,
343        declarations: &SealedDeclarationSnapshot,
344        pool: &crate::MemoryAllocationPool,
345        policy_identity: &PolicyIdentity,
346    ) -> Result<(), RuntimeBootstrapError<P>> {
347        if &self.source != declarations {
348            return Err(RuntimeBootstrapError::DeclarationSnapshotMismatch);
349        }
350        if &self.pool != pool {
351            return Err(RuntimeBootstrapError::AllocationPoolMismatch);
352        }
353        if &self.policy_identity != policy_identity {
354            return Err(RuntimeBootstrapError::PolicyIdentityMismatch {
355                established: self.policy_identity.clone(),
356                requested: policy_identity.clone(),
357            });
358        }
359        Ok(())
360    }
361}
362
363fn write_ledger_cell<M: Memory, P>(
364    memory: RuntimeMemory<M>,
365    record: &LedgerCommitStore,
366) -> Result<(), RuntimeBootstrapError<P>> {
367    let encoded = EncodedLedgerRecord::new(record).map_err(|value_size| {
368        RuntimeBootstrapError::StableCellLedgerWriteTooLarge { value_size }
369    })?;
370    let value_size = encoded.len();
371    // The record limit is below u32::MAX, including the eight-byte cell header.
372    let required_bytes = STABLE_CELL_VALUE_OFFSET + value_size as u64;
373    let available_bytes = memory.size().saturating_mul(crate::WASM_PAGE_SIZE_BYTES);
374    if required_bytes > available_bytes {
375        let grow_by = (required_bytes - available_bytes).div_ceil(crate::WASM_PAGE_SIZE_BYTES);
376        memory.grow(grow_by)?;
377    }
378    drop(Cell::new(memory, encoded));
379    Ok(())
380}
381
382const fn check_bucket_size(
383    actual: u16,
384    requested: MemoryManagerConfig,
385) -> Result<(), RuntimeConstructionError> {
386    if actual != requested.bucket_size_pages() {
387        return Err(RuntimeConstructionError::BucketSizeMismatch {
388            persisted: actual,
389            requested: requested.bucket_size_pages(),
390        });
391    }
392    Ok(())
393}
ic_memory/runtime/mod.rs

ic_memory/runtime/
mod.rs
