ic_memory/runtime/
mod.rs

1mod admission;
2#[cfg(test)]
3mod admission_tests;
4mod adoption;
5#[cfg(test)]
6mod adoption_tests;
7mod allocations;
8mod backing;
9mod config;
10mod default;
11mod diagnostics;
12mod error;
13mod layout;
14mod policy;
15
16#[cfg(test)]
17mod allocation_tests;
18#[cfg(test)]
19mod growth_tests;
20#[cfg(test)]
21#[expect(
22    unsafe_code,
23    reason = "exercise raw reads with valid uninitialized destinations"
24)]
25mod read_tests;
26#[cfg(test)]
27mod request_tests;
28#[cfg(test)]
29mod tests;
30
31pub use admission::{BootstrapAdmission, BootstrapAdmissionError, RecoveredAllocationMetadata};
32pub use adoption::RuntimeAdoptionError;
33
34pub use allocations::{
35    AllocationBinding, MemoryAllocation, MemoryAllocationSummary, MemoryAllocations,
36    MemoryBindingSummary,
37};
38pub use backing::RuntimeMemory;
39pub use config::MemoryManagerConfig;
40pub use default::{
41    bootstrap_default_memory_manager, bootstrap_default_memory_manager_with_config,
42    bootstrap_default_memory_manager_with_policy, committed_allocations,
43    default_memory_manager_commit_recovery_diagnostic, default_memory_manager_diagnostic_export,
44    default_memory_manager_doctor_report, default_memory_manager_doctor_report_with_policy,
45    default_memory_manager_memory_allocation_summary, default_memory_manager_memory_allocations,
46    default_memory_manager_memory_id, is_default_memory_manager_bootstrapped,
47    open_default_memory_manager_memory, verify_default_memory_manager_authority,
48};
49pub use error::{
50    MemoryResolutionError, RuntimeBootstrapError, RuntimeConstructionError, RuntimeDiagnosticError,
51    RuntimeGrowError, RuntimeOpenError, RuntimeStateError,
52};
53pub use layout::MemoryManagerLayoutError;
54pub use policy::GenericAllocationPolicy;
55
56use self::policy::{RuntimeMemoryManagerPolicy, runtime_bootstrap_error_from_bootstrap};
57use crate::{
58    AllocationLedger, CommittedAllocations, PolicyIdentity, RuntimeBootstrapPolicy,
59    STABLE_CELL_VALUE_OFFSET, StableCellLedgerError, StableCellLedgerRecord,
60    requests::SealedDeclarationSnapshot,
61    slot::MEMORY_MANAGER_LEDGER_ID,
62    stable_cell::{EncodedLedgerRecord, decode_stable_cell_ledger_record_from_memory},
63};
64use ic_stable_structures::{
65    Cell, Memory,
66    memory_manager::{MemoryId, MemoryManager},
67};
68
69use std::rc::Rc;
70
71enum RuntimeLifecycle {
72    Unbootstrapped,
73    Bootstrapped {
74        committed_allocations: CommittedAllocations,
75        binding: RuntimeBootstrapBinding,
76    },
77}
78
79struct RuntimeBootstrapBinding {
80    source: SealedDeclarationSnapshot,
81    policy_identity: PolicyIdentity,
82    pool: crate::MemoryAllocationPool,
83}
84
85///
86/// MemoryRuntime
87///
88/// Canonical owner of allocation bootstrap state for one backing memory.
89///
90/// The runtime owns its `MemoryManager`, allocation-ledger persistence, bootstrap
91/// lifecycle, committed allocation capability, opens, and diagnostics. Static
92/// linked-program declarations are supplied separately as one immutable
93/// [`SealedDeclarationSnapshot`].
94///
95/// Each bootstrap attempt fallibly decodes its ledger record from persisted
96/// memory. Capacity-checked writes use `Cell`; diagnostics also read persisted
97/// memory directly.
98///
99/// `M` needs only [`Memory`]. The runtime does not require the backing memory
100/// to be `Send`, `Sync`, `Clone`, or `'static`.
101///
102
103pub struct MemoryRuntime<M: Memory> {
104    memory_manager: MemoryManager<Rc<M>>,
105    // Shared backing, immutable geometry and live accounting belong to growth.
106    // Handles reserve physical capacity; the manager owns bucket metadata.
107    growth: Rc<backing::GrowthState<M>>,
108    lifecycle: RuntimeLifecycle,
109}
110
111impl<M: Memory> MemoryRuntime<M> {
112    /// Construct an unbootstrapped runtime without overwriting foreign memory.
113    ///
114    /// Empty backing memory is initialized as an
115    /// `ic_stable_structures::MemoryManager`. Nonempty memory must pass bounded
116    /// validation of the current manager header, bucket table, and extents; otherwise
117    /// construction returns a typed error before the manager can
118    /// write its header or allocation table. A pre-grown blank memory is
119    /// nonempty and is therefore rejected rather than assumed disposable.
120    ///
121    /// # Errors
122    ///
123    /// Returns [`RuntimeConstructionError::ForeignMemory`] for nonempty memory
124    /// without `MemoryManager` magic, or
125    /// [`RuntimeConstructionError::UnsupportedMemoryManagerVersion`] when the
126    /// magic is recognized but the layout version is not current. Invalid
127    /// metadata returns [`RuntimeConstructionError::Layout`]. Refused fresh
128    /// metadata growth returns [`RuntimeConstructionError::Growth`] without
129    /// writes, allowing the same backing memory to be retried. Reopening honors
130    /// the actual persisted bucket size; only fresh memory uses 128 pages.
131    pub fn new(memory: M) -> Result<Self, RuntimeConstructionError> {
132        Self::construct(memory, None)
133    }
134
135    /// Construct with an explicit immutable bucket policy. Existing memory must
136    /// match exactly; mismatches fail before manager initialization or writes.
137    ///
138    /// # Errors
139    ///
140    /// Returns the construction errors described by [`Self::new`], or
141    /// [`RuntimeConstructionError::BucketSizeMismatch`] when existing geometry
142    /// differs from `config`.
143    pub fn new_with_config(
144        memory: M,
145        config: MemoryManagerConfig,
146    ) -> Result<Self, RuntimeConstructionError> {
147        Self::construct(memory, Some(config))
148    }
149
150    fn construct(
151        memory: M,
152        requested: Option<MemoryManagerConfig>,
153    ) -> Result<Self, RuntimeConstructionError> {
154        if cfg!(target_endian = "big") {
155            return Err(MemoryManagerLayoutError::UnsupportedByteOrder.into());
156        }
157        let (bucket_size_pages, allocated_buckets) = if memory.size() == 0 {
158            // Reserve the metadata page before the dependency writes its header.
159            // This fresh allocation is owned here; caller-supplied nonempty
160            // memory still passes the layout checks below before any writes.
161            if memory.grow(1) == -1 {
162                return Err(RuntimeGrowError::BackingRefused {
163                    additional_pages: 1,
164                }
165                .into());
166            }
167            (requested.unwrap_or_default().bucket_size_pages(), 0)
168        } else {
169            let measured = layout::read(&memory)?;
170            let actual = measured.bucket_pages;
171            if let Some(config) = requested {
172                check_bucket_size(actual, config)?;
173            }
174            (actual, measured.allocated_buckets)
175        };
176        let backing = Rc::new(memory);
177        let growth = Rc::new(backing::GrowthState {
178            backing: Rc::clone(&backing),
179            bucket_size_pages,
180            allocated_buckets: std::cell::RefCell::new(allocated_buckets),
181        });
182        Ok(Self {
183            memory_manager: MemoryManager::init_with_bucket_size(
184                Rc::clone(&backing),
185                bucket_size_pages,
186            ),
187            growth,
188            lifecycle: RuntimeLifecycle::Unbootstrapped,
189        })
190    }
191
192    /// Return the immutable bucket configuration bound to this runtime's manager.
193    #[must_use]
194    pub fn memory_manager_config(&self) -> MemoryManagerConfig {
195        // Construction has already validated the nonzero persisted setting.
196        MemoryManagerConfig::from_validated(self.growth.bucket_size_pages)
197    }
198
199    /// Return whether this runtime has published committed allocation authority.
200    #[must_use]
201    pub const fn is_bootstrapped(&self) -> bool {
202        matches!(self.lifecycle, RuntimeLifecycle::Bootstrapped { .. })
203    }
204
205    /// Bootstrap this backing memory from one immutable declaration snapshot.
206    ///
207    /// Recovery, metadata admission, logical resolution, policy evaluation,
208    /// staging, persistence and capability publication are local to this runtime.
209    /// A repeated call is idempotent only when the sealed declaration snapshot,
210    /// canonical host pool and [`RuntimeBootstrapPolicy::runtime_bootstrap_identity`] match the
211    /// successful bootstrap. A mismatch returns a typed error without
212    /// advancing the durable generation or re-evaluating policy.
213    /// Independently sealed snapshots match when their canonical contents are equal.
214    ///
215    /// # Panics
216    ///
217    /// Panics if a private runtime or encoding invariant is broken, or backing
218    /// memory or a policy callback panics.
219    pub fn bootstrap<P: RuntimeBootstrapPolicy>(
220        &mut self,
221        declarations: &SealedDeclarationSnapshot,
222        pool: &crate::MemoryAllocationPool,
223        policy: &P,
224    ) -> Result<&CommittedAllocations, RuntimeBootstrapError<P::Error>> {
225        let policy_identity = policy.runtime_bootstrap_identity()?;
226        match &self.lifecycle {
227            RuntimeLifecycle::Unbootstrapped => {
228                self.bootstrap_unbootstrapped(declarations, pool, policy, policy_identity)?;
229            }
230            RuntimeLifecycle::Bootstrapped { binding, .. } => {
231                binding.validate(declarations, pool, &policy_identity)?;
232            }
233        }
234        match &self.lifecycle {
235            RuntimeLifecycle::Bootstrapped {
236                committed_allocations,
237                ..
238            } => Ok(committed_allocations),
239            RuntimeLifecycle::Unbootstrapped => {
240                unreachable!("successful bootstrap publishes committed allocations")
241            }
242        }
243    }
244
245    fn validate_pool_custody(
246        &self,
247        ledger: &AllocationLedger,
248        pool: &crate::MemoryAllocationPool,
249    ) -> Result<(), super::MemoryResolutionError> {
250        for id in 0..crate::MEMORY_MANAGER_INVALID_ID {
251            if pool.contains(id)
252                && self.memory_size_pages(id) != 0
253                && !ledger
254                    .records()
255                    .iter()
256                    .any(|record| record.slot().id() == id)
257            {
258                return Err(super::MemoryResolutionError::UnmanagedAllocation { id });
259            }
260        }
261        Ok(())
262    }
263
264    fn bootstrap_unbootstrapped<P: RuntimeBootstrapPolicy>(
265        &mut self,
266        declarations: &SealedDeclarationSnapshot,
267        pool: &crate::MemoryAllocationPool,
268        policy: &P,
269        policy_identity: PolicyIdentity,
270    ) -> Result<(), RuntimeBootstrapError<P::Error>> {
271        let memory = self.memory(MEMORY_MANAGER_LEDGER_ID);
272        let mut record = decode_stable_cell_ledger_record_from_memory(&memory)?;
273        let genesis = AllocationLedger::empty_genesis();
274        let recovered = record.store_mut().recover_or_initialize(&genesis)?;
275        self.validate_pool_custody(recovered.ledger(), pool)?;
276        if memory.size() == 0 {
277            // Empty memory decodes to an uninitialized record. Preserve fresh
278            // cell acquisition before admission without persisting genesis.
279            write_ledger_cell(memory.clone(), &StableCellLedgerRecord::default())?;
280        }
281        let mut admission = BootstrapAdmission::new(recovered.ledger(), declarations, pool);
282        let preparation = policy.prepare_bootstrap(&mut admission);
283        let historical = admission.complete()?;
284        preparation.map_err(RuntimeBootstrapError::AdmissionPolicy)?;
285        let resolved = declarations.resolve(recovered.ledger(), historical, pool)?;
286        let runtime_policy = RuntimeMemoryManagerPolicy {
287            custom_policy: policy,
288        };
289        let commit = crate::bootstrap::prepare_commit(
290            record.store_mut(),
291            recovered,
292            resolved,
293            &runtime_policy,
294        )
295        .map_err(runtime_bootstrap_error_from_bootstrap)?;
296        write_ledger_cell(memory, &record)?;
297        let committed = commit.confirm_persisted().into_application_allocations();
298        self.lifecycle = RuntimeLifecycle::Bootstrapped {
299            committed_allocations: committed,
300            binding: RuntimeBootstrapBinding {
301                source: declarations.clone(),
302                policy_identity,
303                pool: pool.clone(),
304            },
305        };
306        Ok(())
307    }
308
309    /// Borrow this runtime's committed allocation-open capability.
310    pub const fn committed_allocations(&self) -> Result<&CommittedAllocations, RuntimeOpenError> {
311        match &self.lifecycle {
312            RuntimeLifecycle::Unbootstrapped => Err(RuntimeOpenError::NotBootstrapped),
313            RuntimeLifecycle::Bootstrapped {
314                committed_allocations,
315                ..
316            } => Ok(committed_allocations),
317        }
318    }
319
320    /// Open by durable key using only this runtime's persisted current capability.
321    pub fn open_memory(&self, stable_key: &str) -> Result<RuntimeMemory<M>, RuntimeOpenError> {
322        Ok(self.memory(self.memory_id(stable_key)?))
323    }
324
325    /// Resolve an application key's committed ID without opening memory,
326    /// reading history, or changing the host's policy or bucket configuration.
327    pub fn memory_id(&self, stable_key: &str) -> Result<u8, RuntimeOpenError> {
328        crate::key::validate(stable_key)?;
329        if crate::is_ic_memory_stable_key(stable_key) {
330            return Err(RuntimeOpenError::ReservedStableKey {
331                stable_key: stable_key.to_string(),
332            });
333        }
334        let declaration = crate::capability::declaration_for_key(
335            self.committed_allocations()?.declarations(),
336            stable_key,
337        )
338        .ok_or_else(|| RuntimeOpenError::StableKeyNotCommitted(stable_key.to_string()))?;
339        Ok(declaration.slot().id())
340    }
341
342    fn memory(&self, id: u8) -> RuntimeMemory<M> {
343        RuntimeMemory {
344            memory: self.memory_manager.get(MemoryId::new(id)),
345            growth: Rc::clone(&self.growth),
346        }
347    }
348
349    // Size-only observations need no handle carrying runtime growth authority.
350    fn memory_size_pages(&self, id: u8) -> u64 {
351        self.memory_manager.get(MemoryId::new(id)).size()
352    }
353
354    fn ledger_record_from_memory(&self) -> Result<StableCellLedgerRecord, StableCellLedgerError> {
355        // Decoding only reads; the manager handle retains backing lifetime
356        // without carrying the growth owner needed by writable runtime handles.
357        let memory = self
358            .memory_manager
359            .get(MemoryId::new(MEMORY_MANAGER_LEDGER_ID));
360        decode_stable_cell_ledger_record_from_memory(&memory)
361    }
362}
363
364impl RuntimeBootstrapBinding {
365    fn validate<P>(
366        &self,
367        declarations: &SealedDeclarationSnapshot,
368        pool: &crate::MemoryAllocationPool,
369        policy_identity: &PolicyIdentity,
370    ) -> Result<(), RuntimeBootstrapError<P>> {
371        if &self.source != declarations {
372            return Err(RuntimeBootstrapError::DeclarationSnapshotMismatch);
373        }
374        if &self.pool != pool {
375            return Err(RuntimeBootstrapError::AllocationPoolMismatch);
376        }
377        if &self.policy_identity != policy_identity {
378            return Err(RuntimeBootstrapError::PolicyIdentityMismatch {
379                established: self.policy_identity.clone(),
380                requested: policy_identity.clone(),
381            });
382        }
383        Ok(())
384    }
385}
386
387fn write_ledger_cell<M: Memory, P>(
388    memory: RuntimeMemory<M>,
389    record: &StableCellLedgerRecord,
390) -> Result<(), RuntimeBootstrapError<P>> {
391    let encoded = EncodedLedgerRecord::new(record).map_err(|value_size| {
392        RuntimeBootstrapError::StableCellLedgerWriteTooLarge { value_size }
393    })?;
394    let value_size = encoded.len();
395    // The record limit is below u32::MAX, including the eight-byte cell header.
396    let required_bytes = STABLE_CELL_VALUE_OFFSET + value_size as u64;
397    let available_bytes = memory.size().saturating_mul(crate::WASM_PAGE_SIZE_BYTES);
398    if required_bytes > available_bytes {
399        let grow_by = (required_bytes - available_bytes).div_ceil(crate::WASM_PAGE_SIZE_BYTES);
400        memory.grow(grow_by)?;
401    }
402    drop(Cell::new(memory, encoded));
403    Ok(())
404}
405
406const fn check_bucket_size(
407    actual: u16,
408    requested: MemoryManagerConfig,
409) -> Result<(), RuntimeConstructionError> {
410    if actual != requested.bucket_size_pages() {
411        return Err(RuntimeConstructionError::BucketSizeMismatch {
412            persisted: actual,
413            requested: requested.bucket_size_pages(),
414        });
415    }
416    Ok(())
417}
ic_memory/runtime/mod.rs

ic_memory/runtime/
mod.rs
