ic_memory/runtime/
mod.rs

1mod admission;
2#[cfg(test)]
3mod admission_tests;
4mod adoption;
5#[cfg(test)]
6mod adoption_tests;
7mod allocations;
8mod backing;
9mod config;
10mod default;
11mod diagnostics;
12mod error;
13mod layout;
14mod policy;
15
16#[cfg(test)]
17mod allocation_tests;
18#[cfg(test)]
19mod growth_tests;
20#[cfg(test)]
21#[expect(
22    unsafe_code,
23    reason = "exercise raw reads with valid uninitialized destinations"
24)]
25mod read_tests;
26#[cfg(test)]
27mod request_tests;
28#[cfg(test)]
29mod tests;
30
31pub use admission::{BootstrapAdmission, BootstrapAdmissionError, RecoveredAllocationMetadata};
32pub use adoption::RuntimeAdoptionError;
33
34pub use allocations::{
35    AllocationBinding, MemoryAllocation, MemoryAllocationSummary, MemoryAllocations,
36    MemoryBindingSummary,
37};
38pub use backing::RuntimeMemory;
39pub use config::MemoryManagerConfig;
40pub use default::{
41    bootstrap_default_memory_manager, bootstrap_default_memory_manager_with_config,
42    bootstrap_default_memory_manager_with_policy, committed_allocations,
43    default_memory_manager_commit_recovery_diagnostic, default_memory_manager_diagnostic_export,
44    default_memory_manager_doctor_report, default_memory_manager_doctor_report_with_policy,
45    default_memory_manager_memory_allocation_summary, default_memory_manager_memory_allocations,
46    default_memory_manager_memory_id, is_default_memory_manager_bootstrapped,
47    open_default_memory_manager_memory, verify_default_memory_manager_authority,
48};
49pub use error::{
50    MemoryResolutionError, RuntimeBootstrapError, RuntimeConstructionError, RuntimeDiagnosticError,
51    RuntimeGrowError, RuntimeOpenError, RuntimeStateError,
52};
53pub use layout::MemoryManagerLayoutError;
54pub use policy::GenericAllocationPolicy;
55
56use self::policy::{RuntimeMemoryManagerPolicy, runtime_bootstrap_error_from_bootstrap};
57use crate::{
58    AllocationBootstrap, AllocationLedger, CommittedAllocations, PolicyIdentity,
59    RuntimeBootstrapPolicy, STABLE_CELL_VALUE_OFFSET, StableCellLedgerError,
60    StableCellLedgerRecord,
61    requests::SealedDeclarationSnapshot,
62    slot::MEMORY_MANAGER_LEDGER_ID,
63    stable_cell::{EncodedLedgerRecord, decode_stable_cell_ledger_record_from_memory},
64};
65use ic_stable_structures::{
66    Cell, Memory,
67    memory_manager::{MemoryId, MemoryManager},
68};
69
70use std::rc::Rc;
71
72enum RuntimeLifecycle {
73    Unbootstrapped,
74    Bootstrapped {
75        committed_allocations: CommittedAllocations,
76        binding: RuntimeBootstrapBinding,
77    },
78}
79
80struct RuntimeBootstrapBinding {
81    source: SealedDeclarationSnapshot,
82    policy_identity: PolicyIdentity,
83    pool: crate::MemoryAllocationPool,
84}
85
86///
87/// MemoryRuntime
88///
89/// Canonical owner of allocation bootstrap state for one backing memory.
90///
91/// The runtime owns its `MemoryManager`, allocation-ledger persistence, bootstrap
92/// lifecycle, committed allocation capability, opens, and diagnostics. Static
93/// linked-program declarations are supplied separately as one immutable
94/// [`SealedDeclarationSnapshot`].
95///
96/// Each bootstrap attempt fallibly decodes its ledger record from persisted
97/// memory. Capacity-checked writes use `Cell`; diagnostics also read persisted
98/// memory directly.
99///
100/// `M` needs only [`Memory`]. The runtime does not require the backing memory
101/// to be `Send`, `Sync`, `Clone`, or `'static`.
102///
103
104pub struct MemoryRuntime<M: Memory> {
105    memory_manager: MemoryManager<Rc<M>>,
106    // Shared backing, immutable geometry and live accounting belong to growth.
107    // Handles reserve physical capacity; the manager owns bucket metadata.
108    growth: Rc<backing::GrowthState<M>>,
109    lifecycle: RuntimeLifecycle,
110}
111
112impl<M: Memory> MemoryRuntime<M> {
113    /// Construct an unbootstrapped runtime without overwriting foreign memory.
114    ///
115    /// Empty backing memory is initialized as an
116    /// `ic_stable_structures::MemoryManager`. Nonempty memory must pass bounded
117    /// validation of the current manager header, bucket table, and extents; otherwise
118    /// construction returns a typed error before the manager can
119    /// write its header or allocation table. A pre-grown blank memory is
120    /// nonempty and is therefore rejected rather than assumed disposable.
121    ///
122    /// # Errors
123    ///
124    /// Returns [`RuntimeConstructionError::ForeignMemory`] for nonempty memory
125    /// without `MemoryManager` magic, or
126    /// [`RuntimeConstructionError::UnsupportedMemoryManagerVersion`] when the
127    /// magic is recognized but the layout version is not current. Invalid
128    /// metadata returns [`RuntimeConstructionError::Layout`]. Refused fresh
129    /// metadata growth returns [`RuntimeConstructionError::Growth`] without
130    /// writes, allowing the same backing memory to be retried. Reopening honors
131    /// the actual persisted bucket size; only fresh memory uses 128 pages.
132    pub fn new(memory: M) -> Result<Self, RuntimeConstructionError> {
133        Self::construct(memory, None)
134    }
135
136    /// Construct with an explicit immutable bucket policy. Existing memory must
137    /// match exactly; mismatches fail before manager initialization or writes.
138    ///
139    /// # Errors
140    ///
141    /// Returns the construction errors described by [`Self::new`], or
142    /// [`RuntimeConstructionError::BucketSizeMismatch`] when existing geometry
143    /// differs from `config`.
144    pub fn new_with_config(
145        memory: M,
146        config: MemoryManagerConfig,
147    ) -> Result<Self, RuntimeConstructionError> {
148        Self::construct(memory, Some(config))
149    }
150
151    fn construct(
152        memory: M,
153        requested: Option<MemoryManagerConfig>,
154    ) -> Result<Self, RuntimeConstructionError> {
155        if cfg!(target_endian = "big") {
156            return Err(MemoryManagerLayoutError::UnsupportedByteOrder.into());
157        }
158        let (bucket_size_pages, allocated_buckets) = if memory.size() == 0 {
159            // Reserve the metadata page before the dependency writes its header.
160            // This fresh allocation is owned here; caller-supplied nonempty
161            // memory still passes the layout checks below before any writes.
162            if memory.grow(1) == -1 {
163                return Err(RuntimeGrowError::BackingRefused {
164                    additional_pages: 1,
165                }
166                .into());
167            }
168            (requested.unwrap_or_default().bucket_size_pages(), 0)
169        } else {
170            let measured = layout::read(&memory)?;
171            let actual = measured.bucket_pages;
172            if let Some(config) = requested {
173                check_bucket_size(actual, config)?;
174            }
175            (actual, measured.allocated_buckets)
176        };
177        let backing = Rc::new(memory);
178        let growth = Rc::new(backing::GrowthState {
179            backing: Rc::clone(&backing),
180            bucket_size_pages,
181            allocated_buckets: std::cell::RefCell::new(allocated_buckets),
182        });
183        Ok(Self {
184            memory_manager: MemoryManager::init_with_bucket_size(
185                Rc::clone(&backing),
186                bucket_size_pages,
187            ),
188            growth,
189            lifecycle: RuntimeLifecycle::Unbootstrapped,
190        })
191    }
192
193    /// Return the immutable bucket configuration bound to this runtime's manager.
194    #[must_use]
195    pub fn memory_manager_config(&self) -> MemoryManagerConfig {
196        // Construction has already validated the nonzero persisted setting.
197        MemoryManagerConfig::from_validated(self.growth.bucket_size_pages)
198    }
199
200    /// Return whether this runtime has published committed allocation authority.
201    #[must_use]
202    pub const fn is_bootstrapped(&self) -> bool {
203        matches!(self.lifecycle, RuntimeLifecycle::Bootstrapped { .. })
204    }
205
206    /// Bootstrap this backing memory from one immutable declaration snapshot.
207    ///
208    /// Recovery, metadata admission, logical resolution, policy evaluation,
209    /// staging, persistence and capability publication are local to this runtime.
210    /// A repeated call is idempotent only when the sealed declaration snapshot,
211    /// canonical host pool and [`RuntimeBootstrapPolicy::runtime_bootstrap_identity`] match the
212    /// successful bootstrap. A mismatch returns a typed error without
213    /// advancing the durable generation or re-evaluating policy.
214    /// Independently sealed snapshots match when their canonical contents are equal.
215    ///
216    /// # Panics
217    ///
218    /// Panics if a private runtime or encoding invariant is broken, or backing
219    /// memory or a policy callback panics.
220    pub fn bootstrap<P: RuntimeBootstrapPolicy>(
221        &mut self,
222        declarations: &SealedDeclarationSnapshot,
223        pool: &crate::MemoryAllocationPool,
224        policy: &P,
225    ) -> Result<&CommittedAllocations, RuntimeBootstrapError<P::Error>> {
226        let policy_identity = policy.runtime_bootstrap_identity()?;
227        match &self.lifecycle {
228            RuntimeLifecycle::Unbootstrapped => {
229                self.bootstrap_unbootstrapped(declarations, pool, policy, policy_identity)?;
230            }
231            RuntimeLifecycle::Bootstrapped { binding, .. } => {
232                binding.validate(declarations, pool, &policy_identity)?;
233            }
234        }
235        match &self.lifecycle {
236            RuntimeLifecycle::Bootstrapped {
237                committed_allocations,
238                ..
239            } => Ok(committed_allocations),
240            RuntimeLifecycle::Unbootstrapped => {
241                unreachable!("successful bootstrap publishes committed allocations")
242            }
243        }
244    }
245
246    fn validate_pool_custody(
247        &self,
248        ledger: &AllocationLedger,
249        pool: &crate::MemoryAllocationPool,
250    ) -> Result<(), super::MemoryResolutionError> {
251        for id in 0..crate::MEMORY_MANAGER_INVALID_ID {
252            if pool.contains(id)
253                && self.memory_size_pages(id) != 0
254                && !ledger
255                    .records()
256                    .iter()
257                    .any(|record| record.slot().id() == id)
258            {
259                return Err(super::MemoryResolutionError::UnmanagedAllocation { id });
260            }
261        }
262        Ok(())
263    }
264
265    fn bootstrap_unbootstrapped<P: RuntimeBootstrapPolicy>(
266        &mut self,
267        declarations: &SealedDeclarationSnapshot,
268        pool: &crate::MemoryAllocationPool,
269        policy: &P,
270        policy_identity: PolicyIdentity,
271    ) -> Result<(), RuntimeBootstrapError<P::Error>> {
272        let memory = self.memory(MEMORY_MANAGER_LEDGER_ID);
273        let mut record = decode_stable_cell_ledger_record_from_memory(&memory)?;
274        let genesis = AllocationLedger::empty_genesis();
275        let recovered = record.store_mut().recover_or_initialize(&genesis)?;
276        self.validate_pool_custody(recovered.ledger(), pool)?;
277        if memory.size() == 0 {
278            // Empty memory decodes to an uninitialized record. Preserve fresh
279            // cell acquisition before admission without persisting genesis.
280            write_ledger_cell(memory.clone(), &StableCellLedgerRecord::default())?;
281        }
282        let mut admission = BootstrapAdmission::new(recovered.ledger(), declarations, pool);
283        let preparation = policy.prepare_bootstrap(&mut admission);
284        let historical = admission.complete()?;
285        preparation.map_err(RuntimeBootstrapError::AdmissionPolicy)?;
286        let resolved = declarations.resolve(recovered.ledger(), historical, pool)?;
287        let runtime_policy = RuntimeMemoryManagerPolicy {
288            custom_policy: policy,
289        };
290        let commit = AllocationBootstrap::new(record.store_mut())
291            .validate_against(recovered, resolved, &runtime_policy)
292            .map_err(runtime_bootstrap_error_from_bootstrap)?;
293        write_ledger_cell(memory, &record)?;
294        let committed = commit.confirm_persisted().into_application_allocations();
295        self.lifecycle = RuntimeLifecycle::Bootstrapped {
296            committed_allocations: committed,
297            binding: RuntimeBootstrapBinding {
298                source: declarations.clone(),
299                policy_identity,
300                pool: pool.clone(),
301            },
302        };
303        Ok(())
304    }
305
306    /// Borrow this runtime's committed allocation-open capability.
307    pub const fn committed_allocations(&self) -> Result<&CommittedAllocations, RuntimeOpenError> {
308        match &self.lifecycle {
309            RuntimeLifecycle::Unbootstrapped => Err(RuntimeOpenError::NotBootstrapped),
310            RuntimeLifecycle::Bootstrapped {
311                committed_allocations,
312                ..
313            } => Ok(committed_allocations),
314        }
315    }
316
317    /// Open by durable key using only this runtime's persisted current capability.
318    pub fn open_memory(&self, stable_key: &str) -> Result<RuntimeMemory<M>, RuntimeOpenError> {
319        Ok(self.memory(self.memory_id(stable_key)?))
320    }
321
322    /// Resolve an application key's committed ID without opening memory,
323    /// reading history, or changing the host's policy or bucket configuration.
324    pub fn memory_id(&self, stable_key: &str) -> Result<u8, RuntimeOpenError> {
325        crate::key::validate(stable_key)?;
326        if crate::is_ic_memory_stable_key(stable_key) {
327            return Err(RuntimeOpenError::ReservedStableKey {
328                stable_key: stable_key.to_string(),
329            });
330        }
331        let declaration = crate::capability::declaration_for_key(
332            self.committed_allocations()?.declarations(),
333            stable_key,
334        )
335        .ok_or_else(|| RuntimeOpenError::StableKeyNotCommitted(stable_key.to_string()))?;
336        Ok(declaration.slot().id())
337    }
338
339    fn memory(&self, id: u8) -> RuntimeMemory<M> {
340        RuntimeMemory {
341            memory: self.memory_manager.get(MemoryId::new(id)),
342            growth: Rc::clone(&self.growth),
343        }
344    }
345
346    // Size-only observations need no handle carrying runtime growth authority.
347    fn memory_size_pages(&self, id: u8) -> u64 {
348        self.memory_manager.get(MemoryId::new(id)).size()
349    }
350
351    fn ledger_record_from_memory(&self) -> Result<StableCellLedgerRecord, StableCellLedgerError> {
352        // Decoding only reads; the manager handle retains backing lifetime
353        // without carrying the growth owner needed by writable runtime handles.
354        let memory = self
355            .memory_manager
356            .get(MemoryId::new(MEMORY_MANAGER_LEDGER_ID));
357        decode_stable_cell_ledger_record_from_memory(&memory)
358    }
359}
360
361impl RuntimeBootstrapBinding {
362    fn validate<P>(
363        &self,
364        declarations: &SealedDeclarationSnapshot,
365        pool: &crate::MemoryAllocationPool,
366        policy_identity: &PolicyIdentity,
367    ) -> Result<(), RuntimeBootstrapError<P>> {
368        if &self.source != declarations {
369            return Err(RuntimeBootstrapError::DeclarationSnapshotMismatch);
370        }
371        if &self.pool != pool {
372            return Err(RuntimeBootstrapError::AllocationPoolMismatch);
373        }
374        if &self.policy_identity != policy_identity {
375            return Err(RuntimeBootstrapError::PolicyIdentityMismatch {
376                established: self.policy_identity.clone(),
377                requested: policy_identity.clone(),
378            });
379        }
380        Ok(())
381    }
382}
383
384fn write_ledger_cell<M: Memory, P>(
385    memory: RuntimeMemory<M>,
386    record: &StableCellLedgerRecord,
387) -> Result<(), RuntimeBootstrapError<P>> {
388    let encoded = EncodedLedgerRecord::new(record).map_err(|value_size| {
389        RuntimeBootstrapError::StableCellLedgerWriteTooLarge { value_size }
390    })?;
391    let value_size = encoded.len();
392    // The record limit is below u32::MAX, including the eight-byte cell header.
393    let required_bytes = STABLE_CELL_VALUE_OFFSET + value_size as u64;
394    let available_bytes = memory.size().saturating_mul(crate::WASM_PAGE_SIZE_BYTES);
395    if required_bytes > available_bytes {
396        let grow_by = (required_bytes - available_bytes).div_ceil(crate::WASM_PAGE_SIZE_BYTES);
397        memory.grow(grow_by)?;
398    }
399    drop(Cell::new(memory, encoded));
400    Ok(())
401}
402
403const fn check_bucket_size(
404    actual: u16,
405    requested: MemoryManagerConfig,
406) -> Result<(), RuntimeConstructionError> {
407    if actual != requested.bucket_size_pages() {
408        return Err(RuntimeConstructionError::BucketSizeMismatch {
409            persisted: actual,
410            requested: requested.bucket_size_pages(),
411        });
412    }
413    Ok(())
414}
ic_memory/runtime/mod.rs

ic_memory/runtime/
mod.rs
