ic_memory/runtime/
mod.rs

1mod admission;
2#[cfg(test)]
3mod admission_tests;
4mod adoption;
5#[cfg(test)]
6mod adoption_tests;
7mod allocations;
8mod backing;
9mod config;
10mod default;
11mod diagnostics;
12mod error;
13mod layout;
14mod policy;
15
16#[cfg(test)]
17mod allocation_tests;
18#[cfg(test)]
19mod growth_tests;
20#[cfg(test)]
21#[expect(
22    unsafe_code,
23    reason = "exercise raw reads with valid uninitialized destinations"
24)]
25mod read_tests;
26#[cfg(test)]
27mod request_tests;
28#[cfg(test)]
29mod tests;
30
31pub use admission::{BootstrapAdmission, BootstrapAdmissionError, RecoveredAllocationMetadata};
32pub use adoption::RuntimeAdoptionError;
33
34pub use allocations::{
35    AllocationBinding, AllocationRangeClaim, MemoryAllocation, MemoryAllocationSummary,
36    MemoryAllocations, MemoryBindingSummary,
37};
38pub use backing::RuntimeMemory;
39pub use config::MemoryManagerConfig;
40pub use default::{
41    bootstrap_default_memory_manager, bootstrap_default_memory_manager_with_config,
42    bootstrap_default_memory_manager_with_policy, committed_allocations,
43    default_memory_manager_commit_recovery_diagnostic, default_memory_manager_diagnostic_export,
44    default_memory_manager_doctor_report, default_memory_manager_doctor_report_with_policy,
45    default_memory_manager_memory_allocation_summary, default_memory_manager_memory_allocations,
46    default_memory_manager_memory_id, is_default_memory_manager_bootstrapped,
47    open_default_memory_manager_memory, open_default_memory_manager_memory_by_key,
48    verify_default_memory_manager_authority,
49};
50pub use error::{
51    MemoryResolutionError, RuntimeBootstrapError, RuntimeConstructionError, RuntimeDiagnosticError,
52    RuntimeGrowError, RuntimeOpenError, RuntimePolicyError, RuntimeStateError,
53};
54pub use layout::MemoryManagerLayoutError;
55pub use policy::GenericRangePolicy;
56
57use self::policy::{RuntimeMemoryManagerPolicy, runtime_bootstrap_error_from_bootstrap};
58use crate::{
59    AllocationBootstrap, AllocationHistory, AllocationLedger, CommittedAllocations, PolicyIdentity,
60    RuntimeBootstrapPolicy, STABLE_CELL_VALUE_OFFSET, StableCellLedgerError,
61    StableCellLedgerRecord, StableKey, registry::SealedDeclarationSnapshot,
62    slot::MEMORY_MANAGER_LEDGER_ID, stable_cell::decode_stable_cell_ledger_record_from_memory,
63};
64use ic_stable_structures::{
65    Cell, Memory,
66    memory_manager::{MemoryId, MemoryManager},
67};
68
69use std::rc::Rc;
70
71type LedgerCell<M> = Cell<StableCellLedgerRecord, RuntimeMemory<M>>;
72
73enum RuntimeLifecycle {
74    Unbootstrapped,
75    Bootstrapped {
76        committed_allocations: CommittedAllocations,
77        binding: RuntimeBootstrapBinding,
78    },
79}
80
81struct RuntimeBootstrapBinding {
82    source: SealedDeclarationSnapshot,
83    declarations: SealedDeclarationSnapshot,
84    policy_identity: PolicyIdentity,
85}
86
87///
88/// MemoryRuntime
89///
90/// Canonical owner of allocation bootstrap state for one backing memory.
91///
92/// The runtime owns its `MemoryManager`, allocation-ledger cell, bootstrap
93/// lifecycle, committed allocation capability, opens, and diagnostics. Static
94/// linked-program declarations are supplied separately as one immutable
95/// [`SealedDeclarationSnapshot`].
96///
97/// `M` needs only [`Memory`]. The runtime does not require the backing memory
98/// to be `Send`, `Sync`, `Clone`, or `'static`.
99///
100
101pub struct MemoryRuntime<M: Memory> {
102    memory_manager: MemoryManager<Rc<M>>,
103    // Shared backing, immutable geometry and live accounting belong to growth.
104    // Handles reserve physical capacity; the manager owns bucket metadata.
105    growth: Rc<backing::GrowthState<M>>,
106    ledger_cell: Option<LedgerCell<M>>,
107    lifecycle: RuntimeLifecycle,
108}
109
110impl<M: Memory> MemoryRuntime<M> {
111    /// Construct an unbootstrapped runtime without overwriting foreign memory.
112    ///
113    /// Empty backing memory is initialized as an
114    /// `ic_stable_structures::MemoryManager`. Nonempty memory must pass bounded
115    /// validation of the current manager header, bucket table, and extents; otherwise
116    /// construction returns a typed error before the manager can
117    /// write its header or allocation table. A pre-grown blank memory is
118    /// nonempty and is therefore rejected rather than assumed disposable.
119    ///
120    /// # Errors
121    ///
122    /// Returns [`RuntimeConstructionError::ForeignMemory`] for nonempty memory
123    /// without `MemoryManager` magic, or
124    /// [`RuntimeConstructionError::UnsupportedMemoryManagerVersion`] when the
125    /// magic is recognized but the layout version is not current. Invalid
126    /// metadata returns [`RuntimeConstructionError::Layout`]. Refused fresh
127    /// metadata growth returns [`RuntimeConstructionError::Growth`] without
128    /// writes, allowing the same backing memory to be retried. Reopening honors
129    /// the actual persisted bucket size; only fresh memory uses 128 pages.
130    pub fn new(memory: M) -> Result<Self, RuntimeConstructionError> {
131        Self::construct(memory, None)
132    }
133
134    /// Construct with an explicit immutable bucket policy. Existing memory must
135    /// match exactly; mismatches fail before manager initialization or writes.
136    ///
137    /// # Errors
138    ///
139    /// Returns the construction errors described by [`Self::new`], or
140    /// [`RuntimeConstructionError::BucketSizeMismatch`] when existing geometry
141    /// differs from `config`.
142    pub fn new_with_config(
143        memory: M,
144        config: MemoryManagerConfig,
145    ) -> Result<Self, RuntimeConstructionError> {
146        Self::construct(memory, Some(config))
147    }
148
149    fn construct(
150        memory: M,
151        requested: Option<MemoryManagerConfig>,
152    ) -> Result<Self, RuntimeConstructionError> {
153        if cfg!(target_endian = "big") {
154            return Err(MemoryManagerLayoutError::UnsupportedByteOrder.into());
155        }
156        let (bucket_size_pages, allocated_buckets) = if memory.size() == 0 {
157            // Reserve the metadata page before the dependency writes its header.
158            // This fresh allocation is owned here; caller-supplied nonempty
159            // memory still passes the layout checks below before any writes.
160            if memory.grow(1) == -1 {
161                return Err(RuntimeGrowError::BackingRefused {
162                    additional_pages: 1,
163                }
164                .into());
165            }
166            (requested.unwrap_or_default().bucket_size_pages(), 0)
167        } else {
168            let measured = layout::read(&memory)?;
169            let actual = measured.bucket_pages;
170            if let Some(config) = requested {
171                check_bucket_size(actual, config)?;
172            }
173            (actual, measured.allocated_buckets)
174        };
175        let backing = Rc::new(memory);
176        let growth = Rc::new(backing::GrowthState {
177            backing: Rc::clone(&backing),
178            bucket_size_pages,
179            allocated_buckets: std::cell::RefCell::new(allocated_buckets),
180        });
181        Ok(Self {
182            memory_manager: MemoryManager::init_with_bucket_size(
183                Rc::clone(&backing),
184                bucket_size_pages,
185            ),
186            growth,
187            ledger_cell: None,
188            lifecycle: RuntimeLifecycle::Unbootstrapped,
189        })
190    }
191
192    /// Return the immutable bucket configuration bound to this runtime's manager.
193    #[must_use]
194    pub fn memory_manager_config(&self) -> MemoryManagerConfig {
195        // Construction has already validated the nonzero persisted setting.
196        MemoryManagerConfig::from_validated(self.growth.bucket_size_pages)
197    }
198
199    /// Return whether this runtime has published committed allocation authority.
200    #[must_use]
201    pub const fn is_bootstrapped(&self) -> bool {
202        matches!(self.lifecycle, RuntimeLifecycle::Bootstrapped { .. })
203    }
204
205    /// Bootstrap this backing memory from one immutable declaration snapshot.
206    ///
207    /// Recovery, metadata admission, logical resolution, policy evaluation,
208    /// staging, persistence and capability publication are local to this runtime.
209    /// A repeated call is idempotent only when the sealed declaration snapshot and
210    /// [`RuntimeBootstrapPolicy::runtime_bootstrap_identity`] match the
211    /// successful bootstrap. A mismatch returns a typed error without
212    /// advancing the durable generation or re-evaluating policy.
213    /// Independently sealed snapshots match when their canonical contents are equal.
214    pub fn bootstrap<P: RuntimeBootstrapPolicy>(
215        &mut self,
216        declarations: &SealedDeclarationSnapshot,
217        policy: &P,
218    ) -> Result<&CommittedAllocations, RuntimeBootstrapError<P::Error>> {
219        let policy_identity = policy.runtime_bootstrap_identity()?;
220        let already_bootstrapped = match &self.lifecycle {
221            RuntimeLifecycle::Unbootstrapped => false,
222            RuntimeLifecycle::Bootstrapped { binding, .. } => {
223                binding.validate(declarations, &policy_identity)?;
224                true
225            }
226        };
227        if !already_bootstrapped {
228            self.bootstrap_unbootstrapped(declarations, policy, policy_identity)?;
229        }
230        match &self.lifecycle {
231            RuntimeLifecycle::Bootstrapped {
232                committed_allocations,
233                ..
234            } => Ok(committed_allocations),
235            RuntimeLifecycle::Unbootstrapped => Err(RuntimeBootstrapError::State(
236                RuntimeStateError::InconsistentLifecycle,
237            )),
238        }
239    }
240
241    fn bootstrap_unbootstrapped<P: RuntimeBootstrapPolicy>(
242        &mut self,
243        declarations: &SealedDeclarationSnapshot,
244        policy: &P,
245        policy_identity: PolicyIdentity,
246    ) -> Result<(), RuntimeBootstrapError<P::Error>> {
247        self.initialize_ledger_cell()?;
248        let mut record = self
249            .ledger_cell
250            .as_ref()
251            .map(|cell| cell.get().clone())
252            .ok_or(RuntimeStateError::InconsistentLifecycle)?;
253        let genesis = AllocationLedger::new(0, AllocationHistory::default())?;
254        let recovered = record.store_mut().recover_or_initialize(&genesis)?;
255        let mut admission = BootstrapAdmission::new(recovered.ledger(), declarations);
256        let preparation = policy.prepare_bootstrap(&mut admission);
257        let historical = admission.complete()?;
258        preparation.map_err(RuntimeBootstrapError::AdmissionPolicy)?;
259        let resolved = declarations.resolve(recovered.ledger(), historical)?;
260        let runtime_policy = RuntimeMemoryManagerPolicy {
261            declarations: &resolved,
262            custom_policy: policy,
263        };
264        let commit = AllocationBootstrap::new(record.store_mut())
265            .validate_against(
266                recovered,
267                resolved.allocation_snapshot().clone(),
268                &runtime_policy,
269                None,
270            )
271            .map_err(runtime_bootstrap_error_from_bootstrap)?;
272        self.persist_ledger_record(record)?;
273        let committed = external_runtime_allocations(commit.confirm_persisted());
274        self.lifecycle = RuntimeLifecycle::Bootstrapped {
275            committed_allocations: committed,
276            binding: RuntimeBootstrapBinding {
277                source: declarations.clone(),
278                declarations: resolved,
279                policy_identity,
280            },
281        };
282        Ok(())
283    }
284
285    /// Borrow this runtime's committed allocation-open capability.
286    pub const fn committed_allocations(&self) -> Result<&CommittedAllocations, RuntimeOpenError> {
287        match &self.lifecycle {
288            RuntimeLifecycle::Unbootstrapped => Err(RuntimeOpenError::NotBootstrapped),
289            RuntimeLifecycle::Bootstrapped {
290                committed_allocations,
291                ..
292            } => Ok(committed_allocations),
293        }
294    }
295
296    /// Open by durable key using only this runtime's persisted current capability.
297    pub fn open_memory_by_key(
298        &self,
299        stable_key: &str,
300    ) -> Result<RuntimeMemory<M>, RuntimeOpenError> {
301        Ok(self.memory(self.memory_id(stable_key)?))
302    }
303
304    /// Open this runtime's committed memory by stable key and expected ID.
305    pub fn open_memory(
306        &self,
307        stable_key: &str,
308        expected_id: u8,
309    ) -> Result<RuntimeMemory<M>, RuntimeOpenError> {
310        let committed_id = self.memory_id(stable_key)?;
311        if committed_id != expected_id {
312            return Err(RuntimeOpenError::MemoryIdMismatch {
313                stable_key: stable_key.to_string(),
314                committed_id,
315                requested_id: expected_id,
316            });
317        }
318        Ok(self.memory(committed_id))
319    }
320
321    /// Resolve an application key's committed ID without opening memory,
322    /// reading history, or changing the host's policy or bucket configuration.
323    pub fn memory_id(&self, stable_key: &str) -> Result<u8, RuntimeOpenError> {
324        let key = StableKey::parse(stable_key)?;
325        if crate::is_ic_memory_stable_key(key.as_str()) {
326            return Err(RuntimeOpenError::ReservedStableKey {
327                stable_key: stable_key.to_string(),
328            });
329        }
330        let slot = self
331            .committed_allocations()?
332            .slot_for(&key)
333            .ok_or_else(|| RuntimeOpenError::StableKeyNotCommitted(stable_key.to_string()))?;
334        Ok(slot.memory_manager_id()?)
335    }
336
337    fn initialize_ledger_cell<P>(&mut self) -> Result<(), RuntimeBootstrapError<P>> {
338        if self.ledger_cell.is_some() {
339            return Ok(());
340        }
341        let memory = self.memory(MEMORY_MANAGER_LEDGER_ID);
342        crate::validate_stable_cell_ledger_memory(&memory)?;
343        ensure_ledger_cell_capacity(&memory, &StableCellLedgerRecord::default())?;
344        self.ledger_cell = Some(Cell::init(memory, StableCellLedgerRecord::default()));
345        Ok(())
346    }
347
348    fn persist_ledger_record<P>(
349        &mut self,
350        record: StableCellLedgerRecord,
351    ) -> Result<(), RuntimeBootstrapError<P>> {
352        let memory = self.memory(MEMORY_MANAGER_LEDGER_ID);
353        ensure_ledger_cell_capacity(&memory, &record)?;
354        let cell = self
355            .ledger_cell
356            .as_mut()
357            .ok_or(RuntimeStateError::InconsistentLifecycle)?;
358        let _previous = cell.set(record);
359        Ok(())
360    }
361
362    fn memory(&self, id: u8) -> RuntimeMemory<M> {
363        RuntimeMemory {
364            memory: self.memory_manager.get(MemoryId::new(id)),
365            growth: Rc::clone(&self.growth),
366        }
367    }
368
369    fn ledger_record_from_memory(&self) -> Result<StableCellLedgerRecord, StableCellLedgerError> {
370        decode_stable_cell_ledger_record_from_memory(&self.memory(MEMORY_MANAGER_LEDGER_ID))
371    }
372}
373
374impl RuntimeBootstrapBinding {
375    fn validate<P>(
376        &self,
377        declarations: &SealedDeclarationSnapshot,
378        policy_identity: &PolicyIdentity,
379    ) -> Result<(), RuntimeBootstrapError<P>> {
380        if &self.source != declarations {
381            return Err(RuntimeBootstrapError::DeclarationSnapshotMismatch);
382        }
383        if &self.policy_identity != policy_identity {
384            return Err(RuntimeBootstrapError::PolicyIdentityMismatch {
385                established: self.policy_identity.clone(),
386                requested: policy_identity.clone(),
387            });
388        }
389        Ok(())
390    }
391}
392
393fn ensure_ledger_cell_capacity<M: Memory, P>(
394    memory: &RuntimeMemory<M>,
395    record: &StableCellLedgerRecord,
396) -> Result<(), RuntimeBootstrapError<P>> {
397    let value_size = record.encoded_size();
398    if value_size > crate::constants::MAX_LEDGER_RECORD_BYTES {
399        return Err(RuntimeBootstrapError::StableCellLedgerWriteTooLarge { value_size });
400    }
401    // The record limit is below u32::MAX, including the eight-byte cell header.
402    let required_bytes = STABLE_CELL_VALUE_OFFSET + value_size as u64;
403    let available_bytes = memory.size().saturating_mul(crate::WASM_PAGE_SIZE_BYTES);
404    if required_bytes <= available_bytes {
405        return Ok(());
406    }
407    let grow_by = (required_bytes - available_bytes).div_ceil(crate::WASM_PAGE_SIZE_BYTES);
408    memory.grow(grow_by)?;
409    Ok(())
410}
411
412fn external_runtime_allocations(committed: CommittedAllocations) -> CommittedAllocations {
413    committed.without_stable_key_prefix(crate::IC_MEMORY_STABLE_KEY_PREFIX)
414}
415
416const fn check_bucket_size(
417    actual: u16,
418    requested: MemoryManagerConfig,
419) -> Result<(), RuntimeConstructionError> {
420    if actual != requested.bucket_size_pages() {
421        return Err(RuntimeConstructionError::BucketSizeMismatch {
422            persisted: actual,
423            requested: requested.bucket_size_pages(),
424        });
425    }
426    Ok(())
427}
ic_memory/runtime/mod.rs

ic_memory/runtime/
mod.rs
