ic_memory/runtime/
mod.rs

1mod admission;
2#[cfg(test)]
3mod admission_tests;
4mod adoption;
5#[cfg(test)]
6mod adoption_tests;
7mod allocations;
8mod backing;
9mod config;
10mod default;
11mod diagnostics;
12mod error;
13mod layout;
14mod policy;
15
16#[cfg(test)]
17mod allocation_tests;
18#[cfg(test)]
19mod growth_tests;
20#[cfg(test)]
21#[expect(
22    unsafe_code,
23    reason = "exercise raw reads with valid uninitialized destinations"
24)]
25mod read_tests;
26#[cfg(test)]
27mod request_tests;
28#[cfg(test)]
29mod tests;
30
31pub use admission::{BootstrapAdmission, BootstrapAdmissionError, RecoveredAllocationMetadata};
32pub use adoption::RuntimeAdoptionError;
33
34pub use allocations::{
35    AllocationBinding, AllocationRangeClaim, MemoryAllocation, MemoryAllocationSummary,
36    MemoryAllocations, MemoryBindingSummary,
37};
38pub use backing::RuntimeMemory;
39pub use config::MemoryManagerConfig;
40pub use default::{
41    bootstrap_default_memory_manager, bootstrap_default_memory_manager_with_config,
42    bootstrap_default_memory_manager_with_policy, committed_allocations,
43    default_memory_manager_commit_recovery_diagnostic, default_memory_manager_diagnostic_export,
44    default_memory_manager_doctor_report, default_memory_manager_doctor_report_with_policy,
45    default_memory_manager_memory_allocation_summary, default_memory_manager_memory_allocations,
46    default_memory_manager_memory_id, is_default_memory_manager_bootstrapped,
47    open_default_memory_manager_memory, open_default_memory_manager_memory_by_key,
48    verify_default_memory_manager_authority,
49};
50pub use error::{
51    MemoryResolutionError, RuntimeBootstrapError, RuntimeConstructionError, RuntimeDiagnosticError,
52    RuntimeGrowError, RuntimeOpenError, RuntimePolicyError, RuntimeStateError,
53};
54pub use layout::MemoryManagerLayoutError;
55pub use policy::GenericRangePolicy;
56
57use self::policy::{RuntimeMemoryManagerPolicy, runtime_bootstrap_error_from_bootstrap};
58use crate::{
59    AllocationBootstrap, AllocationHistory, AllocationLedger, CommittedAllocations, PolicyIdentity,
60    RuntimeBootstrapPolicy, STABLE_CELL_VALUE_OFFSET, StableCellLedgerError,
61    StableCellLedgerRecord, StableKey, registry::SealedDeclarationSnapshot,
62    slot::MEMORY_MANAGER_LEDGER_ID, stable_cell::decode_stable_cell_ledger_record_from_memory,
63};
64use ic_stable_structures::{
65    Cell, Memory, Storable,
66    memory_manager::{MemoryId, MemoryManager},
67};
68
69use std::rc::Rc;
70
71type LedgerCell<M> = Cell<StableCellLedgerRecord, RuntimeMemory<M>>;
72
73enum RuntimeLifecycle {
74    Unbootstrapped,
75    Bootstrapped {
76        committed_allocations: CommittedAllocations,
77        binding: RuntimeBootstrapBinding,
78    },
79}
80
81struct RuntimeBootstrapBinding {
82    source: SealedDeclarationSnapshot,
83    declarations: SealedDeclarationSnapshot,
84    policy_identity: PolicyIdentity,
85}
86
87///
88/// MemoryRuntime
89///
90/// Canonical owner of allocation bootstrap state for one backing memory.
91///
92/// The runtime owns its `MemoryManager`, allocation-ledger cell, bootstrap
93/// lifecycle, committed allocation capability, opens, and diagnostics. Static
94/// linked-program declarations are supplied separately as one immutable
95/// [`SealedDeclarationSnapshot`].
96///
97/// `M` needs only [`Memory`]. The runtime does not require the backing memory
98/// to be `Send`, `Sync`, `Clone`, or `'static`.
99///
100
101pub struct MemoryRuntime<M: Memory> {
102    memory_manager: MemoryManager<Rc<M>>,
103    // Share the owned backing using upstream's Memory implementation for Rc.
104    // Handles reserve physical capacity; the manager owns bucket metadata.
105    // Attribution borrows the backing read-only.
106    backing: Rc<M>,
107    bucket_size_pages: u16,
108    growth: Rc<backing::GrowthState<M>>,
109    ledger_cell: Option<LedgerCell<M>>,
110    lifecycle: RuntimeLifecycle,
111}
112
113impl<M: Memory> MemoryRuntime<M> {
114    /// Construct an unbootstrapped runtime without overwriting foreign memory.
115    ///
116    /// Empty backing memory is initialized as an
117    /// `ic_stable_structures::MemoryManager`. Nonempty memory must pass bounded
118    /// validation of the current manager header, bucket table, and extents; otherwise
119    /// construction returns a typed error before the manager can
120    /// write its header or allocation table. A pre-grown blank memory is
121    /// nonempty and is therefore rejected rather than assumed disposable.
122    ///
123    /// # Errors
124    ///
125    /// Returns [`RuntimeConstructionError::ForeignMemory`] for nonempty memory
126    /// without `MemoryManager` magic, or
127    /// [`RuntimeConstructionError::UnsupportedMemoryManagerVersion`] when the
128    /// magic is recognized but the layout version is not current. Invalid
129    /// metadata returns [`RuntimeConstructionError::Layout`]. Reopening honors
130    /// the actual persisted bucket size; only fresh memory uses 128 pages.
131    pub fn new(memory: M) -> Result<Self, RuntimeConstructionError> {
132        Self::construct(memory, None)
133    }
134
135    /// Construct with an explicit immutable bucket policy. Existing memory must
136    /// match exactly; mismatches fail before manager initialization or writes.
137    pub fn new_with_config(
138        memory: M,
139        config: MemoryManagerConfig,
140    ) -> Result<Self, RuntimeConstructionError> {
141        Self::construct(memory, Some(config))
142    }
143
144    fn construct(
145        memory: M,
146        requested: Option<MemoryManagerConfig>,
147    ) -> Result<Self, RuntimeConstructionError> {
148        if cfg!(target_endian = "big") {
149            return Err(MemoryManagerLayoutError::UnsupportedByteOrder.into());
150        }
151        let (bucket_size_pages, allocated_buckets) = if memory.size() == 0 {
152            (requested.unwrap_or_default().bucket_size_pages(), 0)
153        } else {
154            let measured = layout::read(&memory)?;
155            let actual = measured.bucket_pages;
156            if let Some(config) = requested {
157                check_bucket_size(actual, config)?;
158            }
159            (actual, measured.allocated_buckets)
160        };
161        let backing = Rc::new(memory);
162        let growth = Rc::new(backing::GrowthState {
163            backing: Rc::clone(&backing),
164            bucket_size_pages,
165            allocated_buckets: std::cell::RefCell::new(allocated_buckets),
166        });
167        Ok(Self {
168            memory_manager: MemoryManager::init_with_bucket_size(
169                Rc::clone(&backing),
170                bucket_size_pages,
171            ),
172            backing,
173            bucket_size_pages,
174            growth,
175            ledger_cell: None,
176            lifecycle: RuntimeLifecycle::Unbootstrapped,
177        })
178    }
179
180    /// Return the actual policy bound to this runtime's sole manager.
181    #[must_use]
182    pub const fn memory_manager_config(&self) -> MemoryManagerConfig {
183        // Construction has already validated the nonzero persisted setting.
184        MemoryManagerConfig::from_validated(self.bucket_size_pages)
185    }
186
187    /// Return whether this runtime has published committed allocation authority.
188    #[must_use]
189    pub const fn is_bootstrapped(&self) -> bool {
190        matches!(self.lifecycle, RuntimeLifecycle::Bootstrapped { .. })
191    }
192
193    /// Bootstrap this backing memory from one immutable declaration snapshot.
194    ///
195    /// Recovery, metadata admission, policy evaluation, staging, persistence,
196    /// and capability publication are local to this runtime. A repeated call is idempotent
197    /// only when the sealed declaration snapshot and
198    /// [`RuntimeBootstrapPolicy::runtime_bootstrap_identity`] match the
199    /// successful bootstrap. A mismatch returns a typed error without
200    /// advancing the durable generation or re-evaluating policy.
201    pub fn bootstrap<P: RuntimeBootstrapPolicy>(
202        &mut self,
203        declarations: &SealedDeclarationSnapshot,
204        policy: &P,
205    ) -> Result<&CommittedAllocations, RuntimeBootstrapError<P::Error>> {
206        let policy_identity = policy.runtime_bootstrap_identity()?;
207        let already_bootstrapped = match &self.lifecycle {
208            RuntimeLifecycle::Unbootstrapped => false,
209            RuntimeLifecycle::Bootstrapped { binding, .. } => {
210                binding.validate(declarations, &policy_identity)?;
211                true
212            }
213        };
214        if !already_bootstrapped {
215            self.bootstrap_unbootstrapped(declarations, policy, policy_identity)?;
216        }
217        match &self.lifecycle {
218            RuntimeLifecycle::Bootstrapped {
219                committed_allocations,
220                ..
221            } => Ok(committed_allocations),
222            RuntimeLifecycle::Unbootstrapped => Err(RuntimeBootstrapError::State(
223                RuntimeStateError::InconsistentLifecycle,
224            )),
225        }
226    }
227
228    fn bootstrap_unbootstrapped<P: RuntimeBootstrapPolicy>(
229        &mut self,
230        declarations: &SealedDeclarationSnapshot,
231        policy: &P,
232        policy_identity: PolicyIdentity,
233    ) -> Result<(), RuntimeBootstrapError<P::Error>> {
234        self.initialize_ledger_cell()?;
235        let mut record = self
236            .ledger_cell
237            .as_ref()
238            .map(|cell| cell.get().clone())
239            .ok_or(RuntimeStateError::InconsistentLifecycle)?;
240        let genesis = AllocationLedger::new(0, AllocationHistory::default())?;
241        let recovered = record.store_mut().recover_or_initialize(&genesis)?;
242        let mut admission = BootstrapAdmission::new(recovered.ledger(), declarations);
243        let preparation = policy.prepare_bootstrap(&mut admission);
244        let historical = admission.complete()?;
245        preparation.map_err(RuntimeBootstrapError::AdmissionPolicy)?;
246        let resolved = declarations.resolve(recovered.ledger(), historical)?;
247        let runtime_policy = RuntimeMemoryManagerPolicy {
248            declarations: &resolved,
249            custom_policy: policy,
250        };
251        let commit = AllocationBootstrap::new(record.store_mut())
252            .validate_against(
253                recovered,
254                resolved.allocation_snapshot().clone(),
255                &runtime_policy,
256                None,
257            )
258            .map_err(runtime_bootstrap_error_from_bootstrap)?;
259        let (ledger, validated) = commit.into_parts();
260
261        self.persist_ledger_record(record)?;
262        let committed =
263            external_runtime_allocations(validated.confirm_persisted(ledger.current_generation()));
264        self.lifecycle = RuntimeLifecycle::Bootstrapped {
265            committed_allocations: committed,
266            binding: RuntimeBootstrapBinding {
267                source: declarations.clone(),
268                declarations: resolved,
269                policy_identity,
270            },
271        };
272        Ok(())
273    }
274
275    /// Borrow this runtime's committed allocation-open capability.
276    pub const fn committed_allocations(&self) -> Result<&CommittedAllocations, RuntimeOpenError> {
277        match &self.lifecycle {
278            RuntimeLifecycle::Unbootstrapped => Err(RuntimeOpenError::NotBootstrapped),
279            RuntimeLifecycle::Bootstrapped {
280                committed_allocations,
281                ..
282            } => Ok(committed_allocations),
283        }
284    }
285
286    /// Open by durable key using only this runtime's persisted current capability.
287    pub fn open_memory_by_key(
288        &self,
289        stable_key: &str,
290    ) -> Result<RuntimeMemory<M>, RuntimeOpenError> {
291        Ok(self.memory(self.memory_id(stable_key)?))
292    }
293
294    /// Open this runtime's committed memory by stable key and expected ID.
295    pub fn open_memory(
296        &self,
297        stable_key: &str,
298        expected_id: u8,
299    ) -> Result<RuntimeMemory<M>, RuntimeOpenError> {
300        let committed_id = self.memory_id(stable_key)?;
301        if committed_id != expected_id {
302            return Err(RuntimeOpenError::MemoryIdMismatch {
303                stable_key: stable_key.to_string(),
304                committed_id,
305                requested_id: expected_id,
306            });
307        }
308        Ok(self.memory(committed_id))
309    }
310
311    /// Resolve an application key's committed ID without opening memory,
312    /// reading history, or changing the host's policy or bucket configuration.
313    pub fn memory_id(&self, stable_key: &str) -> Result<u8, RuntimeOpenError> {
314        let key = StableKey::parse(stable_key)?;
315        if crate::is_ic_memory_stable_key(key.as_str()) {
316            return Err(RuntimeOpenError::ReservedStableKey {
317                stable_key: stable_key.to_string(),
318            });
319        }
320        let slot = self
321            .committed_allocations()?
322            .slot_for(&key)
323            .ok_or_else(|| RuntimeOpenError::StableKeyNotCommitted(stable_key.to_string()))?;
324        Ok(slot.memory_manager_id()?)
325    }
326
327    fn initialize_ledger_cell<P>(&mut self) -> Result<(), RuntimeBootstrapError<P>> {
328        if self.ledger_cell.is_some() {
329            return Ok(());
330        }
331        let memory = self.memory(MEMORY_MANAGER_LEDGER_ID);
332        crate::validate_stable_cell_ledger_memory(&memory)?;
333        ensure_ledger_cell_capacity(&memory, &StableCellLedgerRecord::default())?;
334        self.ledger_cell = Some(Cell::init(memory, StableCellLedgerRecord::default()));
335        Ok(())
336    }
337
338    fn persist_ledger_record<P>(
339        &mut self,
340        record: StableCellLedgerRecord,
341    ) -> Result<(), RuntimeBootstrapError<P>> {
342        let memory = self.memory(MEMORY_MANAGER_LEDGER_ID);
343        ensure_ledger_cell_capacity(&memory, &record)?;
344        let cell = self
345            .ledger_cell
346            .as_mut()
347            .ok_or(RuntimeStateError::InconsistentLifecycle)?;
348        let _previous = cell.set(record);
349        Ok(())
350    }
351
352    fn memory(&self, id: u8) -> RuntimeMemory<M> {
353        RuntimeMemory {
354            memory: self.memory_manager.get(MemoryId::new(id)),
355            growth: Rc::clone(&self.growth),
356        }
357    }
358
359    fn ledger_record_from_memory(&self) -> Result<StableCellLedgerRecord, StableCellLedgerError> {
360        decode_stable_cell_ledger_record_from_memory(&self.memory(MEMORY_MANAGER_LEDGER_ID))
361    }
362}
363
364impl RuntimeBootstrapBinding {
365    fn validate<P>(
366        &self,
367        declarations: &SealedDeclarationSnapshot,
368        policy_identity: &PolicyIdentity,
369    ) -> Result<(), RuntimeBootstrapError<P>> {
370        if !self.source.shares_storage_with(declarations) {
371            return Err(RuntimeBootstrapError::DeclarationSnapshotMismatch);
372        }
373        if &self.policy_identity != policy_identity {
374            return Err(RuntimeBootstrapError::PolicyIdentityMismatch {
375                established: self.policy_identity.clone(),
376                requested: policy_identity.clone(),
377            });
378        }
379        Ok(())
380    }
381}
382
383fn ensure_ledger_cell_capacity<M: Memory, P>(
384    memory: &RuntimeMemory<M>,
385    record: &StableCellLedgerRecord,
386) -> Result<(), RuntimeBootstrapError<P>> {
387    let value_size = record.to_bytes().len();
388    if value_size > crate::constants::MAX_LEDGER_RECORD_BYTES {
389        return Err(RuntimeBootstrapError::StableCellLedgerWriteTooLarge { value_size });
390    }
391    let value_size_u32 = u32::try_from(value_size)
392        .map_err(|_| RuntimeBootstrapError::StableCellLedgerWriteTooLarge { value_size })?;
393    let required_bytes = STABLE_CELL_VALUE_OFFSET
394        .checked_add(u64::from(value_size_u32))
395        .ok_or(RuntimeBootstrapError::StableCellLedgerWriteTooLarge { value_size })?;
396    let available_bytes = memory.size().saturating_mul(crate::WASM_PAGE_SIZE_BYTES);
397    if required_bytes <= available_bytes {
398        return Ok(());
399    }
400    let grow_by = required_bytes
401        .saturating_sub(available_bytes)
402        .div_ceil(crate::WASM_PAGE_SIZE_BYTES);
403    memory.grow(grow_by)?;
404    Ok(())
405}
406
407fn external_runtime_allocations(committed: CommittedAllocations) -> CommittedAllocations {
408    committed.without_stable_key_prefix(crate::IC_MEMORY_STABLE_KEY_PREFIX)
409}
410
411const fn check_bucket_size(
412    actual: u16,
413    requested: MemoryManagerConfig,
414) -> Result<(), RuntimeConstructionError> {
415    if actual != requested.bucket_size_pages() {
416        return Err(RuntimeConstructionError::BucketSizeMismatch {
417            persisted: actual,
418            requested: requested.bucket_size_pages(),
419        });
420    }
421    Ok(())
422}
ic_memory/runtime/mod.rs

ic_memory/runtime/
mod.rs
