ic_memory/runtime/
mod.rs

1mod allocations;
2mod backing;
3mod config;
4mod default;
5mod diagnostics;
6mod error;
7mod layout;
8mod policy;
9
10#[cfg(test)]
11mod allocation_tests;
12#[cfg(test)]
13#[allow(
14    unsafe_code,
15    reason = "exercise raw reads with valid uninitialized destinations"
16)]
17mod read_tests;
18#[cfg(test)]
19mod tests;
20
21pub use allocations::{
22    AllocationBinding, AllocationRangeClaim, MemoryAllocation, MemoryAllocations,
23};
24pub use backing::RuntimeMemory;
25pub use config::MemoryManagerConfig;
26pub use default::{
27    bootstrap_default_memory_manager, bootstrap_default_memory_manager_with_config,
28    bootstrap_default_memory_manager_with_policy, committed_allocations,
29    default_memory_manager_commit_recovery_diagnostic, default_memory_manager_diagnostic_export,
30    default_memory_manager_doctor_report, default_memory_manager_doctor_report_with_policy,
31    default_memory_manager_memory_allocations, is_default_memory_manager_bootstrapped,
32    open_default_memory_manager_memory,
33};
34pub use error::{
35    RuntimeBootstrapError, RuntimeConstructionError, RuntimeDiagnosticError, RuntimeOpenError,
36    RuntimePolicyError, RuntimeStateError,
37};
38pub use layout::MemoryManagerLayoutError;
39pub use policy::GenericRangePolicy;
40
41use self::policy::{RuntimeMemoryManagerPolicy, runtime_bootstrap_error_from_bootstrap};
42use crate::{
43    AllocationBootstrap, AllocationHistory, AllocationLedger, AllocationPolicy,
44    CommittedAllocations, PolicyIdentity, RuntimeBootstrapPolicy, STABLE_CELL_VALUE_OFFSET,
45    StableCellLedgerError, StableCellLedgerRecord, StableKey, registry::SealedDeclarationSnapshot,
46    slot::MEMORY_MANAGER_LEDGER_ID, stable_cell::decode_stable_cell_ledger_record_from_memory,
47};
48use ic_stable_structures::{
49    Cell, Memory, Storable,
50    memory_manager::{MemoryId, MemoryManager},
51};
52
53use std::rc::Rc;
54
55type LedgerCell<M> = Cell<StableCellLedgerRecord, RuntimeMemory<M>>;
56
57enum RuntimeLifecycle {
58    Unbootstrapped,
59    Bootstrapped {
60        committed_allocations: CommittedAllocations,
61        binding: RuntimeBootstrapBinding,
62    },
63}
64
65struct RuntimeBootstrapBinding {
66    declarations: SealedDeclarationSnapshot,
67    policy_identity: PolicyIdentity,
68}
69
70///
71/// MemoryRuntime
72///
73/// Canonical owner of allocation bootstrap state for one backing memory.
74///
75/// The runtime owns its `MemoryManager`, allocation-ledger cell, bootstrap
76/// lifecycle, committed allocation capability, opens, and diagnostics. Static
77/// linked-program declarations are supplied separately as one immutable
78/// [`SealedDeclarationSnapshot`].
79///
80/// `M` needs only [`Memory`]. The runtime does not require the backing memory
81/// to be `Send`, `Sync`, `Clone`, or `'static`.
82///
83
84pub struct MemoryRuntime<M: Memory> {
85    memory_manager: MemoryManager<Rc<M>>,
86    // Share the owned backing using upstream's Memory implementation for Rc.
87    // Only the manager writes it; attribution borrows it read-only.
88    backing: Rc<M>,
89    bucket_size_pages: u16,
90    ledger_cell: Option<LedgerCell<M>>,
91    lifecycle: RuntimeLifecycle,
92}
93
94impl<M: Memory> MemoryRuntime<M> {
95    /// Construct an unbootstrapped runtime without overwriting foreign memory.
96    ///
97    /// Empty backing memory is initialized as an
98    /// `ic_stable_structures::MemoryManager`. Nonempty memory must pass bounded
99    /// validation of the current manager header, bucket table, and extents; otherwise
100    /// construction returns a typed error before the manager can
101    /// write its header or allocation table. A pre-grown blank memory is
102    /// nonempty and is therefore rejected rather than assumed disposable.
103    ///
104    /// # Errors
105    ///
106    /// Returns [`RuntimeConstructionError::ForeignMemory`] for nonempty memory
107    /// without `MemoryManager` magic, or
108    /// [`RuntimeConstructionError::UnsupportedMemoryManagerVersion`] when the
109    /// magic is recognized but the layout version is not current. Invalid
110    /// metadata returns [`RuntimeConstructionError::Layout`]. Reopening honors
111    /// the actual persisted bucket size; only fresh memory uses 128 pages.
112    pub fn new(memory: M) -> Result<Self, RuntimeConstructionError> {
113        Self::construct(memory, None)
114    }
115
116    /// Construct with an explicit immutable bucket policy. Existing memory must
117    /// match exactly; mismatches fail before manager initialization or writes.
118    pub fn new_with_config(
119        memory: M,
120        config: MemoryManagerConfig,
121    ) -> Result<Self, RuntimeConstructionError> {
122        Self::construct(memory, Some(config))
123    }
124
125    fn construct(
126        memory: M,
127        requested: Option<MemoryManagerConfig>,
128    ) -> Result<Self, RuntimeConstructionError> {
129        if cfg!(target_endian = "big") {
130            return Err(MemoryManagerLayoutError::UnsupportedByteOrder.into());
131        }
132        let bucket_size_pages = if memory.size() == 0 {
133            requested.unwrap_or_default().bucket_size_pages()
134        } else {
135            let actual = layout::read(&memory)?.bucket_pages;
136            if let Some(config) = requested {
137                check_bucket_size(actual, config)?;
138            }
139            actual
140        };
141        let backing = Rc::new(memory);
142        Ok(Self {
143            memory_manager: MemoryManager::init_with_bucket_size(
144                Rc::clone(&backing),
145                bucket_size_pages,
146            ),
147            backing,
148            bucket_size_pages,
149            ledger_cell: None,
150            lifecycle: RuntimeLifecycle::Unbootstrapped,
151        })
152    }
153
154    /// Return the actual policy bound to this runtime's sole manager.
155    #[must_use]
156    pub const fn memory_manager_config(&self) -> MemoryManagerConfig {
157        // Construction has already validated the nonzero persisted setting.
158        MemoryManagerConfig::from_validated(self.bucket_size_pages)
159    }
160
161    /// Return whether this runtime has published committed allocation authority.
162    #[must_use]
163    pub const fn is_bootstrapped(&self) -> bool {
164        matches!(self.lifecycle, RuntimeLifecycle::Bootstrapped { .. })
165    }
166
167    /// Bootstrap this backing memory from one immutable declaration snapshot.
168    ///
169    /// Recovery, policy evaluation, staging, persistence, and capability
170    /// publication are local to this runtime. A repeated call is idempotent
171    /// only when the sealed declaration snapshot and
172    /// [`RuntimeBootstrapPolicy::runtime_bootstrap_identity`] match the
173    /// successful bootstrap. A mismatch returns a typed error without
174    /// advancing the durable generation or re-evaluating policy.
175    pub fn bootstrap<P: RuntimeBootstrapPolicy>(
176        &mut self,
177        declarations: &SealedDeclarationSnapshot,
178        policy: &P,
179    ) -> Result<&CommittedAllocations, RuntimeBootstrapError<P::Error>> {
180        let policy_identity = policy.runtime_bootstrap_identity()?;
181        let already_bootstrapped = match &self.lifecycle {
182            RuntimeLifecycle::Unbootstrapped => false,
183            RuntimeLifecycle::Bootstrapped { binding, .. } => {
184                binding.validate(declarations, &policy_identity)?;
185                true
186            }
187        };
188        if !already_bootstrapped {
189            self.bootstrap_unbootstrapped(declarations, policy, policy_identity)?;
190        }
191        match &self.lifecycle {
192            RuntimeLifecycle::Bootstrapped {
193                committed_allocations,
194                ..
195            } => Ok(committed_allocations),
196            RuntimeLifecycle::Unbootstrapped => Err(RuntimeBootstrapError::State(
197                RuntimeStateError::InconsistentLifecycle,
198            )),
199        }
200    }
201
202    fn bootstrap_unbootstrapped<P: AllocationPolicy>(
203        &mut self,
204        declarations: &SealedDeclarationSnapshot,
205        policy: &P,
206        policy_identity: PolicyIdentity,
207    ) -> Result<(), RuntimeBootstrapError<P::Error>> {
208        self.initialize_ledger_cell()?;
209        let mut record = self
210            .ledger_cell
211            .as_ref()
212            .map(|cell| cell.get().clone())
213            .ok_or(RuntimeStateError::InconsistentLifecycle)?;
214        let runtime_policy = RuntimeMemoryManagerPolicy {
215            declarations,
216            custom_policy: policy,
217        };
218        let genesis = AllocationLedger::new(0, AllocationHistory::default())?;
219        let commit = AllocationBootstrap::new(record.store_mut())
220            .initialize_validate_and_commit(
221                &genesis,
222                declarations.allocation_snapshot().clone(),
223                &runtime_policy,
224                None,
225            )
226            .map_err(runtime_bootstrap_error_from_bootstrap)?;
227        let (ledger, validated) = commit.into_parts();
228
229        self.persist_ledger_record(record)?;
230        let committed =
231            external_runtime_allocations(validated.confirm_persisted(ledger.current_generation()));
232        self.lifecycle = RuntimeLifecycle::Bootstrapped {
233            committed_allocations: committed,
234            binding: RuntimeBootstrapBinding {
235                declarations: declarations.clone(),
236                policy_identity,
237            },
238        };
239        Ok(())
240    }
241
242    /// Borrow this runtime's committed allocation-open capability.
243    pub const fn committed_allocations(&self) -> Result<&CommittedAllocations, RuntimeOpenError> {
244        match &self.lifecycle {
245            RuntimeLifecycle::Unbootstrapped => Err(RuntimeOpenError::NotBootstrapped),
246            RuntimeLifecycle::Bootstrapped {
247                committed_allocations,
248                ..
249            } => Ok(committed_allocations),
250        }
251    }
252
253    /// Open this runtime's committed memory by stable key and expected ID.
254    pub fn open_memory(
255        &self,
256        stable_key: &str,
257        expected_id: u8,
258    ) -> Result<RuntimeMemory<M>, RuntimeOpenError> {
259        let key = StableKey::parse(stable_key)?;
260        if crate::is_ic_memory_stable_key(key.as_str()) {
261            return Err(RuntimeOpenError::ReservedStableKey {
262                stable_key: stable_key.to_string(),
263            });
264        }
265        let committed = self.committed_allocations()?;
266        let slot = committed
267            .slot_for(&key)
268            .ok_or_else(|| RuntimeOpenError::StableKeyNotCommitted(stable_key.to_string()))?;
269        let committed_id = slot.memory_manager_id()?;
270        if committed_id != expected_id {
271            return Err(RuntimeOpenError::MemoryIdMismatch {
272                stable_key: stable_key.to_string(),
273                committed_id,
274                requested_id: expected_id,
275            });
276        }
277        Ok(self.memory(expected_id))
278    }
279
280    fn initialize_ledger_cell<P>(&mut self) -> Result<(), RuntimeBootstrapError<P>> {
281        if self.ledger_cell.is_some() {
282            return Ok(());
283        }
284        let memory = self.memory(MEMORY_MANAGER_LEDGER_ID);
285        crate::validate_stable_cell_ledger_memory(&memory)?;
286        ensure_ledger_cell_capacity(&memory, &StableCellLedgerRecord::default())?;
287        self.ledger_cell = Some(Cell::init(memory, StableCellLedgerRecord::default()));
288        Ok(())
289    }
290
291    fn persist_ledger_record<P>(
292        &mut self,
293        record: StableCellLedgerRecord,
294    ) -> Result<(), RuntimeBootstrapError<P>> {
295        let memory = self.memory(MEMORY_MANAGER_LEDGER_ID);
296        ensure_ledger_cell_capacity(&memory, &record)?;
297        let cell = self
298            .ledger_cell
299            .as_mut()
300            .ok_or(RuntimeStateError::InconsistentLifecycle)?;
301        let _previous = cell.set(record);
302        Ok(())
303    }
304
305    fn memory(&self, id: u8) -> RuntimeMemory<M> {
306        RuntimeMemory(self.memory_manager.get(MemoryId::new(id)))
307    }
308
309    fn ledger_record_from_memory(&self) -> Result<StableCellLedgerRecord, StableCellLedgerError> {
310        decode_stable_cell_ledger_record_from_memory(&self.memory(MEMORY_MANAGER_LEDGER_ID))
311    }
312}
313
314impl RuntimeBootstrapBinding {
315    fn validate<P>(
316        &self,
317        declarations: &SealedDeclarationSnapshot,
318        policy_identity: &PolicyIdentity,
319    ) -> Result<(), RuntimeBootstrapError<P>> {
320        if !self.declarations.shares_storage_with(declarations) {
321            return Err(RuntimeBootstrapError::DeclarationSnapshotMismatch);
322        }
323        if &self.policy_identity != policy_identity {
324            return Err(RuntimeBootstrapError::PolicyIdentityMismatch {
325                established: self.policy_identity.clone(),
326                requested: policy_identity.clone(),
327            });
328        }
329        Ok(())
330    }
331}
332
333fn ensure_ledger_cell_capacity<M: Memory, P>(
334    memory: &RuntimeMemory<M>,
335    record: &StableCellLedgerRecord,
336) -> Result<(), RuntimeBootstrapError<P>> {
337    let value_size = record.to_bytes().len();
338    let value_size_u32 = u32::try_from(value_size)
339        .map_err(|_| RuntimeBootstrapError::StableCellLedgerWriteTooLarge { value_size })?;
340    let required_bytes = STABLE_CELL_VALUE_OFFSET
341        .checked_add(u64::from(value_size_u32))
342        .ok_or(RuntimeBootstrapError::StableCellLedgerWriteTooLarge { value_size })?;
343    let available_bytes = memory.size().saturating_mul(crate::WASM_PAGE_SIZE_BYTES);
344    if required_bytes <= available_bytes {
345        return Ok(());
346    }
347    let grow_by = required_bytes
348        .saturating_sub(available_bytes)
349        .div_ceil(crate::WASM_PAGE_SIZE_BYTES);
350    if memory.grow(grow_by) < 0 {
351        return Err(RuntimeBootstrapError::StableCellLedgerWriteTooLarge { value_size });
352    }
353    Ok(())
354}
355
356fn external_runtime_allocations(committed: CommittedAllocations) -> CommittedAllocations {
357    committed.without_stable_key_prefix(crate::IC_MEMORY_STABLE_KEY_PREFIX)
358}
359
360const fn check_bucket_size(
361    actual: u16,
362    requested: MemoryManagerConfig,
363) -> Result<(), RuntimeConstructionError> {
364    if actual != requested.bucket_size_pages() {
365        return Err(RuntimeConstructionError::BucketSizeMismatch {
366            persisted: actual,
367            requested: requested.bucket_size_pages(),
368        });
369    }
370    Ok(())
371}
ic_memory/runtime/mod.rs

ic_memory/runtime/
mod.rs
