ic_memory/runtime/
mod.rs

1mod allocations;
2mod backing;
3mod config;
4mod default;
5mod diagnostics;
6mod error;
7mod layout;
8mod policy;
9
10#[cfg(test)]
11mod allocation_tests;
12#[cfg(test)]
13#[allow(
14    unsafe_code,
15    reason = "exercise raw reads with valid uninitialized destinations"
16)]
17mod read_tests;
18#[cfg(test)]
19mod tests;
20
21pub use allocations::{
22    AllocationBinding, AllocationRangeClaim, MemoryAllocation, MemoryAllocations,
23};
24pub use backing::RuntimeMemory;
25pub use config::MemoryManagerConfig;
26pub use default::{
27    bootstrap_default_memory_manager, bootstrap_default_memory_manager_with_config,
28    bootstrap_default_memory_manager_with_policy, committed_allocations,
29    default_memory_manager_commit_recovery_diagnostic, default_memory_manager_diagnostic_export,
30    default_memory_manager_doctor_report, default_memory_manager_doctor_report_with_policy,
31    default_memory_manager_memory_allocations, is_default_memory_manager_bootstrapped,
32    open_default_memory_manager_memory,
33};
34pub use error::{
35    RuntimeBootstrapError, RuntimeConstructionError, RuntimeDiagnosticError, RuntimeOpenError,
36    RuntimePolicyError, RuntimeStateError,
37};
38pub use layout::MemoryManagerLayoutError;
39
40use self::policy::{RuntimeMemoryManagerPolicy, runtime_bootstrap_error_from_bootstrap};
41use crate::{
42    AllocationBootstrap, AllocationHistory, AllocationLedger, AllocationPolicy,
43    CommittedAllocations, PolicyIdentity, RuntimeBootstrapPolicy, STABLE_CELL_VALUE_OFFSET,
44    StableCellLedgerError, StableCellLedgerRecord, StableKey, registry::SealedDeclarationSnapshot,
45    slot::MEMORY_MANAGER_LEDGER_ID, stable_cell::decode_stable_cell_ledger_record_from_memory,
46};
47use ic_stable_structures::{
48    Cell, Memory, Storable,
49    memory_manager::{MemoryId, MemoryManager},
50};
51
52use std::rc::Rc;
53
54type LedgerCell<M> = Cell<StableCellLedgerRecord, RuntimeMemory<M>>;
55
56enum RuntimeLifecycle {
57    Unbootstrapped,
58    Bootstrapped {
59        committed_allocations: CommittedAllocations,
60        binding: RuntimeBootstrapBinding,
61    },
62}
63
64struct RuntimeBootstrapBinding {
65    declarations: SealedDeclarationSnapshot,
66    policy_identity: PolicyIdentity,
67}
68
69///
70/// MemoryRuntime
71///
72/// Canonical owner of allocation bootstrap state for one backing memory.
73///
74/// The runtime owns its `MemoryManager`, allocation-ledger cell, bootstrap
75/// lifecycle, committed allocation capability, opens, and diagnostics. Static
76/// linked-program declarations are supplied separately as one immutable
77/// [`SealedDeclarationSnapshot`].
78///
79/// `M` needs only [`Memory`]. The runtime does not require the backing memory
80/// to be `Send`, `Sync`, `Clone`, or `'static`.
81///
82
83pub struct MemoryRuntime<M: Memory> {
84    memory_manager: MemoryManager<Rc<M>>,
85    // Share the owned backing using upstream's Memory implementation for Rc.
86    // Only the manager writes it; attribution borrows it read-only.
87    backing: Rc<M>,
88    bucket_size_pages: u16,
89    ledger_cell: Option<LedgerCell<M>>,
90    lifecycle: RuntimeLifecycle,
91}
92
93impl<M: Memory> MemoryRuntime<M> {
94    /// Construct an unbootstrapped runtime without overwriting foreign memory.
95    ///
96    /// Empty backing memory is initialized as an
97    /// `ic_stable_structures::MemoryManager`. Nonempty memory must pass bounded
98    /// validation of the current manager header, bucket table, and extents; otherwise
99    /// construction returns a typed error before the manager can
100    /// write its header or allocation table. A pre-grown blank memory is
101    /// nonempty and is therefore rejected rather than assumed disposable.
102    ///
103    /// # Errors
104    ///
105    /// Returns [`RuntimeConstructionError::ForeignMemory`] for nonempty memory
106    /// without `MemoryManager` magic, or
107    /// [`RuntimeConstructionError::UnsupportedMemoryManagerVersion`] when the
108    /// magic is recognized but the layout version is not current. Invalid
109    /// metadata returns [`RuntimeConstructionError::Layout`]. Reopening honors
110    /// the actual persisted bucket size; only fresh memory uses 128 pages.
111    pub fn new(memory: M) -> Result<Self, RuntimeConstructionError> {
112        Self::construct(memory, None)
113    }
114
115    /// Construct with an explicit immutable bucket policy. Existing memory must
116    /// match exactly; mismatches fail before manager initialization or writes.
117    pub fn new_with_config(
118        memory: M,
119        config: MemoryManagerConfig,
120    ) -> Result<Self, RuntimeConstructionError> {
121        Self::construct(memory, Some(config))
122    }
123
124    fn construct(
125        memory: M,
126        requested: Option<MemoryManagerConfig>,
127    ) -> Result<Self, RuntimeConstructionError> {
128        if cfg!(target_endian = "big") {
129            return Err(MemoryManagerLayoutError::UnsupportedByteOrder.into());
130        }
131        let bucket_size_pages = if memory.size() == 0 {
132            requested.unwrap_or_default().bucket_size_pages()
133        } else {
134            let actual = layout::read(&memory)?.bucket_pages;
135            if let Some(config) = requested {
136                check_bucket_size(actual, config)?;
137            }
138            actual
139        };
140        let backing = Rc::new(memory);
141        Ok(Self {
142            memory_manager: MemoryManager::init_with_bucket_size(
143                Rc::clone(&backing),
144                bucket_size_pages,
145            ),
146            backing,
147            bucket_size_pages,
148            ledger_cell: None,
149            lifecycle: RuntimeLifecycle::Unbootstrapped,
150        })
151    }
152
153    /// Return the actual policy bound to this runtime's sole manager.
154    #[must_use]
155    pub const fn memory_manager_config(&self) -> MemoryManagerConfig {
156        // Construction has already validated the nonzero persisted setting.
157        MemoryManagerConfig::from_validated(self.bucket_size_pages)
158    }
159
160    /// Return whether this runtime has published committed allocation authority.
161    #[must_use]
162    pub const fn is_bootstrapped(&self) -> bool {
163        matches!(self.lifecycle, RuntimeLifecycle::Bootstrapped { .. })
164    }
165
166    /// Bootstrap this backing memory from one immutable declaration snapshot.
167    ///
168    /// Recovery, policy evaluation, staging, persistence, and capability
169    /// publication are local to this runtime. A repeated call is idempotent
170    /// only when the sealed declaration snapshot and
171    /// [`RuntimeBootstrapPolicy::runtime_bootstrap_identity`] match the
172    /// successful bootstrap. A mismatch returns a typed error without
173    /// advancing the durable generation or re-evaluating policy.
174    pub fn bootstrap<P: RuntimeBootstrapPolicy>(
175        &mut self,
176        declarations: &SealedDeclarationSnapshot,
177        policy: &P,
178    ) -> Result<&CommittedAllocations, RuntimeBootstrapError<P::Error>> {
179        let policy_identity = policy.runtime_bootstrap_identity()?;
180        let already_bootstrapped = match &self.lifecycle {
181            RuntimeLifecycle::Unbootstrapped => false,
182            RuntimeLifecycle::Bootstrapped { binding, .. } => {
183                binding.validate(declarations, &policy_identity)?;
184                true
185            }
186        };
187        if !already_bootstrapped {
188            self.bootstrap_unbootstrapped(declarations, policy, policy_identity)?;
189        }
190        match &self.lifecycle {
191            RuntimeLifecycle::Bootstrapped {
192                committed_allocations,
193                ..
194            } => Ok(committed_allocations),
195            RuntimeLifecycle::Unbootstrapped => Err(RuntimeBootstrapError::State(
196                RuntimeStateError::InconsistentLifecycle,
197            )),
198        }
199    }
200
201    fn bootstrap_unbootstrapped<P: AllocationPolicy>(
202        &mut self,
203        declarations: &SealedDeclarationSnapshot,
204        policy: &P,
205        policy_identity: PolicyIdentity,
206    ) -> Result<(), RuntimeBootstrapError<P::Error>> {
207        self.initialize_ledger_cell()?;
208        let mut record = self
209            .ledger_cell
210            .as_ref()
211            .map(|cell| cell.get().clone())
212            .ok_or(RuntimeStateError::InconsistentLifecycle)?;
213        let runtime_policy = RuntimeMemoryManagerPolicy {
214            declarations,
215            custom_policy: policy,
216        };
217        let genesis = AllocationLedger::new(0, AllocationHistory::default())?;
218        let commit = AllocationBootstrap::new(record.store_mut())
219            .initialize_validate_and_commit(
220                &genesis,
221                declarations.allocation_snapshot().clone(),
222                &runtime_policy,
223                None,
224            )
225            .map_err(runtime_bootstrap_error_from_bootstrap)?;
226        let (ledger, validated) = commit.into_parts();
227
228        self.persist_ledger_record(record)?;
229        let committed =
230            external_runtime_allocations(validated.confirm_persisted(ledger.current_generation()));
231        self.lifecycle = RuntimeLifecycle::Bootstrapped {
232            committed_allocations: committed,
233            binding: RuntimeBootstrapBinding {
234                declarations: declarations.clone(),
235                policy_identity,
236            },
237        };
238        Ok(())
239    }
240
241    /// Borrow this runtime's committed allocation-open capability.
242    pub const fn committed_allocations(&self) -> Result<&CommittedAllocations, RuntimeOpenError> {
243        match &self.lifecycle {
244            RuntimeLifecycle::Unbootstrapped => Err(RuntimeOpenError::NotBootstrapped),
245            RuntimeLifecycle::Bootstrapped {
246                committed_allocations,
247                ..
248            } => Ok(committed_allocations),
249        }
250    }
251
252    /// Open this runtime's committed memory by stable key and expected ID.
253    pub fn open_memory(
254        &self,
255        stable_key: &str,
256        expected_id: u8,
257    ) -> Result<RuntimeMemory<M>, RuntimeOpenError> {
258        let key = StableKey::parse(stable_key)?;
259        if crate::is_ic_memory_stable_key(key.as_str()) {
260            return Err(RuntimeOpenError::ReservedStableKey {
261                stable_key: stable_key.to_string(),
262            });
263        }
264        let committed = self.committed_allocations()?;
265        let slot = committed
266            .slot_for(&key)
267            .ok_or_else(|| RuntimeOpenError::StableKeyNotCommitted(stable_key.to_string()))?;
268        let committed_id = slot.memory_manager_id()?;
269        if committed_id != expected_id {
270            return Err(RuntimeOpenError::MemoryIdMismatch {
271                stable_key: stable_key.to_string(),
272                committed_id,
273                requested_id: expected_id,
274            });
275        }
276        Ok(self.memory(expected_id))
277    }
278
279    fn initialize_ledger_cell<P>(&mut self) -> Result<(), RuntimeBootstrapError<P>> {
280        if self.ledger_cell.is_some() {
281            return Ok(());
282        }
283        let memory = self.memory(MEMORY_MANAGER_LEDGER_ID);
284        crate::validate_stable_cell_ledger_memory(&memory)?;
285        ensure_ledger_cell_capacity(&memory, &StableCellLedgerRecord::default())?;
286        self.ledger_cell = Some(Cell::init(memory, StableCellLedgerRecord::default()));
287        Ok(())
288    }
289
290    fn persist_ledger_record<P>(
291        &mut self,
292        record: StableCellLedgerRecord,
293    ) -> Result<(), RuntimeBootstrapError<P>> {
294        let memory = self.memory(MEMORY_MANAGER_LEDGER_ID);
295        ensure_ledger_cell_capacity(&memory, &record)?;
296        let cell = self
297            .ledger_cell
298            .as_mut()
299            .ok_or(RuntimeStateError::InconsistentLifecycle)?;
300        let _previous = cell.set(record);
301        Ok(())
302    }
303
304    fn memory(&self, id: u8) -> RuntimeMemory<M> {
305        RuntimeMemory(self.memory_manager.get(MemoryId::new(id)))
306    }
307
308    fn ledger_record_from_memory(&self) -> Result<StableCellLedgerRecord, StableCellLedgerError> {
309        decode_stable_cell_ledger_record_from_memory(&self.memory(MEMORY_MANAGER_LEDGER_ID))
310    }
311}
312
313impl RuntimeBootstrapBinding {
314    fn validate<P>(
315        &self,
316        declarations: &SealedDeclarationSnapshot,
317        policy_identity: &PolicyIdentity,
318    ) -> Result<(), RuntimeBootstrapError<P>> {
319        if !self.declarations.shares_storage_with(declarations) {
320            return Err(RuntimeBootstrapError::DeclarationSnapshotMismatch);
321        }
322        if &self.policy_identity != policy_identity {
323            return Err(RuntimeBootstrapError::PolicyIdentityMismatch {
324                established: self.policy_identity.clone(),
325                requested: policy_identity.clone(),
326            });
327        }
328        Ok(())
329    }
330}
331
332fn ensure_ledger_cell_capacity<M: Memory, P>(
333    memory: &RuntimeMemory<M>,
334    record: &StableCellLedgerRecord,
335) -> Result<(), RuntimeBootstrapError<P>> {
336    let value_size = record.to_bytes().len();
337    let value_size_u32 = u32::try_from(value_size)
338        .map_err(|_| RuntimeBootstrapError::StableCellLedgerWriteTooLarge { value_size })?;
339    let required_bytes = STABLE_CELL_VALUE_OFFSET
340        .checked_add(u64::from(value_size_u32))
341        .ok_or(RuntimeBootstrapError::StableCellLedgerWriteTooLarge { value_size })?;
342    let available_bytes = memory.size().saturating_mul(crate::WASM_PAGE_SIZE_BYTES);
343    if required_bytes <= available_bytes {
344        return Ok(());
345    }
346    let grow_by = required_bytes
347        .saturating_sub(available_bytes)
348        .div_ceil(crate::WASM_PAGE_SIZE_BYTES);
349    if memory.grow(grow_by) < 0 {
350        return Err(RuntimeBootstrapError::StableCellLedgerWriteTooLarge { value_size });
351    }
352    Ok(())
353}
354
355fn external_runtime_allocations(committed: CommittedAllocations) -> CommittedAllocations {
356    committed.without_stable_key_prefix(crate::IC_MEMORY_STABLE_KEY_PREFIX)
357}
358
359const fn check_bucket_size(
360    actual: u16,
361    requested: MemoryManagerConfig,
362) -> Result<(), RuntimeConstructionError> {
363    if actual != requested.bucket_size_pages() {
364        return Err(RuntimeConstructionError::BucketSizeMismatch {
365            persisted: actual,
366            requested: requested.bucket_size_pages(),
367        });
368    }
369    Ok(())
370}
ic_memory/runtime/mod.rs

ic_memory/runtime/
mod.rs
