ic_memory/runtime/
mod.rs

1mod admission;
2#[cfg(test)]
3mod admission_tests;
4mod adoption;
5#[cfg(test)]
6mod adoption_tests;
7mod allocations;
8mod backing;
9mod config;
10mod diagnostics;
11mod error;
12mod layout;
13mod policy;
14
15#[cfg(test)]
16mod allocation_tests;
17#[cfg(test)]
18mod growth_tests;
19#[cfg(test)]
20#[expect(
21    unsafe_code,
22    reason = "exercise raw reads with valid uninitialized destinations"
23)]
24mod read_tests;
25#[cfg(test)]
26mod request_tests;
27#[cfg(test)]
28mod tests;
29
30pub use admission::{BootstrapAdmission, BootstrapAdmissionError};
31pub use adoption::RuntimeAdoptionError;
32
33pub use allocations::{
34    AllocationBinding, MemoryAllocation, MemoryAllocationSummary, MemoryAllocations,
35    MemoryBindingSummary,
36};
37pub use backing::RuntimeMemory;
38pub use config::MemoryManagerConfig;
39pub use error::{
40    MemoryResolutionError, RuntimeBootstrapError, RuntimeConstructionError, RuntimeDiagnosticError,
41    RuntimeGrowError, RuntimeOpenError,
42};
43pub use layout::MemoryManagerLayoutError;
44pub use policy::GenericAllocationPolicy;
45
46use crate::{
47    CommittedAllocations, LedgerCommitStore, PolicyIdentity, RuntimeBootstrapPolicy,
48    STABLE_CELL_VALUE_OFFSET, StableCellLedgerError,
49    requests::SealedDeclarationSnapshot,
50    slot::MEMORY_MANAGER_LEDGER_ID,
51    stable_cell::{EncodedLedgerRecord, decode_stable_cell_commit_store_from_memory},
52};
53use ic_stable_structures::{
54    Cell, Memory,
55    memory_manager::{MemoryId, MemoryManager},
56};
57
58use std::rc::Rc;
59
60enum RuntimeLifecycle {
61    Unbootstrapped,
62    Bootstrapped {
63        committed_allocations: CommittedAllocations,
64        binding: RuntimeBootstrapBinding,
65    },
66}
67
68struct RuntimeBootstrapBinding {
69    source: SealedDeclarationSnapshot,
70    policy_identity: PolicyIdentity,
71    pool: crate::MemoryAllocationPool,
72}
73
74///
75/// MemoryRuntime
76///
77/// Canonical owner of allocation bootstrap state for one backing memory.
78///
79/// The runtime owns its `MemoryManager`, allocation-ledger persistence, bootstrap
80/// lifecycle, committed allocation capability, opens, and diagnostics. Static
81/// linked-program declarations are supplied separately as one immutable
82/// [`SealedDeclarationSnapshot`].
83///
84/// Each bootstrap attempt fallibly decodes its ledger record from persisted
85/// memory. Capacity-checked writes use `Cell`; diagnostics also read persisted
86/// memory directly.
87///
88/// `M` needs only [`Memory`]. The runtime does not require the backing memory
89/// to be `Send`, `Sync`, `Clone`, or `'static`.
90///
91
92pub struct MemoryRuntime<M: Memory> {
93    memory_manager: MemoryManager<Rc<M>>,
94    // Shared backing, immutable geometry and live accounting belong to growth.
95    // Handles reserve physical capacity; the manager owns bucket metadata.
96    growth: Rc<backing::GrowthState<M>>,
97    lifecycle: RuntimeLifecycle,
98}
99
100impl<M: Memory> MemoryRuntime<M> {
101    /// Construct an unbootstrapped runtime without overwriting foreign memory.
102    ///
103    /// Empty backing memory is initialized as an
104    /// `ic_stable_structures::MemoryManager`. Nonempty memory must pass bounded
105    /// validation of the current manager header, bucket table, and extents; otherwise
106    /// construction returns a typed error before the manager can
107    /// write its header or allocation table. A pre-grown blank memory is
108    /// nonempty and is therefore rejected rather than assumed disposable.
109    ///
110    /// # Errors
111    ///
112    /// Returns [`RuntimeConstructionError::ForeignMemory`] for nonempty memory
113    /// without `MemoryManager` magic, or
114    /// [`RuntimeConstructionError::UnsupportedMemoryManagerVersion`] when the
115    /// magic is recognized but the layout version is not current. Invalid
116    /// metadata returns [`RuntimeConstructionError::Layout`]. Refused fresh
117    /// metadata growth returns [`RuntimeConstructionError::Growth`] without
118    /// writes, allowing the same backing memory to be retried. Reopening honors
119    /// the actual persisted bucket size; only fresh memory uses 128 pages.
120    pub fn new(memory: M) -> Result<Self, RuntimeConstructionError> {
121        Self::construct(memory, None)
122    }
123
124    /// Construct with an explicit immutable bucket policy. Existing memory must
125    /// match exactly; mismatches fail before manager initialization or writes.
126    ///
127    /// # Errors
128    ///
129    /// Returns the construction errors described by [`Self::new`], or
130    /// [`RuntimeConstructionError::BucketSizeMismatch`] when existing geometry
131    /// differs from `config`.
132    pub fn new_with_config(
133        memory: M,
134        config: MemoryManagerConfig,
135    ) -> Result<Self, RuntimeConstructionError> {
136        Self::construct(memory, Some(config))
137    }
138
139    fn construct(
140        memory: M,
141        requested: Option<MemoryManagerConfig>,
142    ) -> Result<Self, RuntimeConstructionError> {
143        if cfg!(target_endian = "big") {
144            return Err(MemoryManagerLayoutError::UnsupportedByteOrder.into());
145        }
146        let (bucket_size_pages, allocated_buckets) = if memory.size() == 0 {
147            // Reserve the metadata page before the dependency writes its header.
148            // This fresh allocation is owned here; caller-supplied nonempty
149            // memory still passes the layout checks below before any writes.
150            if memory.grow(1) == -1 {
151                return Err(RuntimeGrowError::BackingRefused {
152                    additional_pages: 1,
153                }
154                .into());
155            }
156            (requested.unwrap_or_default().bucket_size_pages(), 0)
157        } else {
158            let measured = layout::read(&memory)?;
159            let actual = measured.bucket_pages;
160            if let Some(config) = requested {
161                check_bucket_size(actual, config)?;
162            }
163            (actual, measured.allocated_buckets)
164        };
165        let backing = Rc::new(memory);
166        let growth = Rc::new(backing::GrowthState {
167            backing: Rc::clone(&backing),
168            bucket_size_pages,
169            allocated_buckets: std::cell::RefCell::new(allocated_buckets),
170        });
171        Ok(Self {
172            memory_manager: MemoryManager::init_with_bucket_size(
173                Rc::clone(&backing),
174                bucket_size_pages,
175            ),
176            growth,
177            lifecycle: RuntimeLifecycle::Unbootstrapped,
178        })
179    }
180
181    /// Return the immutable bucket configuration bound to this runtime's manager.
182    #[must_use]
183    pub fn memory_manager_config(&self) -> MemoryManagerConfig {
184        // Construction has already validated the nonzero persisted setting.
185        MemoryManagerConfig::from_validated(self.growth.bucket_size_pages)
186    }
187
188    /// Return whether this runtime has published committed allocation authority.
189    #[must_use]
190    pub const fn is_bootstrapped(&self) -> bool {
191        matches!(self.lifecycle, RuntimeLifecycle::Bootstrapped { .. })
192    }
193
194    /// Bootstrap this backing memory from one immutable declaration snapshot.
195    ///
196    /// Recovery, metadata admission, logical resolution, policy evaluation,
197    /// staging, persistence and capability publication are local to this runtime.
198    /// A repeated call is idempotent only when the sealed declaration snapshot,
199    /// canonical host pool and [`RuntimeBootstrapPolicy::runtime_bootstrap_identity`] match the
200    /// successful bootstrap. A mismatch returns a typed error without
201    /// advancing the durable generation or re-evaluating policy.
202    /// Independently sealed snapshots match when their canonical contents are equal.
203    ///
204    /// # Panics
205    ///
206    /// Panics if a private runtime or encoding invariant is broken, or backing
207    /// memory or a policy callback panics.
208    pub fn bootstrap<P: RuntimeBootstrapPolicy>(
209        &mut self,
210        declarations: &SealedDeclarationSnapshot,
211        pool: &crate::MemoryAllocationPool,
212        policy: &P,
213    ) -> Result<&CommittedAllocations, RuntimeBootstrapError<P::Error>> {
214        let policy_identity = policy.runtime_bootstrap_identity()?;
215        match &self.lifecycle {
216            RuntimeLifecycle::Unbootstrapped => {
217                self.bootstrap_unbootstrapped(declarations, pool, policy, policy_identity)?;
218            }
219            RuntimeLifecycle::Bootstrapped { binding, .. } => {
220                binding.validate(declarations, pool, &policy_identity)?;
221            }
222        }
223        match &self.lifecycle {
224            RuntimeLifecycle::Bootstrapped {
225                committed_allocations,
226                ..
227            } => Ok(committed_allocations),
228            RuntimeLifecycle::Unbootstrapped => {
229                unreachable!("successful bootstrap publishes committed allocations")
230            }
231        }
232    }
233
234    fn validate_pool_custody(
235        &self,
236        occupied: &[bool; crate::constants::MAX_ALLOCATIONS],
237        pool: &crate::MemoryAllocationPool,
238    ) -> Result<(), super::MemoryResolutionError> {
239        for id in 0..crate::MEMORY_MANAGER_INVALID_ID {
240            if pool.contains(id) && self.memory_size_pages(id) != 0 && !occupied[usize::from(id)] {
241                return Err(super::MemoryResolutionError::UnmanagedAllocation { id });
242            }
243        }
244        Ok(())
245    }
246
247    fn bootstrap_unbootstrapped<P: RuntimeBootstrapPolicy>(
248        &mut self,
249        declarations: &SealedDeclarationSnapshot,
250        pool: &crate::MemoryAllocationPool,
251        policy: &P,
252        policy_identity: PolicyIdentity,
253    ) -> Result<(), RuntimeBootstrapError<P::Error>> {
254        let memory = self.memory(MEMORY_MANAGER_LEDGER_ID);
255        let mut record = decode_stable_cell_commit_store_from_memory(&memory)?;
256        let recovered = record.recover_or_genesis()?;
257        let occupied = recovered.ledger().occupied_slots();
258        self.validate_pool_custody(&occupied, pool)?;
259        if memory.size() == 0 {
260            // Empty memory decodes to an uninitialized record. Preserve fresh
261            // cell acquisition before admission without persisting genesis.
262            write_ledger_cell(memory.clone(), &LedgerCommitStore::default())?;
263        }
264        let mut admission = BootstrapAdmission::new(recovered.ledger(), declarations, pool);
265        let preparation = policy.prepare_bootstrap(&mut admission);
266        let historical = admission.complete()?;
267        preparation.map_err(RuntimeBootstrapError::AdmissionPolicy)?;
268        let resolved = declarations.resolve(&recovered, &occupied, historical, pool)?;
269        let (generation, resolved) = {
270            let (staged, resolved) = crate::ledger::stage_generation(recovered, resolved, policy)?;
271            record.commit_generation(&staged)?;
272            (staged.current_generation(), resolved)
273        };
274        write_ledger_cell(memory, &record)?;
275        let committed = CommittedAllocations::from_persisted(resolved, generation);
276        self.lifecycle = RuntimeLifecycle::Bootstrapped {
277            committed_allocations: committed,
278            binding: RuntimeBootstrapBinding {
279                source: declarations.clone(),
280                policy_identity,
281                pool: pool.clone(),
282            },
283        };
284        Ok(())
285    }
286
287    /// Borrow this runtime's committed allocation-open capability.
288    pub const fn committed_allocations(&self) -> Result<&CommittedAllocations, RuntimeOpenError> {
289        match &self.lifecycle {
290            RuntimeLifecycle::Unbootstrapped => Err(RuntimeOpenError::NotBootstrapped),
291            RuntimeLifecycle::Bootstrapped {
292                committed_allocations,
293                ..
294            } => Ok(committed_allocations),
295        }
296    }
297
298    /// Open by durable key using only this runtime's persisted current capability.
299    pub fn open_memory(&self, stable_key: &str) -> Result<RuntimeMemory<M>, RuntimeOpenError> {
300        Ok(self.memory(self.memory_id(stable_key)?))
301    }
302
303    /// Resolve an application key's committed ID without opening memory,
304    /// reading history, or changing the host's policy or bucket configuration.
305    pub fn memory_id(&self, stable_key: &str) -> Result<u8, RuntimeOpenError> {
306        crate::key::validate(stable_key)?;
307        if crate::is_ic_memory_stable_key(stable_key) {
308            return Err(RuntimeOpenError::ReservedStableKey {
309                stable_key: stable_key.to_string(),
310            });
311        }
312        let declaration = crate::capability::declaration_for_key(
313            self.committed_allocations()?.declarations(),
314            stable_key,
315        )
316        .ok_or_else(|| RuntimeOpenError::StableKeyNotCommitted(stable_key.to_string()))?;
317        Ok(declaration.slot().id())
318    }
319
320    fn memory(&self, id: u8) -> RuntimeMemory<M> {
321        RuntimeMemory {
322            memory: self.memory_manager.get(MemoryId::new(id)),
323            growth: Rc::clone(&self.growth),
324        }
325    }
326
327    // Size-only observations need no handle carrying runtime growth authority.
328    fn memory_size_pages(&self, id: u8) -> u64 {
329        self.memory_manager.get(MemoryId::new(id)).size()
330    }
331
332    fn ledger_record_from_memory(&self) -> Result<LedgerCommitStore, StableCellLedgerError> {
333        // Decoding only reads; the manager handle retains backing lifetime
334        // without carrying the growth owner needed by writable runtime handles.
335        let memory = self
336            .memory_manager
337            .get(MemoryId::new(MEMORY_MANAGER_LEDGER_ID));
338        decode_stable_cell_commit_store_from_memory(&memory)
339    }
340}
341
342impl RuntimeBootstrapBinding {
343    fn validate<P>(
344        &self,
345        declarations: &SealedDeclarationSnapshot,
346        pool: &crate::MemoryAllocationPool,
347        policy_identity: &PolicyIdentity,
348    ) -> Result<(), RuntimeBootstrapError<P>> {
349        if &self.source != declarations {
350            return Err(RuntimeBootstrapError::DeclarationSnapshotMismatch);
351        }
352        if &self.pool != pool {
353            return Err(RuntimeBootstrapError::AllocationPoolMismatch);
354        }
355        if &self.policy_identity != policy_identity {
356            return Err(RuntimeBootstrapError::PolicyIdentityMismatch {
357                established: self.policy_identity.clone(),
358                requested: policy_identity.clone(),
359            });
360        }
361        Ok(())
362    }
363}
364
365fn write_ledger_cell<M: Memory, P>(
366    memory: RuntimeMemory<M>,
367    record: &LedgerCommitStore,
368) -> Result<(), RuntimeBootstrapError<P>> {
369    let encoded = EncodedLedgerRecord::new(record).map_err(|value_size| {
370        RuntimeBootstrapError::StableCellLedgerWriteTooLarge { value_size }
371    })?;
372    let value_size = encoded.len();
373    // The record limit is below u32::MAX, including the eight-byte cell header.
374    let required_bytes = STABLE_CELL_VALUE_OFFSET + value_size as u64;
375    let available_bytes = memory.size().saturating_mul(crate::WASM_PAGE_SIZE_BYTES);
376    if required_bytes > available_bytes {
377        let grow_by = (required_bytes - available_bytes).div_ceil(crate::WASM_PAGE_SIZE_BYTES);
378        memory.grow(grow_by)?;
379    }
380    drop(Cell::new(memory, encoded));
381    Ok(())
382}
383
384const fn check_bucket_size(
385    actual: u16,
386    requested: MemoryManagerConfig,
387) -> Result<(), RuntimeConstructionError> {
388    if actual != requested.bucket_size_pages() {
389        return Err(RuntimeConstructionError::BucketSizeMismatch {
390            persisted: actual,
391            requested: requested.bucket_size_pages(),
392        });
393    }
394    Ok(())
395}
ic_memory/runtime/mod.rs

ic_memory/runtime/
mod.rs
