1use crate::{
2 constants::DIAGNOSTIC_STRING_MAX_BYTES,
3 declaration::{AllocationDeclaration, DeclarationSnapshot},
4 schema::SchemaMetadata,
5 slot::{
6 IC_MEMORY_AUTHORITY_OWNER, IC_MEMORY_AUTHORITY_PURPOSE, IC_MEMORY_LEDGER_LABEL,
7 IC_MEMORY_LEDGER_STABLE_KEY, MEMORY_MANAGER_GOVERNANCE_MAX_ID, MEMORY_MANAGER_LEDGER_ID,
8 MemoryManagerAuthorityRecord, MemoryManagerIdRange, MemoryManagerRangeAuthority,
9 MemoryManagerRangeAuthorityError, MemoryManagerRangeMode, is_ic_memory_stable_key,
10 },
11};
12use std::{
13 collections::BTreeMap,
14 panic::{AssertUnwindSafe, catch_unwind},
15 sync::{Arc, Mutex, MutexGuard},
16 thread::ThreadId,
17};
18
19#[cfg(test)]
20pub static TEST_REGISTRY_LOCK: Mutex<()> = Mutex::new(());
21
22#[derive(Clone, Debug, Eq, PartialEq)]
33pub struct StaticMemoryDeclaration {
34 authority: String,
35 declaration: AllocationDeclaration,
36}
37
38impl StaticMemoryDeclaration {
39 pub fn new(
41 authority: impl Into<String>,
42 declaration: AllocationDeclaration,
43 ) -> Result<Self, StaticMemoryDeclarationError> {
44 let authority = authority.into();
45 validate_external_authority(&authority)?;
46 declaration.validate()?;
47 if is_ic_memory_stable_key(declaration.stable_key().as_str()) {
48 return Err(StaticMemoryDeclarationError::ReservedStableKey {
49 stable_key: declaration.stable_key().as_str().to_string(),
50 });
51 }
52 Ok(Self {
53 authority,
54 declaration,
55 })
56 }
57
58 #[must_use]
60 pub fn authority(&self) -> &str {
61 &self.authority
62 }
63
64 #[must_use]
66 pub const fn declaration(&self) -> &AllocationDeclaration {
67 &self.declaration
68 }
69
70 #[must_use]
72 pub fn into_declaration(self) -> AllocationDeclaration {
73 self.declaration
74 }
75}
76
77#[derive(Clone, Debug, Eq, PartialEq)]
86pub struct StaticMemoryRangeDeclaration {
87 record: MemoryManagerAuthorityRecord,
88}
89
90impl StaticMemoryRangeDeclaration {
91 pub fn new(record: MemoryManagerAuthorityRecord) -> Result<Self, StaticMemoryDeclarationError> {
93 validate_external_authority(record.authority())?;
94 record.validate()?;
95 Ok(Self { record })
96 }
97
98 #[must_use]
100 pub fn authority(&self) -> &str {
101 self.record.authority()
102 }
103
104 #[must_use]
106 pub const fn record(&self) -> &MemoryManagerAuthorityRecord {
107 &self.record
108 }
109
110 #[must_use]
112 pub fn into_record(self) -> MemoryManagerAuthorityRecord {
113 self.record
114 }
115}
116
117#[non_exhaustive]
122#[derive(Clone, Debug, Eq, thiserror::Error, PartialEq)]
123pub enum StaticMemoryDeclarationError {
124 #[error("static memory declaration registry lock poisoned")]
126 RegistryPoisoned,
127 #[error("static memory declaration registry is already sealed")]
129 RegistrySealed,
130 #[error("static memory declaration snapshot sealing is already active on this thread")]
132 ReentrantSealing,
133 #[error("static memory declaration registry lifecycle is internally inconsistent")]
135 InconsistentLifecycle,
136 #[error("static memory declaration eager-init hook panicked")]
138 EagerInitPanicked,
139 #[error(transparent)]
141 Declaration(#[from] crate::DeclarationSnapshotError),
142 #[error(transparent)]
144 Range(#[from] MemoryManagerRangeAuthorityError),
145 #[error("authority {reason}")]
147 InvalidAuthority {
148 reason: &'static str,
150 },
151 #[error("authority '{authority}' is reserved for ic-memory runtime internals")]
153 ReservedAuthority {
154 authority: String,
156 },
157 #[error("stable key '{stable_key}' is reserved for ic-memory runtime internals")]
159 ReservedStableKey {
160 stable_key: String,
162 },
163}
164
165#[derive(Clone, Debug, Eq, PartialEq)]
178pub struct SealedDeclarationSnapshot {
179 inner: Arc<SealedDeclarationSnapshotInner>,
180}
181
182#[derive(Debug, Eq, PartialEq)]
183struct SealedDeclarationSnapshotInner {
184 allocation_snapshot: DeclarationSnapshot,
185 registered_declarations: Vec<StaticMemoryDeclaration>,
186 registered_ranges: Vec<StaticMemoryRangeDeclaration>,
187 range_authority: MemoryManagerRangeAuthority,
188 declaration_authority: BTreeMap<String, RuntimeDeclarationAuthority>,
189}
190
191#[derive(Clone, Debug, Eq, PartialEq)]
192pub enum RuntimeDeclarationAuthority {
193 Internal,
194 External(String),
195}
196
197impl SealedDeclarationSnapshot {
198 #[must_use]
200 pub fn allocation_snapshot(&self) -> &DeclarationSnapshot {
201 &self.inner.allocation_snapshot
202 }
203
204 #[must_use]
206 pub fn registered_declarations(&self) -> &[StaticMemoryDeclaration] {
207 &self.inner.registered_declarations
208 }
209
210 #[must_use]
212 pub fn registered_ranges(&self) -> &[StaticMemoryRangeDeclaration] {
213 &self.inner.registered_ranges
214 }
215
216 #[must_use]
218 pub fn range_authority(&self) -> &MemoryManagerRangeAuthority {
219 &self.inner.range_authority
220 }
221
222 pub(crate) fn declaration_authority(&self) -> &BTreeMap<String, RuntimeDeclarationAuthority> {
223 &self.inner.declaration_authority
224 }
225
226 pub(crate) fn user_ranges_registered(&self) -> bool {
227 !self.inner.registered_ranges.is_empty()
228 }
229
230 pub(crate) fn shares_storage_with(&self, other: &Self) -> bool {
231 Arc::ptr_eq(&self.inner, &other.inner)
232 }
233}
234
235type StaticRegistrationHook = fn() -> Result<(), StaticMemoryDeclarationError>;
236
237#[derive(Debug)]
238struct StaticMemoryDeclarationRegistry {
239 declarations: Vec<StaticMemoryDeclaration>,
240 ranges: Vec<StaticMemoryRangeDeclaration>,
241 registration_hooks: Vec<StaticRegistrationHook>,
242 eager_init_hooks: Vec<fn()>,
243 lifecycle: StaticRegistryLifecycle,
244}
245
246#[derive(Debug)]
247enum StaticRegistryLifecycle {
248 Open,
249 Sealing {
250 owner: ThreadId,
251 deferred_error: Option<StaticMemoryDeclarationError>,
252 },
253 Sealed(SealedDeclarationSnapshot),
254 Failed(StaticMemoryDeclarationError),
255}
256
257static STATIC_MEMORY_DECLARATIONS: Mutex<StaticMemoryDeclarationRegistry> =
258 Mutex::new(StaticMemoryDeclarationRegistry {
259 declarations: Vec::new(),
260 ranges: Vec::new(),
261 registration_hooks: Vec::new(),
262 eager_init_hooks: Vec::new(),
263 lifecycle: StaticRegistryLifecycle::Open,
264 });
265
266static STATIC_MEMORY_SEAL: Mutex<()> = Mutex::new(());
267
268fn lock_registry()
269-> Result<MutexGuard<'static, StaticMemoryDeclarationRegistry>, StaticMemoryDeclarationError> {
270 STATIC_MEMORY_DECLARATIONS
271 .lock()
272 .map_err(|_| StaticMemoryDeclarationError::RegistryPoisoned)
273}
274
275fn ensure_registration_open(
276 registry: &StaticMemoryDeclarationRegistry,
277) -> Result<(), StaticMemoryDeclarationError> {
278 match ®istry.lifecycle {
279 StaticRegistryLifecycle::Open => Ok(()),
280 StaticRegistryLifecycle::Sealing { owner, .. } if *owner == std::thread::current().id() => {
281 Ok(())
282 }
283 StaticRegistryLifecycle::Sealing { .. }
284 | StaticRegistryLifecycle::Sealed(_)
285 | StaticRegistryLifecycle::Failed(_) => Err(StaticMemoryDeclarationError::RegistrySealed),
286 }
287}
288
289fn with_unsealed_registry(
290 op: impl FnOnce(&mut StaticMemoryDeclarationRegistry),
291) -> Result<(), StaticMemoryDeclarationError> {
292 let mut registry = lock_registry()?;
293 ensure_registration_open(®istry)?;
294 op(&mut registry);
295 Ok(())
296}
297
298#[doc(hidden)]
303pub fn defer_static_memory_registration(hook: StaticRegistrationHook) {
304 defer_constructor_registration(|registry| {
305 registry.registration_hooks.push(hook);
306 });
307}
308
309#[doc(hidden)]
314pub fn defer_eager_init(hook: fn()) {
315 defer_constructor_registration(|registry| {
316 registry.eager_init_hooks.push(hook);
317 });
318}
319
320fn defer_constructor_registration(op: impl FnOnce(&mut StaticMemoryDeclarationRegistry)) {
321 let Ok(mut registry) = STATIC_MEMORY_DECLARATIONS.lock() else {
322 return;
325 };
326 if matches!(registry.lifecycle, StaticRegistryLifecycle::Open) {
327 op(&mut registry);
328 return;
329 }
330 match &mut registry.lifecycle {
331 StaticRegistryLifecycle::Sealing { deferred_error, .. } => {
332 if deferred_error.is_none() {
333 *deferred_error = Some(StaticMemoryDeclarationError::RegistrySealed);
334 }
335 }
336 StaticRegistryLifecycle::Sealed(_) => {
337 registry.lifecycle =
338 StaticRegistryLifecycle::Failed(StaticMemoryDeclarationError::RegistrySealed);
339 }
340 StaticRegistryLifecycle::Failed(_) | StaticRegistryLifecycle::Open => {}
341 }
342}
343
344pub fn register_static_memory_declaration(
346 authority: impl Into<String>,
347 declaration: AllocationDeclaration,
348) -> Result<(), StaticMemoryDeclarationError> {
349 let registration = StaticMemoryDeclaration::new(authority, declaration)?;
350 with_unsealed_registry(|registry| {
351 registry.declarations.push(registration);
352 })
353}
354
355pub fn register_static_memory_manager_range(
357 start: u8,
358 end: u8,
359 authority: impl Into<String>,
360 mode: MemoryManagerRangeMode,
361 purpose: Option<String>,
362) -> Result<(), StaticMemoryDeclarationError> {
363 let authority = authority.into();
364 let record = MemoryManagerAuthorityRecord::new(
365 MemoryManagerIdRange::new(start, end).map_err(MemoryManagerRangeAuthorityError::Range)?,
366 authority,
367 mode,
368 purpose,
369 )?;
370 register_static_memory_range_declaration(StaticMemoryRangeDeclaration::new(record)?)
371}
372
373pub fn register_static_memory_range_declaration(
375 declaration: StaticMemoryRangeDeclaration,
376) -> Result<(), StaticMemoryDeclarationError> {
377 validate_external_authority(declaration.authority())?;
378 with_unsealed_registry(|registry| {
379 registry.ranges.push(declaration);
380 })
381}
382
383fn validate_external_authority(value: &str) -> Result<(), StaticMemoryDeclarationError> {
384 if value == IC_MEMORY_AUTHORITY_OWNER {
385 return Err(StaticMemoryDeclarationError::ReservedAuthority {
386 authority: value.to_string(),
387 });
388 }
389 if value.is_empty() {
390 return Err(StaticMemoryDeclarationError::InvalidAuthority {
391 reason: "must not be empty",
392 });
393 }
394 if value.len() > DIAGNOSTIC_STRING_MAX_BYTES {
395 return Err(StaticMemoryDeclarationError::InvalidAuthority {
396 reason: "must be at most 256 bytes",
397 });
398 }
399 if !value.is_ascii() {
400 return Err(StaticMemoryDeclarationError::InvalidAuthority {
401 reason: "must be ASCII",
402 });
403 }
404 if value.bytes().any(|byte| byte.is_ascii_control()) {
405 return Err(StaticMemoryDeclarationError::InvalidAuthority {
406 reason: "must not contain ASCII control characters",
407 });
408 }
409 Ok(())
410}
411
412pub fn register_static_memory_manager_declaration(
414 id: u8,
415 authority: impl Into<String>,
416 label: impl Into<String>,
417 stable_key: impl AsRef<str>,
418) -> Result<(), StaticMemoryDeclarationError> {
419 register_static_memory_manager_declaration_with_schema(
420 id,
421 authority,
422 label,
423 stable_key,
424 SchemaMetadata::default(),
425 )
426}
427
428pub fn register_static_memory_manager_declaration_with_schema(
430 id: u8,
431 authority: impl Into<String>,
432 label: impl Into<String>,
433 stable_key: impl AsRef<str>,
434 schema: SchemaMetadata,
435) -> Result<(), StaticMemoryDeclarationError> {
436 let declaration =
437 AllocationDeclaration::memory_manager_with_schema(stable_key, id, label, schema)?;
438 register_static_memory_declaration(authority, declaration)
439}
440
441pub fn sealed_declaration_snapshot()
448-> Result<SealedDeclarationSnapshot, StaticMemoryDeclarationError> {
449 {
450 let registry = lock_registry()?;
451 match ®istry.lifecycle {
452 StaticRegistryLifecycle::Sealed(snapshot) => return Ok(snapshot.clone()),
453 StaticRegistryLifecycle::Failed(err) => return Err(err.clone()),
454 StaticRegistryLifecycle::Sealing { owner, .. }
455 if *owner == std::thread::current().id() =>
456 {
457 return Err(StaticMemoryDeclarationError::ReentrantSealing);
458 }
459 StaticRegistryLifecycle::Open | StaticRegistryLifecycle::Sealing { .. } => {}
460 }
461 }
462
463 let _seal = STATIC_MEMORY_SEAL
464 .lock()
465 .map_err(|_| StaticMemoryDeclarationError::RegistryPoisoned)?;
466 let (registration_hooks, eager_init_hooks) = {
467 let mut registry = lock_registry()?;
468 match ®istry.lifecycle {
469 StaticRegistryLifecycle::Sealed(snapshot) => return Ok(snapshot.clone()),
470 StaticRegistryLifecycle::Failed(err) => return Err(err.clone()),
471 StaticRegistryLifecycle::Sealing { .. } => {
472 return Err(StaticMemoryDeclarationError::ReentrantSealing);
473 }
474 StaticRegistryLifecycle::Open => {}
475 }
476 registry.lifecycle = StaticRegistryLifecycle::Sealing {
477 owner: std::thread::current().id(),
478 deferred_error: None,
479 };
480 (
481 std::mem::take(&mut registry.registration_hooks),
482 std::mem::take(&mut registry.eager_init_hooks),
483 )
484 };
485
486 for hook in registration_hooks {
487 let result = catch_unwind(AssertUnwindSafe(hook))
488 .map_err(|_| StaticMemoryDeclarationError::EagerInitPanicked)
489 .and_then(std::convert::identity);
490 if let Err(err) = result {
491 return fail_sealing(err);
492 }
493 }
494 for hook in eager_init_hooks {
495 if catch_unwind(AssertUnwindSafe(hook)).is_err() {
496 return fail_sealing(StaticMemoryDeclarationError::EagerInitPanicked);
497 }
498 }
499
500 let mut registry = lock_registry()?;
501 let deferred_error = match ®istry.lifecycle {
502 StaticRegistryLifecycle::Sealing { deferred_error, .. } => deferred_error.clone(),
503 StaticRegistryLifecycle::Failed(err) => return Err(err.clone()),
504 StaticRegistryLifecycle::Open | StaticRegistryLifecycle::Sealed(_) => {
505 return Err(StaticMemoryDeclarationError::InconsistentLifecycle);
506 }
507 };
508 if let Some(err) = deferred_error {
509 registry.lifecycle = StaticRegistryLifecycle::Failed(err.clone());
510 return Err(err);
511 }
512 let snapshot = match build_sealed_snapshot(®istry.declarations, ®istry.ranges) {
513 Ok(snapshot) => snapshot,
514 Err(err) => {
515 registry.lifecycle = StaticRegistryLifecycle::Failed(err.clone());
516 return Err(err);
517 }
518 };
519 registry.lifecycle = StaticRegistryLifecycle::Sealed(snapshot.clone());
520 Ok(snapshot)
521}
522
523fn fail_sealing<T>(err: StaticMemoryDeclarationError) -> Result<T, StaticMemoryDeclarationError> {
524 let mut registry = lock_registry()?;
525 let failure = match ®istry.lifecycle {
526 StaticRegistryLifecycle::Sealing {
527 deferred_error: Some(deferred_error),
528 ..
529 } => deferred_error.clone(),
530 StaticRegistryLifecycle::Open
531 | StaticRegistryLifecycle::Sealing {
532 deferred_error: None,
533 ..
534 }
535 | StaticRegistryLifecycle::Sealed(_) => err,
536 StaticRegistryLifecycle::Failed(failure) => failure.clone(),
537 };
538 registry.lifecycle = StaticRegistryLifecycle::Failed(failure.clone());
539 Err(failure)
540}
541
542fn build_sealed_snapshot(
543 declarations: &[StaticMemoryDeclaration],
544 ranges: &[StaticMemoryRangeDeclaration],
545) -> Result<SealedDeclarationSnapshot, StaticMemoryDeclarationError> {
546 let mut registered_declarations = declarations.to_vec();
547 registered_declarations.sort_by(|left, right| {
548 left.declaration()
549 .stable_key()
550 .cmp(right.declaration().stable_key())
551 .then_with(|| left.declaration().slot().cmp(right.declaration().slot()))
552 .then_with(|| left.authority().cmp(right.authority()))
553 });
554
555 let mut registered_ranges = ranges.to_vec();
556 registered_ranges.sort_by(|left, right| {
557 let left = left.record();
558 let right = right.record();
559 left.range()
560 .start()
561 .cmp(&right.range().start())
562 .then_with(|| left.range().end().cmp(&right.range().end()))
563 .then_with(|| left.authority().cmp(right.authority()))
564 .then_with(|| range_mode_order(left.mode()).cmp(&range_mode_order(right.mode())))
565 .then_with(|| left.purpose().cmp(&right.purpose()))
566 });
567
568 let mut allocation_declarations = Vec::with_capacity(registered_declarations.len() + 1);
569 allocation_declarations.push(internal_ledger_declaration()?);
570 allocation_declarations.extend(
571 registered_declarations
572 .iter()
573 .map(|registration| registration.declaration().clone()),
574 );
575 let allocation_snapshot = DeclarationSnapshot::new(allocation_declarations)?;
576
577 let mut authority_records = Vec::with_capacity(registered_ranges.len() + 1);
578 authority_records.push(internal_ledger_range()?);
579 authority_records.extend(
580 registered_ranges
581 .iter()
582 .map(|registration| registration.record().clone()),
583 );
584 let range_authority = MemoryManagerRangeAuthority::from_records(authority_records)?;
585
586 let mut declaration_authority = BTreeMap::new();
587 declaration_authority.insert(
588 IC_MEMORY_LEDGER_STABLE_KEY.to_string(),
589 RuntimeDeclarationAuthority::Internal,
590 );
591 for registration in ®istered_declarations {
592 declaration_authority.insert(
593 registration.declaration().stable_key().as_str().to_string(),
594 RuntimeDeclarationAuthority::External(registration.authority().to_string()),
595 );
596 }
597
598 Ok(SealedDeclarationSnapshot {
599 inner: Arc::new(SealedDeclarationSnapshotInner {
600 allocation_snapshot,
601 registered_declarations,
602 registered_ranges,
603 range_authority,
604 declaration_authority,
605 }),
606 })
607}
608
609const fn range_mode_order(mode: MemoryManagerRangeMode) -> u8 {
610 match mode {
611 MemoryManagerRangeMode::Reserved => 0,
612 MemoryManagerRangeMode::Allowed => 1,
613 }
614}
615
616fn internal_ledger_declaration() -> Result<AllocationDeclaration, crate::DeclarationSnapshotError> {
617 AllocationDeclaration::memory_manager(
618 IC_MEMORY_LEDGER_STABLE_KEY,
619 MEMORY_MANAGER_LEDGER_ID,
620 IC_MEMORY_LEDGER_LABEL,
621 )
622}
623
624fn internal_ledger_range() -> Result<MemoryManagerAuthorityRecord, MemoryManagerRangeAuthorityError>
625{
626 MemoryManagerAuthorityRecord::new(
627 MemoryManagerIdRange::new(MEMORY_MANAGER_LEDGER_ID, MEMORY_MANAGER_GOVERNANCE_MAX_ID)?,
628 IC_MEMORY_AUTHORITY_OWNER,
629 MemoryManagerRangeMode::Reserved,
630 Some(IC_MEMORY_AUTHORITY_PURPOSE.to_string()),
631 )
632}
633
634#[cfg(test)]
635pub fn reset_static_memory_declarations_for_tests() {
636 let mut registry = STATIC_MEMORY_DECLARATIONS
637 .lock()
638 .expect("static memory declaration registry poisoned");
639 registry.declarations.clear();
640 registry.ranges.clear();
641 registry.registration_hooks.clear();
642 registry.eager_init_hooks.clear();
643 registry.lifecycle = StaticRegistryLifecycle::Open;
644}
645
646#[cfg(test)]
647mod tests;