1mod creation_fee;
8pub(super) mod recovery;
9pub(super) mod reinstall;
10pub(in crate::fleet_ensure) mod root_reinstall;
11pub(super) mod startup_funding;
12
13use crate::{
14 component_topology::{
15 RootPoolImportCapacityError, RootPoolImportCapacityInput,
16 validate_root_pool_import_capacity,
17 },
18 fleet_ensure::model::{
19 CanisterCyclePolicy, CanisterDisposition, CanisterPlan, CanisterRuntimeStatus,
20 CurrentFleetProtocolAction, CycleConservation, DesiredCanisterKind, DesiredFleet,
21 DesiredFleetArtifacts, DesiredFleetBootstrapRoot, DesiredPresence, EnsureAction,
22 EstateFundingDomainPlan, EstatePoolAssetLifecycle, FLEET_ENSURE_SCHEMA_VERSION,
23 FleetEnsurePlan, FleetEnsurePlanScope, FleetEnsureStateRecord, FleetObservation,
24 InstallMode, LiveCanister, MAX_FLEET_ENSURE_CANISTERS, MAX_FLEET_ENSURE_PROTOCOL_STEPS,
25 PoolFundingAuthority, RetainedRootStartAuthorityRecord, RootManagementBinding,
26 RootManagementObservation, RootOwnedCanisterLifecycle,
27 },
28};
29use candid::Principal;
30use canic_core::{
31 cdk::types::Cycles,
32 control_plane_support::model::fleet_funding_policy::{
33 FleetFundingPolicyValidationError, validate_coordinator_root_funding_policy,
34 validate_fleet_root_funding_capacity, validate_fleet_subnet_root_funding_authority,
35 },
36 ids::{FleetName, FleetSubnetCanisterPoolConfig},
37};
38use sha2::{Digest, Sha256};
39use std::collections::{BTreeMap, BTreeSet};
40use thiserror::Error as ThisError;
41
42pub(super) use creation_fee::validate_creation_fee_scope;
43
44#[derive(Debug, Eq, PartialEq, ThisError)]
47pub enum EnsurePolicyError {
48 #[error(
49 "Root {root} startup role {role} cannot receive {shortfall_cycles} required cycles under its configured top-up/lifetime policy"
50 )]
51 StartupRoleFundingUnavailable {
52 root: String,
53 role: String,
54 shortfall_cycles: u128,
55 },
56 #[error(
57 "this Fleet operation creates on multiple subnets {subnets:?}, but management_creation_fee_cycles supplies only one exact fee; mixed-subnet creation is unsupported and no effect was authorized; retain this operation's evidence and review placement so new creations use one subnet (existing identities on other subnets may be reused or reinstalled)"
58 )]
59 MixedSubnetCreationFees { subnets: Vec<String> },
60
61 #[error(
62 "Root {root} reset omits known controlled assets {missing_principals:?}; refresh the estate seed and matching policy imports, regenerate desired state, and review before reinstall"
63 )]
64 IncompleteRootEstate {
65 root: String,
66 missing_principals: Vec<String>,
67 },
68
69 #[error(
70 "Root {root} requires {workloads} Workloads plus {ready_floor} Ready assets, exceeding pool maximum {maximum_size}"
71 )]
72 TerminalPoolCapacity {
73 root: String,
74 workloads: u32,
75 ready_floor: u32,
76 maximum_size: u32,
77 },
78 #[error("invalid protected funding policy: {0}")]
79 FundingPolicy(#[from] FleetFundingPolicyValidationError),
80
81 #[error("cycle arithmetic overflow while compiling {field}")]
82 ArithmeticOverflow { field: &'static str },
83
84 #[error(
85 "Fleet cycle-conservation headroom is insufficient: available {available} cycles after reviewed funding, required maximum burn {required}, shortfall {shortfall}, actions {action_count}; no plan or effect was authorized"
86 )]
87 InsufficientCycleConservation {
88 action_count: usize,
89 available: u128,
90 required: u128,
91 shortfall: u128,
92 },
93
94 #[error(
95 "Root reinstall prerequisite for {name} ({principal}) has insufficient native headroom for its unfunded management effects: available {available} cycles, required conservative maximum burn {required}, shortfall {shortfall}, effects {action_count}; headroom is not expected expenditure or a complete rollout quote; no plan or effect was authorized"
96 )]
97 RootReinstallHeadroom {
98 name: String,
99 principal: String,
100 action_count: usize,
101 available: u128,
102 required: u128,
103 shortfall: u128,
104 },
105
106 #[error(
107 "authority seal for {name} ({principal}) has insufficient cycle headroom: available {available} cycles, required conservative maximum burn {required}, shortfall {shortfall}; this review transfers no funding and another authority's balance cannot cover the shortfall; no plan or effect was authorized"
108 )]
109 AuthoritySealHeadroom {
110 name: String,
111 principal: String,
112 available: u128,
113 required: u128,
114 shortfall: u128,
115 },
116
117 #[error("controlled canister {name} has duplicate name or principal authority")]
118 DuplicateAuthority { name: String },
119
120 #[error("controlled canister {name} references unavailable controller canister {controller}")]
121 ControllerCanisterUnavailable { controller: String, name: String },
122
123 #[error("desired Fleet has invalid or anonymous Principal for {field}: {value}")]
124 InvalidPrincipal { field: String, value: String },
125
126 #[error("desired Fleet contains an empty canister or drain-method name")]
127 EmptyName,
128
129 #[error("desired Fleet has unsafe {field} path label: {value}")]
130 UnsafePathLabel { field: &'static str, value: String },
131
132 #[error("controlled canister {name} has invalid {field} cycle value: {value}")]
133 InvalidCanisterCycles {
134 field: &'static str,
135 name: String,
136 value: String,
137 },
138
139 #[error("desired Fleet has invalid {field} cycle value: {value}")]
140 InvalidFleetCycles { field: &'static str, value: String },
141
142 #[error("desired Fleet estate-funding topology is invalid: {reason}")]
143 EstateFundingTopology { reason: String },
144
145 #[error(
146 "Root {root} pool cannot satisfy current desired demand: occupied {occupied_assets}/{maximum_size}, eligible Ready {eligible_ready_assets}, allocated Workloads {allocated_workloads}, pending creations {pending_creations}, required new creations {required_creation_count}, available slots {available_slots}, capacity shortfall {capacity_shortfall}; repair or retire retained pool assets before funding"
147 )]
148 EstatePoolCapacity {
149 allocated_workloads: u32,
150 available_slots: u32,
151 capacity_shortfall: u32,
152 eligible_ready_assets: u32,
153 maximum_size: u32,
154 occupied_assets: u32,
155 pending_creations: u32,
156 required_creation_count: u32,
157 root: String,
158 },
159
160 #[error("live Cycles Ledger fee is {actual}, but reviewed desired fee is {expected}")]
161 LedgerFeeDrift { actual: u128, expected: u128 },
162
163 #[error("desired Fleet has {actual} controlled canisters, above limit {maximum}")]
164 TooManyCanisters { actual: usize, maximum: usize },
165
166 #[error("desired Fleet has {actual} protocol steps, above limit {maximum}")]
167 TooManyProtocolSteps { actual: usize, maximum: usize },
168
169 #[error("desired Fleet maximum_stalled_observations must be greater than zero")]
170 InvalidStallBound,
171
172 #[error("controlled canister {name} initial cycles are below its minimum cycles")]
173 InitialCyclesBelowMinimum { name: String },
174
175 #[error(
176 "fresh pool canister {name} requests {creation_funding_cycles} creation cycles for readiness floor {readiness_floor_cycles}, but its bounded pre-import burn is {admissible_burn_cycles} cycles; required funding {required_creation_funding_cycles}, shortfall {shortfall_cycles}; no effect was authorized"
177 )]
178 FreshPoolCreationFundingInsufficient {
179 admissible_burn_cycles: u128,
180 creation_funding_cycles: u128,
181 name: String,
182 readiness_floor_cycles: u128,
183 required_creation_funding_cycles: u128,
184 shortfall_cycles: u128,
185 },
186
187 #[error(
188 "live observation for controlled canister {name} returned Principal {actual}, expected {expected}"
189 )]
190 ObservationAuthorityMismatch {
191 actual: String,
192 expected: String,
193 name: String,
194 },
195
196 #[error("desired Fleet name {actual} does not match requested Fleet {expected}")]
197 FleetMismatch { actual: String, expected: String },
198
199 #[error("controlled canister {name} is missing exact live observation authority")]
200 MissingObservation { name: String },
201
202 #[error("Fleet treasury {treasury} is not an explicitly controlled present canister")]
203 MissingTreasury { treasury: String },
204
205 #[error("Fleet treasury {treasury} must be reused and cannot be replaced")]
206 TreasuryReplacement { treasury: String },
207
208 #[error(
209 "present controlled canister {name} does not retain the configured operator as controller"
210 )]
211 MissingOperatorController { name: String },
212
213 #[error(
214 "controlled canister {name} retains {cycles} transferable cycles but has no exact treasury-bound drain authority; canister left untouched"
215 )]
216 NoSafeDrain { name: String, cycles: u128 },
217
218 #[error("controlled canister {name} names a drain destination other than Fleet treasury")]
219 WrongDrainDestination { name: String },
220
221 #[error("desired Fleet schema {actual} is unsupported; current schema is {expected}")]
222 WrongSchema { actual: u16, expected: u16 },
223
224 #[error("controlled canister {name} has no resolved current Wasm identity")]
225 MissingWasmIdentity { name: String },
226
227 #[error("controlled canister {name} has no resolved {kind} identity")]
228 MissingArtifactIdentity { kind: &'static str, name: String },
229
230 #[error("protocol step {step} targets unknown or absent controlled canister {canister}")]
231 InvalidProtocolTarget { canister: String, step: String },
232
233 #[error("protocol step {0} has duplicate or empty authority")]
234 InvalidProtocolStep(String),
235
236 #[error("protocol step {0} is missing an exact live terminal observation")]
237 MissingProtocolObservation(String),
238
239 #[error("controlled canister {0} must configure init_arg and init_candid together")]
240 IncompleteInitTemplate(String),
241
242 #[error("Canic infrastructure canister {name} requires one exact typed bootstrap initializer")]
243 MissingCanicInitializer { name: String },
244
245 #[error("Canic infrastructure canister {name} has conflicting typed and generic initializers")]
246 ConflictingCanicInitializer { name: String },
247
248 #[error("retained pool asset {name} is missing; no replacement was created")]
249 MissingPoolAsset { name: String },
250
251 #[error("retained pool asset {name} has no exact Root-owned lifecycle observation")]
252 MissingPoolLifecycle { name: String },
253
254 #[error("canister {name} is not an exact present infrastructure reinstall target")]
255 InvalidInfrastructureReinstall { name: String },
256
257 #[error("Root-owned canister {name} is still awaiting exact current balance observation")]
258 PendingRootOwnedBalance { name: String },
259
260 #[error("desired Fleet topology is invalid for {name}: {reason}")]
261 InvalidTopology { name: String, reason: &'static str },
262
263 #[error("management observation for retained Root {name} is missing")]
264 MissingRootManagementObservation { name: String },
265
266 #[error("management observation for retained Root {name} has invalid {field}")]
267 RootManagementAuthorityMismatch { field: &'static str, name: String },
268
269 #[error("retained Root {name} is stopping; wait for terminal Stopped state")]
270 RootStopping { name: String },
271
272 #[error(transparent)]
273 PoolImportCapacity(#[from] RootPoolImportCapacityError),
274}
275
276#[derive(Clone, Copy)]
277struct CycleBounds {
278 ledger_fee: u128,
279 management_creation_fee: u128,
280 material_threshold: u128,
281 observation_burn: u128,
282 update_burn: u128,
283}
284
285#[derive(Clone)]
286struct PlanAccumulator {
287 canisters: Vec<CanisterPlan>,
288 execution_burn: u128,
289 fees: u128,
290 new_funding: u128,
291 retained: u128,
292 transfers: u128,
293}
294
295impl PlanAccumulator {
296 const fn new() -> Self {
297 Self {
298 canisters: Vec::new(),
299 execution_burn: 0,
300 fees: 0,
301 new_funding: 0,
302 retained: 0,
303 transfers: 0,
304 }
305 }
306
307 fn add_burn(&mut self, value: u128) -> Result<(), EnsurePolicyError> {
308 self.execution_burn = checked_add(self.execution_burn, value, "execution burn")?;
309 Ok(())
310 }
311
312 fn add_fee(&mut self, value: u128) -> Result<(), EnsurePolicyError> {
313 self.fees = checked_add(self.fees, value, "unavoidable fees")?;
314 Ok(())
315 }
316
317 fn add_funding(&mut self, value: u128) -> Result<(), EnsurePolicyError> {
318 self.new_funding = checked_add(self.new_funding, value, "new funding")?;
319 Ok(())
320 }
321}
322
323#[expect(
325 clippy::too_many_lines,
326 reason = "one pure compiler keeps the complete reviewed conservation and action authority visible"
327)]
328#[expect(
329 clippy::too_many_arguments,
330 reason = "explicit operation identity also binds deliberate same-release resets"
331)]
332pub fn compile_plan(
333 desired: &DesiredFleet,
334 artifacts: &DesiredFleetArtifacts,
335 compiled_protocol_actions: &[EnsureAction],
336 desired_sha256: &str,
337 requested_fleet: &str,
338 observation: &FleetObservation,
339 created_at_time: u64,
340 operation_id: &str,
341 reinstall: Option<&crate::fleet_ensure::model::FleetReinstallRecord>,
342) -> Result<FleetEnsurePlan, EnsurePolicyError> {
343 validate_authority(desired, requested_fleet)?;
344 validate_observation_authority(desired, observation)?;
345 let bounds = cycle_bounds(desired)?;
346 if observation.ledger_fee_cycles != bounds.ledger_fee {
347 return Err(EnsurePolicyError::LedgerFeeDrift {
348 actual: observation.ledger_fee_cycles,
349 expected: bounds.ledger_fee,
350 });
351 }
352 let operation_id = operation_id.to_string();
353 let mut accumulator = PlanAccumulator::new();
354 let reviewed_reinstalls = if let Some(intent) = reinstall {
355 reinstall::validate_reset(desired, artifacts, observation, intent, &operation_id)?
356 } else {
357 reviewed_reinstall_canisters(desired, observation)?
358 };
359
360 for (index, configured) in desired.canisters.iter().enumerate() {
361 let observed = observation
362 .canisters
363 .get(&configured.name)
364 .ok_or_else(|| EnsurePolicyError::MissingObservation {
365 name: configured.name.clone(),
366 })?
367 .as_ref();
368 let cycle_policy = canister_cycle_policy(configured)?;
369 let action_time = created_at_time
370 .checked_add(u64::try_from(index).unwrap_or(u64::MAX))
371 .ok_or(EnsurePolicyError::ArithmeticOverflow {
372 field: "action timestamp",
373 })?;
374 let mut plan = compile_canister(
375 desired,
376 artifacts,
377 configured,
378 observed,
379 observation,
380 cycle_policy,
381 bounds,
382 action_time,
383 reviewed_reinstalls.contains(&configured.name),
384 &mut accumulator,
385 )?;
386 if let Some(intent) = reinstall {
387 reinstall::bind_history_witness(desired, artifacts, intent, configured, &mut plan)?;
388 }
389 accumulator.canisters.push(plan);
390 }
391 for name in &reviewed_reinstalls {
392 let has_exact_reinstall = accumulator.canisters.iter().any(|canister| {
393 canister.name == *name
394 && canister.actions.iter().any(|action| {
395 matches!(
396 action,
397 EnsureAction::Install {
398 mode: InstallMode::Reinstall,
399 ..
400 }
401 )
402 })
403 });
404 if !has_exact_reinstall {
405 return Err(EnsurePolicyError::InvalidInfrastructureReinstall { name: name.clone() });
406 }
407 }
408
409 let mut protocol_actions = Vec::new();
410 let mut protocol_names = BTreeSet::new();
411 for step in &desired.protocol_steps {
412 let ready = observation
413 .protocol_ready
414 .get(&step.name)
415 .copied()
416 .ok_or_else(|| EnsurePolicyError::MissingProtocolObservation(step.name.clone()))?;
417 if ready {
418 continue;
419 }
420 if !protocol_names.insert(step.name.as_str()) {
421 return Err(EnsurePolicyError::InvalidProtocolStep(step.name.clone()));
422 }
423 let configured = desired
424 .canisters
425 .iter()
426 .find(|canister| canister.name == step.canister)
427 .ok_or_else(|| EnsurePolicyError::InvalidProtocolTarget {
428 canister: step.canister.clone(),
429 step: step.name.clone(),
430 })?;
431 let principal = observation
432 .canisters
433 .get(&step.canister)
434 .and_then(Option::as_ref)
435 .map_or_else(
436 || format!("created:{}", step.canister),
437 |live| live.principal.clone(),
438 );
439 if configured.presence != DesiredPresence::Present {
440 return Err(EnsurePolicyError::InvalidProtocolTarget {
441 canister: step.canister.clone(),
442 step: step.name.clone(),
443 });
444 }
445 let identities = artifacts.protocol_by_step.get(&step.name).ok_or_else(|| {
446 EnsurePolicyError::MissingArtifactIdentity {
447 kind: "protocol contract",
448 name: step.name.clone(),
449 }
450 })?;
451 let maximum_execution_burn_cycles = parse_canister_cycles(
452 &step.canister,
453 "protocol.maximum_execution_burn_cycles",
454 &step.maximum_execution_burn_cycles,
455 )?;
456 protocol_actions.push(EnsureAction::Protocol {
457 candid: step.candid.clone(),
458 candid_sha256: identities.candid_sha256.clone(),
459 command_args: step.command_args.clone(),
460 command_args_sha256: identities.command_args_sha256.clone(),
461 command_method: step.command_method.clone(),
462 expected_status: step.expected_status.clone(),
463 expected_status_sha256: identities.expected_status_sha256.clone(),
464 maximum_execution_burn_cycles,
465 name: step.name.clone(),
466 principal,
467 status_args: step.status_args.clone(),
468 status_args_sha256: identities.status_args_sha256.clone(),
469 status_method: step.status_method.clone(),
470 });
471 }
472 let expected_typed_protocol = desired.protocol.as_ref();
473 for action in compiled_protocol_actions {
474 let EnsureAction::FleetProtocol {
475 action: current_action,
476 candid,
477 name,
478 principal,
479 ..
480 } = action
481 else {
482 return Err(EnsurePolicyError::InvalidProtocolStep(
483 action.name().to_string(),
484 ));
485 };
486 let target_kind = current_action.target_kind();
487 let target_is_exact = desired.canisters.iter().any(|canister| {
488 canister.kind == target_kind
489 && canister.presence == DesiredPresence::Present
490 && observation
491 .canisters
492 .get(&canister.name)
493 .and_then(Option::as_ref)
494 .is_some_and(|live| live.principal == *principal)
495 });
496 let operation_matches = match current_action.as_ref() {
497 CurrentFleetProtocolAction::MaintainPoolReadiness {
498 maximum_updates,
499 minimum_ready,
500 readiness_floor,
501 } => pool_readiness_authority_matches(
502 desired,
503 observation,
504 principal,
505 *minimum_ready,
506 readiness_floor,
507 Some(*maximum_updates),
508 ),
509 CurrentFleetProtocolAction::ObservePoolReadiness {
510 minimum_ready,
511 readiness_floor,
512 } => pool_readiness_authority_matches(
513 desired,
514 observation,
515 principal,
516 *minimum_ready,
517 readiness_floor,
518 None,
519 ),
520 CurrentFleetProtocolAction::ReconcilePoolAsset {
521 request,
522 minimum_cycles,
523 } => observation
524 .estate_funding_domains
525 .iter()
526 .any(|(_root, domain)| {
527 domain.root_principal.as_deref() == Some(principal)
528 && domain.pool.as_ref().is_some_and(|pool| {
529 minimum_cycles.to_u128() == pool.readiness_floor_cycles
530 && pool.assets.iter().any(|asset| {
531 asset.principal == request.canister_id.to_text()
532 && matches!(
533 asset.lifecycle,
534 EstatePoolAssetLifecycle::PendingReset
535 | EstatePoolAssetLifecycle::Failed
536 )
537 })
538 })
539 }),
540 CurrentFleetProtocolAction::ProvisionComponents { request, .. } => {
541 canic_core::cdk::utils::hash::decode_hex(&operation_id)
542 .ok()
543 .and_then(|bytes| <[u8; 32]>::try_from(bytes).ok())
544 .is_some_and(|operation_id| request.operation_id == operation_id)
545 }
546 current_action => current_action
547 .operation_id()
548 .is_none_or(|identity| identity != [0; 32]),
549 };
550 let protocol_matches = expected_typed_protocol.is_some_and(|protocol| {
551 let expected_candid = match target_kind {
552 DesiredCanisterKind::Coordinator => protocol.coordinator_candid.as_str(),
553 DesiredCanisterKind::Root => protocol.root_candid.as_str(),
554 DesiredCanisterKind::Store => protocol.store_candid.as_str(),
555 DesiredCanisterKind::Auxiliary
556 | DesiredCanisterKind::Component
557 | DesiredCanisterKind::Pool => return false,
558 };
559 candid == expected_candid
560 });
561 if name.is_empty()
562 || !protocol_names.insert(name.as_str())
563 || !target_is_exact
564 || !operation_matches
565 || !protocol_matches
566 {
567 return Err(EnsurePolicyError::InvalidProtocolStep(name.clone()));
568 }
569 protocol_actions.push(action.clone());
570 }
571 if protocol_actions.len() > MAX_FLEET_ENSURE_PROTOCOL_STEPS {
572 return Err(EnsurePolicyError::TooManyProtocolSteps {
573 actual: protocol_actions.len(),
574 maximum: MAX_FLEET_ENSURE_PROTOCOL_STEPS,
575 });
576 }
577 if reviewed_reinstalls.is_empty() && protocol_actions.is_empty() {
578 let pending = desired.canisters.iter().find_map(|configured| {
579 observation
580 .canisters
581 .get(&configured.name)
582 .and_then(Option::as_ref)
583 .is_some_and(|live| {
584 live.root_owned_lifecycle == Some(RootOwnedCanisterLifecycle::Retained)
585 })
586 .then(|| configured.name.clone())
587 });
588 if let Some(name) = pending
589 && !retained_root_observation_is_start_only(
590 desired,
591 &accumulator.canisters,
592 observation,
593 )
594 {
595 return Err(EnsurePolicyError::PendingRootOwnedBalance { name });
596 }
597 }
598
599 let observed_native_cycles = observation
600 .canisters
601 .values()
602 .filter_map(Option::as_ref)
603 .try_fold(0_u128, |total, live| {
604 checked_add(total, live.cycles, "observed controlled cycles")
605 })?;
606 let observed_native_cycles = observation
607 .additional_controlled_cycles
608 .values()
609 .try_fold(observed_native_cycles, |total, cycles| {
610 checked_add(total, *cycles, "observed controlled cycles")
611 })?;
612 startup_funding::planning::tranche(
613 desired,
614 artifacts,
615 observation,
616 bounds,
617 observed_native_cycles,
618 created_at_time,
619 &mut accumulator,
620 &mut protocol_actions,
621 )?;
622
623 let continuation = artifacts.continuation.clone().filter(|_| {
624 !accumulator.canisters.is_empty()
625 && (accumulator
626 .canisters
627 .iter()
628 .all(|canister| canister.disposition == CanisterDisposition::Create)
629 || (accumulator
630 .canisters
631 .iter()
632 .any(|canister| canister.disposition == CanisterDisposition::Reinstall)
633 && accumulator.canisters.iter().all(|canister| {
634 matches!(
635 canister.disposition,
636 CanisterDisposition::Reuse | CanisterDisposition::Reinstall
637 )
638 })))
639 && desired.protocol_steps.is_empty()
640 && protocol_actions.is_empty()
641 });
642 let reconciliation_only = !protocol_actions.is_empty() && protocol_actions.iter().all(|action| matches!(action,
643 EnsureAction::FleetProtocol { action, .. } if matches!(action.as_ref(), CurrentFleetProtocolAction::ReconcilePoolAsset { .. })
644 ));
645 if continuation.is_some() {
646 startup_funding::planning::prepay_continuation(
647 desired,
648 artifacts,
649 observation,
650 bounds,
651 created_at_time,
652 &mut accumulator,
653 )?;
654 }
655 append_pool_reconciliation_funding(
656 desired,
657 observation,
658 &protocol_actions,
659 bounds,
660 created_at_time,
661 &mut accumulator,
662 )?;
663 let defer_pool_creation = reconciliation_only || continuation.is_some();
666 let estate_funding_domains =
667 compile_estate_funding_domains(desired, observation, bounds, defer_pool_creation)?;
668 validate_creation_fee_scope(desired, &accumulator.canisters, &estate_funding_domains)?;
669 append_estate_funding_actions(
670 &estate_funding_domains,
671 created_at_time,
672 desired.canisters.len(),
673 &mut accumulator,
674 )?;
675 let observation_count = maximum_observation_count(
676 desired,
677 &accumulator.canisters,
678 &protocol_actions,
679 observation.additional_controlled_cycles.len(),
680 )?;
681 let observation_count = checked_add(
682 observation_count,
683 reinstall::funding_observation_count(desired, reinstall, &accumulator.canisters)?,
684 "reinstall funding observations",
685 )?;
686 accumulator.add_burn(
687 bounds
688 .observation_burn
689 .checked_mul(observation_count)
690 .ok_or(EnsurePolicyError::ArithmeticOverflow {
691 field: "observation burn",
692 })?,
693 )?;
694
695 let observed_estate_funding_cycles =
696 estate_funding_domains
697 .iter()
698 .try_fold(0_u128, |total, domain| {
699 checked_add(
700 total,
701 domain.available_cycles.unwrap_or_default(),
702 "observed estate funding cycles",
703 )
704 })?;
705 let maximum_estate_creation_fee_cycles =
706 estate_funding_domains
707 .iter()
708 .try_fold(0_u128, |total, domain| {
709 checked_add(
710 total,
711 domain.maximum_creation_fee_cycles,
712 "maximum estate creation fees",
713 )
714 })?;
715 let observed_controlled_cycles = checked_add(
716 observed_native_cycles,
717 observed_estate_funding_cycles,
718 "observed controlled cycles",
719 )?;
720 let maximum_operator_debit_cycles = checked_add(
721 accumulator.new_funding,
722 accumulator.fees,
723 "maximum operator debit",
724 )?;
725 if observation.operator_cycles < maximum_operator_debit_cycles {
726 }
729 let available_after_operator_funding = checked_add(
730 observed_controlled_cycles,
731 accumulator.new_funding,
732 "cycle conservation available balance",
733 )?;
734 let available_after_estate_fees = available_after_operator_funding
735 .checked_sub(maximum_estate_creation_fee_cycles)
736 .ok_or(EnsurePolicyError::ArithmeticOverflow {
737 field: "estate creation fees",
738 })?;
739 let recovery_review = continuation
740 .as_ref()
741 .map(|authority| {
742 let mut review = recovery::review(
743 observation,
744 bounds,
745 authority.maximum_successor_actions,
746 authority.fixture_publication_retry_attempts,
747 accumulator.execution_burn,
748 available_after_estate_fees,
749 )?;
750 review.startup_funding = recovery::startup_forecasts(desired, artifacts, bounds)?;
751 Ok::<_, EnsurePolicyError>(review)
752 })
753 .transpose()?
754 .map(Box::new);
755 if let Some(review) = &recovery_review {
756 accumulator.add_burn(review.continuation_reserve_cycles)?;
757 }
758 let expected_post_operation_cycles = available_after_estate_fees
759 .checked_sub(accumulator.execution_burn)
760 .ok_or_else(|| {
761 insufficient_cycle_conservation(
762 &accumulator,
763 protocol_actions.len(),
764 available_after_operator_funding,
765 accumulator.execution_burn,
766 )
767 })?;
768 let conservation = CycleConservation {
769 estate_funding_domains,
770 expected_post_operation_cycles,
771 maximum_execution_burn_cycles: accumulator.execution_burn,
772 maximum_new_funding_cycles: accumulator.new_funding,
773 maximum_operator_debit_cycles,
774 maximum_unavoidable_fee_cycles: accumulator.fees,
775 observed_controlled_cycles,
776 retained_in_reused_canisters_cycles: accumulator.retained,
777 scheduled_transfer_cycles: accumulator.transfers,
778 };
779 let mut plan = FleetEnsurePlan {
780 continuation,
781 canisters: accumulator.canisters,
782 conservation,
783 desired_sha256: desired_sha256.to_string(),
784 environment: desired.environment.clone(),
785 fleet: requested_fleet.to_string(),
786 operation_id,
787 plan_sha256: String::new(),
788 planned_at_time: created_at_time,
789 protocol_actions,
790 recovery_review,
791 reinstall: reinstall.cloned().map(Box::new),
792 root_reinstall_bindings: Vec::new(),
793 root_start_authority: None,
794 reviewed_desired: Some(Box::new(
795 crate::fleet_ensure::model::ReviewedDesiredFleetRecord::capture(desired),
796 )),
797 schema_version: FLEET_ENSURE_SCHEMA_VERSION,
798 scope: FleetEnsurePlanScope::Full,
799 terminal_inventory_operation_id: None,
800 };
801 plan.plan_sha256 = expected_plan_sha256(&plan);
802 Ok(plan)
803}
804
805fn pool_readiness_authority_matches(
806 desired: &DesiredFleet,
807 observation: &FleetObservation,
808 principal: &str,
809 minimum_ready: u32,
810 readiness_floor: &Cycles,
811 maximum_updates: Option<u32>,
812) -> bool {
813 let network_matches = desired.bootstrap.as_ref().is_some_and(|bootstrap| {
814 let mainnet =
815 bootstrap.canonical_network_id == canic_core::ids::CanonicalNetworkId::ic_mainnet();
816 (mainnet && !bootstrap.fresh_estate) == maximum_updates.is_some()
817 });
818 network_matches
819 && observation
820 .estate_funding_domains
821 .iter()
822 .any(|(root, domain)| {
823 domain.root_principal.as_deref() == Some(principal)
824 && desired_root_pool_policy(desired, root).is_ok_and(|pool| {
825 pool.minimum_size == minimum_ready
826 && pool.canister_cycles == *readiness_floor
827 && maximum_updates.is_none_or(|maximum| {
828 maximum == pool_maintenance_update_bound(minimum_ready)
829 })
830 })
831 })
832}
833
834fn append_pool_reconciliation_funding(
836 desired: &DesiredFleet,
837 observation: &FleetObservation,
838 protocol_actions: &[EnsureAction],
839 bounds: CycleBounds,
840 created_at_time: u64,
841 accumulator: &mut PlanAccumulator,
842) -> Result<(), EnsurePolicyError> {
843 for (index, action) in protocol_actions.iter().enumerate() {
844 let EnsureAction::FleetProtocol {
845 action,
846 principal: root,
847 name,
848 ..
849 } = action
850 else {
851 continue;
852 };
853 let CurrentFleetProtocolAction::ReconcilePoolAsset { request, .. } = action.as_ref() else {
854 continue;
855 };
856 let principal = request.canister_id.to_text();
857 let pool = observation
858 .estate_funding_domains
859 .values()
860 .find(|domain| domain.root_principal.as_deref() == Some(root))
861 .and_then(|domain| domain.pool.as_ref())
862 .ok_or_else(|| EnsurePolicyError::InvalidProtocolStep(name.clone()))?;
863 let asset = pool
864 .assets
865 .iter()
866 .find(|asset| asset.principal == principal)
867 .ok_or_else(|| EnsurePolicyError::InvalidProtocolStep(name.clone()))?;
868 let Some(funding) = recovery::pool_funding(pool, asset, root, bounds)? else {
869 continue;
870 };
871 let amount = funding.amount_cycles;
872 let deficit = funding.funding_deficit_cycles;
873 let margin = funding.funding_margin_cycles;
874 let expected_post_cycles = funding.expected_post_cycles;
875 let offset = desired
876 .canisters
877 .len()
878 .checked_add(index)
879 .and_then(|value| u64::try_from(value).ok())
880 .ok_or(EnsurePolicyError::ArithmeticOverflow {
881 field: "pool funding timestamp",
882 })?;
883 let action_time =
884 created_at_time
885 .checked_add(offset)
886 .ok_or(EnsurePolicyError::ArithmeticOverflow {
887 field: "pool funding timestamp",
888 })?;
889 let plan_index = if let Some(index) = accumulator
890 .canisters
891 .iter()
892 .position(|plan| plan.principal.as_deref() == Some(&principal))
893 {
894 index
895 } else {
896 accumulator.canisters.push(CanisterPlan {
897 actions: Vec::new(),
898 disposition: CanisterDisposition::Reuse,
899 name: name.clone(),
900 observed_cycles: asset.cycles,
901 principal: Some(principal.clone()),
902 });
903 accumulator.canisters.len() - 1
904 };
905 let target_name = accumulator.canisters[plan_index].name.clone();
906 if accumulator.canisters[plan_index]
908 .actions
909 .iter()
910 .any(|action| matches!(action, EnsureAction::Fund { .. }))
911 {
912 return Err(EnsurePolicyError::InvalidProtocolStep(name.clone()));
913 }
914 accumulator.canisters[plan_index]
915 .actions
916 .push(EnsureAction::Fund {
917 pool_funding: Some(PoolFundingAuthority {
918 root: root.clone(),
919 lifecycle: asset.lifecycle,
920 }),
921 amount,
922 created_at_time: action_time,
923 expected_post_cycles,
924 funding_deficit_cycles: deficit,
925 funding_margin_cycles: margin,
926 ledger: desired.cycles_ledger.clone(),
927 name: target_name,
928 principal,
929 });
930 accumulator.add_funding(amount)?;
931 accumulator.add_fee(bounds.ledger_fee)?;
932 }
933 Ok(())
934}
935
936fn append_estate_funding_actions(
937 domains: &[EstateFundingDomainPlan],
938 created_at_time: u64,
939 canister_count: usize,
940 accumulator: &mut PlanAccumulator,
941) -> Result<(), EnsurePolicyError> {
942 for (index, domain) in domains.iter().enumerate() {
943 if domain.shortfall_cycles == 0 {
944 continue;
945 }
946 let offset = canister_count
947 .checked_add(index)
948 .and_then(|value| u64::try_from(value).ok())
949 .ok_or(EnsurePolicyError::ArithmeticOverflow {
950 field: "estate funding action timestamp",
951 })?;
952 let action_time =
953 created_at_time
954 .checked_add(offset)
955 .ok_or(EnsurePolicyError::ArithmeticOverflow {
956 field: "estate funding action timestamp",
957 })?;
958 let expected_post_cycles = domain
959 .available_cycles
960 .unwrap_or_default()
961 .checked_add(domain.shortfall_cycles)
962 .ok_or(EnsurePolicyError::ArithmeticOverflow {
963 field: "estate funding destination balance",
964 })?;
965 let principal = domain
966 .root_principal
967 .clone()
968 .unwrap_or_else(|| format!("created:{}", domain.root));
969 let root_plan = accumulator
970 .canisters
971 .iter_mut()
972 .find(|canister| canister.name == domain.root)
973 .ok_or_else(|| EnsurePolicyError::EstateFundingTopology {
974 reason: format!("Root {} has no canister plan", domain.root),
975 })?;
976 root_plan.actions.push(EnsureAction::FundEstate {
977 amount: domain.shortfall_cycles,
978 created_at_time: action_time,
979 expected_post_cycles,
980 ledger: domain.cycles_ledger.clone(),
981 ledger_fee_cycles: domain.ledger_fee_cycles,
982 name: domain.root.clone(),
983 principal,
984 });
985 accumulator.add_funding(domain.shortfall_cycles)?;
986 accumulator.add_fee(domain.ledger_fee_cycles)?;
987 }
988 Ok(())
989}
990
991pub(crate) struct RootStartPlanInput<'a> {
993 pub state: &'a FleetEnsureStateRecord,
994 pub authority: Option<&'a RetainedRootStartAuthorityRecord>,
995 pub created_at_time: u64,
996 pub desired: &'a DesiredFleet,
997 pub desired_sha256: &'a str,
998 pub observation: &'a RootManagementObservation,
999 pub requested_fleet: &'a str,
1000}
1001
1002fn root_management_principal<'a>(
1003 configured: &'a crate::fleet_ensure::model::DesiredCanister,
1004 state: &'a FleetEnsureStateRecord,
1005 fleet: &str,
1006) -> Result<&'a str, EnsurePolicyError> {
1007 let retained = state
1008 .pending_principals
1009 .get(&configured.name)
1010 .or_else(|| state.principals.get(&configured.name))
1011 .map(String::as_str);
1012 let configured_principal = configured.principal.as_deref();
1013 let conflict = state.fleet != fleet
1014 || matches!((configured_principal, retained), (Some(configured), Some(retained)) if configured != retained);
1015 if conflict {
1016 return Err(EnsurePolicyError::RootManagementAuthorityMismatch {
1017 field: "retained Principal",
1018 name: configured.name.clone(),
1019 });
1020 }
1021 configured_principal.or(retained).ok_or_else(|| {
1022 EnsurePolicyError::RootManagementAuthorityMismatch {
1023 field: "Principal",
1024 name: configured.name.clone(),
1025 }
1026 })
1027}
1028
1029pub(crate) fn compile_root_start_prerequisite_plan(
1031 input: RootStartPlanInput<'_>,
1032) -> Result<Option<FleetEnsurePlan>, EnsurePolicyError> {
1033 compile_root_start_plan(&input, None)
1034}
1035
1036pub(crate) fn recompile_root_start_prerequisite_plan(
1038 input: RootStartPlanInput<'_>,
1039 reviewed_targets: &BTreeSet<String>,
1040) -> Result<FleetEnsurePlan, EnsurePolicyError> {
1041 compile_root_start_plan(&input, Some(reviewed_targets))?.ok_or_else(|| {
1042 EnsurePolicyError::MissingRootManagementObservation {
1043 name: "reviewed Root-start target".to_string(),
1044 }
1045 })
1046}
1047
1048#[expect(
1049 clippy::too_many_lines,
1050 reason = "one pure boundary validates and compiles the complete Root-start authority"
1051)]
1052fn compile_root_start_plan(
1053 input: &RootStartPlanInput<'_>,
1054 reviewed_targets: Option<&BTreeSet<String>>,
1055) -> Result<Option<FleetEnsurePlan>, EnsurePolicyError> {
1056 let RootStartPlanInput {
1057 state,
1058 authority,
1059 created_at_time,
1060 desired,
1061 desired_sha256,
1062 observation,
1063 requested_fleet,
1064 } = *input;
1065 validate_authority(desired, requested_fleet)?;
1066 let bounds = cycle_bounds(desired)?;
1067 let configured_roots = desired
1068 .canisters
1069 .iter()
1070 .filter(|configured| {
1071 configured.kind == DesiredCanisterKind::Root
1072 && configured.presence == DesiredPresence::Present
1073 })
1074 .collect::<Vec<_>>();
1075 if observation.roots.len() != configured_roots.len() {
1076 return Err(EnsurePolicyError::MissingRootManagementObservation {
1077 name: "configured Root set".to_string(),
1078 });
1079 }
1080
1081 let mut canisters = Vec::new();
1082 let mut root_start_bindings = Vec::new();
1083 let mut observed_controlled_cycles = 0_u128;
1084 for configured in configured_roots {
1085 let observed = observation.roots.get(&configured.name).ok_or_else(|| {
1086 EnsurePolicyError::MissingRootManagementObservation {
1087 name: configured.name.clone(),
1088 }
1089 })?;
1090 let expected_principal = root_management_principal(configured, state, requested_fleet)?;
1091 if observed.name != configured.name || observed.live.principal != expected_principal {
1092 return Err(EnsurePolicyError::RootManagementAuthorityMismatch {
1093 field: "Principal",
1094 name: configured.name.clone(),
1095 });
1096 }
1097 if observed.subnet != configured.subnet {
1098 return Err(EnsurePolicyError::RootManagementAuthorityMismatch {
1099 field: "Subnet",
1100 name: configured.name.clone(),
1101 });
1102 }
1103 if observed.live.status == CanisterRuntimeStatus::Stopping {
1104 return Err(EnsurePolicyError::RootStopping {
1105 name: configured.name.clone(),
1106 });
1107 }
1108 let reviewed = reviewed_targets.is_some_and(|targets| targets.contains(&configured.name));
1109 let needs_start = observed.live.status == CanisterRuntimeStatus::Stopped;
1110 if reviewed_targets.is_some() && needs_start && !reviewed {
1111 return Err(EnsurePolicyError::RootManagementAuthorityMismatch {
1112 field: "reviewed runtime transition",
1113 name: configured.name.clone(),
1114 });
1115 }
1116 if !needs_start && !reviewed {
1117 continue;
1118 }
1119 if !configured.controller_canisters.is_empty() {
1120 return Err(EnsurePolicyError::RootManagementAuthorityMismatch {
1121 field: "controller authority",
1122 name: configured.name.clone(),
1123 });
1124 }
1125 let mut expected_controllers = configured.controllers.clone();
1126 expected_controllers.sort();
1127 let mut actual_controllers = observed.live.controllers.clone();
1128 actual_controllers.sort();
1129 if actual_controllers != expected_controllers {
1130 return Err(EnsurePolicyError::RootManagementAuthorityMismatch {
1131 field: "controllers",
1132 name: configured.name.clone(),
1133 });
1134 }
1135 let observed_module = observed.live.module_sha256.as_deref().ok_or_else(|| {
1136 EnsurePolicyError::RootManagementAuthorityMismatch {
1137 field: "module SHA-256",
1138 name: configured.name.clone(),
1139 }
1140 })?;
1141 root_start_bindings.push(RootManagementBinding {
1142 controllers: actual_controllers.clone(),
1143 name: configured.name.clone(),
1144 module_sha256: observed_module.to_string(),
1145 principal: observed.live.principal.clone(),
1146 subnet: observed.subnet.clone(),
1147 });
1148 if observed.live.root_owned_lifecycle.is_some() || observed.live.reinstall_required {
1149 return Err(EnsurePolicyError::RootManagementAuthorityMismatch {
1150 field: "runtime authority",
1151 name: configured.name.clone(),
1152 });
1153 }
1154 observed_controlled_cycles = checked_add(
1155 observed_controlled_cycles,
1156 observed.live.cycles,
1157 "Root-start observed cycles",
1158 )?;
1159 let actions = vec![EnsureAction::Start {
1160 name: configured.name.clone(),
1161 principal: observed.live.principal.clone(),
1162 }];
1163 canisters.push(CanisterPlan {
1164 actions,
1165 disposition: CanisterDisposition::Reuse,
1166 name: configured.name.clone(),
1167 observed_cycles: observed.live.cycles,
1168 principal: Some(observed.live.principal.clone()),
1169 });
1170 }
1171 if canisters.is_empty() {
1172 return Ok(None);
1173 }
1174 let retained_authority = validate_root_start_authority(
1175 desired,
1176 requested_fleet,
1177 authority,
1178 &root_start_bindings,
1179 true,
1180 )?;
1181 if let Some(reviewed_targets) = reviewed_targets
1182 && (reviewed_targets.len() != canisters.len()
1183 || canisters
1184 .iter()
1185 .any(|canister| !reviewed_targets.contains(&canister.name)))
1186 {
1187 return Err(EnsurePolicyError::RootManagementAuthorityMismatch {
1188 field: "reviewed Root set",
1189 name: "Root-start prerequisite".to_string(),
1190 });
1191 }
1192 let target_count = u128::try_from(
1193 canisters
1194 .iter()
1195 .map(|canister| canister.actions.len())
1196 .sum::<usize>(),
1197 )
1198 .map_err(|_| EnsurePolicyError::ArithmeticOverflow {
1199 field: "Root-start target count",
1200 })?;
1201 let maximum_execution_burn_cycles = bounds.update_burn.checked_mul(target_count).ok_or(
1202 EnsurePolicyError::ArithmeticOverflow {
1203 field: "Root-start execution burn",
1204 },
1205 )?;
1206 let expected_post_operation_cycles = observed_controlled_cycles
1207 .checked_sub(maximum_execution_burn_cycles)
1208 .ok_or(EnsurePolicyError::ArithmeticOverflow {
1209 field: "Root-start conservation equation",
1210 })?;
1211 let mut plan = FleetEnsurePlan {
1212 canisters,
1213 continuation: None,
1214 conservation: CycleConservation {
1215 estate_funding_domains: Vec::new(),
1216 expected_post_operation_cycles,
1217 maximum_execution_burn_cycles,
1218 maximum_new_funding_cycles: 0,
1219 maximum_operator_debit_cycles: 0,
1220 maximum_unavoidable_fee_cycles: 0,
1221 observed_controlled_cycles,
1222 retained_in_reused_canisters_cycles: observed_controlled_cycles,
1223 scheduled_transfer_cycles: 0,
1224 },
1225 desired_sha256: desired_sha256.to_string(),
1226 environment: desired.environment.clone(),
1227 fleet: requested_fleet.to_string(),
1228 operation_id: operation_id(desired_sha256, &desired.environment, requested_fleet),
1229 plan_sha256: String::new(),
1230 planned_at_time: created_at_time,
1231 protocol_actions: Vec::new(),
1232 recovery_review: None,
1233 reinstall: None,
1234 root_reinstall_bindings: Vec::new(),
1235 root_start_authority: retained_authority.map(|authority| Box::new(authority.clone())),
1236 reviewed_desired: Some(Box::new(
1237 crate::fleet_ensure::model::ReviewedDesiredFleetRecord::capture(desired),
1238 )),
1239 schema_version: FLEET_ENSURE_SCHEMA_VERSION,
1240 scope: FleetEnsurePlanScope::RootStartPrerequisite,
1241 terminal_inventory_operation_id: None,
1242 };
1243 plan.plan_sha256 = expected_plan_sha256(&plan);
1244 Ok(Some(plan))
1245}
1246
1247fn validate_root_start_authority<'a>(
1248 desired: &DesiredFleet,
1249 requested_fleet: &str,
1250 authority: Option<&'a RetainedRootStartAuthorityRecord>,
1251 root_start_bindings: &[RootManagementBinding],
1252 authority_required: bool,
1253) -> Result<Option<&'a RetainedRootStartAuthorityRecord>, EnsurePolicyError> {
1254 if !authority_required {
1255 return Ok(None);
1256 }
1257 if root_start_bindings.is_empty() {
1258 return Err(EnsurePolicyError::RootManagementAuthorityMismatch {
1259 field: "retained module authority",
1260 name: "Root-start prerequisite".to_string(),
1261 });
1262 }
1263 let authority =
1264 authority.ok_or_else(|| EnsurePolicyError::RootManagementAuthorityMismatch {
1265 field: "retained module authority",
1266 name: root_start_bindings[0].name.clone(),
1267 })?;
1268 let bootstrap = desired.bootstrap.as_ref().ok_or_else(|| {
1269 EnsurePolicyError::RootManagementAuthorityMismatch {
1270 field: "Fleet bootstrap authority",
1271 name: root_start_bindings[0].name.clone(),
1272 }
1273 })?;
1274 let mut expected_bindings = root_start_bindings.to_vec();
1275 expected_bindings.sort_by(|left, right| left.name.cmp(&right.name));
1276 let mut actual_bindings = authority.roots.clone();
1277 actual_bindings.sort_by(|left, right| left.name.cmp(&right.name));
1278 let path_identity_matches =
1279 authority.environment == desired.environment && authority.fleet == requested_fleet;
1280 let fleet_identity_matches = authority.fleet_id == bootstrap.fleet_id;
1281 let record_identity_matches =
1282 authority.schema_version == FLEET_ENSURE_SCHEMA_VERSION && authority.has_valid_digest();
1283 let bindings_match = actual_bindings == expected_bindings;
1284 if !(path_identity_matches
1285 && fleet_identity_matches
1286 && record_identity_matches
1287 && bindings_match)
1288 {
1289 return Err(EnsurePolicyError::RootManagementAuthorityMismatch {
1290 field: "retained module authority",
1291 name: root_start_bindings[0].name.clone(),
1292 });
1293 }
1294 Ok(Some(authority))
1295}
1296
1297fn retained_root_observation_is_start_only(
1298 desired: &DesiredFleet,
1299 plans: &[CanisterPlan],
1300 observation: &FleetObservation,
1301) -> bool {
1302 let only_start_effects = plans
1303 .iter()
1304 .flat_map(|plan| &plan.actions)
1305 .all(|action| matches!(action, EnsureAction::Start { .. }));
1306 only_start_effects
1307 && desired
1308 .canisters
1309 .iter()
1310 .filter(|configured| {
1311 observation
1312 .canisters
1313 .get(&configured.name)
1314 .and_then(Option::as_ref)
1315 .is_some_and(|live| {
1316 live.root_owned_lifecycle == Some(RootOwnedCanisterLifecycle::Retained)
1317 })
1318 })
1319 .all(|configured| {
1320 configured.parent.as_ref().is_some_and(|parent| {
1321 plans.iter().any(|plan| {
1322 plan.name == *parent
1323 && plan.actions.iter().any(|action| {
1324 matches!(action, EnsureAction::Start { name, .. } if name == parent)
1325 })
1326 })
1327 })
1328 })
1329}
1330
1331pub(crate) fn validate_path_identity(
1333 desired: &DesiredFleet,
1334 requested_fleet: &str,
1335) -> Result<(), EnsurePolicyError> {
1336 if desired.schema_version != FLEET_ENSURE_SCHEMA_VERSION {
1337 return Err(EnsurePolicyError::WrongSchema {
1338 actual: desired.schema_version,
1339 expected: FLEET_ENSURE_SCHEMA_VERSION,
1340 });
1341 }
1342 if desired.fleet != requested_fleet {
1343 return Err(EnsurePolicyError::FleetMismatch {
1344 actual: desired.fleet.clone(),
1345 expected: requested_fleet.to_string(),
1346 });
1347 }
1348 validate_path_labels(&desired.environment, requested_fleet)
1349}
1350
1351pub(crate) fn validate_path_labels(
1352 environment: &str,
1353 requested_fleet: &str,
1354) -> Result<(), EnsurePolicyError> {
1355 requested_fleet
1356 .parse::<FleetName>()
1357 .map_err(|_| EnsurePolicyError::UnsafePathLabel {
1358 field: "fleet",
1359 value: requested_fleet.to_string(),
1360 })?;
1361 if environment.is_empty()
1362 || environment.len() > 64
1363 || !environment
1364 .bytes()
1365 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_'))
1366 {
1367 return Err(EnsurePolicyError::UnsafePathLabel {
1368 field: "environment",
1369 value: environment.to_string(),
1370 });
1371 }
1372 Ok(())
1373}
1374
1375#[expect(
1376 clippy::too_many_arguments,
1377 reason = "pure compiler receives one complete authority tuple"
1378)]
1379fn compile_canister(
1380 desired: &DesiredFleet,
1381 artifacts: &DesiredFleetArtifacts,
1382 configured: &crate::fleet_ensure::model::DesiredCanister,
1383 observed: Option<&LiveCanister>,
1384 observation: &FleetObservation,
1385 cycle_policy: CanisterCyclePolicy,
1386 bounds: CycleBounds,
1387 created_at_time: u64,
1388 force_reinstall: bool,
1389 accumulator: &mut PlanAccumulator,
1390) -> Result<CanisterPlan, EnsurePolicyError> {
1391 match (configured.presence, observed) {
1392 (DesiredPresence::Absent, None) => Ok(CanisterPlan {
1393 actions: Vec::new(),
1394 disposition: CanisterDisposition::Delete,
1395 name: configured.name.clone(),
1396 observed_cycles: 0,
1397 principal: configured.principal.clone(),
1398 }),
1399 (DesiredPresence::Absent, Some(live)) => retire_plan(
1400 desired,
1401 artifacts,
1402 configured,
1403 live,
1404 bounds,
1405 CanisterDisposition::Delete,
1406 accumulator,
1407 ),
1408 (DesiredPresence::Present, None) => {
1409 let fresh_pool = configured.kind == DesiredCanisterKind::Pool
1410 && desired
1411 .bootstrap
1412 .as_ref()
1413 .is_some_and(|bootstrap| bootstrap.fresh_estate)
1414 && configured.principal.is_none();
1415 if configured.kind == DesiredCanisterKind::Pool && !fresh_pool {
1416 return Err(EnsurePolicyError::MissingPoolAsset {
1417 name: configured.name.clone(),
1418 });
1419 }
1420 if configured.principal.is_some() {
1421 return Err(EnsurePolicyError::MissingObservation {
1422 name: configured.name.clone(),
1423 });
1424 }
1425 create_plan(
1426 desired,
1427 artifacts,
1428 configured,
1429 cycle_policy,
1430 bounds,
1431 created_at_time,
1432 CanisterDisposition::Create,
1433 accumulator,
1434 None,
1435 )
1436 }
1437 (DesiredPresence::Present, Some(live))
1438 if configured.replace
1439 && configured.principal.as_deref() == Some(live.principal.as_str()) =>
1440 {
1441 create_plan(
1442 desired,
1443 artifacts,
1444 configured,
1445 cycle_policy,
1446 bounds,
1447 created_at_time,
1448 CanisterDisposition::Replace,
1449 accumulator,
1450 Some(live),
1451 )
1452 }
1453 (DesiredPresence::Present, Some(live)) => reuse_plan(
1454 desired,
1455 artifacts,
1456 configured,
1457 live,
1458 observation,
1459 cycle_policy,
1460 bounds,
1461 created_at_time,
1462 force_reinstall,
1463 accumulator,
1464 ),
1465 }
1466}
1467
1468#[expect(
1469 clippy::too_many_arguments,
1470 reason = "the pure create compiler receives the complete reviewed authority tuple"
1471)]
1472fn create_plan(
1473 desired: &DesiredFleet,
1474 artifacts: &DesiredFleetArtifacts,
1475 configured: &crate::fleet_ensure::model::DesiredCanister,
1476 cycle_policy: CanisterCyclePolicy,
1477 bounds: CycleBounds,
1478 created_at_time: u64,
1479 disposition: CanisterDisposition,
1480 accumulator: &mut PlanAccumulator,
1481 replaced: Option<&LiveCanister>,
1482) -> Result<CanisterPlan, EnsurePolicyError> {
1483 let temporary_pool_observation_controller = configured.kind == DesiredCanisterKind::Pool
1484 && desired
1485 .bootstrap
1486 .as_ref()
1487 .is_some_and(|bootstrap| bootstrap.fresh_estate)
1488 && configured.principal.is_none()
1489 && configured.controllers.is_empty()
1490 && configured.parent.as_ref().is_some_and(|parent| {
1491 configured.controller_canisters.as_slice() == std::slice::from_ref(parent)
1492 });
1493 if temporary_pool_observation_controller {
1494 validate_fresh_pool_creation_funding(&configured.name, cycle_policy, bounds)?;
1495 }
1496 accumulator.add_burn(bounds.update_burn)?;
1499 accumulator.add_funding(cycle_policy.initial_cycles)?;
1500 accumulator.add_fee(bounds.ledger_fee)?;
1501 accumulator.add_fee(bounds.management_creation_fee)?;
1502 let symbolic = format!("created:{}", configured.name);
1503 let mut creation_controllers = configured.controllers.clone();
1504 if temporary_pool_observation_controller {
1505 creation_controllers.push(desired.operator.clone());
1506 creation_controllers.sort();
1507 creation_controllers.dedup();
1508 }
1509 let mut actions = vec![EnsureAction::Create {
1510 controller_canisters: configured.controller_canisters.clone(),
1511 controllers: creation_controllers,
1512 created_at_time,
1513 ledger: desired.cycles_ledger.clone(),
1514 name: configured.name.clone(),
1515 requested_initial_cycles: cycle_policy.initial_cycles,
1516 subnet: configured.subnet.clone(),
1517 }];
1518 if let Some(wasm) = &configured.wasm {
1519 require_install_initializer(desired, configured)?;
1520 let wasm_sha256 = wasm_sha256(artifacts, &configured.name)?;
1521 actions.push(EnsureAction::Install {
1522 canic_init: configured.canic_init.clone(),
1523 reinstall_witness: None,
1524 init_arg: configured.init_arg.clone(),
1525 init_arg_sha256: optional_init_arg_sha256(artifacts, configured)?,
1526 init_candid: configured.init_candid.clone(),
1527 init_candid_sha256: optional_init_candid_sha256(artifacts, configured)?,
1528 mode: InstallMode::Install,
1529 name: configured.name.clone(),
1530 principal: symbolic.clone(),
1531 wasm: wasm.clone(),
1532 wasm_sha256,
1533 });
1534 accumulator.add_burn(bounds.update_burn)?;
1535 }
1536 if temporary_pool_observation_controller {
1537 actions.push(EnsureAction::SetControllers {
1538 controller_canisters: configured.controller_canisters.clone(),
1539 controllers: configured.controllers.clone(),
1540 name: configured.name.clone(),
1541 principal: symbolic,
1542 });
1543 accumulator.add_burn(bounds.update_burn)?;
1544 }
1545 if let Some(live) = replaced {
1546 append_retirement_actions(
1547 desired,
1548 artifacts,
1549 configured,
1550 live,
1551 bounds,
1552 &mut actions,
1553 accumulator,
1554 )?;
1555 }
1556 Ok(CanisterPlan {
1557 actions,
1558 disposition,
1559 name: configured.name.clone(),
1560 observed_cycles: replaced.map_or(0, |live| live.cycles),
1561 principal: replaced
1562 .map(|live| live.principal.clone())
1563 .or_else(|| configured.principal.clone()),
1564 })
1565}
1566
1567fn validate_fresh_pool_creation_funding(
1568 name: &str,
1569 cycle_policy: CanisterCyclePolicy,
1570 bounds: CycleBounds,
1571) -> Result<(), EnsurePolicyError> {
1572 let creation_burn = crate::fleet_ensure::model::creation_observation_burn(
1573 bounds.observation_burn,
1574 bounds.update_burn,
1575 )
1576 .ok_or(EnsurePolicyError::ArithmeticOverflow {
1577 field: "Create execution and observation burn",
1578 })?;
1579 let admissible_burn_cycles = checked_add(
1580 creation_burn,
1581 bounds.update_burn,
1582 "fresh pool pre-import burn",
1583 )?;
1584 let required_creation_funding_cycles = checked_add(
1585 cycle_policy.minimum_cycles,
1586 admissible_burn_cycles,
1587 "fresh pool creation funding",
1588 )?;
1589 if cycle_policy.initial_cycles < required_creation_funding_cycles {
1590 return Err(EnsurePolicyError::FreshPoolCreationFundingInsufficient {
1591 admissible_burn_cycles,
1592 creation_funding_cycles: cycle_policy.initial_cycles,
1593 name: name.to_string(),
1594 readiness_floor_cycles: cycle_policy.minimum_cycles,
1595 required_creation_funding_cycles,
1596 shortfall_cycles: required_creation_funding_cycles - cycle_policy.initial_cycles,
1597 });
1598 }
1599 Ok(())
1600}
1601
1602#[expect(
1603 clippy::too_many_arguments,
1604 reason = "the pure reuse compiler receives the complete reviewed authority tuple"
1605)]
1606fn reuse_plan(
1607 desired: &DesiredFleet,
1608 artifacts: &DesiredFleetArtifacts,
1609 configured: &crate::fleet_ensure::model::DesiredCanister,
1610 live: &LiveCanister,
1611 observation: &FleetObservation,
1612 cycle_policy: CanisterCyclePolicy,
1613 bounds: CycleBounds,
1614 created_at_time: u64,
1615 force_reinstall: bool,
1616 accumulator: &mut PlanAccumulator,
1617) -> Result<CanisterPlan, EnsurePolicyError> {
1618 accumulator.retained = checked_add(accumulator.retained, live.cycles, "retained cycles")?;
1619 let mut actions = Vec::new();
1620 if configured.kind == DesiredCanisterKind::Pool && live.root_owned_lifecycle.is_none() {
1621 return Err(EnsurePolicyError::MissingPoolLifecycle {
1622 name: configured.name.clone(),
1623 });
1624 }
1625 let active_pool_asset = configured.kind == DesiredCanisterKind::Pool
1626 && matches!(
1627 live.root_owned_lifecycle,
1628 Some(RootOwnedCanisterLifecycle::Claimed | RootOwnedCanisterLifecycle::Workload)
1629 );
1630 let separately_managed_pool_balance = matches!(
1631 live.root_owned_lifecycle,
1632 Some(RootOwnedCanisterLifecycle::Reconciling | RootOwnedCanisterLifecycle::Retained)
1633 );
1634 let mut disposition = CanisterDisposition::Reuse;
1635 if let Some(wasm) = &configured.wasm {
1636 let wasm_sha256 = wasm_sha256(artifacts, &configured.name)?;
1637 if force_reinstall || live.module_sha256.as_deref() != Some(wasm_sha256.as_str()) {
1638 require_install_initializer(desired, configured)?;
1639 actions.push(EnsureAction::Install {
1640 canic_init: configured.canic_init.clone(),
1641 reinstall_witness: None,
1642 init_arg: configured.init_arg.clone(),
1643 init_arg_sha256: optional_init_arg_sha256(artifacts, configured)?,
1644 init_candid: configured.init_candid.clone(),
1645 init_candid_sha256: optional_init_candid_sha256(artifacts, configured)?,
1646 mode: if force_reinstall || live.module_sha256.is_some() {
1647 InstallMode::Reinstall
1648 } else {
1649 InstallMode::Install
1650 },
1651 name: configured.name.clone(),
1652 principal: live.principal.clone(),
1653 wasm: wasm.clone(),
1654 wasm_sha256,
1655 });
1656 accumulator.add_burn(bounds.update_burn)?;
1657 disposition = CanisterDisposition::Reinstall;
1658 }
1659 }
1660 if configured.kind != DesiredCanisterKind::Pool && live.status != CanisterRuntimeStatus::Running
1661 {
1662 actions.push(EnsureAction::Start {
1663 name: configured.name.clone(),
1664 principal: live.principal.clone(),
1665 });
1666 accumulator.add_burn(bounds.update_burn)?;
1667 }
1668 let mut actual_controllers = live.controllers.clone();
1669 actual_controllers.sort();
1670 let mut desired_controllers = resolved_controllers(configured, observation)?;
1671 desired_controllers.sort();
1672 if configured.kind != DesiredCanisterKind::Pool && actual_controllers != desired_controllers {
1673 actions.push(EnsureAction::SetControllers {
1674 controller_canisters: configured.controller_canisters.clone(),
1675 controllers: desired_controllers,
1676 name: configured.name.clone(),
1677 principal: live.principal.clone(),
1678 });
1679 accumulator.add_burn(bounds.update_burn)?;
1680 }
1681 append_target_funding(
1682 desired,
1683 configured,
1684 live,
1685 cycle_policy,
1686 bounds,
1687 created_at_time,
1688 active_pool_asset || separately_managed_pool_balance,
1689 &mut actions,
1690 accumulator,
1691 )?;
1692 Ok(CanisterPlan {
1693 actions,
1694 disposition,
1695 name: configured.name.clone(),
1696 observed_cycles: live.cycles,
1697 principal: Some(live.principal.clone()),
1698 })
1699}
1700
1701fn resolved_controllers(
1702 configured: &crate::fleet_ensure::model::DesiredCanister,
1703 observation: &FleetObservation,
1704) -> Result<Vec<String>, EnsurePolicyError> {
1705 let mut controllers = configured.controllers.clone();
1706 for controller in &configured.controller_canisters {
1707 let principal = observation
1708 .canisters
1709 .get(controller)
1710 .and_then(Option::as_ref)
1711 .map(|live| live.principal.clone())
1712 .ok_or_else(|| EnsurePolicyError::ControllerCanisterUnavailable {
1713 controller: controller.clone(),
1714 name: configured.name.clone(),
1715 })?;
1716 controllers.push(principal);
1717 }
1718 controllers.sort();
1719 controllers.dedup();
1720 Ok(controllers)
1721}
1722
1723#[expect(
1724 clippy::too_many_arguments,
1725 reason = "the funding step receives one complete target-local economic authority tuple"
1726)]
1727fn append_target_funding(
1728 desired: &DesiredFleet,
1729 configured: &crate::fleet_ensure::model::DesiredCanister,
1730 live: &LiveCanister,
1731 cycle_policy: CanisterCyclePolicy,
1732 bounds: CycleBounds,
1733 created_at_time: u64,
1734 funding_fenced: bool,
1735 actions: &mut Vec<EnsureAction>,
1736 accumulator: &mut PlanAccumulator,
1737) -> Result<(), EnsurePolicyError> {
1738 if funding_fenced || live.cycles >= cycle_policy.minimum_cycles {
1739 return Ok(());
1740 }
1741 let target_updates =
1742 u128::try_from(actions.len()).map_err(|_| EnsurePolicyError::ArithmeticOverflow {
1743 field: "target update count",
1744 })?;
1745 let update_margin = bounds.update_burn.checked_mul(target_updates).ok_or(
1746 EnsurePolicyError::ArithmeticOverflow {
1747 field: "target update margin",
1748 },
1749 )?;
1750 let target_margin = bounds.observation_burn.checked_add(update_margin).ok_or(
1751 EnsurePolicyError::ArithmeticOverflow {
1752 field: "target funding margin",
1753 },
1754 )?;
1755 let funding_deficit_cycles = cycle_policy.minimum_cycles - live.cycles;
1756 let amount = funding_deficit_cycles.checked_add(target_margin).ok_or(
1757 EnsurePolicyError::ArithmeticOverflow {
1758 field: "funding observation margin",
1759 },
1760 )?;
1761 let expected_post_cycles =
1762 live.cycles
1763 .checked_add(amount)
1764 .ok_or(EnsurePolicyError::ArithmeticOverflow {
1765 field: "funding post-balance",
1766 })?;
1767 actions.insert(
1768 0,
1769 EnsureAction::Fund {
1770 pool_funding: if configured.kind == DesiredCanisterKind::Pool {
1771 live.controllers
1772 .first()
1773 .cloned()
1774 .map(|root| PoolFundingAuthority {
1775 root,
1776 lifecycle: EstatePoolAssetLifecycle::Ready,
1777 })
1778 } else {
1779 None
1780 },
1781 amount,
1782 created_at_time,
1783 expected_post_cycles,
1784 funding_deficit_cycles,
1785 funding_margin_cycles: target_margin,
1786 ledger: desired.cycles_ledger.clone(),
1787 name: configured.name.clone(),
1788 principal: live.principal.clone(),
1789 },
1790 );
1791 accumulator.add_funding(amount)?;
1792 accumulator.add_fee(bounds.ledger_fee)
1793}
1794
1795fn reviewed_reinstall_canisters(
1796 desired: &DesiredFleet,
1797 observation: &FleetObservation,
1798) -> Result<BTreeSet<String>, EnsurePolicyError> {
1799 let mut reinstalls = BTreeSet::new();
1800 for configured in &desired.canisters {
1801 let required = observation
1802 .canisters
1803 .get(&configured.name)
1804 .and_then(Option::as_ref)
1805 .is_some_and(|live| live.reinstall_required);
1806 if !required {
1807 continue;
1808 }
1809 let exact_infrastructure = matches!(
1810 configured.kind,
1811 DesiredCanisterKind::Coordinator
1812 | DesiredCanisterKind::Root
1813 | DesiredCanisterKind::Store
1814 ) && configured.presence == DesiredPresence::Present;
1815 if !exact_infrastructure {
1816 return Err(EnsurePolicyError::InvalidInfrastructureReinstall {
1817 name: configured.name.clone(),
1818 });
1819 }
1820 reinstalls.insert(configured.name.clone());
1821 }
1822 Ok(reinstalls)
1823}
1824
1825fn retire_plan(
1826 desired: &DesiredFleet,
1827 artifacts: &DesiredFleetArtifacts,
1828 configured: &crate::fleet_ensure::model::DesiredCanister,
1829 live: &LiveCanister,
1830 bounds: CycleBounds,
1831 disposition: CanisterDisposition,
1832 accumulator: &mut PlanAccumulator,
1833) -> Result<CanisterPlan, EnsurePolicyError> {
1834 let mut actions = Vec::new();
1835 append_retirement_actions(
1836 desired,
1837 artifacts,
1838 configured,
1839 live,
1840 bounds,
1841 &mut actions,
1842 accumulator,
1843 )?;
1844 Ok(CanisterPlan {
1845 actions,
1846 disposition,
1847 name: configured.name.clone(),
1848 observed_cycles: live.cycles,
1849 principal: Some(live.principal.clone()),
1850 })
1851}
1852
1853fn append_retirement_actions(
1854 desired: &DesiredFleet,
1855 artifacts: &DesiredFleetArtifacts,
1856 configured: &crate::fleet_ensure::model::DesiredCanister,
1857 live: &LiveCanister,
1858 bounds: CycleBounds,
1859 actions: &mut Vec<EnsureAction>,
1860 accumulator: &mut PlanAccumulator,
1861) -> Result<(), EnsurePolicyError> {
1862 if live.cycles > bounds.material_threshold {
1863 let drain = configured
1864 .drain
1865 .as_ref()
1866 .ok_or_else(|| EnsurePolicyError::NoSafeDrain {
1867 name: configured.name.clone(),
1868 cycles: live.cycles,
1869 })?;
1870 if drain.destination != desired.treasury {
1871 return Err(EnsurePolicyError::WrongDrainDestination {
1872 name: configured.name.clone(),
1873 });
1874 }
1875 let maximum_execution_burn_cycles = parse_canister_cycles(
1876 &configured.name,
1877 "drain.maximum_execution_burn_cycles",
1878 &drain.maximum_execution_burn_cycles,
1879 )?;
1880 let reserved = checked_add(
1881 bounds.material_threshold,
1882 maximum_execution_burn_cycles,
1883 "retirement reserve",
1884 )?;
1885 let amount =
1886 live.cycles
1887 .checked_sub(reserved)
1888 .ok_or_else(|| EnsurePolicyError::NoSafeDrain {
1889 name: configured.name.clone(),
1890 cycles: live.cycles,
1891 })?;
1892 if amount == 0 {
1893 return Err(EnsurePolicyError::NoSafeDrain {
1894 name: configured.name.clone(),
1895 cycles: live.cycles,
1896 });
1897 }
1898 actions.push(EnsureAction::Transfer {
1899 amount,
1900 candid: drain.candid.clone(),
1901 candid_sha256: artifacts
1902 .drain_candid_sha256_by_canister
1903 .get(&configured.name)
1904 .cloned()
1905 .ok_or_else(|| EnsurePolicyError::MissingArtifactIdentity {
1906 kind: "drain Candid",
1907 name: configured.name.clone(),
1908 })?,
1909 destination: drain.destination.clone(),
1910 maximum_execution_burn_cycles,
1911 method: drain.method.clone(),
1912 name: configured.name.clone(),
1913 principal: live.principal.clone(),
1914 });
1915 accumulator.transfers = checked_add(accumulator.transfers, amount, "scheduled transfer")?;
1916 accumulator.add_burn(maximum_execution_burn_cycles)?;
1917 }
1918 actions.push(EnsureAction::Stop {
1919 name: configured.name.clone(),
1920 principal: live.principal.clone(),
1921 });
1922 actions.push(EnsureAction::Delete {
1923 maximum_remaining_cycles: bounds.material_threshold,
1924 name: configured.name.clone(),
1925 principal: live.principal.clone(),
1926 });
1927 accumulator.add_burn(bounds.material_threshold)?;
1928 accumulator.add_burn(bounds.update_burn)?;
1929 accumulator.add_burn(bounds.update_burn)?;
1930 Ok(())
1931}
1932
1933#[expect(
1934 clippy::too_many_lines,
1935 reason = "one pure validation boundary checks the complete desired Fleet authority"
1936)]
1937fn validate_authority(
1938 desired: &DesiredFleet,
1939 requested_fleet: &str,
1940) -> Result<(), EnsurePolicyError> {
1941 validate_path_identity(desired, requested_fleet)?;
1942 validate_funding_policy(desired)?;
1943 validate_terminal_pool_capacity(desired)?;
1944 if let Some(bootstrap) = &desired.bootstrap {
1945 for root in &bootstrap.roots {
1946 validate_bootstrap_root_pool_import_capacity(root)?;
1947 }
1948 }
1949 if desired.canisters.len() > MAX_FLEET_ENSURE_CANISTERS {
1950 return Err(EnsurePolicyError::TooManyCanisters {
1951 actual: desired.canisters.len(),
1952 maximum: MAX_FLEET_ENSURE_CANISTERS,
1953 });
1954 }
1955 if desired.protocol_steps.len() > MAX_FLEET_ENSURE_PROTOCOL_STEPS {
1956 return Err(EnsurePolicyError::TooManyProtocolSteps {
1957 actual: desired.protocol_steps.len(),
1958 maximum: MAX_FLEET_ENSURE_PROTOCOL_STEPS,
1959 });
1960 }
1961 if desired.maximum_stalled_observations == 0 {
1962 return Err(EnsurePolicyError::InvalidStallBound);
1963 }
1964 validate_principal("operator", &desired.operator)?;
1965 validate_principal("cycles_ledger", &desired.cycles_ledger)?;
1966 let mut names = BTreeSet::new();
1967 let mut principals = BTreeSet::new();
1968 for configured in &desired.canisters {
1969 if configured.canic_init.is_some()
1970 && (configured.init_arg.is_some() || configured.init_candid.is_some())
1971 {
1972 return Err(EnsurePolicyError::ConflictingCanicInitializer {
1973 name: configured.name.clone(),
1974 });
1975 }
1976 if configured.init_arg.is_some() != configured.init_candid.is_some() {
1977 return Err(EnsurePolicyError::IncompleteInitTemplate(
1978 configured.name.clone(),
1979 ));
1980 }
1981 if configured.name.is_empty()
1982 || configured
1983 .drain
1984 .as_ref()
1985 .is_some_and(|drain| drain.method.is_empty())
1986 {
1987 return Err(EnsurePolicyError::EmptyName);
1988 }
1989 validate_principal(
1990 &format!("canisters.{}.subnet", configured.name),
1991 &configured.subnet,
1992 )?;
1993 for controller in &configured.controllers {
1994 validate_principal(
1995 &format!("canisters.{}.controllers", configured.name),
1996 controller,
1997 )?;
1998 }
1999 for controller in &configured.controller_canisters {
2000 if controller.is_empty() {
2001 return Err(EnsurePolicyError::InvalidTopology {
2002 name: configured.name.clone(),
2003 reason: "controller canister name must not be empty",
2004 });
2005 }
2006 }
2007 if let Some(principal) = &configured.principal {
2008 validate_principal(
2009 &format!("canisters.{}.principal", configured.name),
2010 principal,
2011 )?;
2012 }
2013 let unique_name = names.insert(configured.name.as_str());
2014 let unique_principal = configured
2015 .principal
2016 .as_deref()
2017 .is_none_or(|principal| principals.insert(principal));
2018 if !unique_name || !unique_principal {
2019 return Err(EnsurePolicyError::DuplicateAuthority {
2020 name: configured.name.clone(),
2021 });
2022 }
2023 if configured.presence == DesiredPresence::Present
2024 && !configured.controllers.contains(&desired.operator)
2025 && configured.kind != DesiredCanisterKind::Pool
2026 {
2027 return Err(EnsurePolicyError::MissingOperatorController {
2028 name: configured.name.clone(),
2029 });
2030 }
2031 validate_canic_initializer(desired, configured)?;
2032 let fresh_pool = configured.kind == DesiredCanisterKind::Pool
2033 && desired
2034 .bootstrap
2035 .as_ref()
2036 .is_some_and(|bootstrap| bootstrap.fresh_estate)
2037 && configured.principal.is_none();
2038 if configured.kind == DesiredCanisterKind::Pool
2039 && (configured.presence != DesiredPresence::Present
2040 || (configured.principal.is_none() && !fresh_pool)
2041 || configured.replace
2042 || configured.wasm.is_some()
2043 || configured.drain.is_some()
2044 || configured.init_arg.is_some()
2045 || configured.init_candid.is_some()
2046 || configured.canic_init.is_some())
2047 {
2048 return Err(EnsurePolicyError::InvalidTopology {
2049 name: configured.name.clone(),
2050 reason: "pool assets must be exact present non-replaceable identities without independent runtime authority",
2051 });
2052 }
2053 }
2054 for (index, configured) in desired.canisters.iter().enumerate() {
2055 for controller in &configured.controller_canisters {
2056 let Some(controller_index) = desired
2057 .canisters
2058 .iter()
2059 .position(|candidate| candidate.name == *controller)
2060 else {
2061 return Err(EnsurePolicyError::ControllerCanisterUnavailable {
2062 controller: controller.clone(),
2063 name: configured.name.clone(),
2064 });
2065 };
2066 if controller_index >= index || controller == &configured.name {
2067 return Err(EnsurePolicyError::InvalidTopology {
2068 name: configured.name.clone(),
2069 reason: "controller canister must be a distinct earlier desired role",
2070 });
2071 }
2072 }
2073 }
2074 validate_typed_topology(desired)?;
2075 let mut protocol_names = BTreeSet::new();
2076 for step in &desired.protocol_steps {
2077 if step.name.is_empty()
2078 || step.command_method.is_empty()
2079 || step.status_method.is_empty()
2080 || !protocol_names.insert(step.name.as_str())
2081 || !desired.canisters.iter().any(|canister| {
2082 canister.name == step.canister && canister.presence == DesiredPresence::Present
2083 })
2084 {
2085 return Err(EnsurePolicyError::InvalidProtocolStep(step.name.clone()));
2086 }
2087 }
2088 if !desired.canisters.iter().any(|configured| {
2089 configured.presence == DesiredPresence::Present && configured.name == desired.treasury
2090 }) {
2091 return Err(EnsurePolicyError::MissingTreasury {
2092 treasury: desired.treasury.clone(),
2093 });
2094 }
2095 if desired
2096 .canisters
2097 .iter()
2098 .any(|configured| configured.name == desired.treasury && configured.replace)
2099 {
2100 return Err(EnsurePolicyError::TreasuryReplacement {
2101 treasury: desired.treasury.clone(),
2102 });
2103 }
2104 Ok(())
2105}
2106
2107pub fn validate_funding_policy(desired: &DesiredFleet) -> Result<(), EnsurePolicyError> {
2109 let Some(bootstrap) = &desired.bootstrap else {
2110 return Ok(());
2111 };
2112 let ic_mainnet =
2113 bootstrap.canonical_network_id == canic_core::ids::CanonicalNetworkId::ic_mainnet();
2114 for root in &bootstrap.roots {
2115 validate_fleet_subnet_root_funding_authority(&root.funding, ic_mainnet)?;
2116 }
2117 if let Some(coordinator) = &bootstrap.root_funding {
2118 validate_coordinator_root_funding_policy(coordinator)?;
2119 validate_fleet_root_funding_capacity(
2120 coordinator,
2121 bootstrap.roots.iter().map(|root| &root.funding),
2122 )?;
2123 }
2124 Ok(())
2125}
2126
2127pub fn validate_terminal_pool_capacity(desired: &DesiredFleet) -> Result<(), EnsurePolicyError> {
2129 let Some(bootstrap) = &desired.bootstrap else {
2130 return Ok(());
2131 };
2132 if desired.protocol.is_none() {
2133 return Ok(());
2134 }
2135 for root in &bootstrap.roots {
2136 let workloads = desired_initial_root_workloads(desired, &root.root)?;
2137 let pool = &root.limits.canister_pool;
2138 terminal_pool_supply(&root.root, workloads, pool.minimum_size, pool.maximum_size)?;
2139 }
2140 Ok(())
2141}
2142
2143pub(super) fn terminal_pool_supply(
2145 root: &str,
2146 workloads: u32,
2147 ready_floor: u32,
2148 maximum_size: u32,
2149) -> Result<u32, EnsurePolicyError> {
2150 workloads
2151 .checked_add(ready_floor)
2152 .filter(|total| *total <= maximum_size)
2153 .ok_or_else(|| EnsurePolicyError::TerminalPoolCapacity {
2154 root: root.to_string(),
2155 workloads,
2156 ready_floor,
2157 maximum_size,
2158 })
2159}
2160
2161fn validate_bootstrap_root_pool_import_capacity(
2162 root: &crate::fleet_ensure::model::DesiredFleetBootstrapRoot,
2163) -> Result<(), EnsurePolicyError> {
2164 validate_root_pool_import_capacity(&RootPoolImportCapacityInput {
2165 import_count: root.canister_pool_imports.len(),
2166 maximum_size: root.limits.canister_pool.maximum_size,
2167 root: root.root.clone(),
2168 })?;
2169 Ok(())
2170}
2171
2172fn validate_canic_initializer(
2173 desired: &DesiredFleet,
2174 configured: &crate::fleet_ensure::model::DesiredCanister,
2175) -> Result<(), EnsurePolicyError> {
2176 use crate::fleet_ensure::model::DesiredCanisterInit;
2177
2178 let matches_kind = matches!(
2179 (&configured.kind, &configured.canic_init),
2180 (_, None)
2181 | (
2182 DesiredCanisterKind::Coordinator,
2183 Some(DesiredCanisterInit::Coordinator)
2184 )
2185 | (
2186 DesiredCanisterKind::Root,
2187 Some(DesiredCanisterInit::Root { .. })
2188 )
2189 | (
2190 DesiredCanisterKind::Store,
2191 Some(DesiredCanisterInit::Store { .. })
2192 )
2193 );
2194 let infrastructure_artifact = desired.protocol.is_some()
2195 && configured.wasm.is_some()
2196 && matches!(
2197 configured.kind,
2198 DesiredCanisterKind::Coordinator
2199 | DesiredCanisterKind::Root
2200 | DesiredCanisterKind::Store
2201 );
2202 if !matches_kind
2203 || (configured.canic_init.is_some() && desired.bootstrap.is_none())
2204 || (infrastructure_artifact && configured.canic_init.is_none())
2205 {
2206 return Err(EnsurePolicyError::MissingCanicInitializer {
2207 name: configured.name.clone(),
2208 });
2209 }
2210 if let (Some(bootstrap), Some(initializer)) = (&desired.bootstrap, &configured.canic_init) {
2211 let bound = match initializer {
2212 DesiredCanisterInit::Coordinator => configured.name == bootstrap.coordinator,
2213 DesiredCanisterInit::Root { root } => {
2214 root == &configured.name && bootstrap.roots.iter().any(|entry| entry.root == *root)
2215 }
2216 DesiredCanisterInit::Store { root } => bootstrap
2217 .roots
2218 .iter()
2219 .any(|entry| entry.root == *root && entry.store == configured.name),
2220 };
2221 if !bound {
2222 return Err(EnsurePolicyError::MissingCanicInitializer {
2223 name: configured.name.clone(),
2224 });
2225 }
2226 }
2227 Ok(())
2228}
2229
2230fn require_install_initializer(
2231 desired: &DesiredFleet,
2232 configured: &crate::fleet_ensure::model::DesiredCanister,
2233) -> Result<(), EnsurePolicyError> {
2234 if desired.protocol.is_some()
2235 && matches!(
2236 configured.kind,
2237 DesiredCanisterKind::Coordinator
2238 | DesiredCanisterKind::Root
2239 | DesiredCanisterKind::Store
2240 )
2241 && configured.canic_init.is_none()
2242 {
2243 return Err(EnsurePolicyError::MissingCanicInitializer {
2244 name: configured.name.clone(),
2245 });
2246 }
2247 Ok(())
2248}
2249
2250fn validate_typed_topology(desired: &DesiredFleet) -> Result<(), EnsurePolicyError> {
2251 let coordinators = desired
2252 .canisters
2253 .iter()
2254 .filter(|canister| {
2255 canister.presence == DesiredPresence::Present
2256 && canister.kind == DesiredCanisterKind::Coordinator
2257 })
2258 .collect::<Vec<_>>();
2259 let [coordinator] = coordinators.as_slice() else {
2260 return Err(EnsurePolicyError::InvalidTopology {
2261 name: desired.fleet.clone(),
2262 reason: "exactly one present Coordinator role is required",
2263 });
2264 };
2265 if coordinator.parent.is_some() {
2266 return Err(EnsurePolicyError::InvalidTopology {
2267 name: coordinator.name.clone(),
2268 reason: "Coordinator cannot have a topology parent",
2269 });
2270 }
2271 let has_managed_topology = desired.canisters.iter().any(|canister| {
2272 canister.presence == DesiredPresence::Present
2273 && matches!(
2274 canister.kind,
2275 DesiredCanisterKind::Root
2276 | DesiredCanisterKind::Store
2277 | DesiredCanisterKind::Component
2278 )
2279 });
2280 if has_managed_topology && desired.protocol.is_none() && desired.protocol_steps.is_empty() {
2281 return Err(EnsurePolicyError::InvalidTopology {
2282 name: desired.fleet.clone(),
2283 reason: "managed Fleet roles require complete typed protocol intent",
2284 });
2285 }
2286 for canister in desired
2287 .canisters
2288 .iter()
2289 .filter(|canister| canister.presence == DesiredPresence::Present)
2290 {
2291 let required_parent_kind = match canister.kind {
2292 DesiredCanisterKind::Root => Some(DesiredCanisterKind::Coordinator),
2293 DesiredCanisterKind::Store | DesiredCanisterKind::Component => {
2294 Some(DesiredCanisterKind::Root)
2295 }
2296 DesiredCanisterKind::Pool => Some(DesiredCanisterKind::Root),
2297 DesiredCanisterKind::Auxiliary | DesiredCanisterKind::Coordinator => None,
2298 };
2299 let Some(required_parent_kind) = required_parent_kind else {
2300 continue;
2301 };
2302 let Some(parent) = canister.parent.as_deref().and_then(|parent| {
2303 desired.canisters.iter().find(|candidate| {
2304 candidate.name == parent && candidate.presence == DesiredPresence::Present
2305 })
2306 }) else {
2307 return Err(EnsurePolicyError::InvalidTopology {
2308 name: canister.name.clone(),
2309 reason: "required present topology parent is missing",
2310 });
2311 };
2312 if parent.kind != required_parent_kind
2313 || (canister.kind == DesiredCanisterKind::Root && parent.name != coordinator.name)
2314 {
2315 return Err(EnsurePolicyError::InvalidTopology {
2316 name: canister.name.clone(),
2317 reason: "topology parent has the wrong Canic role",
2318 });
2319 }
2320 }
2321 Ok(())
2322}
2323
2324fn validate_principal(field: &str, value: &str) -> Result<(), EnsurePolicyError> {
2325 let principal =
2326 Principal::from_text(value).map_err(|_| EnsurePolicyError::InvalidPrincipal {
2327 field: field.to_string(),
2328 value: value.to_string(),
2329 })?;
2330 if principal == Principal::anonymous() {
2331 return Err(EnsurePolicyError::InvalidPrincipal {
2332 field: field.to_string(),
2333 value: value.to_string(),
2334 });
2335 }
2336 Ok(())
2337}
2338
2339fn validate_observation_authority(
2340 desired: &DesiredFleet,
2341 observation: &FleetObservation,
2342) -> Result<(), EnsurePolicyError> {
2343 let mut principals = BTreeSet::new();
2344 for configured in &desired.canisters {
2345 let Some(live) = observation
2346 .canisters
2347 .get(&configured.name)
2348 .ok_or_else(|| EnsurePolicyError::MissingObservation {
2349 name: configured.name.clone(),
2350 })?
2351 .as_ref()
2352 else {
2353 continue;
2354 };
2355 if !configured.replace
2356 && let Some(expected) = &configured.principal
2357 && expected != &live.principal
2358 {
2359 return Err(EnsurePolicyError::ObservationAuthorityMismatch {
2360 actual: live.principal.clone(),
2361 expected: expected.clone(),
2362 name: configured.name.clone(),
2363 });
2364 }
2365 if !principals.insert(live.principal.as_str()) {
2366 return Err(EnsurePolicyError::DuplicateAuthority {
2367 name: configured.name.clone(),
2368 });
2369 }
2370 }
2371 for principal in observation.additional_controlled_cycles.keys() {
2372 validate_principal("additional_controlled_cycles", principal)?;
2373 if !principals.insert(principal.as_str()) {
2374 return Err(EnsurePolicyError::DuplicateAuthority {
2375 name: principal.clone(),
2376 });
2377 }
2378 }
2379 Ok(())
2380}
2381
2382fn cycle_bounds(desired: &DesiredFleet) -> Result<CycleBounds, EnsurePolicyError> {
2383 Ok(CycleBounds {
2384 ledger_fee: parse_fleet_cycles("ledger_fee_cycles", &desired.ledger_fee_cycles)?,
2385 management_creation_fee: parse_fleet_cycles(
2386 "management_creation_fee_cycles",
2387 &desired.management_creation_fee_cycles,
2388 )?,
2389 material_threshold: parse_fleet_cycles(
2390 "material_cycle_threshold",
2391 &desired.material_cycle_threshold,
2392 )?,
2393 observation_burn: parse_fleet_cycles(
2394 "maximum_observation_burn_cycles",
2395 &desired.maximum_observation_burn_cycles,
2396 )?,
2397 update_burn: parse_fleet_cycles(
2398 "maximum_update_burn_cycles",
2399 &desired.maximum_update_burn_cycles,
2400 )?,
2401 })
2402}
2403
2404fn compile_estate_funding_domains(
2405 desired: &DesiredFleet,
2406 observation: &FleetObservation,
2407 bounds: CycleBounds,
2408 defer_pool_creation: bool,
2409) -> Result<Vec<EstateFundingDomainPlan>, EnsurePolicyError> {
2410 let Some(bootstrap) = desired.bootstrap.as_ref() else {
2411 if observation.estate_funding_domains.is_empty() {
2412 return Ok(Vec::new());
2413 }
2414 return Err(EnsurePolicyError::EstateFundingTopology {
2415 reason: "observed Root funding accounts without bootstrap authority".to_string(),
2416 });
2417 };
2418 if observation.estate_funding_domains.len() != bootstrap.roots.len() {
2419 return Err(EnsurePolicyError::EstateFundingTopology {
2420 reason: "observed Root funding-account set differs from bootstrap authority"
2421 .to_string(),
2422 });
2423 }
2424
2425 let mut domains = Vec::with_capacity(bootstrap.roots.len());
2426 for root in &bootstrap.roots {
2427 domains.push(compile_estate_funding_domain(
2428 desired,
2429 root,
2430 observation,
2431 bounds,
2432 defer_pool_creation,
2433 )?);
2434 }
2435 Ok(domains)
2436}
2437
2438fn compile_estate_funding_domain(
2439 desired: &DesiredFleet,
2440 root: &DesiredFleetBootstrapRoot,
2441 observation: &FleetObservation,
2442 bounds: CycleBounds,
2443 defer_pool_creation: bool,
2444) -> Result<EstateFundingDomainPlan, EnsurePolicyError> {
2445 let observed = observation
2446 .estate_funding_domains
2447 .get(&root.root)
2448 .ok_or_else(|| EnsurePolicyError::EstateFundingTopology {
2449 reason: format!("Root {} has no funding-account observation", root.root),
2450 })?;
2451 validate_estate_funding_owner(desired, observation, root, observed)?;
2452 let asset_cycles = root.limits.canister_pool.canister_cycles.to_u128();
2453 let pool_forecast = required_root_pool_creations(
2454 desired,
2455 &root.root,
2456 root.limits.canister_pool.minimum_size,
2457 asset_cycles,
2458 bounds.management_creation_fee,
2459 observation,
2460 defer_pool_creation,
2461 )?;
2462 let creation_execution_margin_cycles = root
2463 .limits
2464 .canister_pool
2465 .creation_execution_margin
2466 .to_u128();
2467 if creation_execution_margin_cycles == 0 {
2468 return Err(EnsurePolicyError::EstateFundingTopology {
2469 reason: format!(
2470 "Root {} has no autonomous-creation execution margin",
2471 root.root
2472 ),
2473 });
2474 }
2475 let creation_amount_cycles = asset_cycles
2476 .checked_add(creation_execution_margin_cycles)
2477 .ok_or(EnsurePolicyError::ArithmeticOverflow {
2478 field: "estate creation margin",
2479 })?
2480 .checked_add(bounds.management_creation_fee)
2481 .ok_or(EnsurePolicyError::ArithmeticOverflow {
2482 field: "estate creation amount",
2483 })?;
2484 let debit_per_creation = creation_amount_cycles
2485 .checked_add(bounds.ledger_fee)
2486 .ok_or(EnsurePolicyError::ArithmeticOverflow {
2487 field: "estate creation debit",
2488 })?;
2489 let fee_per_creation = bounds
2490 .management_creation_fee
2491 .checked_add(bounds.ledger_fee)
2492 .ok_or(EnsurePolicyError::ArithmeticOverflow {
2493 field: "estate creation fees",
2494 })?;
2495 let count = u128::from(pool_forecast.required_creation_count);
2496 let maximum_creation_debit_cycles =
2497 debit_per_creation
2498 .checked_mul(count)
2499 .ok_or(EnsurePolicyError::ArithmeticOverflow {
2500 field: "estate creation debit",
2501 })?;
2502 let maximum_creation_fee_cycles =
2503 fee_per_creation
2504 .checked_mul(count)
2505 .ok_or(EnsurePolicyError::ArithmeticOverflow {
2506 field: "estate creation fees",
2507 })?;
2508 let available_cycles = observed.balance_cycles;
2509 let shortfall_cycles =
2510 maximum_creation_debit_cycles.saturating_sub(available_cycles.unwrap_or_default());
2511 let initial_pool_assets = initial_pool_asset_authority(desired, root, observed);
2512 Ok(EstateFundingDomainPlan {
2513 allocated_workloads: pool_forecast.allocated_workloads,
2514 available_cycles,
2515 available_pool_slots: pool_forecast.available_slots,
2516 creation_amount_cycles,
2517 creation_execution_margin_cycles,
2518 readiness_floor_cycles: asset_cycles,
2519 cycles_ledger: observed.cycles_ledger.clone(),
2520 eligible_ready_pool_assets: pool_forecast.eligible_ready_assets,
2521 initial_pool_assets,
2522 ledger_fee_cycles: bounds.ledger_fee,
2523 management_creation_fee_cycles: bounds.management_creation_fee,
2524 maximum_creation_debit_cycles,
2525 maximum_creation_fee_cycles,
2526 maximum_funding_cycles: shortfall_cycles,
2527 occupied_pool_assets: pool_forecast.occupied_assets,
2528 pending_creation_count: pool_forecast.pending_creations,
2529 pending_creation: pool_forecast.pending_creation,
2530 planned_initial_workloads: pool_forecast.planned_workloads,
2531 pool_maximum_size: pool_forecast.maximum_size,
2532 required_creation_count: pool_forecast.required_creation_count,
2533 root: root.root.clone(),
2534 root_principal: observed.root_principal.clone(),
2535 shortfall_cycles,
2536 })
2537}
2538
2539fn initial_pool_asset_authority(
2540 desired: &DesiredFleet,
2541 root: &DesiredFleetBootstrapRoot,
2542 observed: &crate::fleet_ensure::model::EstateFundingDomainObservation,
2543) -> Vec<String> {
2544 let mut assets = desired
2545 .canisters
2546 .iter()
2547 .filter(|canister| {
2548 canister.kind == DesiredCanisterKind::Pool
2549 && canister.presence == DesiredPresence::Present
2550 && canister.parent.as_deref() == Some(&root.root)
2551 })
2552 .map(|canister| {
2553 canister
2554 .principal
2555 .clone()
2556 .unwrap_or_else(|| format!("created:{}", canister.name))
2557 })
2558 .collect::<BTreeSet<_>>();
2559 if let Some(pool) = observed.pool.as_ref() {
2560 assets.extend(pool.assets.iter().map(|asset| asset.principal.clone()));
2561 if let Some(created) = pool
2562 .pending_creation
2563 .as_ref()
2564 .and_then(|pending| pending.created_principal.clone())
2565 {
2566 assets.insert(created);
2567 }
2568 }
2569 assets.into_iter().collect()
2570}
2571
2572fn validate_estate_funding_owner(
2573 desired: &DesiredFleet,
2574 observation: &FleetObservation,
2575 root: &DesiredFleetBootstrapRoot,
2576 observed: &crate::fleet_ensure::model::EstateFundingDomainObservation,
2577) -> Result<(), EnsurePolicyError> {
2578 if observed.cycles_ledger != desired.cycles_ledger {
2579 return Err(EnsurePolicyError::EstateFundingTopology {
2580 reason: format!("Root {} funding-account Ledger differs", root.root),
2581 });
2582 }
2583 let Some(principal) = observed.root_principal.as_deref() else {
2584 return Ok(());
2585 };
2586 validate_principal("estate_funding_domains.root_principal", principal)?;
2587 let configured = desired
2588 .canisters
2589 .iter()
2590 .find(|canister| canister.name == root.root)
2591 .ok_or_else(|| EnsurePolicyError::EstateFundingTopology {
2592 reason: format!("Root {} is absent from desired canisters", root.root),
2593 })?;
2594 let live = observation
2595 .canisters
2596 .get(&root.root)
2597 .and_then(Option::as_ref);
2598 if live.is_some_and(|live| live.principal != principal)
2599 || configured
2600 .principal
2601 .as_deref()
2602 .is_some_and(|expected| expected != principal)
2603 {
2604 return Err(EnsurePolicyError::EstateFundingTopology {
2605 reason: format!("Root {} funding-account owner differs", root.root),
2606 });
2607 }
2608 Ok(())
2609}
2610
2611fn required_root_pool_creations(
2612 desired: &DesiredFleet,
2613 root: &str,
2614 ready_floor: u32,
2615 required_asset_cycles: u128,
2616 management_creation_fee: u128,
2617 observation: &FleetObservation,
2618 defer_pool_creation: bool,
2619) -> Result<RootPoolCreationForecast, EnsurePolicyError> {
2620 let workload_count = desired_initial_root_workloads(desired, root)?;
2621 let desired_pool = desired_root_pool_policy(desired, root)?;
2622 let domain = observation
2623 .estate_funding_domains
2624 .get(root)
2625 .ok_or_else(|| EnsurePolicyError::EstateFundingTopology {
2626 reason: format!("Root {root} has no funding-account observation"),
2627 })?;
2628 if let Some(pool) = domain.pool.as_ref() {
2629 if defer_pool_creation {
2630 let pending = pending_pool_creation_forecast(
2631 root,
2632 pool,
2633 desired_pool,
2634 required_asset_cycles,
2635 management_creation_fee,
2636 )?;
2637 let counts = observed_pool_counts(pool, pending.unmaterialized, required_asset_cycles)?;
2638 if counts.occupied_assets > desired_pool.maximum_size {
2639 return Err(EnsurePolicyError::EstatePoolCapacity {
2640 allocated_workloads: counts.allocated_workloads,
2641 available_slots: 0,
2642 capacity_shortfall: counts.occupied_assets - desired_pool.maximum_size,
2643 eligible_ready_assets: counts.eligible_ready_assets,
2644 maximum_size: desired_pool.maximum_size,
2645 occupied_assets: counts.occupied_assets,
2646 pending_creations: pending.unmaterialized,
2647 required_creation_count: 0,
2648 root: root.to_string(),
2649 });
2650 }
2651 return Ok(RootPoolCreationForecast {
2654 allocated_workloads: counts.allocated_workloads,
2655 available_slots: desired_pool.maximum_size - counts.occupied_assets,
2656 eligible_ready_assets: counts.eligible_ready_assets,
2657 maximum_size: desired_pool.maximum_size,
2658 occupied_assets: counts.occupied_assets,
2659 pending_creations: pending.unmaterialized,
2660 pending_creation: pool.pending_creation.clone(),
2661 planned_workloads: workload_count,
2662 required_creation_count: 0,
2663 });
2664 }
2665 return forecast_observed_root_pool(
2666 root,
2667 pool,
2668 desired_pool,
2669 workload_count,
2670 ready_floor,
2671 required_asset_cycles,
2672 management_creation_fee,
2673 );
2674 }
2675 forecast_configured_root_pool(
2676 desired,
2677 observation,
2678 root,
2679 workload_count,
2680 ready_floor,
2681 required_asset_cycles,
2682 desired_pool.maximum_size,
2683 )
2684}
2685
2686fn desired_root_pool_policy<'a>(
2687 desired: &'a DesiredFleet,
2688 root: &str,
2689) -> Result<&'a FleetSubnetCanisterPoolConfig, EnsurePolicyError> {
2690 desired
2691 .bootstrap
2692 .as_ref()
2693 .and_then(|bootstrap| {
2694 bootstrap
2695 .roots
2696 .iter()
2697 .find(|candidate| candidate.root == root)
2698 })
2699 .map(|root| &root.limits.canister_pool)
2700 .ok_or_else(|| EnsurePolicyError::EstateFundingTopology {
2701 reason: format!("Root {root} has no desired pool authority"),
2702 })
2703}
2704
2705fn forecast_configured_root_pool(
2706 desired: &DesiredFleet,
2707 observation: &FleetObservation,
2708 root: &str,
2709 workload_count: u32,
2710 _ready_floor: u32,
2711 required_asset_cycles: u128,
2712 maximum_size: u32,
2713) -> Result<RootPoolCreationForecast, EnsurePolicyError> {
2714 let (reusable_ready_assets, completed_workloads) = desired
2715 .canisters
2716 .iter()
2717 .filter(|canister| {
2718 canister.kind == DesiredCanisterKind::Pool
2719 && canister.parent.as_deref() == Some(root)
2720 && canister.presence == DesiredPresence::Present
2721 })
2722 .try_fold(
2723 (0_u32, 0_u32),
2724 |(ready, workload), canister| -> Result<(u32, u32), EnsurePolicyError> {
2725 let live = observation
2726 .canisters
2727 .get(&canister.name)
2728 .and_then(Option::as_ref);
2729 match live.map(|live| (live.root_owned_lifecycle, live.cycles)) {
2730 Some((Some(RootOwnedCanisterLifecycle::Idle), cycles))
2731 if cycles >= required_asset_cycles =>
2732 {
2733 Ok((
2734 ready
2735 .checked_add(1)
2736 .ok_or(EnsurePolicyError::ArithmeticOverflow {
2737 field: "reusable Ready-pool asset count",
2738 })?,
2739 workload,
2740 ))
2741 }
2742 Some((
2743 Some(
2744 RootOwnedCanisterLifecycle::Claimed
2745 | RootOwnedCanisterLifecycle::Workload,
2746 ),
2747 _,
2748 )) => Ok((
2749 ready,
2750 workload
2751 .checked_add(1)
2752 .ok_or(EnsurePolicyError::ArithmeticOverflow {
2753 field: "completed Root workload count",
2754 })?,
2755 )),
2756 _ => Ok((ready, workload)),
2757 }
2758 },
2759 )?;
2760 let required_creation_count = 0;
2763 let occupied_assets = reusable_ready_assets
2764 .checked_add(completed_workloads)
2765 .ok_or(EnsurePolicyError::ArithmeticOverflow {
2766 field: "configured Root pool asset count",
2767 })?;
2768 Ok(RootPoolCreationForecast {
2769 allocated_workloads: completed_workloads,
2770 available_slots: maximum_size.saturating_sub(occupied_assets),
2771 eligible_ready_assets: reusable_ready_assets,
2772 maximum_size,
2773 occupied_assets,
2774 pending_creations: 0,
2775 pending_creation: None,
2776 planned_workloads: workload_count,
2777 required_creation_count,
2778 })
2779}
2780
2781fn forecast_observed_root_pool(
2782 root: &str,
2783 pool: &crate::fleet_ensure::model::EstatePoolInventoryObservation,
2784 desired_pool: &FleetSubnetCanisterPoolConfig,
2785 workload_count: u32,
2786 ready_floor: u32,
2787 required_asset_cycles: u128,
2788 management_creation_fee: u128,
2789) -> Result<RootPoolCreationForecast, EnsurePolicyError> {
2790 let pending = pending_pool_creation_forecast(
2791 root,
2792 pool,
2793 desired_pool,
2794 required_asset_cycles,
2795 management_creation_fee,
2796 )?;
2797 let counts = observed_pool_counts(pool, pending.unmaterialized, required_asset_cycles)?;
2798 let available_supply = counts
2799 .eligible_ready_assets
2800 .checked_add(pending.unmaterialized)
2801 .ok_or(EnsurePolicyError::ArithmeticOverflow {
2802 field: "available Root pool supply",
2803 })?;
2804 let required_additional_creations = required_estate_creation_count(
2805 workload_count,
2806 ready_floor,
2807 available_supply,
2808 counts.allocated_workloads,
2809 )?;
2810 let required_creation_count = required_additional_creations
2811 .checked_add(pending.debit)
2812 .ok_or(EnsurePolicyError::ArithmeticOverflow {
2813 field: "required Root pool creation debit count",
2814 })?;
2815 let available_slots = desired_pool
2816 .maximum_size
2817 .saturating_sub(counts.occupied_assets);
2818 let inventory_exceeds_capacity = counts.occupied_assets > desired_pool.maximum_size;
2819 let demand_exceeds_capacity = required_additional_creations > available_slots;
2820 if inventory_exceeds_capacity || demand_exceeds_capacity {
2821 return Err(EnsurePolicyError::EstatePoolCapacity {
2822 allocated_workloads: counts.allocated_workloads,
2823 available_slots,
2824 capacity_shortfall: required_additional_creations.saturating_sub(available_slots),
2825 eligible_ready_assets: counts.eligible_ready_assets,
2826 maximum_size: desired_pool.maximum_size,
2827 occupied_assets: counts.occupied_assets,
2828 pending_creations: pending.unmaterialized,
2829 required_creation_count: required_additional_creations,
2830 root: root.to_string(),
2831 });
2832 }
2833 Ok(RootPoolCreationForecast {
2834 allocated_workloads: counts.allocated_workloads,
2835 available_slots,
2836 eligible_ready_assets: counts.eligible_ready_assets,
2837 maximum_size: desired_pool.maximum_size,
2838 occupied_assets: counts.occupied_assets,
2839 pending_creations: pending.unmaterialized,
2840 pending_creation: pool.pending_creation.clone(),
2841 planned_workloads: workload_count,
2842 required_creation_count,
2843 })
2844}
2845
2846#[derive(Clone, Copy)]
2847struct PendingPoolCreationForecast {
2848 debit: u32,
2849 unmaterialized: u32,
2850}
2851
2852fn pending_pool_creation_forecast(
2853 root: &str,
2854 pool: &crate::fleet_ensure::model::EstatePoolInventoryObservation,
2855 desired: &FleetSubnetCanisterPoolConfig,
2856 required_asset_cycles: u128,
2857 management_creation_fee: u128,
2858) -> Result<PendingPoolCreationForecast, EnsurePolicyError> {
2859 let Some(pending) = pool.pending_creation.as_ref() else {
2860 return Ok(PendingPoolCreationForecast {
2861 debit: 0,
2862 unmaterialized: 0,
2863 });
2864 };
2865 if pending.uncertain_result {
2866 return Err(EnsurePolicyError::EstateFundingTopology {
2867 reason: format!(
2868 "Root {root} has an unresolved creation response; reconcile that exact operation before funding"
2869 ),
2870 });
2871 }
2872 let expected_amount = required_asset_cycles
2873 .checked_add(desired.creation_execution_margin.to_u128())
2874 .and_then(|amount| amount.checked_add(management_creation_fee))
2875 .ok_or(EnsurePolicyError::ArithmeticOverflow {
2876 field: "pending estate creation amount",
2877 })?;
2878 if pending.creation_amount_cycles != expected_amount {
2879 return Err(EnsurePolicyError::EstateFundingTopology {
2880 reason: format!(
2881 "Root {root} pending creation amount {} differs from current authority {expected_amount}",
2882 pending.creation_amount_cycles
2883 ),
2884 });
2885 }
2886 let materialized = pending.created_principal.as_ref().is_some_and(|principal| {
2887 pool.assets
2888 .iter()
2889 .any(|asset| &asset.principal == principal)
2890 });
2891 Ok(PendingPoolCreationForecast {
2892 debit: u32::from(pending.created_principal.is_none()),
2893 unmaterialized: u32::from(!materialized),
2894 })
2895}
2896
2897#[derive(Clone, Copy)]
2898struct ObservedPoolCounts {
2899 allocated_workloads: u32,
2900 eligible_ready_assets: u32,
2901 occupied_assets: u32,
2902}
2903
2904fn observed_pool_counts(
2905 pool: &crate::fleet_ensure::model::EstatePoolInventoryObservation,
2906 pending_unmaterialized: u32,
2907 required_asset_cycles: u128,
2908) -> Result<ObservedPoolCounts, EnsurePolicyError> {
2909 let bounded_count = |count, field| {
2910 u32::try_from(count).map_err(|_| EnsurePolicyError::ArithmeticOverflow { field })
2911 };
2912 let asset_count = bounded_count(pool.assets.len(), "observed Root pool asset count")?;
2913 let occupied_assets = asset_count.checked_add(pending_unmaterialized).ok_or(
2914 EnsurePolicyError::ArithmeticOverflow {
2915 field: "observed Root pool capacity",
2916 },
2917 )?;
2918 let eligible_ready_assets = bounded_count(
2919 pool.assets
2920 .iter()
2921 .filter(|asset| {
2922 asset.lifecycle == EstatePoolAssetLifecycle::Ready
2923 && asset.cycles >= required_asset_cycles
2924 })
2925 .count(),
2926 "eligible Ready-pool asset count",
2927 )?;
2928 let allocated_workloads = bounded_count(
2929 pool.assets
2930 .iter()
2931 .filter(|asset| {
2932 matches!(
2933 asset.lifecycle,
2934 EstatePoolAssetLifecycle::Claimed | EstatePoolAssetLifecycle::Workload
2935 )
2936 })
2937 .count(),
2938 "allocated Root workload count",
2939 )?;
2940 Ok(ObservedPoolCounts {
2941 allocated_workloads,
2942 eligible_ready_assets,
2943 occupied_assets,
2944 })
2945}
2946
2947#[derive(Clone, Debug, Eq, PartialEq)]
2948struct RootPoolCreationForecast {
2949 allocated_workloads: u32,
2950 available_slots: u32,
2951 eligible_ready_assets: u32,
2952 maximum_size: u32,
2953 occupied_assets: u32,
2954 pending_creations: u32,
2955 pending_creation: Option<crate::fleet_ensure::model::EstatePoolPendingCreationObservation>,
2956 planned_workloads: u32,
2957 required_creation_count: u32,
2958}
2959
2960fn desired_initial_root_workloads(
2961 desired: &DesiredFleet,
2962 root: &str,
2963) -> Result<u32, EnsurePolicyError> {
2964 let bootstrap = desired
2965 .bootstrap
2966 .as_ref()
2967 .expect("estate funding is compiled only with bootstrap authority");
2968 let protocol =
2969 desired
2970 .protocol
2971 .as_ref()
2972 .ok_or_else(|| EnsurePolicyError::EstateFundingTopology {
2973 reason: "estate workload forecast requires typed Fleet protocol".to_string(),
2974 })?;
2975 initial_workload_count(
2976 &bootstrap.component_deployment_configuration,
2977 protocol
2978 .component_group_placements
2979 .iter()
2980 .filter(|placement| placement.root == root)
2981 .map(|placement| placement.deployment.as_str()),
2982 root,
2983 )
2984}
2985
2986pub(super) fn initial_workload_count<'a>(
2988 configuration: &canic_core::control_plane_support::config::ComponentDeploymentConfiguration,
2989 mut deployments: impl Iterator<Item = &'a str>,
2990 root: &str,
2991) -> Result<u32, EnsurePolicyError> {
2992 deployments.try_fold(0_u32, |total, deployment_name| {
2993 let deployment = configuration
2994 .deployment_topology
2995 .component_group_deployments
2996 .iter()
2997 .find(|deployment| deployment.deployment.as_str() == deployment_name)
2998 .ok_or_else(|| EnsurePolicyError::EstateFundingTopology {
2999 reason: format!(
3000 "Root {root} placement references unknown deployment {deployment_name}"
3001 ),
3002 })?;
3003 deployment.members.iter().try_fold(total, |total, member| {
3004 let spec = configuration
3005 .component_topology
3006 .component_specs
3007 .iter()
3008 .find(|spec| spec.component_spec == member.component_spec)
3009 .ok_or_else(|| EnsurePolicyError::EstateFundingTopology {
3010 reason: format!(
3011 "Root {root} deployment references unknown Component Spec {}",
3012 member.component_spec
3013 ),
3014 })?;
3015 total.checked_add(initial_role_tree_size(spec)?).ok_or(
3016 EnsurePolicyError::ArithmeticOverflow {
3017 field: "estate initial workload count",
3018 },
3019 )
3020 })
3021 })
3022}
3023
3024fn required_estate_creation_count(
3025 workload_count: u32,
3026 ready_floor: u32,
3027 reusable_ready_assets: u32,
3028 completed_workloads: u32,
3029) -> Result<u32, EnsurePolicyError> {
3030 workload_count
3031 .saturating_sub(completed_workloads)
3032 .checked_add(ready_floor)
3033 .map(|required_assets| required_assets.saturating_sub(reusable_ready_assets))
3034 .ok_or(EnsurePolicyError::ArithmeticOverflow {
3035 field: "estate creation count",
3036 })
3037}
3038
3039fn initial_role_tree_size(
3040 spec: &canic_core::control_plane_support::config::ComponentSpec,
3041) -> Result<u32, EnsurePolicyError> {
3042 initial_role_instances(spec)?
3043 .values()
3044 .try_fold(0_u32, |sum, count| {
3045 sum.checked_add(*count)
3046 .ok_or(EnsurePolicyError::ArithmeticOverflow {
3047 field: "estate initial workload count",
3048 })
3049 })
3050}
3051
3052fn initial_role_instances(
3053 spec: &canic_core::control_plane_support::config::ComponentSpec,
3054) -> Result<BTreeMap<canic_core::ids::CanisterRole, u32>, EnsurePolicyError> {
3055 let mut instances = BTreeMap::<canic_core::ids::CanisterRole, u32>::new();
3056 let mut frontier = BTreeMap::from([(spec.component_role.clone(), 1_u32)]);
3057 for _ in 0..=spec.spawn_grants.len() {
3058 let mut next = BTreeMap::<canic_core::ids::CanisterRole, u32>::new();
3059 for (role, count) in frontier {
3060 let total = instances.entry(role.clone()).or_default();
3061 *total = total
3062 .checked_add(count)
3063 .ok_or(EnsurePolicyError::ArithmeticOverflow {
3064 field: "estate initial workload count",
3065 })?;
3066 for grant in spec
3067 .spawn_grants
3068 .iter()
3069 .filter(|grant| grant.parent_role == role && grant.initial_instances_per_parent > 0)
3070 {
3071 let children = count
3072 .checked_mul(grant.initial_instances_per_parent)
3073 .ok_or(EnsurePolicyError::ArithmeticOverflow {
3074 field: "estate initial child count",
3075 })?;
3076 let total = next.entry(grant.child_role.clone()).or_default();
3077 *total =
3078 total
3079 .checked_add(children)
3080 .ok_or(EnsurePolicyError::ArithmeticOverflow {
3081 field: "estate initial child count",
3082 })?;
3083 }
3084 }
3085 if next.is_empty() {
3086 return Ok(instances);
3087 }
3088 frontier = next;
3089 }
3090 Err(EnsurePolicyError::EstateFundingTopology {
3091 reason: format!(
3092 "Component Spec {} initial child graph did not converge",
3093 spec.component_spec
3094 ),
3095 })
3096}
3097
3098fn tranche_protocol_actions(
3099 desired: &DesiredFleet,
3100 observation: &FleetObservation,
3101 bounds: CycleBounds,
3102 observed_estate_cycles: u128,
3103 accumulator: &mut PlanAccumulator,
3104 protocol_actions: &mut Vec<EnsureAction>,
3105) -> Result<(), EnsurePolicyError> {
3106 let candidates = std::mem::take(protocol_actions);
3107 let mut selected = Vec::with_capacity(candidates.len());
3108 for action in candidates {
3109 let execution_burn = accumulator.execution_burn;
3110 let transfers = accumulator.transfers;
3111
3112 account_protocol_action(accumulator, &action)?;
3113 selected.push(action);
3114
3115 let observation_count = maximum_observation_count(
3116 desired,
3117 &accumulator.canisters,
3118 &selected,
3119 observation.additional_controlled_cycles.len(),
3120 )?;
3121 let observation_burn = bounds
3122 .observation_burn
3123 .checked_mul(observation_count)
3124 .ok_or(EnsurePolicyError::ArithmeticOverflow {
3125 field: "observation burn",
3126 })?;
3127 let required = checked_add(
3128 accumulator.execution_burn,
3129 observation_burn,
3130 "cycle-conservation required burn",
3131 )?;
3132 let available = checked_add(
3133 observed_estate_cycles,
3134 accumulator.new_funding,
3135 "cycle-conservation available balance",
3136 )?;
3137 if required > available {
3138 let error =
3139 insufficient_cycle_conservation(accumulator, selected.len(), available, required);
3140 accumulator.execution_burn = execution_burn;
3141 accumulator.transfers = transfers;
3142 selected.pop();
3143 if selected.is_empty() {
3144 return Err(error);
3145 }
3146 break;
3147 }
3148 }
3149 *protocol_actions = selected;
3150 Ok(())
3151}
3152
3153fn account_protocol_action(
3154 accumulator: &mut PlanAccumulator,
3155 action: &EnsureAction,
3156) -> Result<(), EnsurePolicyError> {
3157 match action {
3158 EnsureAction::FleetProtocol {
3159 maximum_execution_burn_cycles,
3160 ..
3161 }
3162 | EnsureAction::Protocol {
3163 maximum_execution_burn_cycles,
3164 ..
3165 } => accumulator.add_burn(*maximum_execution_burn_cycles),
3166 action => Err(EnsurePolicyError::InvalidProtocolStep(
3167 action.name().to_string(),
3168 )),
3169 }
3170}
3171
3172fn insufficient_cycle_conservation(
3173 accumulator: &PlanAccumulator,
3174 protocol_action_count: usize,
3175 available: u128,
3176 required: u128,
3177) -> EnsurePolicyError {
3178 let canister_action_count = accumulator
3179 .canisters
3180 .iter()
3181 .map(|canister| canister.actions.len())
3182 .sum::<usize>();
3183 EnsurePolicyError::InsufficientCycleConservation {
3184 action_count: canister_action_count.saturating_add(protocol_action_count),
3185 available,
3186 required,
3187 shortfall: required.saturating_sub(available),
3188 }
3189}
3190
3191fn maximum_observation_count(
3192 desired: &DesiredFleet,
3193 canisters: &[CanisterPlan],
3194 protocol_actions: &[EnsureAction],
3195 additional_controlled_canisters: usize,
3196) -> Result<u128, EnsurePolicyError> {
3197 let initially_present = u128::try_from(
3198 canisters
3199 .iter()
3200 .filter(|canister| canister.principal.is_some())
3201 .count(),
3202 )
3203 .map_err(|_| EnsurePolicyError::ArithmeticOverflow {
3204 field: "initial observation count",
3205 })?;
3206 let terminal_present = u128::try_from(
3207 desired
3208 .canisters
3209 .iter()
3210 .filter(|canister| canister.presence == DesiredPresence::Present)
3211 .count(),
3212 )
3213 .map_err(|_| EnsurePolicyError::ArithmeticOverflow {
3214 field: "terminal observation count",
3215 })?;
3216 let action_observations = canisters
3217 .iter()
3218 .flat_map(|canister| &canister.actions)
3219 .try_fold(0_u128, |total, action| {
3220 let count = match action {
3221 EnsureAction::Create { .. } => 1,
3222 EnsureAction::Delete { .. } => 4,
3223 EnsureAction::Fund { .. }
3224 | EnsureAction::FundEstate { .. }
3225 | EnsureAction::Transfer { .. } => 2,
3226 EnsureAction::SealAuthority { .. }
3227 | EnsureAction::Install { .. }
3228 | EnsureAction::FleetProtocol { .. }
3229 | EnsureAction::Protocol { .. }
3230 | EnsureAction::SetControllers { .. }
3231 | EnsureAction::Start { .. }
3232 | EnsureAction::Stop { .. } => 3,
3233 };
3234 checked_add(total, count, "effect observation count")
3235 })?;
3236 let protocol_observations = u128::try_from(protocol_actions.len())
3237 .map_err(|_| EnsurePolicyError::ArithmeticOverflow {
3238 field: "protocol observation count",
3239 })?
3240 .checked_mul(3)
3241 .ok_or(EnsurePolicyError::ArithmeticOverflow {
3242 field: "protocol observation count",
3243 })?;
3244 let publication_retry_observations =
3246 protocol_actions.iter().try_fold(0_u128, |total, action| {
3247 let Some(attempts) = action.fixture_publication_attempt_limit() else {
3248 return Ok(total);
3249 };
3250 let retries = attempts
3251 .checked_sub(1)
3252 .ok_or_else(|| EnsurePolicyError::InvalidProtocolStep(action.name().to_string()))?;
3253 checked_add(
3254 total,
3255 u128::from(retries) * 3,
3256 "publication retry observations",
3257 )
3258 })?;
3259 let terminal_protocol_observations = terminal_protocol_observation_bound(
3260 desired,
3261 protocol_actions,
3262 additional_controlled_canisters,
3263 )?;
3264 let stalled_observations = canisters
3265 .iter()
3266 .flat_map(|canister| &canister.actions)
3267 .chain(protocol_actions.iter())
3268 .filter(|action| {
3269 matches!(action, EnsureAction::FleetProtocol { action, .. }
3270 if matches!(action.as_ref(), CurrentFleetProtocolAction::ProvisionComponents { .. }))
3271 })
3272 .try_fold(desired.maximum_stalled_observations, |bound, action| {
3273 effect_observation_policy(desired, action)
3274 .map(|policy| bound.max(policy.maximum_stalled_observations))
3275 })?;
3276 initially_present
3277 .checked_mul(2)
3278 .and_then(|value| value.checked_add(terminal_present))
3279 .and_then(|value| value.checked_add(action_observations))
3280 .and_then(|value| value.checked_add(protocol_observations))
3281 .and_then(|value| value.checked_add(publication_retry_observations))
3282 .and_then(|value| value.checked_add(terminal_protocol_observations))
3283 .and_then(|value| value.checked_add(u128::from(stalled_observations)))
3284 .ok_or(EnsurePolicyError::ArithmeticOverflow {
3285 field: "maximum observation count",
3286 })
3287}
3288
3289pub(super) fn successor_phase_burn(
3291 desired: &DesiredFleet,
3292 plan: &FleetEnsurePlan,
3293) -> Result<u128, EnsurePolicyError> {
3294 let bounds = cycle_bounds(desired)?;
3295 let observations =
3296 maximum_observation_count(desired, &plan.canisters, &plan.protocol_actions, 0)?;
3297 let mut total = bounds.observation_burn.checked_mul(observations).ok_or(
3298 EnsurePolicyError::ArithmeticOverflow {
3299 field: "successor observation burn",
3300 },
3301 )?;
3302 for action in &plan.protocol_actions {
3303 let EnsureAction::FleetProtocol {
3304 maximum_execution_burn_cycles,
3305 ..
3306 } = action
3307 else {
3308 return Err(EnsurePolicyError::InvalidProtocolStep(
3309 action.name().to_string(),
3310 ));
3311 };
3312 total = checked_add(
3313 total,
3314 *maximum_execution_burn_cycles,
3315 "successor execution burn",
3316 )?;
3317 }
3318 Ok(total)
3319}
3320
3321fn terminal_protocol_observation_bound(
3322 desired: &DesiredFleet,
3323 protocol_actions: &[EnsureAction],
3324 additional_controlled_canisters: usize,
3325) -> Result<u128, EnsurePolicyError> {
3326 if desired.protocol.is_none() {
3327 return Ok(0);
3328 }
3329 let configured = u128::try_from(desired.canisters.len()).map_err(|_| {
3330 EnsurePolicyError::ArithmeticOverflow {
3331 field: "terminal protocol observation count",
3332 }
3333 })?;
3334 let additional = u128::try_from(additional_controlled_canisters).map_err(|_| {
3335 EnsurePolicyError::ArithmeticOverflow {
3336 field: "terminal protocol observation count",
3337 }
3338 })?;
3339 let observed_bound = configured
3340 .checked_mul(4)
3341 .and_then(|count| {
3342 additional
3343 .checked_mul(3)
3344 .and_then(|extra| count.checked_add(extra))
3345 })
3346 .and_then(|count| count.checked_add(4))
3347 .ok_or(EnsurePolicyError::ArithmeticOverflow {
3348 field: "terminal protocol observation count",
3349 })?;
3350 let planned_bound = protocol_actions.iter().try_fold(0_u128, |bound, action| {
3351 let EnsureAction::FleetProtocol {
3352 action: current, ..
3353 } = action
3354 else {
3355 return Ok(bound);
3356 };
3357 let crate::fleet_ensure::model::CurrentFleetProtocolAction::ProvisionComponents {
3358 request,
3359 ..
3360 } = current.as_ref()
3361 else {
3362 return Ok(bound);
3363 };
3364 component_provisioning_observation_bound(&request.plan.batches)
3365 .map(|current_bound| bound.max(current_bound))
3366 })?;
3367 Ok(observed_bound.max(planned_bound))
3368}
3369
3370pub(super) struct EffectObservationPolicy {
3372 pub maximum_stalled_observations: u32,
3373 pub paced: bool,
3374}
3375
3376pub(super) const fn pool_maintenance_update_bound(minimum_ready: u32) -> u32 {
3378 minimum_ready.saturating_mul(2).saturating_add(16)
3379}
3380
3381pub(super) fn effect_observation_policy(
3384 desired: &DesiredFleet,
3385 action: &EnsureAction,
3386) -> Result<EffectObservationPolicy, EnsurePolicyError> {
3387 let EnsureAction::FleetProtocol {
3388 action: current, ..
3389 } = action
3390 else {
3391 return Ok(EffectObservationPolicy {
3392 maximum_stalled_observations: desired.maximum_stalled_observations,
3393 paced: false,
3394 });
3395 };
3396 if matches!(
3397 current.as_ref(),
3398 CurrentFleetProtocolAction::MaintainPoolReadiness { .. }
3399 | CurrentFleetProtocolAction::ObservePoolReadiness { .. }
3400 ) {
3401 return Ok(EffectObservationPolicy {
3402 maximum_stalled_observations: desired
3403 .maximum_stalled_observations
3404 .max(MAXIMUM_PACED_PROTOCOL_STALL_OBSERVATIONS),
3405 paced: true,
3406 });
3407 }
3408 let crate::fleet_ensure::model::CurrentFleetProtocolAction::ProvisionComponents {
3409 request, ..
3410 } = current.as_ref()
3411 else {
3412 return Ok(EffectObservationPolicy {
3413 maximum_stalled_observations: desired.maximum_stalled_observations,
3414 paced: false,
3415 });
3416 };
3417 let topology_bound = component_provisioning_stall_bound(desired, &request.plan.batches)?;
3418 Ok(EffectObservationPolicy {
3419 maximum_stalled_observations: paced_protocol_stall_limit(
3420 desired.maximum_stalled_observations,
3421 topology_bound,
3422 ),
3423 paced: true,
3424 })
3425}
3426
3427const MAXIMUM_PACED_PROTOCOL_STALL_OBSERVATIONS: u32 = 64;
3428
3429fn paced_protocol_stall_limit(configured: u32, topology_bound: u128) -> u32 {
3430 let topology_bound = u32::try_from(topology_bound)
3431 .unwrap_or(MAXIMUM_PACED_PROTOCOL_STALL_OBSERVATIONS)
3432 .min(MAXIMUM_PACED_PROTOCOL_STALL_OBSERVATIONS);
3433 configured.max(topology_bound)
3434}
3435
3436fn component_provisioning_stall_bound(
3437 desired: &DesiredFleet,
3438 batches: &[canic_core::dto::component_provisioning::FleetSubnetRootProvisioningBatch],
3439) -> Result<u128, EnsurePolicyError> {
3440 let base = component_provisioning_observation_bound(batches)?;
3441 if batches.iter().all(|batch| {
3442 batch
3443 .placements
3444 .iter()
3445 .all(|placement| placement.entries.is_empty())
3446 }) {
3447 return Ok(base);
3448 }
3449 let bootstrap =
3450 desired
3451 .bootstrap
3452 .as_ref()
3453 .ok_or_else(|| EnsurePolicyError::EstateFundingTopology {
3454 reason: "Component provisioning has no compiled bootstrap configuration"
3455 .to_string(),
3456 })?;
3457 let specs = &bootstrap
3458 .component_deployment_configuration
3459 .component_topology
3460 .component_specs;
3461 let entries = batches
3462 .iter()
3463 .flat_map(|batch| &batch.placements)
3464 .flat_map(|placement| &placement.entries)
3465 .map(|entry| (&entry.component_spec, &entry.spec_hash));
3466 checked_add(
3467 base,
3468 initial_child_observation_bound(specs, entries)?,
3469 "Component observation count",
3470 )
3471}
3472
3473fn initial_child_observation_bound<'a>(
3474 specs: &[canic_core::control_plane_support::config::ComponentSpec],
3475 entries: impl IntoIterator<Item = (&'a canic_core::ids::ComponentSpecId, &'a [u8; 32])>,
3476) -> Result<u128, EnsurePolicyError> {
3477 entries.into_iter().try_fold(0, |bound, (id, hash)| {
3478 let spec = specs
3479 .iter()
3480 .find(|spec| &spec.component_spec == id && &spec.spec_hash == hash)
3481 .ok_or_else(|| EnsurePolicyError::EstateFundingTopology {
3482 reason: format!("provisioning Component Spec {id} differs from compiled authority"),
3483 })?;
3484 let children = initial_role_tree_size(spec)?.saturating_sub(1);
3485 checked_add(
3486 bound,
3487 u128::from(children) * 3,
3488 "initial child observation count",
3489 )
3490 })
3491}
3492
3493fn component_provisioning_observation_bound(
3494 batches: &[canic_core::dto::component_provisioning::FleetSubnetRootProvisioningBatch],
3495) -> Result<u128, EnsurePolicyError> {
3496 let root_count =
3497 u128::try_from(batches.len()).map_err(|_| EnsurePolicyError::ArithmeticOverflow {
3498 field: "terminal protocol observation count",
3499 })?;
3500 let component_count = batches.iter().try_fold(0_u128, |total, batch| {
3501 batch
3502 .placements
3503 .iter()
3504 .try_fold(total, |subtotal, placement| {
3505 u128::try_from(placement.entries.len())
3506 .ok()
3507 .and_then(|count| subtotal.checked_add(count))
3508 .ok_or(EnsurePolicyError::ArithmeticOverflow {
3509 field: "terminal protocol observation count",
3510 })
3511 })
3512 })?;
3513 component_provisioning_observation_bound_from_counts(root_count, component_count)
3514}
3515
3516fn component_provisioning_observation_bound_from_counts(
3517 root_count: u128,
3518 component_count: u128,
3519) -> Result<u128, EnsurePolicyError> {
3520 let per_component = terminal_initial_component_observation_count(0);
3521 root_count
3522 .checked_mul(5)
3523 .and_then(|root_observations| {
3524 component_count
3525 .checked_mul(per_component)
3526 .and_then(|component_observations| {
3527 root_observations.checked_add(component_observations)
3528 })
3529 })
3530 .and_then(|total| total.checked_add(1))
3531 .ok_or(EnsurePolicyError::ArithmeticOverflow {
3532 field: "terminal protocol observation count",
3533 })
3534}
3535
3536const fn terminal_initial_component_observation_count(_future_descendant_capacity: u32) -> u128 {
3537 3
3541}
3542
3543fn canister_cycle_policy(
3544 configured: &crate::fleet_ensure::model::DesiredCanister,
3545) -> Result<CanisterCyclePolicy, EnsurePolicyError> {
3546 let policy = CanisterCyclePolicy {
3547 initial_cycles: parse_canister_cycles(
3548 &configured.name,
3549 "initial_cycles",
3550 &configured.initial_cycles,
3551 )?,
3552 minimum_cycles: parse_canister_cycles(
3553 &configured.name,
3554 "minimum_cycles",
3555 &configured.minimum_cycles,
3556 )?,
3557 };
3558 if configured.presence == DesiredPresence::Present
3559 && policy.initial_cycles < policy.minimum_cycles
3560 {
3561 return Err(EnsurePolicyError::InitialCyclesBelowMinimum {
3562 name: configured.name.clone(),
3563 });
3564 }
3565 Ok(policy)
3566}
3567
3568fn parse_fleet_cycles(field: &'static str, value: &str) -> Result<u128, EnsurePolicyError> {
3569 value
3570 .parse::<Cycles>()
3571 .map(|cycles| cycles.to_u128())
3572 .map_err(|_| EnsurePolicyError::InvalidFleetCycles {
3573 field,
3574 value: value.to_string(),
3575 })
3576}
3577
3578fn parse_canister_cycles(
3579 name: &str,
3580 field: &'static str,
3581 value: &str,
3582) -> Result<u128, EnsurePolicyError> {
3583 value
3584 .parse::<Cycles>()
3585 .map(|cycles| cycles.to_u128())
3586 .map_err(|_| EnsurePolicyError::InvalidCanisterCycles {
3587 field,
3588 name: name.to_string(),
3589 value: value.to_string(),
3590 })
3591}
3592
3593fn checked_add(left: u128, right: u128, field: &'static str) -> Result<u128, EnsurePolicyError> {
3594 left.checked_add(right)
3595 .ok_or(EnsurePolicyError::ArithmeticOverflow { field })
3596}
3597
3598fn wasm_sha256(artifacts: &DesiredFleetArtifacts, name: &str) -> Result<String, EnsurePolicyError> {
3599 artifacts
3600 .wasm_sha256_by_canister
3601 .get(name)
3602 .cloned()
3603 .ok_or_else(|| EnsurePolicyError::MissingWasmIdentity {
3604 name: name.to_string(),
3605 })
3606}
3607
3608fn optional_init_arg_sha256(
3609 artifacts: &DesiredFleetArtifacts,
3610 configured: &crate::fleet_ensure::model::DesiredCanister,
3611) -> Result<Option<String>, EnsurePolicyError> {
3612 configured
3613 .init_arg
3614 .as_ref()
3615 .map(|_| {
3616 artifacts
3617 .init_arg_sha256_by_canister
3618 .get(&configured.name)
3619 .cloned()
3620 .ok_or_else(|| EnsurePolicyError::MissingArtifactIdentity {
3621 kind: "init argument",
3622 name: configured.name.clone(),
3623 })
3624 })
3625 .transpose()
3626}
3627
3628fn optional_init_candid_sha256(
3629 artifacts: &DesiredFleetArtifacts,
3630 configured: &crate::fleet_ensure::model::DesiredCanister,
3631) -> Result<Option<String>, EnsurePolicyError> {
3632 configured
3633 .init_candid
3634 .as_ref()
3635 .map(|_| {
3636 artifacts
3637 .init_candid_sha256_by_canister
3638 .get(&configured.name)
3639 .cloned()
3640 .ok_or_else(|| EnsurePolicyError::MissingArtifactIdentity {
3641 kind: "init Candid",
3642 name: configured.name.clone(),
3643 })
3644 })
3645 .transpose()
3646}
3647
3648pub(crate) fn operation_id(desired_sha256: &str, environment: &str, fleet: &str) -> String {
3649 let mut hasher = Sha256::new();
3650 hash_field(&mut hasher, b"canic:fleet-ensure:operation:v1");
3651 hash_field(&mut hasher, desired_sha256.as_bytes());
3652 hash_field(&mut hasher, environment.as_bytes());
3653 hash_field(&mut hasher, fleet.as_bytes());
3654 canic_core::cdk::utils::hash::hex_bytes(hasher.finalize())
3655}
3656
3657#[must_use]
3663pub fn expected_plan_sha256(plan: &FleetEnsurePlan) -> String {
3664 let mut canonical = plan.clone();
3665 canonical.plan_sha256.clear();
3666 let bytes = super::json::to_vec(&canonical).expect("Fleet ensure plan is JSON serializable");
3667 let mut hasher = Sha256::new();
3668 hash_field(&mut hasher, b"canic:fleet-ensure:plan:v1");
3669 hash_field(&mut hasher, &bytes);
3670 canic_core::cdk::utils::hash::hex_bytes(hasher.finalize())
3671}
3672
3673fn hash_field(hasher: &mut Sha256, value: &[u8]) {
3674 hasher.update(u64::try_from(value.len()).unwrap_or(u64::MAX).to_be_bytes());
3675 hasher.update(value);
3676}
3677
3678#[cfg(test)]
3679mod tests {
3680 use super::{
3681 CurrentFleetProtocolAction, CycleBounds, DesiredFleet, FLEET_ENSURE_SCHEMA_VERSION,
3682 FleetObservation, append_pool_reconciliation_funding,
3683 };
3684 use std::collections::BTreeMap;
3685
3686 use super::{
3687 EnsurePolicyError, PlanAccumulator, append_estate_funding_actions,
3688 component_provisioning_observation_bound_from_counts, forecast_observed_root_pool,
3689 initial_role_tree_size, paced_protocol_stall_limit, required_estate_creation_count,
3690 terminal_initial_component_observation_count, validate_bootstrap_root_pool_import_capacity,
3691 };
3692 use crate::fleet_ensure::model::{
3693 CanisterDisposition, CanisterPlan, DesiredFleetBootstrapRoot, EnsureAction,
3694 EstateFundingDomainPlan, EstatePoolAssetLifecycle, EstatePoolAssetObservation,
3695 EstatePoolAssetOrigin, EstatePoolInventoryObservation,
3696 EstatePoolPendingCreationObservation, PoolFundingAuthority,
3697 };
3698 use candid::Principal;
3699 use canic_core::{
3700 cdk::types::Cycles,
3701 control_plane_support::config::{ComponentLimits, ComponentSpawnGrant, ComponentSpec},
3702 ids::{
3703 CanisterRole, ComponentSpecId, ComponentTopologyDigest, CyclesFundingBudget,
3704 FleetSubnetCanisterPoolConfig, FleetSubnetRootLimits, SubnetId,
3705 },
3706 };
3707
3708 #[test]
3709 fn continuation_review_reserves_only_available_headroom_and_exposes_pending_discovery() {
3710 let observation = FleetObservation {
3711 additional_controlled_cycles: BTreeMap::new(),
3712 canisters: BTreeMap::new(),
3713 estate_funding_domains: BTreeMap::new(),
3714 ledger_fee_cycles: 0,
3715 operator_cycles: 0,
3716 protocol_ready: BTreeMap::new(),
3717 };
3718 let bounds = CycleBounds {
3719 observation_burn: 3,
3720 update_burn: 5,
3721 ledger_fee: 1,
3722 management_creation_fee: 0,
3723 material_threshold: 1,
3724 };
3725 let review = super::recovery::review(&observation, bounds, 32, 0, 41, 369).unwrap();
3726 assert_eq!(review.base_execution_burn_cycles, 41);
3727 assert_eq!(review.whole_continuation_ceiling_cycles, 448);
3728 assert_eq!(review.continuation_reserve_cycles, 328);
3729 assert_eq!(review.maximum_successor_actions, 32);
3730 assert_eq!(review.fixture_publication_retry_attempts, 0);
3731 assert_eq!(review.per_step_burn_cycles, 14);
3732 assert!(review.known_pool_funding.is_empty());
3733 assert_eq!(
3734 review.discovery,
3735 crate::fleet_ensure::model::RecoveryDiscovery::PendingCurrentProtocol
3736 );
3737 let no_headroom = super::recovery::review(&observation, bounds, 32, 0, 41, 40).unwrap();
3738 assert_eq!(no_headroom.continuation_reserve_cycles, 0);
3739 let fixtures = super::recovery::review(&observation, bounds, 32, 6, 41, 1_000).unwrap();
3740 assert_eq!(fixtures.whole_continuation_ceiling_cycles, 532);
3741 assert_eq!(fixtures.continuation_reserve_cycles, 532);
3742 assert_eq!(fixtures.fixture_publication_retry_attempts, 6);
3743 assert_eq!(fixtures.per_step_burn_cycles, 14);
3744 assert_eq!(
3745 fixtures.base_execution_burn_cycles,
3746 review.base_execution_burn_cycles
3747 );
3748 }
3749
3750 #[test]
3751 fn estate_workload_forecast_includes_recursive_initial_children() {
3752 let mut spec = ComponentSpec {
3753 component_spec: ComponentSpecId::try_from(String::from("hub"))
3754 .expect("hub Component Spec ID"),
3755 spec_hash: [1; 32],
3756 component_role: CanisterRole::from("hub"),
3757 maximum_fleet_instances: 1,
3758 limits: ComponentLimits {
3759 maximum_descendants: 8,
3760 maximum_registry_bytes: 1,
3761 cycles_funding: CyclesFundingBudget {
3762 window_secs: 1,
3763 maximum_cycles: Cycles::new(1),
3764 },
3765 },
3766 children: Vec::new(),
3767 spawn_grants: vec![
3768 ComponentSpawnGrant {
3769 parent_role: CanisterRole::from("hub"),
3770 child_role: CanisterRole::from("shard"),
3771 initial_instances_per_parent: 2,
3772 maximum_instances_per_parent: 2,
3773 },
3774 ComponentSpawnGrant {
3775 parent_role: CanisterRole::from("shard"),
3776 child_role: CanisterRole::from("leaf"),
3777 initial_instances_per_parent: 3,
3778 maximum_instances_per_parent: 3,
3779 },
3780 ],
3781 };
3782
3783 assert_eq!(initial_role_tree_size(&spec), Ok(9));
3784 for capacity in [8, 10_000, u32::MAX] {
3785 spec.limits.maximum_descendants = capacity;
3786 let children = super::initial_child_observation_bound(
3787 std::slice::from_ref(&spec),
3788 [(&spec.component_spec, &spec.spec_hash); 2],
3789 )
3790 .unwrap();
3791 assert_eq!(children, 48);
3792 let base = component_provisioning_observation_bound_from_counts(1, 2).unwrap();
3793 assert_eq!(paced_protocol_stall_limit(8, base + children), 60);
3794 }
3795 assert!(matches!(
3796 super::initial_child_observation_bound(
3797 std::slice::from_ref(&spec),
3798 [(&spec.component_spec, &[2; 32])]
3799 ),
3800 Err(EnsurePolicyError::EstateFundingTopology { .. })
3801 ));
3802 }
3803
3804 #[test]
3805 fn estate_creation_forecast_retains_ready_floor_without_repeating_workloads() {
3806 assert_eq!(required_estate_creation_count(8, 10, 10, 0), Ok(8));
3807 assert_eq!(required_estate_creation_count(8, 10, 10, 8), Ok(0));
3808 assert_eq!(required_estate_creation_count(8, 2, 2, 0), Ok(8));
3809 assert_eq!(required_estate_creation_count(8, 2, 10, 0), Ok(0));
3810 }
3811
3812 #[test]
3813 fn estate_shortfall_compiles_one_exact_plan_owned_ledger_transfer() {
3814 let mut accumulator = PlanAccumulator::new();
3815 accumulator.canisters.push(CanisterPlan {
3816 actions: Vec::new(),
3817 disposition: CanisterDisposition::Reuse,
3818 name: "root-0".to_string(),
3819 observed_cycles: 30,
3820 principal: Some("rrkah-fqaaa-aaaaa-aaaaq-cai".to_string()),
3821 });
3822 let domain = EstateFundingDomainPlan {
3823 allocated_workloads: 1,
3824 available_cycles: Some(40),
3825 available_pool_slots: 1,
3826 creation_amount_cycles: 45,
3827 creation_execution_margin_cycles: 5,
3828 readiness_floor_cycles: 35,
3829 cycles_ledger: "aaaaa-aa".to_string(),
3830 eligible_ready_pool_assets: 1,
3831 initial_pool_assets: vec!["existing-pool".to_string()],
3832 ledger_fee_cycles: 5,
3833 management_creation_fee_cycles: 5,
3834 maximum_creation_debit_cycles: 100,
3835 maximum_creation_fee_cycles: 20,
3836 maximum_funding_cycles: 60,
3837 occupied_pool_assets: 1,
3838 pending_creation_count: 0,
3839 pending_creation: None,
3840 planned_initial_workloads: 2,
3841 pool_maximum_size: 2,
3842 required_creation_count: 2,
3843 root: "root-0".to_string(),
3844 root_principal: Some("rrkah-fqaaa-aaaaa-aaaaq-cai".to_string()),
3845 shortfall_cycles: 60,
3846 };
3847
3848 append_estate_funding_actions(&[domain], 100, 4, &mut accumulator)
3849 .expect("compile exact estate funding action");
3850
3851 assert_eq!(accumulator.new_funding, 60);
3852 assert_eq!(accumulator.fees, 5);
3853 assert!(matches!(
3854 accumulator.canisters[0].actions.as_slice(),
3855 [EnsureAction::FundEstate {
3856 amount: 60,
3857 created_at_time: 104,
3858 expected_post_cycles: 100,
3859 ledger,
3860 ledger_fee_cycles: 5,
3861 name,
3862 principal,
3863 }] if ledger == "aaaaa-aa"
3864 && name == "root-0"
3865 && principal == "rrkah-fqaaa-aaaaa-aaaaq-cai"
3866 ));
3867 }
3868
3869 #[test]
3870 fn complete_pool_inventory_owns_creation_capacity_forecast() {
3871 let policy = pool_policy(4, 8);
3872 let exhausted = observed_pool([
3873 EstatePoolAssetLifecycle::Workload,
3874 EstatePoolAssetLifecycle::Workload,
3875 EstatePoolAssetLifecycle::Workload,
3876 EstatePoolAssetLifecycle::Workload,
3877 EstatePoolAssetLifecycle::Failed,
3878 EstatePoolAssetLifecycle::Failed,
3879 EstatePoolAssetLifecycle::Failed,
3880 EstatePoolAssetLifecycle::Failed,
3881 ]);
3882 assert!(matches!(
3883 forecast_observed_root_pool("root", &exhausted, &policy, 4, 4, 1_900, 500),
3884 Err(EnsurePolicyError::EstatePoolCapacity {
3885 allocated_workloads: 4,
3886 available_slots: 0,
3887 capacity_shortfall: 4,
3888 eligible_ready_assets: 0,
3889 maximum_size: 8,
3890 occupied_assets: 8,
3891 pending_creations: 0,
3892 required_creation_count: 4,
3893 ..
3894 })
3895 ));
3896
3897 let recoverable = observed_pool([
3898 EstatePoolAssetLifecycle::Workload,
3899 EstatePoolAssetLifecycle::Workload,
3900 EstatePoolAssetLifecycle::Workload,
3901 EstatePoolAssetLifecycle::Workload,
3902 EstatePoolAssetLifecycle::Ready,
3903 EstatePoolAssetLifecycle::Ready,
3904 ]);
3905 let forecast = forecast_observed_root_pool("root", &recoverable, &policy, 4, 4, 1_900, 500)
3906 .expect("two free slots can restore the four-Ready floor");
3907 assert_eq!(forecast.occupied_assets, 6);
3908 assert_eq!(forecast.eligible_ready_assets, 2);
3909 assert_eq!(forecast.required_creation_count, 2);
3910 assert_eq!(forecast.available_slots, 2);
3911 }
3912
3913 #[test]
3914 fn every_retained_pool_lifecycle_and_pending_creation_consumes_capacity_once() {
3915 let policy = pool_policy(2, 10);
3916 let mut observed = observed_pool([
3917 EstatePoolAssetLifecycle::Claimed,
3918 EstatePoolAssetLifecycle::Failed,
3919 EstatePoolAssetLifecycle::HandingOff,
3920 EstatePoolAssetLifecycle::PendingReset,
3921 EstatePoolAssetLifecycle::Ready,
3922 EstatePoolAssetLifecycle::Recycling,
3923 EstatePoolAssetLifecycle::Workload,
3924 ]);
3925 observed.minimum_size = 2;
3926 observed.maximum_size = 10;
3927 observed.pending_creation = Some(EstatePoolPendingCreationObservation {
3928 attempt_count: 1,
3929 available_cycles: None,
3930 creation_amount_cycles: 2_500,
3931 created_principal: None,
3932 diagnostic: None,
3933 last_attempt_at_ns: Some(1),
3934 operation_id: "11".repeat(32),
3935 required_cycles: None,
3936 retry_at_ns: Some(2),
3937 shortfall_cycles: Some(2_500),
3938 uncertain_result: false,
3939 });
3940 let forecast = forecast_observed_root_pool("root", &observed, &policy, 2, 2, 1_900, 500)
3941 .expect("all bounded lifecycles fit with one free slot");
3942 assert_eq!(forecast.occupied_assets, 8);
3943 assert_eq!(forecast.allocated_workloads, 2);
3944 assert_eq!(forecast.eligible_ready_assets, 1);
3945 assert_eq!(forecast.pending_creations, 1);
3946 assert_eq!(forecast.required_creation_count, 1);
3947
3948 observed.maximum_size = 9;
3949 let drifted = forecast_observed_root_pool("root", &observed, &policy, 2, 2, 1_900, 500)
3950 .expect("observed policy drift retains exact assets under desired limits");
3951 assert_eq!(drifted, forecast);
3952 }
3953
3954 #[test]
3955 fn unresolved_pool_creation_rejects_before_funding_authority() {
3956 let policy = pool_policy(1, 2);
3957 let mut observed = observed_pool([]);
3958 observed.pending_creation = Some(EstatePoolPendingCreationObservation {
3959 attempt_count: 1,
3960 available_cycles: None,
3961 creation_amount_cycles: 2_500,
3962 created_principal: None,
3963 diagnostic: None,
3964 last_attempt_at_ns: Some(1),
3965 operation_id: "11".repeat(32),
3966 required_cycles: None,
3967 retry_at_ns: None,
3968 shortfall_cycles: None,
3969 uncertain_result: true,
3970 });
3971 assert!(matches!(
3972 forecast_observed_root_pool("root", &observed, &policy, 1, 1, 1_900, 500),
3973 Err(EnsurePolicyError::EstateFundingTopology { reason })
3974 if reason.contains("unresolved creation response")
3975 ));
3976 }
3977
3978 #[test]
3979 #[expect(
3980 clippy::too_many_lines,
3981 reason = "one complete inventory proof binds funding, retained identities and the no-creation outcome"
3982 )]
3983 fn full_failed_inventory_funds_exact_retained_assets_without_creating_replacements() {
3984 let root = Principal::from_slice(&[10]).to_text();
3985 let desired = DesiredFleet {
3986 bootstrap: None,
3987 canisters: Vec::new(),
3988 cycles_ledger: Principal::from_slice(&[11]).to_text(),
3989 environment: "local".into(),
3990 fleet: "repair".into(),
3991 ledger_fee_cycles: "5".into(),
3992 management_creation_fee_cycles: "500".into(),
3993 material_cycle_threshold: "1".into(),
3994 maximum_observation_burn_cycles: "10".into(),
3995 maximum_stalled_observations: 8,
3996 maximum_update_burn_cycles: "20".into(),
3997 operator: Principal::from_slice(&[12]).to_text(),
3998 protocol: None,
3999 protocol_steps: Vec::new(),
4000 schema_version: FLEET_ENSURE_SCHEMA_VERSION,
4001 treasury: "coordinator".into(),
4002 };
4003 let mut pool = observed_pool([
4004 EstatePoolAssetLifecycle::Workload,
4005 EstatePoolAssetLifecycle::Workload,
4006 EstatePoolAssetLifecycle::Workload,
4007 EstatePoolAssetLifecycle::Workload,
4008 EstatePoolAssetLifecycle::Failed,
4009 EstatePoolAssetLifecycle::Failed,
4010 EstatePoolAssetLifecycle::Failed,
4011 EstatePoolAssetLifecycle::Failed,
4012 ]);
4013 let mut actions = Vec::new();
4014 for (index, asset) in pool.assets.iter_mut().enumerate() {
4015 let canister_id =
4016 Principal::from_slice(&[u8::try_from(index + 20).expect("bounded asset")]);
4017 asset.principal = canister_id.to_text();
4018 if asset.lifecycle == EstatePoolAssetLifecycle::Failed {
4019 asset.cycles -= 1;
4020 actions.push(EnsureAction::FleetProtocol {
4021 action: Box::new(CurrentFleetProtocolAction::ReconcilePoolAsset {
4022 request: canic_core::dto::pool::PoolCanisterRequest { canister_id },
4023 minimum_cycles: Cycles::new(pool.readiness_floor_cycles),
4024 }),
4025 candid: "root.did".into(),
4026 candid_sha256: "42".repeat(32),
4027 maximum_execution_burn_cycles: 20,
4028 name: format!("pool-reconcile-{canister_id}"),
4029 principal: root.clone(),
4030 });
4031 }
4032 }
4033 assert!(matches!(
4034 forecast_observed_root_pool("root", &pool, &pool_policy(4, 8), 4, 4, 1_900, 500),
4035 Err(EnsurePolicyError::EstatePoolCapacity {
4036 capacity_shortfall: 4,
4037 ..
4038 })
4039 ));
4040 let observation = FleetObservation {
4041 additional_controlled_cycles: pool
4042 .assets
4043 .iter()
4044 .map(|asset| (asset.principal.clone(), asset.cycles))
4045 .collect(),
4046 canisters: BTreeMap::new(),
4047 estate_funding_domains: BTreeMap::from([(
4048 "root".into(),
4049 crate::fleet_ensure::model::EstateFundingDomainObservation {
4050 balance_cycles: Some(0),
4051 cycles_ledger: desired.cycles_ledger.clone(),
4052 pool: Some(pool.clone()),
4053 root_principal: Some(root.clone()),
4054 },
4055 )]),
4056 ledger_fee_cycles: 5,
4057 operator_cycles: 10_000,
4058 protocol_ready: BTreeMap::new(),
4059 };
4060 let mut accumulator = PlanAccumulator::new();
4061 append_pool_reconciliation_funding(
4062 &desired,
4063 &observation,
4064 &actions,
4065 CycleBounds {
4066 ledger_fee: 5,
4067 management_creation_fee: 500,
4068 material_threshold: 1,
4069 observation_burn: 10,
4070 update_burn: 20,
4071 },
4072 100,
4073 &mut accumulator,
4074 )
4075 .expect("review recovery funding");
4076 let preview = super::recovery::review(
4077 &observation,
4078 CycleBounds {
4079 ledger_fee: 5,
4080 management_creation_fee: 500,
4081 material_threshold: 1,
4082 observation_burn: 10,
4083 update_burn: 20,
4084 },
4085 32,
4086 0,
4087 40,
4088 10_000,
4089 )
4090 .unwrap();
4091 assert_eq!(preview.known_pool_funding.len(), 4);
4092 assert_eq!(
4093 preview
4094 .known_pool_funding
4095 .iter()
4096 .map(|funding| funding.amount_cycles)
4097 .sum::<u128>(),
4098 accumulator.new_funding
4099 );
4100 assert_eq!(
4101 preview
4102 .known_pool_funding
4103 .iter()
4104 .map(|funding| funding.ledger_fee_cycles)
4105 .sum::<u128>(),
4106 accumulator.fees
4107 );
4108 assert_eq!(accumulator.new_funding, 4 * 131);
4109 assert_eq!(accumulator.fees, 4 * 5);
4110 for plan in &accumulator.canisters {
4111 assert!(matches!(plan.actions.as_slice(), [EnsureAction::Fund {
4112 pool_funding: Some(PoolFundingAuthority { root: bound_root, lifecycle: EstatePoolAssetLifecycle::Failed }), principal, amount: 131, expected_post_cycles: 2_030,
4113 funding_deficit_cycles: 101, funding_margin_cycles: 30, ..
4114 }] if bound_root == &root && Some(principal) == plan.principal.as_ref()));
4115 }
4116 for asset in &mut pool.assets {
4117 if asset.lifecycle == EstatePoolAssetLifecycle::Failed {
4118 asset.cycles = 2_000;
4119 asset.lifecycle = EstatePoolAssetLifecycle::Ready;
4120 }
4121 }
4122 let recovered =
4123 forecast_observed_root_pool("root", &pool, &pool_policy(4, 8), 4, 4, 1_900, 500)
4124 .expect("recovered full pool satisfies the same demand");
4125 assert_eq!(recovered.eligible_ready_assets, 4);
4126 assert_eq!(recovered.required_creation_count, 0);
4127 }
4128
4129 fn pool_policy(minimum_size: u32, maximum_size: u32) -> FleetSubnetCanisterPoolConfig {
4130 FleetSubnetCanisterPoolConfig {
4131 minimum_size,
4132 maximum_size,
4133 canister_cycles: Cycles::new(1_900),
4134 creation_execution_margin: Cycles::new(100),
4135 }
4136 }
4137
4138 fn observed_pool(
4139 lifecycles: impl IntoIterator<Item = EstatePoolAssetLifecycle>,
4140 ) -> EstatePoolInventoryObservation {
4141 EstatePoolInventoryObservation {
4142 assets: lifecycles
4143 .into_iter()
4144 .enumerate()
4145 .map(|(index, lifecycle)| EstatePoolAssetObservation {
4146 creation_receipt: None,
4147 cycles: 1_900,
4148 lifecycle,
4149 origin: EstatePoolAssetOrigin::Imported,
4150 principal: format!("asset-{index}"),
4151 })
4152 .collect(),
4153 maximum_size: 8,
4154 minimum_size: 4,
4155 pending_creation: None,
4156 readiness_floor_cycles: 1_900,
4157 creation_execution_margin_cycles: 100,
4158 }
4159 }
4160
4161 #[test]
4162 fn fresh_fleet_descendant_capacity_does_not_multiply_terminal_proof() {
4163 for maximum_descendants in [0, 1, 10_000, u32::MAX] {
4164 assert_eq!(
4165 terminal_initial_component_observation_count(maximum_descendants),
4166 3,
4167 );
4168 }
4169 }
4170
4171 #[test]
4172 fn component_provisioning_stall_floor_scales_and_caps() {
4173 let topology_bound = component_provisioning_observation_bound_from_counts(1, 11)
4174 .expect("bounded one-Root eleven-Component topology");
4175 assert_eq!(paced_protocol_stall_limit(8, topology_bound), 39);
4176 assert_eq!(paced_protocol_stall_limit(8, 10_000), 64);
4177 assert_eq!(paced_protocol_stall_limit(80, topology_bound), 80);
4178 }
4179
4180 #[test]
4181 fn apply_policy_rejects_bootstrap_imports_above_the_root_maximum() {
4182 let root = DesiredFleetBootstrapRoot {
4183 canister_pool_imports: vec![
4184 "pool-0".to_string(),
4185 "pool-1".to_string(),
4186 "pool-2".to_string(),
4187 ],
4188 component_admissions: Vec::new(),
4189 component_topology_digest: ComponentTopologyDigest::from_bytes([1; 32]),
4190 funding: crate::test_support::fleet_subnet_root_funding_authority(),
4191 limits: FleetSubnetRootLimits {
4192 maximum_component_instances: 1,
4193 maximum_registry_bytes: 1,
4194 maximum_wasm_store_bytes: 1,
4195 canister_pool: FleetSubnetCanisterPoolConfig {
4196 minimum_size: 2,
4197 maximum_size: 2,
4198 canister_cycles: Cycles::new(1),
4199 creation_execution_margin: Cycles::new(1),
4200 },
4201 cycles_funding: CyclesFundingBudget {
4202 window_secs: 1,
4203 maximum_cycles: Cycles::new(1),
4204 },
4205 maximum_group_placements: 1,
4206 },
4207 placement_subnet: SubnetId::from_principal(Principal::from_slice(&[2])),
4208 root: "root-0".to_string(),
4209 store: "store-0".to_string(),
4210 };
4211
4212 assert!(matches!(
4213 validate_bootstrap_root_pool_import_capacity(&root),
4214 Err(EnsurePolicyError::PoolImportCapacity(error))
4215 if error.import_count == 3
4216 && error.maximum_size == 2
4217 && error.root == "root-0"
4218 ));
4219 }
4220}