1mod continuation;
8mod funding;
9#[cfg(test)]
10mod funding_tests;
11mod reinstall;
12mod root_reinstall;
13
14use crate::fleet_ensure::{
15 dto::{FleetEnsurePhase, FleetEnsureProgress, FleetEnsureProgressState},
16 inventory::project_current_fleet_inventory,
17 model::{
18 ActualCycleConservation, CanisterDisposition, CurrentFleetProtocolAction,
19 CycleConservation, EffectRecord, EffectState, EnsureAction, EstateFundingDomainPlan,
20 EstateFundingRequiredRecord, FLEET_ENSURE_SCHEMA_VERSION, FleetEnsureCompletion,
21 FleetEnsureJournalRecord, FleetEnsurePlan, FleetEnsurePlanScope, FleetEnsureReport,
22 FleetEnsureStateRecord, FleetObservation, RootManagementObservation,
23 create_balance_is_terminal,
24 },
25 ops::{
26 EffectRetry, EnsurePaths, EnsurePlatform, EnsureStateError, action_sha256,
27 compact_inline_plan, lock_operation, read_journal, read_plan, read_root_start_authority,
28 read_state, resolve_desired_artifacts, write_journal, write_plan, write_state,
29 },
30 policy::{
31 EnsurePolicyError, RootStartPlanInput, compile_plan, compile_root_start_prerequisite_plan,
32 effect_observation_policy, expected_plan_sha256, operation_id,
33 recompile_root_start_prerequisite_plan, validate_path_identity, validate_path_labels,
34 },
35};
36use canic_core::cdk::types::Cycles;
37use std::{
38 collections::{BTreeMap, BTreeSet},
39 path::Path,
40};
41use thiserror::Error as ThisError;
42
43pub use reinstall::plan_reinstall;
44
45#[derive(Debug, ThisError)]
48#[error(
49 "fresh pool canister {canister} requested {requested_creation_funding_cycles} creation cycles for readiness floor {readiness_floor_cycles}; its first live balance is {live_balance_cycles} cycles (readiness shortfall {readiness_shortfall_cycles}) and must retain {remaining_controller_burn_cycles} cycles for controller finalization (required pre-finalization balance {required_pre_finalization_balance_cycles}, pre-finalization shortfall {pre_finalization_shortfall_cycles}; maximum first-observation burn {maximum_observation_burn_cycles}); no controller or protocol action followed"
50)]
51pub struct FreshPoolCreationUnderfundedError {
52 pub canister: String,
53 pub live_balance_cycles: u128,
54 pub maximum_observation_burn_cycles: u128,
55 pub pre_finalization_shortfall_cycles: u128,
56 pub readiness_floor_cycles: u128,
57 pub readiness_shortfall_cycles: u128,
58 pub remaining_controller_burn_cycles: u128,
59 pub requested_creation_funding_cycles: u128,
60 pub required_pre_finalization_balance_cycles: u128,
61}
62
63#[derive(Debug, ThisError)]
65#[error(
66 "Root {root} Cycles Ledger account {root_principal} on {cycles_ledger} has {available_cycles} cycles; {required_creation_count} autonomous creations require at most {maximum_creation_debit_cycles} cycles ({creation_amount_cycles} gross amount containing {readiness_floor_cycles} readiness floor, {management_creation_fee_cycles} management creation fee and {creation_execution_margin_cycles} execution margin, plus {ledger_fee_cycles} Ledger fee each; total reviewed fees {maximum_creation_fee_cycles}), shortfall {shortfall_cycles}; pending_creation={pending_creation_operation_id:?} attempts={attempt_count:?} last_attempt_at_ns={last_attempt_at_ns:?} retry_at_ns={retry_at_ns:?}; preserve this operation, repeat Fleet ensure without --apply and review funding_review, then apply its review_sha256 for the exact shortfall; do not fund the account out of band"
67)]
68pub struct EstateFundingRequiredError {
69 pub attempt_count: Option<u32>,
70 pub available_cycles: u128,
71 pub creation_amount_cycles: u128,
72 pub cycles_ledger: String,
73 pub creation_execution_margin_cycles: u128,
74 pub readiness_floor_cycles: u128,
75 pub ledger_fee_cycles: u128,
76 pub management_creation_fee_cycles: u128,
77 pub maximum_creation_debit_cycles: u128,
78 pub maximum_creation_fee_cycles: u128,
79 pub pending_creation_operation_id: Option<String>,
80 pub required_creation_count: u32,
81 pub root: String,
82 pub root_principal: String,
83 pub last_attempt_at_ns: Option<u64>,
84 pub retry_at_ns: Option<u64>,
85 pub shortfall_cycles: u128,
86}
87
88#[derive(Debug, ThisError)]
90pub enum TerminalInventoryError {
91 #[error("one Principal is retained under more than one desired role")]
92 DuplicateRetainedPrincipal,
93
94 #[error("verified canister {canister} lost its Principal binding")]
95 LostPrincipalBinding { canister: String },
96
97 #[error("canister {canister} names unknown parent {parent}")]
98 UnknownParent { canister: String, parent: String },
99
100 #[error(
101 "canister {canister} retained parent {retained_parent} differs from terminal parent {terminal_parent}"
102 )]
103 RetainedParentConflict {
104 canister: String,
105 retained_parent: String,
106 terminal_parent: String,
107 },
108
109 #[error(
110 "terminal Root-owned Idle canister {canister} cycle observations differ: expected {expected}, observed {observed}"
111 )]
112 IdleCycleObservationConflict {
113 canister: String,
114 expected: u128,
115 observed: u128,
116 },
117
118 #[error(
119 "terminal cycle observation duplicates canister {canister} outside its exact Root-owned Idle or Workload lifecycle"
120 )]
121 CycleObservationLifecycleConflict { canister: String },
122}
123
124#[derive(Debug, ThisError)]
127pub enum EnsureWorkflowError<E>
128where
129 E: std::error::Error + 'static,
130{
131 #[error(
132 "activation {operation_id} has an Issued provisioning action after its Applied host prefix (source document {source_document_sha256}); preserve the plan and journal: recovery of this partial activation is not yet qualified"
133 )]
134 PartialActivationResetUnavailable {
135 operation_id: String,
136 source_document_sha256: String,
137 },
138
139 #[error(
140 "a new reinstall requires the unchanged, fully converged Fleet; preserve the retained plan and journal if activation is blocked, because partial-activation reset is not yet supported"
141 )]
142 ReinstallConflict,
143
144 #[error("reviewed Fleet plan digest changed before its first effect")]
145 DriftedBeforeApply,
146
147 #[error(
148 "in-progress Fleet plan requires exact desired input {expected}; supplied input is {actual}"
149 )]
150 RetainedDesiredUnavailable { actual: String, expected: String },
151
152 #[error(
153 "replan-required Fleet operation owns completed reinstall evidence for desired input {expected}; refusing alternate desired input {actual}"
154 )]
155 RetainedReinstallDesiredConflict { actual: String, expected: String },
156
157 #[error(
158 "Root prerequisite retains activation authority for desired input {expected}; refreshed input {actual} is refused before Store effects; preserve the retained plan and journal for recovery review"
159 )]
160 RootActivationDesiredConflict { actual: String, expected: String },
161
162 #[error("reviewed Fleet plan {expected} does not match retained plan {actual}")]
163 PlanDigestMismatch { actual: String, expected: String },
164
165 #[error("reviewed Fleet plan is missing; run `canic fleet ensure <fleet>` first")]
166 PlanMissing,
167
168 #[error("retained Fleet plan failed its content digest")]
169 PlanIntegrity,
170
171 #[error("retained Fleet journal does not match the reviewed Fleet operation")]
172 JournalIntegrity,
173
174 #[error(
175 "Fleet ensure action {action} made no progress for {observations} consecutive observations (last progress: {progress_identity}); operation remains resumable"
176 )]
177 Stalled {
178 action: String,
179 observations: u32,
180 progress_identity: String,
181 },
182
183 #[error("Root {root} provisioning has made no progress for {observations} observations: stage={stage:?}, target={target}, operation={operation}, diagnostic={diagnostic_code}, retry=Backoff, failed_at_ns={failed_at_ns}; the issued operation remains retained; resume the same reviewed apply after the dependency recovers", operation = canic_core::cdk::utils::hash::hex_bytes(.operation_id))]
184 ProvisioningRetryPending {
185 root: candid::Principal,
186 stage: canic_core::dto::component_provisioning::ProvisioningFailureStage,
187 target: candid::Principal,
188 operation_id: [u8; 32],
189 diagnostic_code: u16,
190 observations: u32,
191 failed_at_ns: u64,
192 },
193
194 #[error(
195 "Root-owned canister {target} remained {last_lifecycle} for {observations} consecutive passive observations; no command was reissued and the operation remains resumable"
196 )]
197 RootOwnedObservationStalled {
198 target: String,
199 observations: u32,
200 last_lifecycle: String,
201 },
202
203 #[error(
204 "selected Cycles Ledger account has {actual} cycles, below reviewed maximum debit {required}"
205 )]
206 InsufficientOperatorCycles { actual: u128, required: u128 },
207
208 #[error("terminal Fleet cycle conservation failed: {0}")]
209 Conservation(String),
210
211 #[error("terminal Fleet inventory failed exact current-authority validation: {0}")]
212 TerminalInventory(#[source] TerminalInventoryError),
213
214 #[error(
215 "reviewed effects completed, but live Fleet state still differs from desired state; review a new plan"
216 )]
217 ConvergenceDrift,
218
219 #[error(
220 "Root pool maintenance {action} exhausted its reviewed limit of {maximum_updates} updates; retained receipts remain authoritative"
221 )]
222 PoolMaintenanceBound {
223 action: String,
224 maximum_updates: u32,
225 },
226
227 #[error(
228 "created canister {canister} retained one exact Ledger creation receipt, but its first live balance is {actual_cycles} cycles instead of requested target {requested_cycles} (deficit {deficit_cycles}; configured management creation fee {configured_fee_cycles}); no controller or protocol action followed, and the immutable operation is now replan-required"
229 )]
230 ReplanRequiredAfterCreateBalanceDrift {
231 actual_cycles: u128,
232 canister: String,
233 configured_fee_cycles: u128,
234 deficit_cycles: u128,
235 requested_cycles: u128,
236 },
237
238 #[error("Fleet successor requires a new review: {reason:?}; {}", .review.as_deref().map(ToString::to_string).unwrap_or_default())]
239 SuccessorReviewRequired {
240 reason: crate::fleet_ensure::model::FleetEnsureSuccessorReviewReason,
241 review: Option<Box<crate::fleet_ensure::model::FleetSuccessorReview>>,
242 },
243
244 #[error(transparent)]
245 FreshPoolCreationUnderfunded(Box<FreshPoolCreationUnderfundedError>),
246
247 #[error(transparent)]
248 EstateFundingRequired(Box<EstateFundingRequiredError>),
249
250 #[error(transparent)]
251 Policy(#[from] EnsurePolicyError),
252
253 #[error(transparent)]
254 Inventory(#[from] crate::fleet_ensure::CurrentFleetInventoryError),
255
256 #[error("Fleet platform operation failed: {0}")]
257 Platform(#[source] E),
258
259 #[error(transparent)]
260 State(#[from] EnsureStateError),
261}
262
263pub fn retained_in_progress_plan<E>(
265 root: &Path,
266 environment: &str,
267 requested_fleet: &str,
268) -> Result<Option<FleetEnsurePlan>, EnsureWorkflowError<E>>
269where
270 E: std::error::Error + 'static,
271{
272 validate_path_labels(environment, requested_fleet)?;
273 let paths = EnsurePaths::under(root, environment, requested_fleet);
274 let _lock = lock_operation(&paths)?;
275 if let Some(review) = crate::fleet_ensure::ops::reinstall::adoption::review(&paths)? {
276 if review.environment != environment || review.fleet != requested_fleet {
277 return Err(EnsureWorkflowError::PlanIntegrity);
278 }
279 return Ok(Some(review));
280 }
281 let Some(journal) = read_journal(&paths)? else {
282 return Ok(None);
283 };
284 if !matches!(
285 journal.completion,
286 FleetEnsureCompletion::InProgress | FleetEnsureCompletion::Prepared
287 ) {
288 return Ok(None);
289 }
290 let retained = verified_plan(read_plan(&paths)?.ok_or(EnsureWorkflowError::PlanMissing)?)?;
291 if retained.environment != environment || retained.fleet != requested_fleet {
292 return Err(EnsureWorkflowError::PlanIntegrity);
293 }
294 let state = read_state(&paths, requested_fleet)?;
295 let prepared_full = journal.completion == FleetEnsureCompletion::Prepared
296 && (retained.scope == FleetEnsurePlanScope::Full
297 || (retained.scope == FleetEnsurePlanScope::RootReinstallPrerequisite
298 && retained.reinstall.as_ref().is_some_and(|intent| {
299 intent
300 .activation_reset
301 .as_ref()
302 .is_some_and(|activation| activation.preparation.is_some())
303 })))
304 && retained.reinstall.as_ref().is_some_and(|intent| {
305 intent.operation_id == journal.operation_id
306 && intent.operation_id == retained.operation_id
307 });
308 if !prepared_full {
309 verify_journal(&journal, &retained, requested_fleet, &state)?;
310 }
311 Ok(Some(retained))
312}
313
314#[expect(
316 clippy::too_many_lines,
317 reason = "one locked read-only transaction reconciles retained authority, observation and the immutable plan"
318)]
319pub fn plan<P>(
320 root: &Path,
321 desired: &crate::fleet_ensure::model::DesiredFleet,
322 desired_sha256: &str,
323 requested_fleet: &str,
324 created_at_time: u64,
325 platform: &mut P,
326) -> Result<FleetEnsureReport, EnsureWorkflowError<P::Error>>
327where
328 P: EnsurePlatform,
329{
330 validate_path_identity(desired, requested_fleet)?;
331 let paths = EnsurePaths::under(root, &desired.environment, requested_fleet);
332 let _lock = lock_operation(&paths)?;
333 if let Some(review) = crate::fleet_ensure::ops::reinstall::adoption::review(&paths)? {
334 if review.desired_sha256 != desired_sha256 {
335 return Err(EnsureWorkflowError::DriftedBeforeApply);
336 }
337 return Ok(FleetEnsureReport {
338 funding_review: None,
339 actual_conservation: None,
340 effects_applied: 0,
341 plan: review,
342 terminal: false,
343 });
344 }
345 if let Some(mut journal) = read_journal(&paths)?
346 && journal.completion == FleetEnsureCompletion::InProgress
347 {
348 let retained = verified_plan(read_plan(&paths)?.ok_or(EnsureWorkflowError::PlanMissing)?)?;
349 let state = read_state(&paths, requested_fleet)?;
350 verify_journal(&journal, &retained, requested_fleet, &state)?;
351 if journal.estate_funding_required.is_some() {
352 if let Some(reviewed) = retained.reviewed_desired.as_deref() {
353 platform
354 .bind_reviewed_desired(reviewed.desired())
355 .map_err(EnsureWorkflowError::Platform)?;
356 } else if retained.desired_sha256 != desired_sha256 {
357 return Err(EnsureWorkflowError::RetainedDesiredUnavailable {
358 actual: desired_sha256.to_string(),
359 expected: retained.desired_sha256.clone(),
360 });
361 }
362 }
363 let funding_review = funding::prepare(
364 &paths,
365 &retained,
366 &mut journal,
367 &state,
368 created_at_time,
369 platform,
370 )?;
371 return Ok(FleetEnsureReport {
372 funding_review,
373 actual_conservation: None,
374 effects_applied: applied_count(&journal),
375 plan: retained,
376 terminal: false,
377 });
378 }
379 let mut state = read_state(&paths, requested_fleet)?;
380 let prior_plan = read_plan(&paths)?.map(verified_plan).transpose()?;
381 let prior_journal = read_journal(&paths)?;
382 if let Some(prior) = &prior_plan {
383 let activation_reset = prior
384 .reinstall
385 .as_ref()
386 .is_some_and(|intent| intent.activation_reset.is_some());
387 if activation_reset
388 && prior.scope == FleetEnsurePlanScope::RootReinstallPrerequisite
389 && let Some(journal) = prior_journal
390 .as_ref()
391 .filter(|journal| journal.completion == FleetEnsureCompletion::Converged)
392 {
393 verify_journal(journal, prior, requested_fleet, &state)?;
394 retain_completed_reinstalls(&mut state, prior, journal);
395 write_state(&paths, &state)?;
396 }
397 let unfinished_reset = prior.reinstall.is_some()
398 && !prior_journal.as_ref().is_some_and(|journal| {
399 journal.plan_sha256 == prior.plan_sha256
400 && journal.completion == FleetEnsureCompletion::Converged
401 });
402 if prior.scope == FleetEnsurePlanScope::ReinstallPreparation
403 || unfinished_reset
404 || (activation_reset && prior.scope == FleetEnsurePlanScope::RootReinstallPrerequisite)
405 {
406 if prior.desired_sha256 != desired_sha256 {
407 return Err(EnsureWorkflowError::ReinstallConflict);
408 }
409 return reinstall::continue_plan(
410 root,
411 &paths,
412 desired,
413 prior,
414 prior_journal.as_ref(),
415 &state,
416 platform,
417 );
418 }
419 }
420 let matching_prior = reconcile_retained_planning_state(
421 &mut state,
422 prior_plan.as_ref(),
423 prior_journal.as_ref(),
424 requested_fleet,
425 &desired.environment,
426 desired_sha256,
427 )?;
428 if let Some(prior) = matching_prior {
429 retain_plan_cycles(&mut state, prior);
430 if let Some(journal) = &prior_journal {
431 retain_completed_reinstalls(&mut state, prior, journal);
432 }
433 }
434 let operation_id = matching_prior
435 .filter(|prior| prior.scope == FleetEnsurePlanScope::Full)
436 .map_or_else(
437 || operation_id(desired_sha256, &desired.environment, requested_fleet),
438 |prior| prior.operation_id.clone(),
439 );
440 if let Some(management) = platform
441 .observe_root_management(&state, &BTreeSet::new())
442 .map_err(EnsureWorkflowError::Platform)?
443 {
444 let root_start_authority = read_root_start_authority(&paths)?;
445 if let Some(plan) = compile_root_start_prerequisite_plan(RootStartPlanInput {
446 state: &state,
447 authority: root_start_authority.as_ref(),
448 created_at_time,
449 desired,
450 desired_sha256,
451 observation: &management,
452 requested_fleet,
453 })? {
454 write_plan(&paths, &plan)?;
455 return Ok(FleetEnsureReport {
456 funding_review: None,
457 actual_conservation: None,
458 effects_applied: 0,
459 plan,
460 terminal: false,
461 });
462 }
463 if let Some(plan) = crate::fleet_ensure::policy::root_reinstall::compile(
464 RootStartPlanInput {
465 state: &state,
466 authority: None,
467 created_at_time,
468 desired,
469 desired_sha256,
470 observation: &management,
471 requested_fleet,
472 },
473 &resolve_desired_artifacts(root, desired)?,
474 )? {
475 write_plan(&paths, &plan)?;
476 return Ok(FleetEnsureReport {
477 funding_review: None,
478 actual_conservation: None,
479 effects_applied: 0,
480 plan,
481 terminal: false,
482 });
483 }
484 }
485 let artifacts = resolve_desired_artifacts(root, desired)?;
486 let mut observation = platform
487 .observe(&operation_id, &state)
488 .map_err(EnsureWorkflowError::Platform)?;
489 retain_observed_cycles(&mut state, &observation);
490 write_state(&paths, &state)?;
493 let terminal_inventory_operation_id = current_terminal_inventory_operation(
494 &state,
495 prior_plan.as_ref(),
496 prior_journal.as_ref(),
497 requested_fleet,
498 )?;
499 if let Some(operation_id) = terminal_inventory_operation_id.as_deref() {
500 attach_terminal_inventory_cycles(operation_id, &state, platform, &mut observation)?;
501 }
502 let mut protocol_actions = platform
503 .protocol_actions(&operation_id, &state)
504 .map_err(EnsureWorkflowError::Platform)?;
505 let mut retained_observations = 0_u32;
506 let mut plan = loop {
507 match compile_plan(
508 desired,
509 &artifacts,
510 &protocol_actions,
511 desired_sha256,
512 requested_fleet,
513 &observation,
514 created_at_time,
515 &operation_id,
516 None,
517 ) {
518 Ok(plan) => break plan,
519 Err(EnsurePolicyError::PendingRootOwnedBalance { name }) => {
520 retained_observations = retained_observations.saturating_add(1);
521 if retained_observations >= desired.maximum_stalled_observations {
522 return Err(EnsureWorkflowError::RootOwnedObservationStalled {
523 last_lifecycle: root_owned_lifecycle_label(&observation, &name).to_string(),
524 observations: retained_observations,
525 target: name,
526 });
527 }
528 platform.pace_root_owned_observation(&name, retained_observations);
529 observation = platform
530 .observe(&operation_id, &state)
531 .map_err(EnsureWorkflowError::Platform)?;
532 retain_observed_cycles(&mut state, &observation);
533 write_state(&paths, &state)?;
534 if let Some(operation_id) = terminal_inventory_operation_id.as_deref() {
535 attach_terminal_inventory_cycles(
536 operation_id,
537 &state,
538 platform,
539 &mut observation,
540 )?;
541 }
542 protocol_actions = platform
543 .protocol_actions(&operation_id, &state)
544 .map_err(EnsureWorkflowError::Platform)?;
545 }
546 Err(error) => return Err(error.into()),
547 }
548 };
549 for action in ordered_actions(&plan) {
550 verify_store_activation_binding(&plan, action, &state, platform)?;
551 }
552 bind_terminal_inventory_operation(&mut plan, terminal_inventory_operation_id);
553 write_plan(&paths, &plan)?;
554 Ok(FleetEnsureReport {
555 funding_review: None,
556 actual_conservation: None,
557 effects_applied: 0,
558 plan,
559 terminal: false,
560 })
561}
562
563fn current_terminal_inventory_operation<E>(
564 state: &FleetEnsureStateRecord,
565 prior_plan: Option<&FleetEnsurePlan>,
566 prior_journal: Option<&FleetEnsureJournalRecord>,
567 requested_fleet: &str,
568) -> Result<Option<String>, EnsureWorkflowError<E>>
569where
570 E: std::error::Error + 'static,
571{
572 if state.active_registry.is_none() {
573 return Ok(None);
574 }
575 let prior_plan = prior_plan.ok_or(EnsureWorkflowError::PlanMissing)?;
576 let prior_journal = prior_journal.ok_or(EnsureWorkflowError::JournalIntegrity)?;
577 verify_journal(prior_journal, prior_plan, requested_fleet, state)?;
578 if prior_journal.completion != FleetEnsureCompletion::Converged
579 || prior_plan.scope != FleetEnsurePlanScope::Full
580 {
581 return Err(EnsureWorkflowError::JournalIntegrity);
582 }
583 let operation_id = prior_plan
584 .terminal_inventory_operation_id
585 .as_ref()
586 .unwrap_or(&prior_plan.operation_id);
587 if operation_id.is_empty() {
588 return Err(EnsureWorkflowError::JournalIntegrity);
589 }
590 Ok(Some(operation_id.clone()))
591}
592
593fn bind_terminal_inventory_operation(plan: &mut FleetEnsurePlan, operation_id: Option<String>) {
594 plan.terminal_inventory_operation_id = operation_id;
595 plan.plan_sha256 = expected_plan_sha256(plan);
596}
597
598fn reconcile_retained_planning_state<'a, E>(
599 state: &mut FleetEnsureStateRecord,
600 prior_plan: Option<&'a FleetEnsurePlan>,
601 prior_journal: Option<&FleetEnsureJournalRecord>,
602 requested_fleet: &str,
603 desired_environment: &str,
604 desired_sha256: &str,
605) -> Result<Option<&'a FleetEnsurePlan>, EnsureWorkflowError<E>>
606where
607 E: std::error::Error + 'static,
608{
609 if let Some((prior, journal)) = prior_plan.zip(prior_journal)
610 && prior.fleet == requested_fleet
611 && prior.environment == desired_environment
612 && retained_reinstall_desired_conflict(
613 state,
614 &prior.fleet,
615 &prior.operation_id,
616 &prior.desired_sha256,
617 desired_sha256,
618 journal,
619 )
620 {
621 return Err(EnsureWorkflowError::RetainedReinstallDesiredConflict {
622 actual: desired_sha256.to_string(),
623 expected: prior.desired_sha256.clone(),
624 });
625 }
626 if let Some((prior, journal)) = prior_plan.zip(prior_journal)
627 && prior.scope == FleetEnsurePlanScope::RootReinstallPrerequisite
628 && prior.desired_sha256 != desired_sha256
629 {
630 verify_journal(journal, prior, requested_fleet, state)?;
631 return Err(EnsureWorkflowError::RootActivationDesiredConflict {
632 actual: desired_sha256.to_string(),
633 expected: prior.desired_sha256.clone(),
634 });
635 }
636 let matching_prior = prior_plan.filter(|prior| {
637 prior.fleet == requested_fleet
638 && prior.environment == desired_environment
639 && prior.desired_sha256 == desired_sha256
640 });
641 let retained_operation_matches =
642 matching_prior
643 .zip(prior_journal)
644 .is_some_and(|(prior, journal)| {
645 retained_reinstall_operation_matches(
646 state,
647 &prior.fleet,
648 &prior.operation_id,
649 journal,
650 )
651 });
652 if !retained_operation_matches {
653 state.completed_reinstall_action_sha256.clear();
654 state.completed_reinstall_operation_id = None;
655 state.completed_reinstalls.clear();
656 }
657 Ok(matching_prior)
658}
659
660fn retain_completed_reinstalls(
661 state: &mut FleetEnsureStateRecord,
662 plan: &FleetEnsurePlan,
663 journal: &FleetEnsureJournalRecord,
664) {
665 if journal.fleet != plan.fleet || journal.operation_id != plan.operation_id {
666 return;
667 }
668 for action in ordered_actions(plan) {
669 let EnsureAction::Install {
670 mode: crate::fleet_ensure::model::InstallMode::Reinstall,
671 name,
672 principal,
673 wasm_sha256,
674 ..
675 } = action
676 else {
677 continue;
678 };
679 let action_sha256 = action_sha256(action);
680 let Some(effect) = journal
681 .effects
682 .iter()
683 .find(|effect| effect.action_sha256 == action_sha256)
684 else {
685 continue;
686 };
687 if effect.state == EffectState::Applied
688 && let Some(pre_canister_version) = effect.pre_canister_version
689 && state
690 .principals
691 .get(name)
692 .is_some_and(|retained| retained == principal)
693 {
694 state.completed_reinstall_operation_id = Some(journal.operation_id.clone());
695 state
696 .completed_reinstall_action_sha256
697 .insert(name.clone(), action_sha256);
698 state
699 .completed_reinstalls
700 .insert(name.clone(), pre_canister_version);
701 if let Some(topology) = state.topology.get_mut(name) {
702 topology.module_hash = Some(wasm_sha256.clone());
703 }
704 }
705 }
706}
707
708fn completed_reinstall_evidence_matches(
709 state: &FleetEnsureStateRecord,
710 journal: &FleetEnsureJournalRecord,
711) -> bool {
712 !state.completed_reinstalls.is_empty()
713 && state
714 .completed_reinstalls
715 .iter()
716 .all(|(name, pre_version)| {
717 state
718 .completed_reinstall_action_sha256
719 .get(name)
720 .and_then(|action_sha256| {
721 journal
722 .effects
723 .iter()
724 .find(|effect| effect.action_sha256 == *action_sha256)
725 })
726 .is_some_and(|effect| {
727 effect.state == EffectState::Applied
728 && effect.pre_canister_version == Some(*pre_version)
729 })
730 })
731 && state.completed_reinstall_action_sha256.len() == state.completed_reinstalls.len()
732}
733
734fn retained_reinstall_operation_matches(
735 state: &FleetEnsureStateRecord,
736 prior_fleet: &str,
737 prior_operation_id: &str,
738 journal: &FleetEnsureJournalRecord,
739) -> bool {
740 journal.completion == FleetEnsureCompletion::ReplanRequired
741 && journal.fleet == prior_fleet
742 && journal.operation_id == prior_operation_id
743 && state.completed_reinstall_operation_id.as_deref() == Some(journal.operation_id.as_str())
744 && completed_reinstall_evidence_matches(state, journal)
745}
746
747fn retained_reinstall_desired_conflict(
748 state: &FleetEnsureStateRecord,
749 prior_fleet: &str,
750 prior_operation_id: &str,
751 prior_desired_sha256: &str,
752 supplied_desired_sha256: &str,
753 journal: &FleetEnsureJournalRecord,
754) -> bool {
755 prior_desired_sha256 != supplied_desired_sha256
756 && retained_reinstall_operation_matches(state, prior_fleet, prior_operation_id, journal)
757}
758
759fn retain_plan_cycles(state: &mut FleetEnsureStateRecord, plan: &FleetEnsurePlan) {
760 for canister in &plan.canisters {
761 if let Some(principal) = &canister.principal {
762 state
763 .retained_cycles_by_principal
764 .entry(principal.clone())
765 .or_insert(canister.observed_cycles);
766 }
767 }
768}
769
770fn retain_observed_cycles(state: &mut FleetEnsureStateRecord, observation: &FleetObservation) {
771 for live in observation.canisters.values().filter_map(Option::as_ref) {
772 state
773 .retained_cycles_by_principal
774 .insert(live.principal.clone(), live.cycles);
775 }
776 state
777 .retained_cycles_by_principal
778 .extend(observation.additional_controlled_cycles.clone());
779}
780
781fn root_owned_lifecycle_label(observation: &FleetObservation, target: &str) -> &'static str {
782 observation
783 .canisters
784 .get(target)
785 .and_then(Option::as_ref)
786 .and_then(|live| live.root_owned_lifecycle)
787 .map_or("unavailable", |lifecycle| lifecycle.label())
788}
789
790#[expect(
792 clippy::too_many_lines,
793 reason = "the sole effect driver keeps intent, reconciliation and terminal publication visibly ordered"
794)]
795pub fn apply<P>(
796 root: &Path,
797 desired: &crate::fleet_ensure::model::DesiredFleet,
798 desired_sha256: &str,
799 requested_fleet: &str,
800 reviewed_plan_sha256: &str,
801 platform: &mut P,
802) -> Result<FleetEnsureReport, EnsureWorkflowError<P::Error>>
803where
804 P: EnsurePlatform,
805{
806 validate_path_identity(desired, requested_fleet)?;
807 let paths = EnsurePaths::under(root, &desired.environment, requested_fleet);
808 let _lock = lock_operation(&paths)?;
809 let pending = crate::fleet_ensure::ops::reinstall::adoption::review(&paths)?;
810 let adopting = pending.is_some();
811 let retained_plan = verified_plan(match pending {
812 Some(plan) => plan,
813 None => read_plan(&paths)?.ok_or(EnsureWorkflowError::PlanMissing)?,
814 })?;
815 let retained_journal = if adopting {
816 None
817 } else {
818 read_journal(&paths)?
819 };
820 if retained_plan.plan_sha256 != reviewed_plan_sha256
821 && !retained_journal
822 .as_ref()
823 .is_some_and(|journal| funding::accepts(&retained_plan, journal, reviewed_plan_sha256))
824 {
825 return Err(EnsureWorkflowError::PlanDigestMismatch {
826 actual: retained_plan.plan_sha256,
827 expected: reviewed_plan_sha256.to_string(),
828 });
829 }
830 let mut state = read_state(&paths, requested_fleet)?;
831 let in_progress = retained_journal
832 .as_ref()
833 .is_some_and(|journal| journal.completion == FleetEnsureCompletion::InProgress);
834 if let Some(journal) = retained_journal.as_ref().filter(|_| in_progress) {
835 verify_journal(journal, &retained_plan, requested_fleet, &state)?;
836 compact_inline_plan(&paths, &retained_plan)?;
837 }
838 if !in_progress && retained_plan.desired_sha256 != desired_sha256 {
839 return Err(EnsureWorkflowError::DriftedBeforeApply);
840 }
841 let operation_desired = if in_progress || retained_plan.reinstall.is_some() {
842 if let Some(reviewed) = retained_plan.reviewed_desired.as_deref() {
843 let reviewed = reviewed.desired();
844 validate_path_identity(reviewed, requested_fleet)?;
845 if reviewed.environment != retained_plan.environment {
846 return Err(EnsureWorkflowError::PlanIntegrity);
847 }
848 reviewed
849 } else if retained_plan.desired_sha256 == desired_sha256 {
850 desired
851 } else {
852 return Err(EnsureWorkflowError::RetainedDesiredUnavailable {
853 actual: desired_sha256.to_string(),
854 expected: retained_plan.desired_sha256.clone(),
855 });
856 }
857 } else {
858 if retained_plan.desired_sha256 != desired_sha256 {
859 return Err(EnsureWorkflowError::DriftedBeforeApply);
860 }
861 desired
862 };
863 platform
864 .bind_reviewed_desired(operation_desired)
865 .map_err(EnsureWorkflowError::Platform)?;
866 let operation_desired_sha256 = retained_plan.desired_sha256.as_str();
867 continuation::verify_inputs(root, operation_desired, &retained_plan)?;
868 if let Some(journal) = &retained_journal {
869 continuation::verify_canonical(&retained_plan, journal, &state, platform)?;
870 }
871 let completed_journal = retained_journal.as_ref().filter(|journal| {
872 (journal.completion == FleetEnsureCompletion::Converged
873 || (journal.completion == FleetEnsureCompletion::Prepared
874 && retained_plan.scope == FleetEnsurePlanScope::ReinstallPreparation))
875 && journal.plan_sha256 == retained_plan.plan_sha256
876 });
877 if retained_plan.scope == FleetEnsurePlanScope::ReinstallPreparation
878 && let Some(journal) = completed_journal
879 {
880 let actual = reinstall::complete(&retained_plan, journal, &state, platform)?;
881 return Ok(FleetEnsureReport {
882 funding_review: None,
883 actual_conservation: Some(actual),
884 effects_applied: 0,
885 plan: retained_plan,
886 terminal: true,
887 });
888 }
889 if retained_plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite
890 && let Some(journal) = completed_journal
891 {
892 verify_journal(journal, &retained_plan, requested_fleet, &state)?;
893 let actual = root_reinstall::complete(&retained_plan, journal, &state, platform)?;
894 report_progress_state(
895 platform,
896 &retained_plan,
897 journal,
898 FleetEnsurePhase::Infrastructure,
899 FleetEnsureProgressState::PrerequisiteComplete,
900 );
901 return Ok(FleetEnsureReport {
902 funding_review: None,
903 actual_conservation: Some(actual),
904 effects_applied: 0,
905 plan: retained_plan,
906 terminal: true,
907 });
908 }
909 if retained_plan.scope == FleetEnsurePlanScope::Full
910 && let Some(journal) = completed_journal
911 {
912 verify_journal(journal, &retained_plan, requested_fleet, &state)?;
913 let actual = continuation::replay(
914 root,
915 operation_desired,
916 &retained_plan,
917 journal,
918 &state,
919 platform,
920 )?;
921 report_progress(
922 platform,
923 &retained_plan,
924 journal,
925 FleetEnsurePhase::Complete,
926 );
927 return Ok(FleetEnsureReport {
928 funding_review: None,
929 actual_conservation: Some(actual),
930 effects_applied: 0,
931 plan: retained_plan,
932 terminal: true,
933 });
934 }
935 if retained_plan.scope == FleetEnsurePlanScope::RootStartPrerequisite
936 && let Some(journal) = completed_journal
937 {
938 verify_terminal_root_start_replay(
939 operation_desired,
940 &retained_plan,
941 journal,
942 requested_fleet,
943 platform,
944 &state,
945 )?;
946 report_progress_state(
947 platform,
948 &retained_plan,
949 journal,
950 FleetEnsurePhase::Infrastructure,
951 FleetEnsureProgressState::PrerequisiteComplete,
952 );
953 return Ok(FleetEnsureReport {
954 funding_review: None,
955 actual_conservation: None,
956 effects_applied: applied_count(journal),
957 plan: retained_plan,
958 terminal: true,
959 });
960 }
961 let mut journal = match retained_journal {
962 Some(journal) if journal.completion == FleetEnsureCompletion::InProgress => {
963 verify_journal(&journal, &retained_plan, requested_fleet, &state)?;
964 journal
965 }
966 _ => {
967 let (observation, initial_controlled_cycles) = verify_fresh_plan(
968 root,
969 operation_desired,
970 operation_desired_sha256,
971 requested_fleet,
972 &retained_plan,
973 platform,
974 &state,
975 )?;
976 let journal = FleetEnsureJournalRecord {
977 funding_reviews: Vec::new(),
978 successor_phases: Vec::new(),
979 completion: FleetEnsureCompletion::InProgress,
980 estate_funding_required: None,
981 effects: Vec::new(),
982 fleet: requested_fleet.to_string(),
983 initial_controlled_cycles,
984 initial_estate_funding_cycles_by_root: retained_plan
985 .conservation
986 .estate_funding_domains
987 .iter()
988 .map(|domain| {
989 (
990 domain.root.clone(),
991 domain.available_cycles.unwrap_or_default(),
992 )
993 })
994 .collect(),
995 initial_operator_cycles: observation.operator_cycles,
996 operation_id: retained_plan.operation_id.clone(),
997 plan_sha256: retained_plan.plan_sha256.clone(),
998 schema_version: FLEET_ENSURE_SCHEMA_VERSION,
999 stalled_observations: 0,
1000 };
1001 if adopting {
1003 crate::fleet_ensure::ops::reinstall::adoption::adopt(
1004 &paths,
1005 &retained_plan,
1006 &journal,
1007 )?;
1008 } else {
1009 write_journal(&paths, &journal)?;
1010 }
1011 journal
1012 }
1013 };
1014
1015 funding::resume(
1016 &paths,
1017 &retained_plan,
1018 &mut journal,
1019 &state,
1020 reviewed_plan_sha256,
1021 platform,
1022 )?;
1023 if journal
1024 .estate_funding_required
1025 .as_ref()
1026 .is_some_and(|pause| funding::covered(&journal, pause))
1027 {
1028 retain_estate_funding_pause(&paths, &mut journal, None)?;
1029 }
1030 let mut terminal_state;
1031 loop {
1032 let execution_actions = continuation::actions(&retained_plan, &journal)
1033 .into_iter()
1034 .cloned()
1035 .collect::<Vec<_>>();
1036 let actions = execution_actions.iter().collect::<Vec<_>>();
1037 let mut replayed_issued_commands = BTreeSet::new();
1038 let mut reported_phase = None;
1039 loop {
1040 let mut deferred_controller_observation = false;
1041 for (index, action) in actions.iter().enumerate() {
1042 let action_hash = action_sha256(action);
1043 let observation_policy = effect_observation_policy(operation_desired, action)?;
1044 let retained_effect = journal.effects.get(index);
1045 if retained_effect.is_none_or(|effect| effect.state != EffectState::Applied) {
1046 let phase = action_progress_phase(action);
1047 if reported_phase != Some(phase) {
1048 report_progress(platform, &retained_plan, &journal, phase);
1049 reported_phase = Some(phase);
1050 }
1051 }
1052 if matches!(action, EnsureAction::FleetProtocol { .. })
1053 && retained_effect.is_none_or(|effect| effect.state == EffectState::Intent)
1054 && !prior_fleet_protocol_effect_started(&actions, &journal, index)
1055 {
1056 let funding_observation = platform
1057 .observe(&journal.operation_id, &state)
1058 .map_err(EnsureWorkflowError::Platform)?;
1059 let funded_plan = funding_plan(&retained_plan, &journal)?;
1060 let required =
1061 estate_funding_requirement(&funded_plan, &state, &funding_observation)?;
1062 retain_estate_funding_pause(&paths, &mut journal, required.as_ref())?;
1063 if let Some(required) = required {
1064 report_progress_state(
1065 platform,
1066 &retained_plan,
1067 &journal,
1068 action_progress_phase(action),
1069 FleetEnsureProgressState::FundingRequired,
1070 );
1071 return Err(EnsureWorkflowError::EstateFundingRequired(Box::new(
1072 estate_funding_error(&required),
1073 )));
1074 }
1075 }
1076 if journal.effects.len() <= index {
1077 let pre_cycles = platform
1078 .action_cycles(action, &state)
1079 .map_err(EnsureWorkflowError::Platform)?;
1080 let destination_pre_cycles = platform
1081 .action_destination_cycles(action, &state)
1082 .map_err(EnsureWorkflowError::Platform)?;
1083 let pre_canister_version = platform
1084 .action_canister_version(action, &state)
1085 .map_err(EnsureWorkflowError::Platform)?;
1086 journal.effects.push(EffectRecord {
1087 maintenance_attempts: 0,
1088 action_sha256: action_hash.clone(),
1089 created_principal: None,
1090 destination_post_cycles: destination_pre_cycles,
1091 destination_pre_cycles,
1092 post_cycles: None,
1093 pre_cycles,
1094 pre_canister_version,
1095 progress_identity: None,
1096 receipt: None,
1097 state: EffectState::Intent,
1098 });
1099 write_journal(&paths, &journal)?;
1100 }
1101 loop {
1102 let record = journal
1103 .effects
1104 .get(index)
1105 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1106 if record.action_sha256 != action_hash {
1107 return Err(EnsureWorkflowError::DriftedBeforeApply);
1108 }
1109 if !create_record_is_exact(action, record, &state) {
1110 return Err(EnsureWorkflowError::JournalIntegrity);
1111 }
1112 if matches!(record.state, EffectState::Applied) {
1113 let retained_create = journal
1114 .effects
1115 .get(index)
1116 .ok_or(EnsureWorkflowError::JournalIntegrity)
1117 .and_then(|record| {
1118 retain_applied_create_authority(
1119 operation_desired,
1120 action,
1121 record,
1122 &mut state,
1123 )
1124 })?;
1125 let retained_funding = journal.effects.get(index).is_some_and(|record| {
1126 retain_applied_funding_cycles(&mut state, action, record)
1127 });
1128 if retained_create || retained_funding {
1129 write_state(&paths, &state)?;
1130 }
1131 break;
1132 }
1133
1134 let observed = platform
1135 .observe_effect(&journal.operation_id, action, record, &state)
1136 .map_err(EnsureWorkflowError::Platform)?;
1137 let source_cycles = if observed.post_cycles.is_some() {
1138 observed.post_cycles
1139 } else {
1140 platform
1141 .action_cycles(action, &state)
1142 .map_err(EnsureWorkflowError::Platform)?
1143 };
1144 let destination_cycles = platform
1145 .action_destination_cycles(action, &state)
1146 .map_err(EnsureWorkflowError::Platform)?;
1147 let replan_after_create_balance =
1148 observed.retry == EffectRetry::ReplanRequiredAfterCreateBalanceDrift;
1149 let retained_create_balance_is_exact = !replan_after_create_balance
1150 || (journal.effects.len() == index + 1
1151 && journal.effects[..index]
1152 .iter()
1153 .all(|effect| effect.state == EffectState::Applied)
1154 && record.state == EffectState::Issued
1155 && create_identity_is_exact(action.name(), record, &state)
1156 && observed.post_cycles.is_some()
1157 && observed.post_cycles == source_cycles);
1158 let protocol_funding_required = if record.state == EffectState::Issued {
1159 observed
1160 .estate_funding_required
1161 .as_ref()
1162 .map(|funding| {
1163 protocol_estate_funding_requirement(&retained_plan, &state, funding)
1164 })
1165 .transpose()?
1166 } else {
1167 None
1168 };
1169 if let Some(required) = protocol_funding_required.as_ref()
1170 && !funding::covered(&journal, required)
1171 {
1172 retain_estate_funding_pause(&paths, &mut journal, Some(required))?;
1173 report_progress_state(
1174 platform,
1175 &retained_plan,
1176 &journal,
1177 action_progress_phase(action),
1178 FleetEnsureProgressState::FundingRequired,
1179 );
1180 return Err(EnsureWorkflowError::EstateFundingRequired(Box::new(
1181 estate_funding_error(required),
1182 )));
1183 }
1184 if journal
1185 .estate_funding_required
1186 .as_ref()
1187 .is_some_and(|required| required.pending_creation_operation_id.is_some())
1188 {
1189 retain_estate_funding_pause(&paths, &mut journal, None)?;
1190 }
1191 let record = journal
1192 .effects
1193 .get_mut(index)
1194 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1195 merge_observed_cycles(record, source_cycles, destination_cycles);
1196 if replan_after_create_balance {
1197 if !retained_create_balance_is_exact {
1198 return Err(EnsureWorkflowError::JournalIntegrity);
1199 }
1200 let EnsureAction::Create {
1201 name,
1202 requested_initial_cycles,
1203 ..
1204 } = action
1205 else {
1206 return Err(EnsureWorkflowError::JournalIntegrity);
1207 };
1208 let actual_cycles = record
1209 .post_cycles
1210 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1211 record.progress_identity = Some(observed.progress_identity);
1212 record.state = EffectState::Applied;
1213 retain_created_canister_for_replan(
1214 operation_desired,
1215 name,
1216 record,
1217 &mut state,
1218 )?;
1219 journal.completion = FleetEnsureCompletion::ReplanRequired;
1220 journal.stalled_observations = 0;
1221 write_journal(&paths, &journal)?;
1222 write_state(&paths, &state)?;
1223 let configured_fee_cycles = operation_desired
1224 .management_creation_fee_cycles
1225 .parse::<Cycles>()
1226 .map(|cycles| cycles.to_u128())
1227 .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
1228 return Err(EnsureWorkflowError::ReplanRequiredAfterCreateBalanceDrift {
1229 actual_cycles,
1230 canister: name.clone(),
1231 configured_fee_cycles,
1232 deficit_cycles: requested_initial_cycles.saturating_sub(actual_cycles),
1233 requested_cycles: *requested_initial_cycles,
1234 });
1235 }
1236 if observed.applied {
1237 record.progress_identity = Some(observed.progress_identity);
1238 record.state = EffectState::Applied;
1239 journal.stalled_observations = 0;
1240 write_journal(&paths, &journal)?;
1241 let record = journal
1242 .effects
1243 .get(index)
1244 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1245 let retained_create = retain_applied_create_authority(
1246 operation_desired,
1247 action,
1248 record,
1249 &mut state,
1250 )?;
1251 let retained_funding =
1252 retain_applied_funding_cycles(&mut state, action, record);
1253 if retained_create || retained_funding {
1254 write_state(&paths, &state)?;
1255 }
1256 break;
1257 }
1258
1259 if observed.retry == EffectRetry::DeferUntilControllerObservation {
1260 if !deferred_create_observation_is_exact(
1261 operation_desired,
1262 &actions,
1263 index,
1264 action,
1265 record,
1266 &state,
1267 ) {
1268 return Err(EnsureWorkflowError::JournalIntegrity);
1269 }
1270 let progress_identity = observed.progress_identity.clone();
1271 if record.progress_identity.as_deref() == Some(&observed.progress_identity)
1272 {
1273 journal.stalled_observations =
1274 journal.stalled_observations.saturating_add(1);
1275 } else {
1276 record.progress_identity = Some(observed.progress_identity);
1277 journal.stalled_observations = 0;
1278 }
1279 write_journal(&paths, &journal)?;
1280 if journal.stalled_observations
1281 >= observation_policy.maximum_stalled_observations
1282 {
1283 return Err(EnsureWorkflowError::Stalled {
1284 action: action.name().to_string(),
1285 observations: journal.stalled_observations,
1286 progress_identity,
1287 });
1288 }
1289 deferred_controller_observation = true;
1290 break;
1291 }
1292
1293 let maintenance_continuation =
1294 observed.retry == EffectRetry::ContinuePoolMaintenance;
1295 if matches!(record.state, EffectState::Intent) || maintenance_continuation {
1296 if let EnsureAction::FleetProtocol {
1297 action: current, ..
1298 } = action
1299 && let CurrentFleetProtocolAction::MaintainPoolReadiness {
1300 maximum_updates,
1301 ..
1302 } = current.as_ref()
1303 {
1304 if record.maintenance_attempts >= *maximum_updates {
1305 return Err(EnsureWorkflowError::PoolMaintenanceBound {
1306 action: action.name().to_string(),
1307 maximum_updates: *maximum_updates,
1308 });
1309 }
1310 if record.progress_identity.as_deref()
1311 == Some(&observed.progress_identity)
1312 {
1313 journal.stalled_observations =
1314 journal.stalled_observations.saturating_add(1);
1315 } else {
1316 journal.stalled_observations = 0;
1317 }
1318 record.maintenance_attempts += 1;
1319 let attempts = record.maintenance_attempts;
1320 write_journal(&paths, &journal)?;
1322 if attempts > 1 {
1323 platform
1324 .pace_effect_observation(action, journal.stalled_observations);
1325 }
1326 } else if maintenance_continuation {
1327 return Err(EnsureWorkflowError::JournalIntegrity);
1328 }
1329 let record = journal
1330 .effects
1331 .get(index)
1332 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1333 if retained_plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite {
1334 root_reinstall::verify_effect_authority(
1335 &retained_plan,
1336 action,
1337 &state,
1338 platform,
1339 )?;
1340 }
1341 verify_store_activation_binding(&retained_plan, action, &state, platform)?;
1342 reinstall::verify_effect_authority(
1343 &retained_plan,
1344 action,
1345 &state,
1346 platform,
1347 )?;
1348 let outcome =
1349 match platform.apply(&journal.operation_id, action, record, &state) {
1350 Ok(outcome) => outcome,
1351 Err(source) => {
1352 let progress_identity = record
1353 .progress_identity
1354 .clone()
1355 .unwrap_or_else(|| "effect-call-failed".to_string());
1356 journal.stalled_observations =
1357 journal.stalled_observations.saturating_add(1);
1358 write_journal(&paths, &journal)?;
1359 if journal.stalled_observations
1360 >= observation_policy.maximum_stalled_observations
1361 {
1362 return Err(EnsureWorkflowError::Stalled {
1363 action: action.name().to_string(),
1364 observations: journal.stalled_observations,
1365 progress_identity,
1366 });
1367 }
1368 return Err(EnsureWorkflowError::Platform(source));
1369 }
1370 };
1371 if !create_outcome_is_exact(action, record, &state, &outcome) {
1372 return Err(EnsureWorkflowError::JournalIntegrity);
1373 }
1374 if let Some(created) = &outcome.created_principal {
1375 state
1376 .pending_principals
1377 .insert(action.name().to_string(), created.clone());
1378 write_state(&paths, &state)?;
1379 }
1380 let record = journal
1381 .effects
1382 .get_mut(index)
1383 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1384 record.created_principal = outcome.created_principal;
1385 record.receipt = outcome.receipt;
1386 record.post_cycles = outcome.post_cycles;
1387 record.progress_identity = Some(observed.progress_identity);
1388 record.state = EffectState::Issued;
1389 if !maintenance_continuation {
1391 journal.stalled_observations = 0;
1392 }
1393 write_journal(&paths, &journal)?;
1394 continue;
1395 }
1396
1397 if observed.retry == EffectRetry::ReplayExactIssuedCommand
1398 && !replayed_issued_commands.contains(&index)
1399 {
1400 if !matches!(
1401 action,
1402 EnsureAction::FleetProtocol { action, .. }
1403 if matches!(
1404 action.as_ref(),
1405 crate::fleet_ensure::model::CurrentFleetProtocolAction::ProvisionComponents { .. }
1406 )
1407 ) {
1408 return Err(EnsureWorkflowError::JournalIntegrity);
1409 }
1410 let retained_receipt = record.receipt.clone();
1411 let outcome = platform
1412 .apply(&journal.operation_id, action, record, &state)
1413 .map_err(EnsureWorkflowError::Platform)?;
1414 if outcome.created_principal.is_some()
1415 || outcome.receipt != retained_receipt
1416 || outcome.post_cycles.is_some()
1417 {
1418 return Err(EnsureWorkflowError::JournalIntegrity);
1419 }
1420 let record = journal
1421 .effects
1422 .get_mut(index)
1423 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1424 record.progress_identity = Some(observed.progress_identity);
1425 journal.stalled_observations = 0;
1426 write_journal(&paths, &journal)?;
1427 replayed_issued_commands.insert(index);
1428 continue;
1429 }
1430
1431 if record.progress_identity.as_deref() == Some(&observed.progress_identity) {
1432 journal.stalled_observations =
1433 journal.stalled_observations.saturating_add(1);
1434 } else {
1435 record.progress_identity = Some(observed.progress_identity);
1436 journal.stalled_observations = 0;
1437 }
1438 let progress_identity = record
1439 .progress_identity
1440 .clone()
1441 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1442 write_journal(&paths, &journal)?;
1443 if journal.stalled_observations
1444 >= observation_policy.maximum_stalled_observations
1445 {
1446 if let Some((failure, origin)) = observed
1447 .provisioning_failure
1448 .and_then(|failure| failure.origin.map(|origin| (failure, origin)))
1449 {
1450 return Err(EnsureWorkflowError::ProvisioningRetryPending {
1451 root: failure.fleet_subnet_root,
1452 stage: origin.stage,
1453 target: origin.target,
1454 operation_id: origin.operation_id,
1455 diagnostic_code: origin.diagnostic_code,
1456 observations: journal.stalled_observations,
1457 failed_at_ns: origin.failed_at_ns,
1458 });
1459 }
1460 return Err(EnsureWorkflowError::Stalled {
1461 action: action.name().to_string(),
1462 observations: journal.stalled_observations,
1463 progress_identity,
1464 });
1465 }
1466 if observation_policy.paced {
1467 report_progress_state(
1468 platform,
1469 &retained_plan,
1470 &journal,
1471 action_progress_phase(action),
1472 FleetEnsureProgressState::AwaitingProgress,
1473 );
1474 platform.pace_effect_observation(action, journal.stalled_observations);
1475 }
1476 }
1477 }
1478 if journal.effects.len() == actions.len()
1479 && journal
1480 .effects
1481 .iter()
1482 .all(|record| record.state == EffectState::Applied)
1483 {
1484 break;
1485 }
1486 if !deferred_controller_observation {
1487 return Err(EnsureWorkflowError::JournalIntegrity);
1488 }
1489 }
1490
1491 report_progress(
1492 platform,
1493 &retained_plan,
1494 &journal,
1495 FleetEnsurePhase::TerminalVerification,
1496 );
1497 if retained_plan.scope == FleetEnsurePlanScope::ReinstallPreparation {
1498 let actual = reinstall::complete(&retained_plan, &journal, &state, platform)?;
1499 journal.completion = FleetEnsureCompletion::Prepared;
1500 journal.stalled_observations = 0;
1501 write_journal(&paths, &journal)?;
1502 report_progress_state(
1503 platform,
1504 &retained_plan,
1505 &journal,
1506 FleetEnsurePhase::Infrastructure,
1507 FleetEnsureProgressState::PrerequisiteComplete,
1508 );
1509 return Ok(FleetEnsureReport {
1510 funding_review: None,
1511 actual_conservation: Some(actual),
1512 effects_applied: applied_count(&journal),
1513 plan: retained_plan,
1514 terminal: true,
1515 });
1516 }
1517 if retained_plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite {
1518 let actual = root_reinstall::complete(&retained_plan, &journal, &state, platform)?;
1519 publish_terminal_state(operation_desired, &retained_plan, &journal, &mut state)
1520 .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
1521 state.active_registry = None;
1522 write_state(&paths, &state)?;
1523 journal.completion = FleetEnsureCompletion::Converged;
1524 journal.stalled_observations = 0;
1525 write_journal(&paths, &journal)?;
1526 report_progress_state(
1527 platform,
1528 &retained_plan,
1529 &journal,
1530 FleetEnsurePhase::Infrastructure,
1531 FleetEnsureProgressState::PrerequisiteComplete,
1532 );
1533 return Ok(FleetEnsureReport {
1534 funding_review: None,
1535 actual_conservation: Some(actual),
1536 effects_applied: applied_count(&journal),
1537 plan: retained_plan,
1538 terminal: true,
1539 });
1540 }
1541 if retained_plan.scope == FleetEnsurePlanScope::RootStartPrerequisite {
1542 let actual_conservation = complete_root_start_prerequisite(
1543 operation_desired,
1544 &retained_plan,
1545 platform,
1546 &state,
1547 &journal,
1548 )?;
1549 journal.completion = FleetEnsureCompletion::Converged;
1550 journal.stalled_observations = 0;
1551 write_journal(&paths, &journal)?;
1552 report_progress_state(
1553 platform,
1554 &retained_plan,
1555 &journal,
1556 FleetEnsurePhase::Infrastructure,
1557 FleetEnsureProgressState::PrerequisiteComplete,
1558 );
1559 return Ok(FleetEnsureReport {
1560 funding_review: None,
1561 actual_conservation: Some(actual_conservation),
1562 effects_applied: applied_count(&journal),
1563 plan: retained_plan,
1564 terminal: true,
1565 });
1566 }
1567
1568 terminal_state = state.clone();
1569 publish_terminal_state(
1570 operation_desired,
1571 &retained_plan,
1572 &journal,
1573 &mut terminal_state,
1574 )
1575 .map_err(|TerminalStatePublicationError::JournalIntegrity| {
1576 EnsureWorkflowError::JournalIntegrity
1577 })?;
1578 if completed_infrastructure_reinstall(operation_desired, &retained_plan, &journal) {
1579 terminal_state.active_registry = None;
1580 }
1581 project_current_fleet_inventory(&terminal_state)?;
1582 let mut terminal_observation = platform
1583 .observe(&retained_plan.operation_id, &terminal_state)
1584 .map_err(EnsureWorkflowError::Platform)?;
1585 {
1586 let artifacts = resolve_desired_artifacts(root, operation_desired)?;
1587 let converged = loop {
1588 let protocol_actions = platform
1589 .protocol_actions(&retained_plan.operation_id, &terminal_state)
1590 .map_err(EnsureWorkflowError::Platform)?;
1591 match compile_plan(
1592 operation_desired,
1593 &artifacts,
1594 &protocol_actions,
1595 operation_desired_sha256,
1596 requested_fleet,
1597 &terminal_observation,
1598 retained_plan.planned_at_time,
1599 &retained_plan.operation_id,
1600 None,
1601 ) {
1602 Ok(plan) => {
1603 if journal.stalled_observations != 0 {
1604 journal.stalled_observations = 0;
1605 write_journal(&paths, &journal)?;
1606 }
1607 break plan;
1608 }
1609 Err(EnsurePolicyError::PendingRootOwnedBalance { name }) => {
1610 journal.stalled_observations =
1611 journal.stalled_observations.saturating_add(1);
1612 retain_observed_cycles(&mut state, &terminal_observation);
1613 write_state(&paths, &state)?;
1614 write_journal(&paths, &journal)?;
1615 if journal.stalled_observations
1616 >= operation_desired.maximum_stalled_observations
1617 {
1618 return Err(EnsureWorkflowError::RootOwnedObservationStalled {
1619 last_lifecycle: root_owned_lifecycle_label(
1620 &terminal_observation,
1621 &name,
1622 )
1623 .to_string(),
1624 observations: journal.stalled_observations,
1625 target: name,
1626 });
1627 }
1628 report_progress_state(
1629 platform,
1630 &retained_plan,
1631 &journal,
1632 FleetEnsurePhase::TerminalVerification,
1633 FleetEnsureProgressState::AwaitingProgress,
1634 );
1635 platform.pace_root_owned_observation(&name, journal.stalled_observations);
1636 terminal_observation = platform
1637 .observe(&retained_plan.operation_id, &terminal_state)
1638 .map_err(EnsureWorkflowError::Platform)?;
1639 }
1640 Err(error @ EnsurePolicyError::EstatePoolCapacity { .. }) => {
1641 retain_observed_cycles(&mut terminal_state, &terminal_observation);
1642 retain_completed_reinstalls(&mut terminal_state, &retained_plan, &journal);
1643 write_state(&paths, &terminal_state)?;
1644 journal.completion = FleetEnsureCompletion::ReplanRequired;
1645 journal.stalled_observations = 0;
1646 write_journal(&paths, &journal)?;
1647 return Err(error.into());
1648 }
1649 Err(error) => return Err(error.into()),
1650 }
1651 };
1652 if converged
1653 .canisters
1654 .iter()
1655 .any(|canister| !canister.actions.is_empty())
1656 || !converged.protocol_actions.is_empty()
1657 {
1658 if retained_plan.continuation.is_some() {
1659 let appended = continuation::append(
1660 &paths,
1661 operation_desired,
1662 &retained_plan,
1663 &mut journal,
1664 &terminal_state,
1665 &terminal_observation,
1666 converged,
1667 platform,
1668 );
1669 match appended {
1670 Ok(()) => {
1671 write_state(&paths, &terminal_state)?;
1672 state = terminal_state.clone();
1673 report_progress_state(
1674 platform,
1675 &retained_plan,
1676 &journal,
1677 FleetEnsurePhase::TerminalVerification,
1678 FleetEnsureProgressState::PrerequisiteComplete,
1679 );
1680 continue;
1681 }
1682 Err(EnsureWorkflowError::SuccessorReviewRequired { reason, review }) => {
1683 retain_observed_cycles(&mut terminal_state, &terminal_observation);
1684 write_state(&paths, &terminal_state)?;
1685 journal.completion = FleetEnsureCompletion::ReplanRequired;
1686 write_journal(&paths, &journal)?;
1687 report_progress_state(
1688 platform,
1689 &retained_plan,
1690 &journal,
1691 FleetEnsurePhase::TerminalVerification,
1692 FleetEnsureProgressState::ReviewRequired {
1693 reason,
1694 review: review.clone(),
1695 },
1696 );
1697 return Err(EnsureWorkflowError::SuccessorReviewRequired {
1698 reason,
1699 review,
1700 });
1701 }
1702 Err(error) => return Err(error),
1703 }
1704 }
1705 retain_observed_cycles(&mut terminal_state, &terminal_observation);
1706 retain_completed_reinstalls(&mut terminal_state, &retained_plan, &journal);
1707 write_state(&paths, &terminal_state)?;
1708 journal.completion = FleetEnsureCompletion::ReplanRequired;
1709 journal.stalled_observations = 0;
1710 write_journal(&paths, &journal)?;
1711 let reason =
1712 crate::fleet_ensure::model::FleetEnsureSuccessorReviewReason::AdditionalEffect;
1713 let review = Some(Box::new(
1714 crate::fleet_ensure::ops::recovery::review_details(&converged),
1715 ));
1716 report_progress_state(
1717 platform,
1718 &retained_plan,
1719 &journal,
1720 FleetEnsurePhase::TerminalVerification,
1721 FleetEnsureProgressState::ReviewRequired {
1722 reason,
1723 review: review.clone(),
1724 },
1725 );
1726 return Err(EnsureWorkflowError::SuccessorReviewRequired { reason, review });
1727 }
1728 }
1729 break;
1730 }
1731 let terminal_inventory_operation_id =
1732 completed_inventory_operation(&retained_plan, &journal, &terminal_state)?;
1733 let terminal_inventory = platform
1734 .terminal_inventory(terminal_inventory_operation_id, &terminal_state)
1735 .map_err(EnsureWorkflowError::Platform)?;
1736 let terminal_cycles = terminal_inventory.controlled_cycles_by_principal.clone();
1737 merge_terminal_inventory(&mut terminal_state, terminal_inventory)?;
1738 project_current_fleet_inventory(&terminal_state)?;
1739 let mut final_observation = platform
1740 .observe(&retained_plan.operation_id, &terminal_state)
1741 .map_err(EnsureWorkflowError::Platform)?;
1742 attach_terminal_cycles(&mut final_observation, terminal_cycles)?;
1743 reinstall::verify_terminal_estate(&retained_plan, &final_observation)?;
1744 reinstall::verify_terminal_authority(&retained_plan, &terminal_state, platform)?;
1745 let actual_conservation = verify_terminal_conservation(
1746 &retained_plan,
1747 &journal,
1748 &terminal_state,
1749 &final_observation,
1750 )?;
1751 terminal_state.completed_reinstall_action_sha256.clear();
1752 terminal_state.completed_reinstall_operation_id = None;
1753 terminal_state.completed_reinstalls.clear();
1754 write_state(&paths, &terminal_state)?;
1756 journal.completion = FleetEnsureCompletion::Converged;
1757 journal.stalled_observations = 0;
1758 write_journal(&paths, &journal)?;
1759 report_progress(
1760 platform,
1761 &retained_plan,
1762 &journal,
1763 FleetEnsurePhase::Complete,
1764 );
1765 Ok(FleetEnsureReport {
1766 funding_review: None,
1767 actual_conservation: Some(actual_conservation),
1768 effects_applied: applied_count(&journal),
1769 plan: retained_plan,
1770 terminal: true,
1771 })
1772}
1773
1774fn action_progress_phase(action: &EnsureAction) -> FleetEnsurePhase {
1775 match action {
1776 EnsureAction::FleetProtocol { action, .. } => match action.as_ref() {
1777 CurrentFleetProtocolAction::ReconcilePoolAsset { .. } => {
1778 FleetEnsurePhase::ImportReconciliation
1779 }
1780 CurrentFleetProtocolAction::ProvisionComponents { .. } => {
1781 FleetEnsurePhase::WorkloadProvisioning
1782 }
1783 CurrentFleetProtocolAction::MaintainPoolReadiness { .. }
1784 | CurrentFleetProtocolAction::ObservePoolReadiness { .. } => {
1785 FleetEnsurePhase::PoolReadiness
1786 }
1787 _ => FleetEnsurePhase::ControlPlane,
1788 },
1789 _ => FleetEnsurePhase::Infrastructure,
1790 }
1791}
1792
1793fn report_progress<P: EnsurePlatform>(
1794 platform: &mut P,
1795 plan: &FleetEnsurePlan,
1796 journal: &FleetEnsureJournalRecord,
1797 phase: FleetEnsurePhase,
1798) {
1799 let state = if phase == FleetEnsurePhase::Complete {
1800 FleetEnsureProgressState::Complete
1801 } else {
1802 FleetEnsureProgressState::Advancing
1803 };
1804 report_progress_state(platform, plan, journal, phase, state);
1805}
1806
1807fn report_progress_state<P: EnsurePlatform>(
1808 platform: &mut P,
1809 plan: &FleetEnsurePlan,
1810 journal: &FleetEnsureJournalRecord,
1811 phase: FleetEnsurePhase,
1812 state: FleetEnsureProgressState,
1813) {
1814 platform.report_progress(FleetEnsureProgress {
1815 state,
1816 operation_id: plan.operation_id.clone(),
1817 plan_sha256: plan.plan_sha256.clone(),
1818 phase,
1819 applied_effects: applied_count(journal),
1820 reviewed_effects: continuation::actions(plan, journal).len()
1821 + journal
1822 .funding_reviews
1823 .iter()
1824 .filter(|review| review.effect.is_some())
1825 .count(),
1826 });
1827}
1828
1829const fn merge_observed_cycles(
1830 record: &mut EffectRecord,
1831 source_cycles: Option<u128>,
1832 destination_cycles: Option<u128>,
1833) {
1834 if let Some(cycles) = source_cycles {
1835 record.post_cycles = Some(cycles);
1836 }
1837 if let Some(cycles) = destination_cycles {
1838 record.destination_post_cycles = Some(cycles);
1839 }
1840}
1841
1842pub(super) fn retain_applied_create_authority<E>(
1843 desired: &crate::fleet_ensure::model::DesiredFleet,
1844 action: &EnsureAction,
1845 record: &EffectRecord,
1846 state: &mut FleetEnsureStateRecord,
1847) -> Result<bool, EnsureWorkflowError<E>>
1848where
1849 E: std::error::Error + 'static,
1850{
1851 let EnsureAction::Create {
1852 name,
1853 requested_initial_cycles,
1854 ..
1855 } = action
1856 else {
1857 return Ok(false);
1858 };
1859 if record.state != EffectState::Applied {
1860 return Err(EnsureWorkflowError::JournalIntegrity);
1861 }
1862 let configured = desired
1863 .canisters
1864 .iter()
1865 .find(|canister| {
1866 canister.name == *name
1867 && canister.presence == crate::fleet_ensure::model::DesiredPresence::Present
1868 })
1869 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1870 let principal = record
1871 .created_principal
1872 .as_deref()
1873 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1874 let cycles = record
1875 .post_cycles
1876 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1877 let maximum_observation_burn_cycles =
1878 crate::fleet_ensure::ops::maximum_creation_observation_burn(desired)
1879 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1880 if !create_identity_is_exact(name, record, state)
1881 || !create_balance_is_terminal(
1882 Some(cycles),
1883 *requested_initial_cycles,
1884 maximum_observation_burn_cycles,
1885 )
1886 || state
1887 .retained_cycles_by_principal
1888 .get(principal)
1889 .is_some_and(|retained| *retained != cycles)
1890 {
1891 return Err(EnsureWorkflowError::JournalIntegrity);
1892 }
1893 validate_fresh_pool_creation_balance(
1894 desired,
1895 configured,
1896 name,
1897 cycles,
1898 maximum_observation_burn_cycles,
1899 *requested_initial_cycles,
1900 )?;
1901
1902 let prior_topology = state.topology.get(name);
1903 let topology_identity_matches = prior_topology.is_some_and(|topology| {
1904 topology.kind == configured.kind
1905 && topology.parent == configured.parent
1906 && topology.protocol_binding == configured.protocol_binding
1907 && topology.role
1908 == configured
1909 .protocol_binding
1910 .as_ref()
1911 .map(|binding| binding.role.to_string())
1912 });
1913 let topology = crate::fleet_ensure::model::FleetEnsureTopologyRecord {
1914 kind: configured.kind,
1915 module_hash: if topology_identity_matches {
1916 prior_topology.and_then(|topology| topology.module_hash.clone())
1917 } else {
1918 None
1919 },
1920 parent: configured.parent.clone(),
1921 protocol_binding: configured.protocol_binding.clone(),
1922 role: configured
1923 .protocol_binding
1924 .as_ref()
1925 .map(|binding| binding.role.to_string()),
1926 };
1927 let changed = state.retained_cycles_by_principal.get(principal) != Some(&cycles)
1928 || state.topology.get(name) != Some(&topology);
1929 state
1930 .retained_cycles_by_principal
1931 .insert(principal.to_string(), cycles);
1932 state.topology.insert(name.clone(), topology);
1933 Ok(changed)
1934}
1935
1936fn validate_fresh_pool_creation_balance<E>(
1937 desired: &crate::fleet_ensure::model::DesiredFleet,
1938 configured: &crate::fleet_ensure::model::DesiredCanister,
1939 name: &str,
1940 cycles: u128,
1941 maximum_observation_burn_cycles: u128,
1942 requested_initial_cycles: u128,
1943) -> Result<(), EnsureWorkflowError<E>>
1944where
1945 E: std::error::Error + 'static,
1946{
1947 if configured.kind == crate::fleet_ensure::model::DesiredCanisterKind::Pool
1948 && desired
1949 .bootstrap
1950 .as_ref()
1951 .is_some_and(|bootstrap| bootstrap.fresh_estate)
1952 && configured.principal.is_none()
1953 && configured.controllers.is_empty()
1954 && configured.parent.as_ref().is_some_and(|parent| {
1955 configured.controller_canisters.as_slice() == std::slice::from_ref(parent)
1956 })
1957 {
1958 let readiness_floor_cycles = configured
1959 .minimum_cycles
1960 .parse::<Cycles>()
1961 .map(|cycles| cycles.to_u128())
1962 .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
1963 let remaining_controller_burn_cycles = desired
1964 .maximum_update_burn_cycles
1965 .parse::<Cycles>()
1966 .map(|cycles| cycles.to_u128())
1967 .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
1968 let required_pre_finalization_balance_cycles = readiness_floor_cycles
1969 .checked_add(remaining_controller_burn_cycles)
1970 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1971 if cycles < required_pre_finalization_balance_cycles {
1972 return Err(EnsureWorkflowError::FreshPoolCreationUnderfunded(Box::new(
1973 FreshPoolCreationUnderfundedError {
1974 canister: name.to_string(),
1975 live_balance_cycles: cycles,
1976 maximum_observation_burn_cycles,
1977 pre_finalization_shortfall_cycles: required_pre_finalization_balance_cycles
1978 - cycles,
1979 readiness_floor_cycles,
1980 readiness_shortfall_cycles: readiness_floor_cycles.saturating_sub(cycles),
1981 remaining_controller_burn_cycles,
1982 requested_creation_funding_cycles: requested_initial_cycles,
1983 required_pre_finalization_balance_cycles,
1984 },
1985 )));
1986 }
1987 }
1988 Ok(())
1989}
1990
1991fn retain_applied_funding_cycles(
1992 state: &mut FleetEnsureStateRecord,
1993 action: &EnsureAction,
1994 record: &EffectRecord,
1995) -> bool {
1996 let EnsureAction::Fund { principal, .. } = action else {
1997 return false;
1998 };
1999 if record.state != EffectState::Applied || record.receipt.is_none() {
2000 return false;
2001 }
2002 let Some(cycles) = record.post_cycles else {
2003 return false;
2004 };
2005 let principal = principal.strip_prefix("created:").map_or_else(
2006 || Some(principal.as_str()),
2007 |name| state.pending_principals.get(name).map(String::as_str),
2008 );
2009 let Some(principal) = principal else {
2010 return false;
2011 };
2012 if state.retained_cycles_by_principal.get(principal) == Some(&cycles) {
2013 return false;
2014 }
2015 state
2016 .retained_cycles_by_principal
2017 .insert(principal.to_string(), cycles);
2018 true
2019}
2020
2021fn retain_created_canister_for_replan<E>(
2022 desired: &crate::fleet_ensure::model::DesiredFleet,
2023 name: &str,
2024 record: &EffectRecord,
2025 state: &mut FleetEnsureStateRecord,
2026) -> Result<(), EnsureWorkflowError<E>>
2027where
2028 E: std::error::Error + 'static,
2029{
2030 let configured = desired
2031 .canisters
2032 .iter()
2033 .find(|canister| {
2034 canister.name == name
2035 && canister.presence == crate::fleet_ensure::model::DesiredPresence::Present
2036 })
2037 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2038 let principal = record
2039 .created_principal
2040 .as_deref()
2041 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2042 let cycles = record
2043 .post_cycles
2044 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2045 if state.pending_principals.get(name).map(String::as_str) != Some(principal) {
2046 return Err(EnsureWorkflowError::JournalIntegrity);
2047 }
2048 state.pending_principals.remove(name);
2049 state
2050 .principals
2051 .insert(name.to_string(), principal.to_string());
2052 state
2053 .retained_cycles_by_principal
2054 .insert(principal.to_string(), cycles);
2055 state.topology.insert(
2056 name.to_string(),
2057 crate::fleet_ensure::model::FleetEnsureTopologyRecord {
2058 kind: configured.kind,
2059 module_hash: None,
2060 parent: configured.parent.clone(),
2061 protocol_binding: configured.protocol_binding.clone(),
2062 role: configured
2063 .protocol_binding
2064 .as_ref()
2065 .map(|binding| binding.role.to_string()),
2066 },
2067 );
2068 Ok(())
2069}
2070
2071fn create_record_is_exact(
2072 action: &EnsureAction,
2073 record: &EffectRecord,
2074 state: &FleetEnsureStateRecord,
2075) -> bool {
2076 let EnsureAction::Create {
2077 name,
2078 requested_initial_cycles,
2079 ..
2080 } = action
2081 else {
2082 return true;
2083 };
2084 match record.state {
2085 EffectState::Intent => {
2086 record.created_principal.is_none()
2087 && record.receipt.is_none()
2088 && record.post_cycles.is_none()
2089 }
2090 EffectState::Issued => {
2091 create_identity_is_exact(name, record, state)
2092 && record.post_cycles == Some(*requested_initial_cycles)
2093 }
2094 EffectState::Applied => create_identity_is_exact(name, record, state),
2095 }
2096}
2097
2098fn create_identity_is_exact(
2099 name: &str,
2100 record: &EffectRecord,
2101 state: &FleetEnsureStateRecord,
2102) -> bool {
2103 let Some(created_principal) = record
2104 .created_principal
2105 .as_deref()
2106 .filter(|value| !value.is_empty())
2107 else {
2108 return false;
2109 };
2110 if record.receipt.as_deref().is_none_or(str::is_empty) {
2111 return false;
2112 }
2113 state_create_principal_matches(name, created_principal, state, true)
2114}
2115
2116fn state_create_principal_matches(
2117 name: &str,
2118 created_principal: &str,
2119 state: &FleetEnsureStateRecord,
2120 require_retained: bool,
2121) -> bool {
2122 let pending = state.pending_principals.get(name).map(String::as_str);
2123 let terminal = state.principals.get(name).map(String::as_str);
2124 let retained = pending.is_some() || terminal.is_some();
2125 let pending_matches = pending.is_none_or(|principal| principal == created_principal);
2126 let terminal_matches = terminal.is_none_or(|principal| principal == created_principal);
2127 (!require_retained || retained) && pending_matches && terminal_matches
2128}
2129
2130fn create_outcome_is_exact(
2131 action: &EnsureAction,
2132 record: &EffectRecord,
2133 state: &FleetEnsureStateRecord,
2134 outcome: &crate::fleet_ensure::ops::EffectOutcome,
2135) -> bool {
2136 if matches!(action, EnsureAction::FundEstate { .. }) {
2137 return outcome.created_principal.is_none()
2138 && outcome.post_cycles.is_none()
2139 && outcome
2140 .receipt
2141 .as_deref()
2142 .is_some_and(|receipt| !receipt.is_empty());
2143 }
2144 let EnsureAction::Create {
2145 name,
2146 requested_initial_cycles,
2147 ..
2148 } = action
2149 else {
2150 return true;
2151 };
2152 let Some(created_principal) = outcome
2153 .created_principal
2154 .as_deref()
2155 .filter(|value| !value.is_empty())
2156 else {
2157 return false;
2158 };
2159 let Some(receipt) = outcome.receipt.as_deref().filter(|value| !value.is_empty()) else {
2160 return false;
2161 };
2162 if outcome.post_cycles != Some(*requested_initial_cycles) {
2163 return false;
2164 }
2165 let principal_matches = record
2166 .created_principal
2167 .as_deref()
2168 .is_none_or(|retained| retained == created_principal);
2169 let receipt_matches = record
2170 .receipt
2171 .as_deref()
2172 .is_none_or(|retained| retained == receipt);
2173 let state_matches = state_create_principal_matches(name, created_principal, state, false);
2174 principal_matches && receipt_matches && state_matches
2175}
2176
2177fn verify_fresh_plan<P>(
2178 root: &Path,
2179 desired: &crate::fleet_ensure::model::DesiredFleet,
2180 desired_sha256: &str,
2181 requested_fleet: &str,
2182 retained_plan: &FleetEnsurePlan,
2183 platform: &mut P,
2184 state: &FleetEnsureStateRecord,
2185) -> Result<(FleetObservation, u128), EnsureWorkflowError<P::Error>>
2186where
2187 P: EnsurePlatform,
2188{
2189 if retained_plan.reinstall.is_some() {
2190 return reinstall::verify_before_apply(root, desired, retained_plan, platform, state);
2191 }
2192 if retained_plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite {
2193 return root_reinstall::verify_before_apply(root, desired, retained_plan, platform, state);
2194 }
2195 if retained_plan.scope == FleetEnsurePlanScope::RootStartPrerequisite {
2196 let targets = reviewed_root_start_targets(retained_plan)?;
2197 let management = platform
2198 .observe_root_management(state, &targets)
2199 .map_err(EnsureWorkflowError::Platform)?
2200 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2201 let current = recompile_root_start_prerequisite_plan(
2202 RootStartPlanInput {
2203 state,
2204 authority: retained_plan.root_start_authority.as_deref(),
2205 created_at_time: retained_plan.planned_at_time,
2206 desired,
2207 desired_sha256,
2208 observation: &management,
2209 requested_fleet,
2210 },
2211 &targets,
2212 )?;
2213 if !compatible_root_start_prerequisite(retained_plan, ¤t, desired) {
2214 return Err(EnsureWorkflowError::DriftedBeforeApply);
2215 }
2216 let observation = root_management_fleet_observation(&management, &targets)?;
2217 return Ok((observation, current.conservation.observed_controlled_cycles));
2218 }
2219 let mut observation = platform
2220 .observe(&retained_plan.operation_id, state)
2221 .map_err(EnsureWorkflowError::Platform)?;
2222 let terminal_inventory_operation_id =
2223 reviewed_terminal_inventory_operation(retained_plan, state)?;
2224 if let Some(operation_id) = terminal_inventory_operation_id {
2225 attach_terminal_inventory_cycles(operation_id, state, platform, &mut observation)?;
2226 }
2227 let artifacts = resolve_desired_artifacts(root, desired)?;
2228 let mut retained_observations = 0_u32;
2229 let mut current = loop {
2230 let protocol_actions = platform
2231 .protocol_actions(&retained_plan.operation_id, state)
2232 .map_err(EnsureWorkflowError::Platform)?;
2233 match compile_plan(
2234 desired,
2235 &artifacts,
2236 &protocol_actions,
2237 desired_sha256,
2238 requested_fleet,
2239 &observation,
2240 retained_plan.planned_at_time,
2241 &retained_plan.operation_id,
2242 None,
2243 ) {
2244 Ok(plan) => break plan,
2245 Err(EnsurePolicyError::PendingRootOwnedBalance { name }) => {
2246 retained_observations = retained_observations.saturating_add(1);
2247 if retained_observations >= desired.maximum_stalled_observations {
2248 return Err(EnsureWorkflowError::RootOwnedObservationStalled {
2249 last_lifecycle: root_owned_lifecycle_label(&observation, &name).to_string(),
2250 observations: retained_observations,
2251 target: name,
2252 });
2253 }
2254 platform.pace_root_owned_observation(&name, retained_observations);
2255 observation = platform
2256 .observe(&retained_plan.operation_id, state)
2257 .map_err(EnsureWorkflowError::Platform)?;
2258 if let Some(operation_id) = terminal_inventory_operation_id {
2259 attach_terminal_inventory_cycles(
2260 operation_id,
2261 state,
2262 platform,
2263 &mut observation,
2264 )?;
2265 }
2266 }
2267 Err(error) => return Err(error.into()),
2268 }
2269 };
2270 if observation.operator_cycles < retained_plan.conservation.maximum_operator_debit_cycles {
2271 return Err(EnsureWorkflowError::InsufficientOperatorCycles {
2272 actual: observation.operator_cycles,
2273 required: retained_plan.conservation.maximum_operator_debit_cycles,
2274 });
2275 }
2276 bind_terminal_inventory_operation(
2277 &mut current,
2278 retained_plan.terminal_inventory_operation_id.clone(),
2279 );
2280 if current.plan_sha256 != retained_plan.plan_sha256
2281 && !compatible_after_bounded_observation(retained_plan, ¤t, desired, &observation)
2282 {
2283 return Err(EnsureWorkflowError::DriftedBeforeApply);
2284 }
2285 Ok((observation, current.conservation.observed_controlled_cycles))
2286}
2287
2288fn estate_funding_requirement<E>(
2289 plan: &FleetEnsurePlan,
2290 state: &FleetEnsureStateRecord,
2291 observation: &FleetObservation,
2292) -> Result<Option<EstateFundingRequiredRecord>, EnsureWorkflowError<E>>
2293where
2294 E: std::error::Error + 'static,
2295{
2296 for reviewed in &plan.conservation.estate_funding_domains {
2297 let observed = observation
2298 .estate_funding_domains
2299 .get(&reviewed.root)
2300 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2301 if observed.cycles_ledger != reviewed.cycles_ledger {
2302 return Err(EnsureWorkflowError::PlanIntegrity);
2303 }
2304 let root_principal = observed
2305 .root_principal
2306 .as_deref()
2307 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2308 if reviewed_estate_root_principal(reviewed, state) != Some(root_principal) {
2309 return Err(EnsureWorkflowError::PlanIntegrity);
2310 }
2311 let available_cycles = observed
2312 .balance_cycles
2313 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2314 let reviewed_available = reviewed.available_cycles.unwrap_or_default();
2315 let maximum_reviewed_balance = reviewed_available
2316 .checked_add(reviewed.maximum_funding_cycles)
2317 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2318 if available_cycles < reviewed_available || available_cycles > maximum_reviewed_balance {
2319 return Err(EnsureWorkflowError::DriftedBeforeApply);
2320 }
2321 if reviewed.required_creation_count == 0 {
2322 if available_cycles != reviewed_available {
2323 return Err(EnsureWorkflowError::DriftedBeforeApply);
2324 }
2325 continue;
2326 }
2327 if available_cycles < reviewed.maximum_creation_debit_cycles {
2328 return Ok(Some(EstateFundingRequiredRecord {
2329 attempt_count: None,
2330 available_cycles,
2331 creation_amount_cycles: reviewed.creation_amount_cycles,
2332 creation_execution_margin_cycles: reviewed.creation_execution_margin_cycles,
2333 readiness_floor_cycles: reviewed.readiness_floor_cycles,
2334 cycles_ledger: reviewed.cycles_ledger.clone(),
2335 ledger_fee_cycles: reviewed.ledger_fee_cycles,
2336 management_creation_fee_cycles: reviewed.management_creation_fee_cycles,
2337 maximum_creation_debit_cycles: reviewed.maximum_creation_debit_cycles,
2338 maximum_creation_fee_cycles: reviewed.maximum_creation_fee_cycles,
2339 operation_id: plan.operation_id.clone(),
2340 pending_creation_operation_id: None,
2341 plan_sha256: plan.plan_sha256.clone(),
2342 required_creation_count: reviewed.required_creation_count,
2343 root: reviewed.root.clone(),
2344 root_principal: root_principal.to_string(),
2345 last_attempt_at_ns: None,
2346 retry_at_ns: None,
2347 shortfall_cycles: reviewed
2348 .maximum_creation_debit_cycles
2349 .saturating_sub(available_cycles),
2350 }));
2351 }
2352 }
2353 Ok(None)
2354}
2355
2356fn protocol_estate_funding_requirement<E>(
2357 plan: &FleetEnsurePlan,
2358 state: &FleetEnsureStateRecord,
2359 funding: &canic_core::dto::component_provisioning::RootEstateFundingRequired,
2360) -> Result<EstateFundingRequiredRecord, EnsureWorkflowError<E>>
2361where
2362 E: std::error::Error + 'static,
2363{
2364 let root_principal = funding.root.to_text();
2365 let reviewed = plan
2366 .conservation
2367 .estate_funding_domains
2368 .iter()
2369 .find(|domain| reviewed_estate_root_principal(domain, state) == Some(&root_principal))
2370 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2371 let creation_amount_cycles = funding.creation_amount.to_u128();
2372 let ledger_fee_cycles = funding.ledger_fee.to_u128();
2373 let management_creation_fee_cycles = funding.management_creation_fee.to_u128();
2374 let required_cycles = creation_amount_cycles
2375 .checked_add(ledger_fee_cycles)
2376 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2377 let maximum_creation_fee_cycles = management_creation_fee_cycles
2378 .checked_add(ledger_fee_cycles)
2379 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2380 let exact_reviewed_authority = reviewed.required_creation_count > 0
2381 && reviewed.cycles_ledger == funding.cycles_ledger.to_text()
2382 && reviewed.creation_amount_cycles == creation_amount_cycles
2383 && reviewed.creation_execution_margin_cycles == funding.execution_margin.to_u128()
2384 && reviewed.readiness_floor_cycles == funding.readiness_floor.to_u128()
2385 && reviewed.ledger_fee_cycles == ledger_fee_cycles
2386 && reviewed.management_creation_fee_cycles == management_creation_fee_cycles;
2387 let exact_runtime_arithmetic = funding.required.to_u128() == required_cycles
2388 && funding.available < funding.required
2389 && funding.shortfall.to_u128()
2390 == required_cycles.saturating_sub(funding.available.to_u128());
2391 if !exact_reviewed_authority || !exact_runtime_arithmetic {
2392 return Err(EnsureWorkflowError::PlanIntegrity);
2393 }
2394 Ok(EstateFundingRequiredRecord {
2395 attempt_count: Some(funding.attempt_count),
2396 available_cycles: funding.available.to_u128(),
2397 creation_amount_cycles,
2398 cycles_ledger: funding.cycles_ledger.to_text(),
2399 creation_execution_margin_cycles: funding.execution_margin.to_u128(),
2400 readiness_floor_cycles: funding.readiness_floor.to_u128(),
2401 ledger_fee_cycles,
2402 management_creation_fee_cycles,
2403 maximum_creation_debit_cycles: required_cycles,
2404 maximum_creation_fee_cycles,
2405 operation_id: plan.operation_id.clone(),
2406 pending_creation_operation_id: Some(canic_core::cdk::utils::hash::hex_bytes(
2407 funding.operation_id,
2408 )),
2409 plan_sha256: plan.plan_sha256.clone(),
2410 required_creation_count: 1,
2411 root: reviewed.root.clone(),
2412 root_principal,
2413 last_attempt_at_ns: funding.last_attempt_at_ns,
2414 retry_at_ns: Some(funding.retry_at_ns),
2415 shortfall_cycles: funding.shortfall.to_u128(),
2416 })
2417}
2418
2419fn estate_funding_error(required: &EstateFundingRequiredRecord) -> EstateFundingRequiredError {
2420 EstateFundingRequiredError {
2421 attempt_count: required.attempt_count,
2422 available_cycles: required.available_cycles,
2423 creation_amount_cycles: required.creation_amount_cycles,
2424 creation_execution_margin_cycles: required.creation_execution_margin_cycles,
2425 readiness_floor_cycles: required.readiness_floor_cycles,
2426 cycles_ledger: required.cycles_ledger.clone(),
2427 ledger_fee_cycles: required.ledger_fee_cycles,
2428 management_creation_fee_cycles: required.management_creation_fee_cycles,
2429 maximum_creation_debit_cycles: required.maximum_creation_debit_cycles,
2430 maximum_creation_fee_cycles: required.maximum_creation_fee_cycles,
2431 pending_creation_operation_id: required.pending_creation_operation_id.clone(),
2432 required_creation_count: required.required_creation_count,
2433 root: required.root.clone(),
2434 root_principal: required.root_principal.clone(),
2435 last_attempt_at_ns: required.last_attempt_at_ns,
2436 retry_at_ns: required.retry_at_ns,
2437 shortfall_cycles: required.shortfall_cycles,
2438 }
2439}
2440
2441fn retain_estate_funding_pause<E>(
2442 paths: &EnsurePaths,
2443 journal: &mut FleetEnsureJournalRecord,
2444 required: Option<&EstateFundingRequiredRecord>,
2445) -> Result<bool, EnsureWorkflowError<E>>
2446where
2447 E: std::error::Error + 'static,
2448{
2449 if journal.estate_funding_required.as_ref() == required {
2450 return Ok(false);
2451 }
2452 journal.estate_funding_required = required.cloned();
2453 write_journal(paths, journal)?;
2454 Ok(true)
2455}
2456
2457fn reviewed_estate_root_principal<'a>(
2458 domain: &'a crate::fleet_ensure::model::EstateFundingDomainPlan,
2459 state: &'a FleetEnsureStateRecord,
2460) -> Option<&'a str> {
2461 domain
2462 .root_principal
2463 .as_deref()
2464 .or_else(|| {
2465 state
2466 .pending_principals
2467 .get(&domain.root)
2468 .map(String::as_str)
2469 })
2470 .or_else(|| state.principals.get(&domain.root).map(String::as_str))
2471}
2472
2473fn prior_fleet_protocol_effect_started(
2474 actions: &[&EnsureAction],
2475 journal: &FleetEnsureJournalRecord,
2476 current_index: usize,
2477) -> bool {
2478 actions
2479 .iter()
2480 .take(current_index)
2481 .zip(&journal.effects)
2482 .any(|(action, effect)| {
2483 matches!(action, EnsureAction::FleetProtocol { .. })
2484 && matches!(effect.state, EffectState::Issued | EffectState::Applied)
2485 })
2486}
2487
2488fn completed_inventory_operation<'a, E>(
2489 plan: &'a FleetEnsurePlan,
2490 journal: &FleetEnsureJournalRecord,
2491 state: &FleetEnsureStateRecord,
2492) -> Result<&'a str, EnsureWorkflowError<E>>
2493where
2494 E: std::error::Error + 'static,
2495{
2496 if plan.protocol_actions.is_empty()
2497 && journal.successor_phases.is_empty()
2498 && state.active_registry.is_some()
2499 {
2500 reviewed_terminal_inventory_operation(plan, state)?
2501 .ok_or(EnsureWorkflowError::PlanIntegrity)
2502 } else {
2503 Ok(&plan.operation_id)
2504 }
2505}
2506
2507fn reviewed_terminal_inventory_operation<'a, E>(
2508 plan: &'a FleetEnsurePlan,
2509 state: &FleetEnsureStateRecord,
2510) -> Result<Option<&'a str>, EnsureWorkflowError<E>>
2511where
2512 E: std::error::Error + 'static,
2513{
2514 match (
2515 state.active_registry.is_some(),
2516 plan.terminal_inventory_operation_id.as_deref(),
2517 ) {
2518 (false, None) => Ok(None),
2519 (true, Some(operation_id)) if !operation_id.is_empty() => Ok(Some(operation_id)),
2520 _ => Err(EnsureWorkflowError::PlanIntegrity),
2521 }
2522}
2523
2524fn complete_root_start_prerequisite<P>(
2525 desired: &crate::fleet_ensure::model::DesiredFleet,
2526 retained_plan: &FleetEnsurePlan,
2527 platform: &mut P,
2528 state: &FleetEnsureStateRecord,
2529 journal: &FleetEnsureJournalRecord,
2530) -> Result<ActualCycleConservation, EnsureWorkflowError<P::Error>>
2531where
2532 P: EnsurePlatform,
2533{
2534 let targets = reviewed_root_start_targets(retained_plan)?;
2535 let management = platform
2536 .observe_root_management(state, &targets)
2537 .map_err(EnsureWorkflowError::Platform)?
2538 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2539 let current = recompile_root_start_prerequisite_plan(
2540 RootStartPlanInput {
2541 state,
2542 authority: retained_plan.root_start_authority.as_deref(),
2543 created_at_time: retained_plan.planned_at_time,
2544 desired,
2545 desired_sha256: &retained_plan.desired_sha256,
2546 observation: &management,
2547 requested_fleet: &retained_plan.fleet,
2548 },
2549 &targets,
2550 )?;
2551 if !compatible_root_start_prerequisite(retained_plan, ¤t, desired) {
2552 return Err(EnsureWorkflowError::ConvergenceDrift);
2553 }
2554 if management.roots.iter().any(|(name, observed)| {
2555 targets.contains(name)
2556 && observed.live.status != crate::fleet_ensure::model::CanisterRuntimeStatus::Running
2557 }) {
2558 return Err(EnsureWorkflowError::ConvergenceDrift);
2559 }
2560 let terminal = root_management_fleet_observation(&management, &targets)?;
2561 verify_terminal_conservation(retained_plan, journal, state, &terminal)
2562}
2563
2564fn verify_terminal_root_start_replay<P>(
2565 desired: &crate::fleet_ensure::model::DesiredFleet,
2566 plan: &FleetEnsurePlan,
2567 journal: &FleetEnsureJournalRecord,
2568 requested_fleet: &str,
2569 platform: &mut P,
2570 state: &FleetEnsureStateRecord,
2571) -> Result<(), EnsureWorkflowError<P::Error>>
2572where
2573 P: EnsurePlatform,
2574{
2575 verify_journal(journal, plan, requested_fleet, state)?;
2576 let actions = ordered_actions(plan);
2577 if actions.len() != journal.effects.len()
2578 || journal
2579 .effects
2580 .iter()
2581 .any(|effect| effect.state != EffectState::Applied)
2582 {
2583 return Err(EnsureWorkflowError::JournalIntegrity);
2584 }
2585 for (action, record) in actions.into_iter().zip(&journal.effects) {
2586 if action_sha256(action) != record.action_sha256
2587 || !platform
2588 .observe_effect(&journal.operation_id, action, record, state)
2589 .map_err(EnsureWorkflowError::Platform)?
2590 .applied
2591 {
2592 return Err(EnsureWorkflowError::ConvergenceDrift);
2593 }
2594 }
2595 let targets = reviewed_root_start_targets(plan)?;
2596 let management = platform
2597 .observe_root_management(state, &targets)
2598 .map_err(EnsureWorkflowError::Platform)?
2599 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2600 let current = recompile_root_start_prerequisite_plan(
2601 RootStartPlanInput {
2602 state,
2603 authority: plan.root_start_authority.as_deref(),
2604 created_at_time: plan.planned_at_time,
2605 desired,
2606 desired_sha256: &plan.desired_sha256,
2607 observation: &management,
2608 requested_fleet: &plan.fleet,
2609 },
2610 &targets,
2611 )?;
2612 if normalized_plan(plan) != normalized_plan(¤t)
2613 || management.roots.iter().any(|(name, observed)| {
2614 targets.contains(name)
2615 && observed.live.status
2616 != crate::fleet_ensure::model::CanisterRuntimeStatus::Running
2617 })
2618 {
2619 return Err(EnsureWorkflowError::ConvergenceDrift);
2620 }
2621 Ok(())
2622}
2623
2624fn reviewed_root_start_targets<E>(
2625 plan: &FleetEnsurePlan,
2626) -> Result<BTreeSet<String>, EnsureWorkflowError<E>>
2627where
2628 E: std::error::Error + 'static,
2629{
2630 if plan.scope != FleetEnsurePlanScope::RootStartPrerequisite
2631 || !plan.protocol_actions.is_empty()
2632 || plan.canisters.is_empty()
2633 || plan.conservation.maximum_new_funding_cycles != 0
2634 || plan.conservation.maximum_operator_debit_cycles != 0
2635 || plan.conservation.maximum_unavoidable_fee_cycles != 0
2636 || !plan.conservation.estate_funding_domains.is_empty()
2637 || plan.conservation.scheduled_transfer_cycles != 0
2638 || plan.conservation.observed_controlled_cycles
2639 != plan.conservation.retained_in_reused_canisters_cycles
2640 || plan
2641 .conservation
2642 .observed_controlled_cycles
2643 .checked_sub(plan.conservation.maximum_execution_burn_cycles)
2644 != Some(plan.conservation.expected_post_operation_cycles)
2645 {
2646 return Err(EnsureWorkflowError::PlanIntegrity);
2647 }
2648 let mut targets = BTreeSet::new();
2649 for canister in &plan.canisters {
2650 if canister.disposition != CanisterDisposition::Reuse
2651 || canister.principal.is_none()
2652 || !matches!(
2653 canister.actions.as_slice(),
2654 [EnsureAction::Start { name, principal }]
2655 if name == &canister.name
2656 && Some(principal) == canister.principal.as_ref()
2657 )
2658 || !targets.insert(canister.name.clone())
2659 {
2660 return Err(EnsureWorkflowError::PlanIntegrity);
2661 }
2662 }
2663 Ok(targets)
2664}
2665
2666fn root_management_fleet_observation<E>(
2667 management: &RootManagementObservation,
2668 targets: &BTreeSet<String>,
2669) -> Result<FleetObservation, EnsureWorkflowError<E>>
2670where
2671 E: std::error::Error + 'static,
2672{
2673 let canisters = targets
2674 .iter()
2675 .map(|name| {
2676 management
2677 .roots
2678 .get(name)
2679 .map(|observed| (name.clone(), Some(observed.live.clone())))
2680 .ok_or(EnsureWorkflowError::PlanIntegrity)
2681 })
2682 .collect::<Result<BTreeMap<_, _>, _>>()?;
2683 Ok(FleetObservation {
2684 additional_controlled_cycles: BTreeMap::new(),
2685 canisters,
2686 estate_funding_domains: BTreeMap::new(),
2687 ledger_fee_cycles: 0,
2688 operator_cycles: management.operator_cycles,
2689 protocol_ready: BTreeMap::new(),
2690 })
2691}
2692
2693fn compatible_root_start_prerequisite(
2694 retained: &FleetEnsurePlan,
2695 current: &FleetEnsurePlan,
2696 desired: &crate::fleet_ensure::model::DesiredFleet,
2697) -> bool {
2698 let Ok(maximum_movement) = desired
2699 .maximum_update_burn_cycles
2700 .parse::<Cycles>()
2701 .map(|cycles| cycles.to_u128())
2702 else {
2703 return false;
2704 };
2705 retained.canisters.len() == current.canisters.len()
2706 && retained
2707 .canisters
2708 .iter()
2709 .zip(¤t.canisters)
2710 .all(|(retained, current)| {
2711 retained.observed_cycles.abs_diff(current.observed_cycles) <= maximum_movement
2712 })
2713 && normalized_plan(retained) == normalized_plan(current)
2714}
2715
2716fn attach_terminal_inventory_cycles<P>(
2717 operation_id: &str,
2718 state: &FleetEnsureStateRecord,
2719 platform: &mut P,
2720 observation: &mut FleetObservation,
2721) -> Result<(), EnsureWorkflowError<P::Error>>
2722where
2723 P: EnsurePlatform,
2724{
2725 let inventory = platform
2726 .terminal_inventory(operation_id, state)
2727 .map_err(EnsureWorkflowError::Platform)?;
2728 attach_terminal_cycles(observation, inventory.controlled_cycles_by_principal)
2729}
2730
2731pub(super) fn completed_infrastructure_reinstall(
2732 desired: &crate::fleet_ensure::model::DesiredFleet,
2733 plan: &FleetEnsurePlan,
2734 journal: &FleetEnsureJournalRecord,
2735) -> bool {
2736 if desired.protocol.is_none() {
2737 return false;
2738 }
2739 let infrastructure = desired.canisters.iter().filter(|canister| {
2740 canister.presence == crate::fleet_ensure::model::DesiredPresence::Present
2741 && matches!(
2742 canister.kind,
2743 crate::fleet_ensure::model::DesiredCanisterKind::Coordinator
2744 | crate::fleet_ensure::model::DesiredCanisterKind::Root
2745 | crate::fleet_ensure::model::DesiredCanisterKind::Store
2746 )
2747 });
2748 let mut count = 0_usize;
2749 for configured in infrastructure {
2750 count += 1;
2751 let Some(action) = plan
2752 .canisters
2753 .iter()
2754 .find(|canister| {
2755 canister.name == configured.name
2756 && canister.disposition == CanisterDisposition::Reinstall
2757 })
2758 .and_then(|canister| {
2759 canister.actions.iter().find(|action| {
2760 matches!(
2761 action,
2762 EnsureAction::Install {
2763 mode: crate::fleet_ensure::model::InstallMode::Reinstall,
2764 name,
2765 ..
2766 } if name == &configured.name
2767 )
2768 })
2769 })
2770 else {
2771 return false;
2772 };
2773 let action_sha256 = action_sha256(action);
2774 if !journal.effects.iter().any(|effect| {
2775 effect.action_sha256 == action_sha256 && effect.state == EffectState::Applied
2776 }) {
2777 return false;
2778 }
2779 }
2780 count > 0
2781}
2782
2783fn compatible_after_bounded_observation(
2784 retained: &FleetEnsurePlan,
2785 current: &FleetEnsurePlan,
2786 desired: &crate::fleet_ensure::model::DesiredFleet,
2787 observation: &FleetObservation,
2788) -> bool {
2789 let Ok(maximum_observation_movement) = desired
2790 .maximum_observation_burn_cycles
2791 .parse::<Cycles>()
2792 .map(|cycles| cycles.to_u128())
2793 else {
2794 return false;
2795 };
2796 if retained.canisters.len() != current.canisters.len() {
2797 return false;
2798 }
2799 for (retained_canister, current_canister) in retained.canisters.iter().zip(¤t.canisters) {
2800 let movement = retained_canister
2801 .observed_cycles
2802 .abs_diff(current_canister.observed_cycles);
2803 if movement > maximum_observation_movement {
2804 return false;
2805 }
2806 }
2807 if !retained_funding_remains_sufficient(retained, desired, observation) {
2808 return false;
2809 }
2810 normalized_plan(retained) == normalized_plan(current)
2811}
2812
2813fn retained_funding_remains_sufficient(
2814 retained: &FleetEnsurePlan,
2815 desired: &crate::fleet_ensure::model::DesiredFleet,
2816 observation: &FleetObservation,
2817) -> bool {
2818 retained
2819 .canisters
2820 .iter()
2821 .flat_map(|canister| &canister.actions)
2822 .filter_map(|action| match action {
2823 EnsureAction::Fund {
2824 amount,
2825 name,
2826 pool_funding,
2827 principal,
2828 ..
2829 } => Some((amount, name, pool_funding, principal)),
2830 _ => None,
2831 })
2832 .all(|(amount, name, pool_funding, principal)| {
2833 if let Some(root) = pool_funding {
2834 return observation
2835 .estate_funding_domains
2836 .values()
2837 .find(|domain| domain.root_principal.as_ref() == Some(&root.root))
2838 .and_then(|domain| domain.pool.as_ref())
2839 .is_some_and(|pool| {
2840 pool.assets.iter().any(|asset| {
2841 asset.principal == *principal
2842 && asset.lifecycle == root.lifecycle
2843 && asset
2844 .cycles
2845 .checked_add(*amount)
2846 .is_some_and(|funded| funded >= pool.readiness_floor_cycles)
2847 })
2848 });
2849 }
2850 let Some(live) = observation.canisters.get(name).and_then(Option::as_ref) else {
2851 return false;
2852 };
2853 let Some(configured) = desired
2854 .canisters
2855 .iter()
2856 .find(|configured| configured.name == *name)
2857 else {
2858 return false;
2859 };
2860 let Ok(minimum) = configured
2861 .minimum_cycles
2862 .parse::<Cycles>()
2863 .map(|cycles| cycles.to_u128())
2864 else {
2865 return false;
2866 };
2867 live.cycles
2868 .checked_add(*amount)
2869 .is_some_and(|funded| funded >= minimum)
2870 })
2871}
2872
2873fn normalized_plan(plan: &FleetEnsurePlan) -> FleetEnsurePlan {
2874 let mut normalized = plan.clone();
2875 normalized.plan_sha256.clear();
2876 normalized.reviewed_desired = None;
2877 if let Some(review) = &mut normalized.recovery_review {
2878 review.continuation_reserve_cycles = 0;
2880 for funding in &mut review.known_pool_funding {
2881 funding.amount_cycles = 0;
2882 funding.funding_deficit_cycles = 0;
2883 }
2884 }
2885 normalized.conservation = CycleConservation {
2886 estate_funding_domains: plan
2887 .conservation
2888 .estate_funding_domains
2889 .iter()
2890 .cloned()
2891 .map(|mut domain| {
2892 domain.available_cycles = None;
2893 domain.maximum_funding_cycles = 0;
2894 domain.root_principal = None;
2895 domain.shortfall_cycles = 0;
2896 domain
2897 })
2898 .collect(),
2899 expected_post_operation_cycles: 0,
2900 maximum_execution_burn_cycles: 0,
2901 maximum_new_funding_cycles: 0,
2902 maximum_operator_debit_cycles: 0,
2903 maximum_unavoidable_fee_cycles: 0,
2904 observed_controlled_cycles: 0,
2905 retained_in_reused_canisters_cycles: 0,
2906 scheduled_transfer_cycles: 0,
2907 };
2908 for canister in &mut normalized.canisters {
2909 canister.observed_cycles = 0;
2910 for action in &mut canister.actions {
2911 if let EnsureAction::Fund {
2912 amount,
2913 funding_deficit_cycles,
2914 ..
2915 } = action
2916 {
2917 *amount = 0;
2918 *funding_deficit_cycles = 0;
2919 }
2920 }
2921 }
2922 normalized
2923}
2924
2925fn attach_terminal_cycles<E>(
2926 observation: &mut FleetObservation,
2927 additional: BTreeMap<String, u128>,
2928) -> Result<(), EnsureWorkflowError<E>>
2929where
2930 E: std::error::Error + 'static,
2931{
2932 let mut retained = BTreeMap::new();
2933 for (principal, cycles) in additional {
2934 let Some(configured) = observation
2935 .canisters
2936 .values_mut()
2937 .filter_map(Option::as_mut)
2938 .find(|canister| canister.principal == principal)
2939 else {
2940 retained.insert(principal, cycles);
2941 continue;
2942 };
2943 match configured.root_owned_lifecycle {
2944 Some(crate::fleet_ensure::model::RootOwnedCanisterLifecycle::Workload) => {
2945 configured.cycles = configured.cycles.min(cycles);
2948 }
2949 Some(crate::fleet_ensure::model::RootOwnedCanisterLifecycle::Idle)
2950 if configured.cycles == cycles => {}
2951 Some(crate::fleet_ensure::model::RootOwnedCanisterLifecycle::Idle) => {
2952 return Err(EnsureWorkflowError::TerminalInventory(
2953 TerminalInventoryError::IdleCycleObservationConflict {
2954 canister: principal,
2955 expected: configured.cycles,
2956 observed: cycles,
2957 },
2958 ));
2959 }
2960 _ => {
2961 return Err(EnsureWorkflowError::TerminalInventory(
2962 TerminalInventoryError::CycleObservationLifecycleConflict {
2963 canister: principal,
2964 },
2965 ));
2966 }
2967 }
2968 }
2969 observation.additional_controlled_cycles = retained;
2970 Ok(())
2971}
2972
2973fn funding_plan<'a, E: std::error::Error + 'static>(
2974 plan: &'a FleetEnsurePlan,
2975 journal: &FleetEnsureJournalRecord,
2976) -> Result<std::borrow::Cow<'a, FleetEnsurePlan>, EnsureWorkflowError<E>> {
2977 if journal.funding_reviews.is_empty() {
2978 return Ok(std::borrow::Cow::Borrowed(plan));
2979 }
2980 let mut funded = plan.clone();
2981 let (amount, fee) = funding::totals::<E>(journal, None)?;
2982 let conservation = &mut funded.conservation;
2983 conservation.maximum_operator_debit_cycles = conservation
2984 .maximum_operator_debit_cycles
2985 .checked_add(amount)
2986 .and_then(|n| n.checked_add(fee))
2987 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2988 conservation.maximum_new_funding_cycles = conservation
2989 .maximum_new_funding_cycles
2990 .checked_add(amount)
2991 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2992 conservation.maximum_unavoidable_fee_cycles = conservation
2993 .maximum_unavoidable_fee_cycles
2994 .checked_add(fee)
2995 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2996 for domain in &mut conservation.estate_funding_domains {
2997 domain.maximum_funding_cycles = domain
2998 .maximum_funding_cycles
2999 .checked_add(funding::totals::<E>(journal, Some(&domain.root))?.0)
3000 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3001 }
3002 Ok(std::borrow::Cow::Owned(funded))
3003}
3004
3005fn verify_terminal_conservation<E>(
3006 plan: &FleetEnsurePlan,
3007 journal: &FleetEnsureJournalRecord,
3008 state: &FleetEnsureStateRecord,
3009 terminal: &FleetObservation,
3010) -> Result<ActualCycleConservation, EnsureWorkflowError<E>>
3011where
3012 E: std::error::Error + 'static,
3013{
3014 let funded = funding_plan(plan, journal)?;
3015 let conservation = &funded.conservation;
3016 let final_controlled_cycles = controlled_cycles(terminal)?;
3017 let operator_debit_cycles = journal
3018 .initial_operator_cycles
3019 .checked_sub(terminal.operator_cycles)
3020 .ok_or_else(|| {
3021 EnsureWorkflowError::Conservation(
3022 "operator balance increased during apply; review a new plan".to_string(),
3023 )
3024 })?;
3025 if operator_debit_cycles > conservation.maximum_operator_debit_cycles {
3026 return Err(EnsureWorkflowError::Conservation(format!(
3027 "operator debit {operator_debit_cycles} exceeded reviewed maximum {}",
3028 conservation.maximum_operator_debit_cycles
3029 )));
3030 }
3031 let received_new_funding_cycles = operator_debit_cycles
3032 .checked_sub(conservation.maximum_unavoidable_fee_cycles)
3033 .ok_or_else(|| {
3034 EnsureWorkflowError::Conservation(
3035 "operator debit was below the exact reviewed fee total".to_string(),
3036 )
3037 })?;
3038 if received_new_funding_cycles > conservation.maximum_new_funding_cycles {
3039 return Err(EnsureWorkflowError::Conservation(format!(
3040 "received funding {received_new_funding_cycles} exceeded reviewed maximum {}",
3041 conservation.maximum_new_funding_cycles
3042 )));
3043 }
3044 let (estate_funding_cycles, exact_estate_creation_fee_cycles) =
3045 reconcile_estate_funding(plan, journal, state, terminal)?;
3046 let available = journal
3047 .initial_controlled_cycles
3048 .checked_add(received_new_funding_cycles)
3049 .ok_or_else(|| {
3050 EnsureWorkflowError::Conservation(
3051 "terminal controlled-cycle arithmetic overflowed".to_string(),
3052 )
3053 })?;
3054 let after_estate_fees = available
3055 .checked_sub(exact_estate_creation_fee_cycles)
3056 .ok_or_else(|| {
3057 EnsureWorkflowError::Conservation(
3058 "terminal controlled cycles cannot cover reviewed estate creation fees".to_string(),
3059 )
3060 })?;
3061 let measured_execution_burn_cycles = after_estate_fees
3062 .checked_sub(final_controlled_cycles)
3063 .ok_or_else(|| {
3064 EnsureWorkflowError::Conservation(
3065 "terminal estate contains more cycles than starting estate plus received funding"
3066 .to_string(),
3067 )
3068 })?;
3069 if measured_execution_burn_cycles > conservation.maximum_execution_burn_cycles {
3070 return Err(EnsureWorkflowError::Conservation(format!(
3071 "measured execution burn {measured_execution_burn_cycles} exceeded reviewed maximum {}",
3072 conservation.maximum_execution_burn_cycles
3073 )));
3074 }
3075 Ok(ActualCycleConservation {
3076 estate_funding_cycles,
3077 exact_estate_creation_fee_cycles,
3078 exact_unavoidable_fee_cycles: conservation.maximum_unavoidable_fee_cycles,
3079 final_controlled_cycles,
3080 measured_execution_burn_cycles,
3081 observed_starting_cycles: journal.initial_controlled_cycles,
3082 operator_debit_cycles,
3083 received_new_funding_cycles,
3084 })
3085}
3086
3087fn resolved_initial_pool_assets<E>(
3088 domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3089 state: &FleetEnsureStateRecord,
3090) -> Result<BTreeSet<String>, EnsureWorkflowError<E>>
3091where
3092 E: std::error::Error + 'static,
3093{
3094 domain
3095 .initial_pool_assets
3096 .iter()
3097 .map(|identity| {
3098 identity.strip_prefix("created:").map_or_else(
3099 || Ok(identity.clone()),
3100 |name| {
3101 state
3102 .pending_principals
3103 .get(name)
3104 .or_else(|| state.principals.get(name))
3105 .cloned()
3106 .ok_or(EnsureWorkflowError::JournalIntegrity)
3107 },
3108 )
3109 })
3110 .collect()
3111}
3112
3113fn exact_estate_creation_costs<E>(
3114 state: &FleetEnsureStateRecord,
3115 terminal: &FleetObservation,
3116 domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3117) -> Result<(u128, u128), EnsureWorkflowError<E>>
3118where
3119 E: std::error::Error + 'static,
3120{
3121 let observed = terminal
3122 .estate_funding_domains
3123 .get(&domain.root)
3124 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3125 let Some(pool) = observed.pool.as_ref() else {
3126 return if domain.required_creation_count == 0 {
3127 Ok((0, 0))
3128 } else {
3129 Err(EnsureWorkflowError::JournalIntegrity)
3130 };
3131 };
3132 if pool.pending_creation.is_some() {
3133 return Err(EnsureWorkflowError::JournalIntegrity);
3134 }
3135
3136 estate_creation_costs(state, terminal, domain)
3137}
3138
3139fn estate_creation_costs<E: std::error::Error + 'static>(
3140 state: &FleetEnsureStateRecord,
3141 observation: &FleetObservation,
3142 domain: &EstateFundingDomainPlan,
3143) -> Result<(u128, u128), EnsureWorkflowError<E>> {
3144 let pool = observation
3145 .estate_funding_domains
3146 .get(&domain.root)
3147 .and_then(|domain| domain.pool.as_ref())
3148 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3149 let initial = resolved_initial_pool_assets(domain, state)?;
3150 let terminal_principals = pool
3151 .assets
3152 .iter()
3153 .map(|asset| asset.principal.clone())
3154 .collect::<BTreeSet<_>>();
3155 if terminal_principals.len() != pool.assets.len() || !initial.is_subset(&terminal_principals) {
3156 return Err(EnsureWorkflowError::JournalIntegrity);
3157 }
3158 let created = pool
3159 .assets
3160 .iter()
3161 .filter(|asset| !initial.contains(&asset.principal))
3162 .collect::<Vec<_>>();
3163 let creation_count_is_bounded = created.len()
3164 <= usize::try_from(domain.required_creation_count)
3165 .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
3166 let pool_policy_is_exact = created.is_empty() || terminal_pool_policy_is_exact(pool, domain);
3169 if !creation_count_is_bounded || !pool_policy_is_exact {
3170 return Err(EnsureWorkflowError::JournalIntegrity);
3171 }
3172
3173 let mut operation_ids = BTreeSet::new();
3174 let mut block_indexes = BTreeSet::new();
3175 let mut creation_debit = 0_u128;
3176 let mut creation_fee = 0_u128;
3177 for asset in created {
3178 let (asset_debit, asset_fee) = exact_estate_creation_receipt_costs(
3179 asset,
3180 domain,
3181 &mut operation_ids,
3182 &mut block_indexes,
3183 )?;
3184 creation_debit = creation_debit.checked_add(asset_debit).ok_or_else(|| {
3185 EnsureWorkflowError::Conservation(
3186 "estate creation-debit arithmetic overflowed".to_string(),
3187 )
3188 })?;
3189 creation_fee = creation_fee.checked_add(asset_fee).ok_or_else(|| {
3190 EnsureWorkflowError::Conservation(
3191 "estate creation-fee arithmetic overflowed".to_string(),
3192 )
3193 })?;
3194 }
3195 if creation_debit > domain.maximum_creation_debit_cycles
3196 || creation_fee > domain.maximum_creation_fee_cycles
3197 {
3198 return Err(EnsureWorkflowError::JournalIntegrity);
3199 }
3200 Ok((creation_debit, creation_fee))
3201}
3202
3203const fn terminal_pool_policy_is_exact(
3204 pool: &crate::fleet_ensure::model::EstatePoolInventoryObservation,
3205 domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3206) -> bool {
3207 let expected_maximum_size = domain.pool_maximum_size;
3208 let expected_readiness_floor = domain.readiness_floor_cycles;
3209 let expected_execution_margin = domain.creation_execution_margin_cycles;
3210 pool.maximum_size == expected_maximum_size
3211 && pool.readiness_floor_cycles == expected_readiness_floor
3212 && pool.creation_execution_margin_cycles == expected_execution_margin
3213}
3214
3215#[derive(Eq, PartialEq)]
3216struct EstateCreationReceiptAuthority<'a> {
3217 creation_execution_margin_cycles: u128,
3218 cycles_ledger: &'a str,
3219 ledger_amount_cycles: u128,
3220 ledger_fee_cycles: u128,
3221 management_creation_fee_cycles: u128,
3222 readiness_floor_cycles: u128,
3223}
3224
3225fn observed_estate_creation_authority(
3226 receipt: &crate::fleet_ensure::model::EstatePoolCreationReceiptObservation,
3227) -> EstateCreationReceiptAuthority<'_> {
3228 EstateCreationReceiptAuthority {
3229 creation_execution_margin_cycles: receipt.creation_execution_margin_cycles,
3230 cycles_ledger: &receipt.cycles_ledger,
3231 ledger_amount_cycles: receipt.ledger_amount_cycles,
3232 ledger_fee_cycles: receipt.ledger_fee_cycles,
3233 management_creation_fee_cycles: receipt.management_creation_fee_cycles,
3234 readiness_floor_cycles: receipt.readiness_floor_cycles,
3235 }
3236}
3237
3238fn reviewed_estate_creation_authority(
3239 domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3240) -> EstateCreationReceiptAuthority<'_> {
3241 EstateCreationReceiptAuthority {
3242 creation_execution_margin_cycles: domain.creation_execution_margin_cycles,
3243 cycles_ledger: &domain.cycles_ledger,
3244 ledger_amount_cycles: domain.creation_amount_cycles,
3245 ledger_fee_cycles: domain.ledger_fee_cycles,
3246 management_creation_fee_cycles: domain.management_creation_fee_cycles,
3247 readiness_floor_cycles: domain.readiness_floor_cycles,
3248 }
3249}
3250
3251fn first_estate_creation_observation_is_exact(
3252 receipt: &crate::fleet_ensure::model::EstatePoolCreationReceiptObservation,
3253) -> bool {
3254 receipt
3255 .ledger_amount_cycles
3256 .checked_sub(receipt.management_creation_fee_cycles)
3257 .zip(receipt.first_observed_cycles)
3258 .is_some_and(|(funded_native_cycles, observed_cycles)| {
3259 observed_cycles >= receipt.readiness_floor_cycles
3260 && observed_cycles <= funded_native_cycles
3261 && funded_native_cycles - observed_cycles
3262 <= receipt.creation_execution_margin_cycles
3263 })
3264}
3265
3266fn exact_estate_creation_receipt_costs<E>(
3267 asset: &crate::fleet_ensure::model::EstatePoolAssetObservation,
3268 domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3269 operation_ids: &mut BTreeSet<String>,
3270 block_indexes: &mut BTreeSet<u64>,
3271) -> Result<(u128, u128), EnsureWorkflowError<E>>
3272where
3273 E: std::error::Error + 'static,
3274{
3275 let receipt = asset
3276 .creation_receipt
3277 .as_ref()
3278 .filter(|_| asset.origin == crate::fleet_ensure::model::EstatePoolAssetOrigin::Created)
3279 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3280 let authority_is_exact =
3281 observed_estate_creation_authority(receipt) == reviewed_estate_creation_authority(domain);
3282 if !is_sha256_hex(&receipt.operation_id)
3283 || !authority_is_exact
3284 || !first_estate_creation_observation_is_exact(receipt)
3285 || !operation_ids.insert(receipt.operation_id.clone())
3286 || !block_indexes.insert(receipt.block_index)
3287 {
3288 return Err(EnsureWorkflowError::JournalIntegrity);
3289 }
3290 let debit = receipt
3291 .ledger_amount_cycles
3292 .checked_add(receipt.ledger_fee_cycles)
3293 .ok_or_else(|| {
3294 EnsureWorkflowError::Conservation(
3295 "estate creation-debit arithmetic overflowed".to_string(),
3296 )
3297 })?;
3298 let fee = receipt
3299 .management_creation_fee_cycles
3300 .checked_add(receipt.ledger_fee_cycles)
3301 .ok_or_else(|| {
3302 EnsureWorkflowError::Conservation(
3303 "estate creation-fee arithmetic overflowed".to_string(),
3304 )
3305 })?;
3306 Ok((debit, fee))
3307}
3308
3309fn reconcile_estate_funding<E>(
3310 plan: &FleetEnsurePlan,
3311 journal: &FleetEnsureJournalRecord,
3312 state: &FleetEnsureStateRecord,
3313 terminal: &FleetObservation,
3314) -> Result<(u128, u128), EnsureWorkflowError<E>>
3315where
3316 E: std::error::Error + 'static,
3317{
3318 plan.conservation
3319 .estate_funding_domains
3320 .iter()
3321 .try_fold((0_u128, 0_u128), |(funding_total, fee_total), domain| {
3322 let initial = journal
3323 .initial_estate_funding_cycles_by_root
3324 .get(&domain.root)
3325 .copied()
3326 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3327 let observed = terminal
3328 .estate_funding_domains
3329 .get(&domain.root)
3330 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3331 if observed.cycles_ledger != domain.cycles_ledger
3332 || reviewed_estate_root_principal(domain, state)
3333 != observed.root_principal.as_deref()
3334 {
3335 return Err(EnsureWorkflowError::JournalIntegrity);
3336 }
3337 let terminal_balance = observed
3338 .balance_cycles
3339 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3340 let (creation_debit, creation_fee) = exact_estate_creation_costs(
3341 state,
3342 terminal,
3343 domain,
3344 )?;
3345 let funding = applied_estate_funding_for_domain(plan, journal, state, domain)?
3346 .checked_add(funding::totals::<E>(journal, Some(&domain.root))?.0)
3347 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3348 let expected_terminal_balance = initial
3349 .checked_add(funding)
3350 .and_then(|available| available.checked_sub(creation_debit))
3351 .ok_or_else(|| {
3352 EnsureWorkflowError::Conservation(format!(
3353 "Root {} funding-account balance cannot reconcile its reviewed funding and creation debit",
3354 domain.root
3355 ))
3356 })?;
3357 if terminal_balance != expected_terminal_balance {
3358 return Err(EnsureWorkflowError::Conservation(format!(
3359 "Root {} funding-account balance {terminal_balance} differs from reviewed terminal balance {expected_terminal_balance}",
3360 domain.root
3361 )));
3362 }
3363 Ok((
3364 funding_total.checked_add(funding).ok_or_else(|| {
3365 EnsureWorkflowError::Conservation(
3366 "estate funding arithmetic overflowed".to_string(),
3367 )
3368 })?,
3369 fee_total
3370 .checked_add(creation_fee)
3371 .ok_or_else(|| {
3372 EnsureWorkflowError::Conservation(
3373 "estate creation-fee arithmetic overflowed".to_string(),
3374 )
3375 })?,
3376 ))
3377 })
3378}
3379
3380fn applied_estate_funding_for_domain<E>(
3381 plan: &FleetEnsurePlan,
3382 journal: &FleetEnsureJournalRecord,
3383 state: &FleetEnsureStateRecord,
3384 domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3385) -> Result<u128, EnsureWorkflowError<E>>
3386where
3387 E: std::error::Error + 'static,
3388{
3389 let action = plan
3390 .canisters
3391 .iter()
3392 .find(|canister| canister.name == domain.root)
3393 .into_iter()
3394 .flat_map(|canister| &canister.actions)
3395 .find(|action| matches!(action, EnsureAction::FundEstate { .. }));
3396 let Some(action) = action else {
3397 return if domain.maximum_funding_cycles == 0 {
3398 Ok(0)
3399 } else {
3400 Err(EnsureWorkflowError::JournalIntegrity)
3401 };
3402 };
3403 let EnsureAction::FundEstate {
3404 amount,
3405 expected_post_cycles,
3406 ledger,
3407 ledger_fee_cycles,
3408 principal,
3409 ..
3410 } = action
3411 else {
3412 unreachable!("filtered estate-funding action")
3413 };
3414 let root_principal = principal.strip_prefix("created:").map_or_else(
3415 || Some(principal.as_str()),
3416 |name| {
3417 state
3418 .pending_principals
3419 .get(name)
3420 .or_else(|| state.principals.get(name))
3421 .map(String::as_str)
3422 },
3423 );
3424 let expected_destination = domain
3425 .available_cycles
3426 .unwrap_or_default()
3427 .checked_add(domain.shortfall_cycles)
3428 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3429 let action_authority_is_exact = *amount == domain.shortfall_cycles
3430 && *expected_post_cycles == expected_destination
3431 && ledger == &domain.cycles_ledger
3432 && *ledger_fee_cycles == domain.ledger_fee_cycles
3433 && root_principal == reviewed_estate_root_principal(domain, state);
3434 let effect = journal
3435 .effects
3436 .iter()
3437 .find(|effect| effect.action_sha256 == action_sha256(action));
3438 let Some(effect) = effect.filter(|effect| {
3439 effect.state == EffectState::Applied
3440 && effect
3441 .receipt
3442 .as_deref()
3443 .is_some_and(|receipt| !receipt.is_empty())
3444 }) else {
3445 return Err(EnsureWorkflowError::JournalIntegrity);
3446 };
3447 let balance_evidence_is_exact = crate::fleet_ensure::ops::estate_funding_applied(
3448 crate::fleet_ensure::ops::EstateFundingObservation {
3449 amount: *amount,
3450 destination_after: effect.destination_post_cycles.unwrap_or_default(),
3451 destination_before: effect.destination_pre_cycles,
3452 expected_destination_after: *expected_post_cycles,
3453 ledger_fee_cycles: *ledger_fee_cycles,
3454 source_after: effect.post_cycles.unwrap_or_default(),
3455 source_before: effect.pre_cycles,
3456 },
3457 );
3458 if !action_authority_is_exact || !balance_evidence_is_exact {
3459 return Err(EnsureWorkflowError::JournalIntegrity);
3460 }
3461 Ok(*amount)
3462}
3463
3464fn controlled_cycles<E>(observation: &FleetObservation) -> Result<u128, EnsureWorkflowError<E>>
3465where
3466 E: std::error::Error + 'static,
3467{
3468 observation
3469 .canisters
3470 .values()
3471 .filter_map(Option::as_ref)
3472 .try_fold(0_u128, |total, live| total.checked_add(live.cycles))
3473 .and_then(|total| {
3474 observation
3475 .additional_controlled_cycles
3476 .values()
3477 .try_fold(total, |subtotal, cycles| subtotal.checked_add(*cycles))
3478 })
3479 .and_then(|total| {
3480 observation
3481 .estate_funding_domains
3482 .values()
3483 .filter_map(|domain| domain.balance_cycles)
3484 .try_fold(total, u128::checked_add)
3485 })
3486 .ok_or_else(|| {
3487 EnsureWorkflowError::Conservation("controlled-cycle total overflowed".to_string())
3488 })
3489}
3490
3491fn verified_plan<E>(plan: FleetEnsurePlan) -> Result<FleetEnsurePlan, EnsureWorkflowError<E>>
3492where
3493 E: std::error::Error + 'static,
3494{
3495 if expected_plan_sha256(&plan) != plan.plan_sha256 {
3496 return Err(EnsureWorkflowError::PlanIntegrity);
3497 }
3498 Ok(plan)
3499}
3500
3501fn verify_journal<E>(
3502 journal: &FleetEnsureJournalRecord,
3503 plan: &FleetEnsurePlan,
3504 requested_fleet: &str,
3505 state: &FleetEnsureStateRecord,
3506) -> Result<(), EnsureWorkflowError<E>>
3507where
3508 E: std::error::Error + 'static,
3509{
3510 if journal.fleet != requested_fleet
3511 || journal.operation_id != plan.operation_id
3512 || journal.plan_sha256 != plan.plan_sha256
3513 || !initial_estate_funding_is_exact(journal, plan)
3514 || !estate_funding_pause_is_exact(journal, plan, state)
3515 {
3516 return Err(EnsureWorkflowError::JournalIntegrity);
3517 }
3518 funding::verify(plan, journal, state)?;
3519 continuation::verify_records(plan, journal)?;
3520 let actions = continuation::actions(plan, journal);
3521 let effect_count_matches = journal.effects.len() <= actions.len();
3522 let action_hashes_match = journal.effects.iter().zip(actions).all(|(effect, action)| {
3523 effect.action_sha256 == action_sha256(action)
3524 && maintenance_attempts_are_exact(effect, action)
3525 });
3526 if !(effect_count_matches && action_hashes_match) {
3527 return Err(EnsureWorkflowError::JournalIntegrity);
3528 }
3529 Ok(())
3530}
3531
3532fn maintenance_attempts_are_exact(effect: &EffectRecord, action: &EnsureAction) -> bool {
3533 if let EnsureAction::FleetProtocol { action, .. } = action
3534 && let CurrentFleetProtocolAction::MaintainPoolReadiness {
3535 maximum_updates, ..
3536 } = action.as_ref()
3537 {
3538 effect.maintenance_attempts <= *maximum_updates
3539 } else {
3540 effect.maintenance_attempts == 0
3541 }
3542}
3543
3544fn initial_estate_funding_is_exact(
3545 journal: &FleetEnsureJournalRecord,
3546 plan: &FleetEnsurePlan,
3547) -> bool {
3548 journal.initial_estate_funding_cycles_by_root.len()
3549 == plan.conservation.estate_funding_domains.len()
3550 && plan
3551 .conservation
3552 .estate_funding_domains
3553 .iter()
3554 .all(|domain| {
3555 journal
3556 .initial_estate_funding_cycles_by_root
3557 .get(&domain.root)
3558 .copied()
3559 == Some(domain.available_cycles.unwrap_or_default())
3560 })
3561}
3562
3563fn estate_funding_pause_is_exact(
3564 journal: &FleetEnsureJournalRecord,
3565 plan: &FleetEnsurePlan,
3566 state: &FleetEnsureStateRecord,
3567) -> bool {
3568 let Some(required) = journal.estate_funding_required.as_ref() else {
3569 return true;
3570 };
3571 journal.completion == FleetEnsureCompletion::InProgress
3572 && estate_funding_record_is_exact(required, plan, state)
3573}
3574
3575fn estate_funding_record_is_exact(
3576 required: &EstateFundingRequiredRecord,
3577 plan: &FleetEnsurePlan,
3578 state: &FleetEnsureStateRecord,
3579) -> bool {
3580 let Some(reviewed) = plan
3581 .conservation
3582 .estate_funding_domains
3583 .iter()
3584 .find(|domain| domain.root == required.root)
3585 else {
3586 return false;
3587 };
3588 let operation_authority_is_exact =
3589 required.operation_id == plan.operation_id && required.plan_sha256 == plan.plan_sha256;
3590 let funding_authority_is_exact =
3591 EstateFundingPauseAuthority::from(required) == EstateFundingPauseAuthority::from(reviewed);
3592 let root_authority_is_exact =
3593 reviewed_estate_root_principal(reviewed, state) == Some(required.root_principal.as_str());
3594 let shortfall_is_exact = required.available_cycles < required.maximum_creation_debit_cycles
3595 && required.shortfall_cycles
3596 == required
3597 .maximum_creation_debit_cycles
3598 .saturating_sub(required.available_cycles);
3599 if !(operation_authority_is_exact
3600 && funding_authority_is_exact
3601 && root_authority_is_exact
3602 && shortfall_is_exact)
3603 {
3604 return false;
3605 }
3606 if required.pending_creation_operation_id.is_some() {
3607 current_creation_funding_pause_is_exact(required, reviewed)
3608 } else {
3609 planned_estate_funding_pause_is_exact(required, reviewed)
3610 }
3611}
3612
3613#[derive(Eq, PartialEq)]
3614struct EstateFundingPauseAuthority<'a> {
3615 creation_amount_cycles: u128,
3616 creation_execution_margin_cycles: u128,
3617 cycles_ledger: &'a str,
3618 ledger_fee_cycles: u128,
3619 management_creation_fee_cycles: u128,
3620 readiness_floor_cycles: u128,
3621 root: &'a str,
3622}
3623
3624impl<'a> From<&'a EstateFundingRequiredRecord> for EstateFundingPauseAuthority<'a> {
3625 fn from(required: &'a EstateFundingRequiredRecord) -> Self {
3626 Self {
3627 creation_amount_cycles: required.creation_amount_cycles,
3628 creation_execution_margin_cycles: required.creation_execution_margin_cycles,
3629 cycles_ledger: &required.cycles_ledger,
3630 ledger_fee_cycles: required.ledger_fee_cycles,
3631 management_creation_fee_cycles: required.management_creation_fee_cycles,
3632 readiness_floor_cycles: required.readiness_floor_cycles,
3633 root: &required.root,
3634 }
3635 }
3636}
3637
3638impl<'a> From<&'a EstateFundingDomainPlan> for EstateFundingPauseAuthority<'a> {
3639 fn from(reviewed: &'a EstateFundingDomainPlan) -> Self {
3640 Self {
3641 creation_amount_cycles: reviewed.creation_amount_cycles,
3642 creation_execution_margin_cycles: reviewed.creation_execution_margin_cycles,
3643 cycles_ledger: &reviewed.cycles_ledger,
3644 ledger_fee_cycles: reviewed.ledger_fee_cycles,
3645 management_creation_fee_cycles: reviewed.management_creation_fee_cycles,
3646 readiness_floor_cycles: reviewed.readiness_floor_cycles,
3647 root: &reviewed.root,
3648 }
3649 }
3650}
3651
3652fn current_creation_funding_pause_is_exact(
3653 required: &EstateFundingRequiredRecord,
3654 reviewed: &EstateFundingDomainPlan,
3655) -> bool {
3656 let operation_id_is_exact = required
3657 .pending_creation_operation_id
3658 .as_deref()
3659 .is_some_and(is_sha256_hex);
3660 let attempt_evidence_is_exact = required.attempt_count.is_some()
3661 && required
3662 .retry_at_ns
3663 .is_some_and(|retry_at_ns| retry_at_ns > 0)
3664 && required
3665 .last_attempt_at_ns
3666 .is_none_or(|attempted_at_ns| attempted_at_ns > 0);
3667 let single_creation_debit = required
3668 .creation_amount_cycles
3669 .checked_add(required.ledger_fee_cycles);
3670 let single_creation_fees = required
3671 .management_creation_fee_cycles
3672 .checked_add(required.ledger_fee_cycles);
3673 let debit_authority_is_exact = Some(required.maximum_creation_debit_cycles)
3674 == single_creation_debit
3675 && Some(required.maximum_creation_fee_cycles) == single_creation_fees;
3676 reviewed.required_creation_count > 0
3677 && required.required_creation_count == 1
3678 && operation_id_is_exact
3679 && attempt_evidence_is_exact
3680 && debit_authority_is_exact
3681}
3682
3683const fn planned_estate_funding_pause_is_exact(
3684 required: &EstateFundingRequiredRecord,
3685 reviewed: &EstateFundingDomainPlan,
3686) -> bool {
3687 let runtime_attempt_is_absent = required.attempt_count.is_none()
3688 && required.last_attempt_at_ns.is_none()
3689 && required.retry_at_ns.is_none();
3690 let reviewed_debit_is_exact = required.maximum_creation_debit_cycles
3691 == reviewed.maximum_creation_debit_cycles
3692 && required.maximum_creation_fee_cycles == reviewed.maximum_creation_fee_cycles
3693 && required.required_creation_count == reviewed.required_creation_count;
3694 runtime_attempt_is_absent && reviewed_debit_is_exact
3695}
3696
3697fn is_sha256_hex(value: &str) -> bool {
3698 value.len() == 64
3699 && value
3700 .bytes()
3701 .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
3702}
3703
3704#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3705enum TerminalStatePublicationError {
3706 JournalIntegrity,
3707}
3708
3709fn publish_terminal_state(
3710 desired: &crate::fleet_ensure::model::DesiredFleet,
3711 plan: &FleetEnsurePlan,
3712 journal: &FleetEnsureJournalRecord,
3713 state: &mut FleetEnsureStateRecord,
3714) -> Result<(), TerminalStatePublicationError> {
3715 let maximum_observation_burn_cycles =
3716 crate::fleet_ensure::ops::maximum_creation_observation_burn(desired)
3717 .ok_or(TerminalStatePublicationError::JournalIntegrity)?;
3718 let actions = continuation::actions(plan, journal);
3719 if actions.len() != journal.effects.len() {
3720 return Err(TerminalStatePublicationError::JournalIntegrity);
3721 }
3722 for (action, record) in actions.into_iter().zip(&journal.effects) {
3723 if action_sha256(action) != record.action_sha256 || record.state != EffectState::Applied {
3724 return Err(TerminalStatePublicationError::JournalIntegrity);
3725 }
3726 let EnsureAction::Create {
3727 name,
3728 requested_initial_cycles,
3729 ..
3730 } = action
3731 else {
3732 continue;
3733 };
3734 let Some(created_principal) = record.created_principal.as_deref() else {
3735 return Err(TerminalStatePublicationError::JournalIntegrity);
3736 };
3737 let Some(post_cycles) = record.post_cycles else {
3738 return Err(TerminalStatePublicationError::JournalIntegrity);
3739 };
3740 if !create_balance_is_terminal(
3741 Some(post_cycles),
3742 *requested_initial_cycles,
3743 maximum_observation_burn_cycles,
3744 ) || !create_identity_is_exact(name, record, state)
3745 || state
3746 .retained_cycles_by_principal
3747 .get(created_principal)
3748 .is_some_and(|retained| *retained != post_cycles)
3749 {
3750 return Err(TerminalStatePublicationError::JournalIntegrity);
3751 }
3752 state
3753 .retained_cycles_by_principal
3754 .insert(created_principal.to_string(), post_cycles);
3755 }
3756
3757 let prior_topology = state.topology.clone();
3758 for canister in &plan.canisters {
3759 if !desired
3760 .canisters
3761 .iter()
3762 .any(|configured| configured.name == canister.name)
3763 {
3764 if !is_reviewed_dynamic_pool_funding(plan, canister) {
3767 return Err(TerminalStatePublicationError::JournalIntegrity);
3768 }
3769 continue;
3770 }
3771
3772 match canister.disposition {
3773 CanisterDisposition::Create | CanisterDisposition::Replace => {
3774 if let Some(principal) = state.pending_principals.remove(&canister.name) {
3775 state.principals.insert(canister.name.clone(), principal);
3776 }
3777 }
3778 CanisterDisposition::Reinstall | CanisterDisposition::Reuse => {
3779 if let Some(principal) = state
3780 .pending_principals
3781 .remove(&canister.name)
3782 .or_else(|| canister.principal.clone())
3783 {
3784 state.principals.insert(canister.name.clone(), principal);
3785 }
3786 }
3787 CanisterDisposition::Delete => {
3788 state.principals.remove(&canister.name);
3789 state.pending_principals.remove(&canister.name);
3790 }
3791 }
3792 }
3793 state.topology = desired
3794 .canisters
3795 .iter()
3796 .filter(|canister| {
3797 canister.presence == crate::fleet_ensure::model::DesiredPresence::Present
3798 })
3799 .map(|canister| {
3800 (
3801 canister.name.clone(),
3802 crate::fleet_ensure::model::FleetEnsureTopologyRecord {
3803 kind: canister.kind,
3804 module_hash: projected_module_hash(plan, &prior_topology, canister),
3805 parent: canister.parent.clone(),
3806 protocol_binding: canister.protocol_binding.clone(),
3807 role: canister
3808 .protocol_binding
3809 .as_ref()
3810 .map(|binding| binding.role.to_string()),
3811 },
3812 )
3813 })
3814 .collect();
3815 Ok(())
3816}
3817
3818fn is_reviewed_dynamic_pool_funding(
3819 plan: &FleetEnsurePlan,
3820 canister: &crate::fleet_ensure::model::CanisterPlan,
3821) -> bool {
3822 canister.disposition == CanisterDisposition::Reuse
3823 && !canister.actions.is_empty()
3824 && canister
3825 .actions
3826 .iter()
3827 .all(|action| is_reviewed_pool_funding(action, plan, canister.principal.as_deref()))
3828}
3829
3830fn is_reviewed_pool_funding(
3831 action: &EnsureAction,
3832 plan: &FleetEnsurePlan,
3833 expected_principal: Option<&str>,
3834) -> bool {
3835 let EnsureAction::Fund {
3836 pool_funding: Some(root),
3837 principal,
3838 ..
3839 } = action
3840 else {
3841 return false;
3842 };
3843 if Some(principal.as_str()) != expected_principal {
3844 return false;
3845 }
3846 plan.protocol_actions.iter().any(|candidate| {
3847 let EnsureAction::FleetProtocol {
3848 principal: target,
3849 action,
3850 ..
3851 } = candidate
3852 else {
3853 return false;
3854 };
3855 let CurrentFleetProtocolAction::ReconcilePoolAsset { request, .. } = action.as_ref() else {
3856 return false;
3857 };
3858 target == &root.root && request.canister_id.to_text() == *principal
3859 })
3860}
3861
3862fn projected_module_hash(
3863 plan: &FleetEnsurePlan,
3864 prior_topology: &BTreeMap<String, crate::fleet_ensure::model::FleetEnsureTopologyRecord>,
3865 canister: &crate::fleet_ensure::model::DesiredCanister,
3866) -> Option<String> {
3867 let installed = plan
3868 .canisters
3869 .iter()
3870 .find(|planned| planned.name == canister.name)
3871 .and_then(|planned| {
3872 planned
3873 .actions
3874 .iter()
3875 .rev()
3876 .find_map(|action| match action {
3877 EnsureAction::Install { wasm_sha256, .. } => Some(wasm_sha256.clone()),
3878 _ => None,
3879 })
3880 });
3881 installed.or_else(|| {
3882 prior_topology
3883 .get(&canister.name)
3884 .filter(|retained| retained.kind == canister.kind && retained.parent == canister.parent)
3885 .and_then(|retained| retained.module_hash.clone())
3886 })
3887}
3888
3889fn merge_terminal_inventory<E>(
3890 state: &mut FleetEnsureStateRecord,
3891 inventory: crate::fleet_ensure::ops::TerminalFleetInventory,
3892) -> Result<(), EnsureWorkflowError<E>>
3893where
3894 E: std::error::Error + 'static,
3895{
3896 if inventory.entries.is_empty() {
3897 state.active_registry = inventory.active_registry;
3898 return Ok(());
3899 }
3900 let entries = inventory.entries;
3901 let mut names_by_principal = state
3902 .principals
3903 .iter()
3904 .map(|(name, principal)| (principal.clone(), name.clone()))
3905 .collect::<BTreeMap<_, _>>();
3906 if names_by_principal.len() != state.principals.len() {
3907 return Err(EnsureWorkflowError::TerminalInventory(
3908 TerminalInventoryError::DuplicateRetainedPrincipal,
3909 ));
3910 }
3911 for entry in &entries {
3912 names_by_principal
3913 .entry(entry.pid.clone())
3914 .or_insert_with(|| format!("observed:{}", entry.pid));
3915 }
3916 for entry in entries {
3917 let name = names_by_principal.get(&entry.pid).cloned().ok_or_else(|| {
3918 EnsureWorkflowError::TerminalInventory(TerminalInventoryError::LostPrincipalBinding {
3919 canister: entry.pid.clone(),
3920 })
3921 })?;
3922 let parent = entry
3923 .parent_pid
3924 .as_ref()
3925 .map(|principal| {
3926 names_by_principal.get(principal).cloned().ok_or_else(|| {
3927 EnsureWorkflowError::TerminalInventory(TerminalInventoryError::UnknownParent {
3928 canister: entry.pid.clone(),
3929 parent: principal.clone(),
3930 })
3931 })
3932 })
3933 .transpose()?;
3934 let existing = state.topology.get(&name);
3935 let kind = terminal_inventory_kind(existing, &entry);
3936 let parent_conflict = existing
3937 .and_then(|topology| topology.parent.as_ref())
3938 .zip(parent.as_ref())
3939 .filter(|(expected, actual)| expected != actual);
3940 if let Some((retained_parent, terminal_parent)) = parent_conflict
3941 && !verified_pool_to_component_parent_transition(existing, kind, &entry)
3942 {
3943 return Err(EnsureWorkflowError::TerminalInventory(
3944 TerminalInventoryError::RetainedParentConflict {
3945 canister: entry.pid,
3946 retained_parent: retained_parent.clone(),
3947 terminal_parent: terminal_parent.clone(),
3948 },
3949 ));
3950 }
3951 state.principals.insert(name.clone(), entry.pid);
3952 state.topology.insert(
3953 name,
3954 crate::fleet_ensure::model::FleetEnsureTopologyRecord {
3955 kind,
3956 module_hash: entry.module_hash,
3957 parent,
3958 protocol_binding: entry.protocol_binding,
3959 role: entry.role,
3960 },
3961 );
3962 }
3963 state.active_registry = inventory.active_registry;
3964 Ok(())
3965}
3966
3967fn terminal_inventory_kind(
3968 existing: Option<&crate::fleet_ensure::model::FleetEnsureTopologyRecord>,
3969 entry: &crate::registry::RegistryEntry,
3970) -> crate::fleet_ensure::model::DesiredCanisterKind {
3971 existing.map_or_else(
3972 || {
3973 if entry.module_hash.is_some() {
3974 crate::fleet_ensure::model::DesiredCanisterKind::Component
3975 } else {
3976 crate::fleet_ensure::model::DesiredCanisterKind::Auxiliary
3977 }
3978 },
3979 |topology| {
3980 if topology.kind == crate::fleet_ensure::model::DesiredCanisterKind::Pool
3981 && entry.module_hash.is_some()
3982 && entry.protocol_binding.is_some()
3983 {
3984 crate::fleet_ensure::model::DesiredCanisterKind::Component
3985 } else {
3986 topology.kind
3987 }
3988 },
3989 )
3990}
3991
3992fn verified_pool_to_component_parent_transition(
3993 existing: Option<&crate::fleet_ensure::model::FleetEnsureTopologyRecord>,
3994 terminal_kind: crate::fleet_ensure::model::DesiredCanisterKind,
3995 entry: &crate::registry::RegistryEntry,
3996) -> bool {
3997 existing.is_some_and(|topology| {
3998 topology.kind == crate::fleet_ensure::model::DesiredCanisterKind::Pool
3999 && terminal_kind == crate::fleet_ensure::model::DesiredCanisterKind::Component
4000 && entry.module_hash.is_some()
4001 && entry.protocol_binding.is_some()
4002 })
4003}
4004
4005fn applied_count(journal: &FleetEnsureJournalRecord) -> u32 {
4006 u32::try_from(
4007 journal
4008 .effects
4009 .iter()
4010 .chain(
4011 journal
4012 .funding_reviews
4013 .iter()
4014 .filter_map(|review| review.effect.as_ref()),
4015 )
4016 .filter(|effect| matches!(effect.state, EffectState::Applied))
4017 .count(),
4018 )
4019 .unwrap_or(u32::MAX)
4020}
4021
4022fn verify_store_activation_binding<P: EnsurePlatform>(
4023 plan: &FleetEnsurePlan,
4024 action: &EnsureAction,
4025 state: &FleetEnsureStateRecord,
4026 platform: &mut P,
4027) -> Result<(), EnsureWorkflowError<P::Error>> {
4028 use crate::fleet_ensure::model::DesiredCanisterInit;
4029 let EnsureAction::Install {
4030 canic_init: Some(DesiredCanisterInit::Store { root }),
4031 ..
4032 } = action
4033 else {
4034 return Ok(());
4035 };
4036 let root_will_be_installed = plan.canisters.iter().flat_map(|canister| &canister.actions)
4037 .any(|action| matches!(action,
4038 EnsureAction::Install { name, canic_init: Some(DesiredCanisterInit::Root { root: installed_root }), .. }
4039 if name == root && installed_root == root));
4040 if !root_will_be_installed {
4041 platform
4042 .require_retained_root_activation(&plan.operation_id, root, state)
4043 .map_err(EnsureWorkflowError::Platform)?;
4044 }
4045 Ok(())
4046}
4047
4048pub(super) fn ordered_actions(plan: &FleetEnsurePlan) -> Vec<&EnsureAction> {
4049 let mut actions = plan
4050 .canisters
4051 .iter()
4052 .flat_map(|canister| canister.actions.iter())
4053 .chain(plan.protocol_actions.iter())
4054 .collect::<Vec<_>>();
4055 if plan.scope == FleetEnsurePlanScope::ReinstallPreparation
4056 && plan
4057 .reinstall
4058 .as_ref()
4059 .is_some_and(|intent| intent.activation_reset.is_some())
4060 {
4061 actions.sort_by_key(|action| matches!(action, EnsureAction::Start { .. }));
4063 return actions;
4064 }
4065 if plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite {
4066 return actions;
4067 }
4068 let temporary_pool_observation_finalizations = plan
4069 .reviewed_desired
4070 .as_deref()
4071 .map(|reviewed| {
4072 let desired = reviewed.desired();
4073 plan.canisters
4074 .iter()
4075 .filter_map(|canister_plan| {
4076 let configured = desired.canisters.iter().find(|configured| {
4077 configured.name == canister_plan.name
4078 && configured.kind
4079 == crate::fleet_ensure::model::DesiredCanisterKind::Pool
4080 })?;
4081 let temporary_create = desired
4082 .bootstrap
4083 .as_ref()
4084 .is_some_and(|bootstrap| bootstrap.fresh_estate)
4085 && configured.principal.is_none()
4086 && configured.controllers.is_empty()
4087 && configured.controller_canisters.len() == 1
4088 && canister_plan.actions.iter().any(|action| {
4089 matches!(
4090 action,
4091 EnsureAction::Create {
4092 controller_canisters,
4093 controllers,
4094 ..
4095 } if controller_canisters == &configured.controller_canisters
4096 && controllers == std::slice::from_ref(&desired.operator)
4097 )
4098 });
4099 temporary_create.then_some(canister_plan.name.as_str())
4100 })
4101 .collect::<BTreeSet<_>>()
4102 })
4103 .unwrap_or_default();
4104 actions.sort_by_key(|action| {
4105 if matches!(action, EnsureAction::SetControllers { name, .. } if temporary_pool_observation_finalizations.contains(name.as_str()))
4106 {
4107 6
4108 } else {
4109 action_order(action)
4110 }
4111 });
4112 actions
4113}
4114
4115fn deferred_create_observation_is_exact(
4116 desired: &crate::fleet_ensure::model::DesiredFleet,
4117 actions: &[&EnsureAction],
4118 index: usize,
4119 action: &EnsureAction,
4120 record: &EffectRecord,
4121 state: &FleetEnsureStateRecord,
4122) -> bool {
4123 let EnsureAction::Create {
4124 controller_canisters,
4125 controllers,
4126 name,
4127 ..
4128 } = action
4129 else {
4130 return false;
4131 };
4132 let Some(configured) = desired.canisters.iter().find(|canister| {
4133 canister.name == *name
4134 && canister.kind == crate::fleet_ensure::model::DesiredCanisterKind::Pool
4135 }) else {
4136 return false;
4137 };
4138 let Some(parent) = configured.parent.as_deref() else {
4139 return false;
4140 };
4141 let fresh_root_only_create = desired
4142 .bootstrap
4143 .as_ref()
4144 .is_some_and(|bootstrap| bootstrap.fresh_estate)
4145 && configured.principal.is_none()
4146 && configured.controllers.is_empty()
4147 && configured.controller_canisters.as_slice() == [parent]
4148 && controller_canisters == &configured.controller_canisters
4149 && controllers.is_empty();
4150 let exact_issued_create = record.state == EffectState::Issued
4151 && create_identity_is_exact(name, record, state)
4152 && record.post_cycles.is_some();
4153 let later_root_install = actions[index.saturating_add(1)..].iter().any(|candidate| {
4154 matches!(
4155 candidate,
4156 EnsureAction::Install {
4157 canic_init: Some(crate::fleet_ensure::model::DesiredCanisterInit::Root { root }),
4158 name: install_name,
4159 ..
4160 } if root == parent && install_name == parent
4161 )
4162 });
4163 let infrastructure_only = actions.iter().all(|candidate| {
4164 matches!(
4165 candidate,
4166 EnsureAction::Create { .. }
4167 | EnsureAction::FundEstate { .. }
4168 | EnsureAction::Install { .. }
4169 | EnsureAction::SetControllers { .. }
4170 )
4171 });
4172 fresh_root_only_create && exact_issued_create && later_root_install && infrastructure_only
4173}
4174
4175pub(super) const fn action_order(action: &EnsureAction) -> u8 {
4176 match action {
4177 EnsureAction::SealAuthority {
4178 authority_kind: crate::fleet_ensure::model::DesiredCanisterKind::Root,
4179 ..
4180 }
4181 | EnsureAction::Create { .. } => 0,
4182 EnsureAction::SealAuthority { .. }
4183 | EnsureAction::Fund { .. }
4184 | EnsureAction::FundEstate { .. } => 1,
4185 EnsureAction::Install {
4186 canic_init: Some(crate::fleet_ensure::model::DesiredCanisterInit::Coordinator),
4187 ..
4188 } => 2,
4189 EnsureAction::SetControllers { .. } => 3,
4190 EnsureAction::Install {
4191 canic_init: Some(crate::fleet_ensure::model::DesiredCanisterInit::Store { .. }),
4192 ..
4193 } => 4,
4194 EnsureAction::Install { .. } => 5,
4195 EnsureAction::Start { .. } => 6,
4196 EnsureAction::FleetProtocol { .. } | EnsureAction::Protocol { .. } => 7,
4197 EnsureAction::Transfer { .. } => 8,
4198 EnsureAction::Stop { .. } => 9,
4199 EnsureAction::Delete { .. } => 10,
4200 }
4201}
4202
4203#[cfg(test)]
4204mod tests {
4205 use super::*;
4206
4207 #[test]
4208 fn store_binding_requires_exact_retained_root_or_reviewed_root_install() {
4209 use crate::fleet_ensure::{
4210 model::{CanisterDisposition, CanisterPlan, DesiredCanisterInit, InstallMode},
4211 tests::MockPlatform,
4212 };
4213 let (state, _) = retained_evidence();
4214 let mut plan = estate_funding_plan();
4215 let desired = serde_json::from_value(serde_json::json!({
4216 "canisters": [], "cycles_ledger": "um5iw-rqaaa-aaaaq-qaaba-cai",
4217 "environment": "local", "fleet": "fleet", "ledger_fee_cycles": "0",
4218 "management_creation_fee_cycles": "0", "material_cycle_threshold": "1",
4219 "maximum_observation_burn_cycles": "1", "maximum_stalled_observations": 2,
4220 "maximum_update_burn_cycles": "2", "operator": "rrkah-fqaaa-aaaaa-aaaaq-cai",
4221 "schema_version": FLEET_ENSURE_SCHEMA_VERSION, "treasury": "root"
4222 }))
4223 .unwrap();
4224 let mut platform = MockPlatform::new(desired, []);
4225 platform.rejected_activation_root = Some("root".into());
4226 let store = EnsureAction::Install {
4227 canic_init: Some(DesiredCanisterInit::Store {
4228 root: "root".into(),
4229 }),
4230 reinstall_witness: None,
4231 init_arg: None,
4232 init_arg_sha256: None,
4233 init_candid: None,
4234 init_candid_sha256: None,
4235 mode: InstallMode::Install,
4236 name: "store".into(),
4237 principal: "store-principal".into(),
4238 wasm: "store.wasm".into(),
4239 wasm_sha256: "store-hash".into(),
4240 };
4241 assert!(matches!(
4242 verify_store_activation_binding(&plan, &store, &state, &mut platform),
4243 Err(EnsureWorkflowError::Platform(_))
4244 ));
4245 assert_eq!(
4246 platform.retained_activation_checks,
4247 [(plan.operation_id.clone(), "root".into())]
4248 );
4249 let mut root_install = store.clone();
4250 if let EnsureAction::Install {
4251 name, canic_init, ..
4252 } = &mut root_install
4253 {
4254 *name = "other-root".into();
4255 *canic_init = Some(DesiredCanisterInit::Root {
4256 root: "other-root".into(),
4257 });
4258 }
4259 plan.canisters.push(CanisterPlan {
4260 actions: vec![root_install],
4261 disposition: CanisterDisposition::Create,
4262 name: "root".into(),
4263 observed_cycles: 0,
4264 principal: None,
4265 });
4266 assert!(verify_store_activation_binding(&plan, &store, &state, &mut platform).is_err());
4267 if let EnsureAction::Install {
4268 name, canic_init, ..
4269 } = &mut plan.canisters.last_mut().unwrap().actions[0]
4270 {
4271 *name = "root".into();
4272 *canic_init = Some(DesiredCanisterInit::Root {
4273 root: "root".into(),
4274 });
4275 }
4276 let reads = platform.retained_activation_checks.len();
4277 verify_store_activation_binding(&plan, &store, &state, &mut platform).unwrap();
4278 assert_eq!(platform.retained_activation_checks.len(), reads);
4279 plan.canisters.pop();
4280 platform.rejected_activation_root = None;
4281 verify_store_activation_binding(&plan, &store, &state, &mut platform).unwrap();
4282 assert_eq!(platform.retained_activation_checks.len(), reads + 1);
4283 }
4284
4285 #[test]
4286 fn root_prerequisite_retains_activation_authority_across_desired_refresh() {
4287 let (mut state, mut journal) = retained_evidence();
4288 let mut plan = estate_funding_plan();
4289 plan.scope = FleetEnsurePlanScope::RootReinstallPrerequisite;
4290 plan.conservation.estate_funding_domains.clear();
4291 journal.effects.clear();
4292 journal.completion = FleetEnsureCompletion::Converged;
4293 let before = state.clone();
4294 assert!(matches!(
4295 reconcile_retained_planning_state::<std::io::Error>(
4296 &mut state,
4297 Some(&plan),
4298 Some(&journal),
4299 "fleet",
4300 "local",
4301 "refreshed-imports"
4302 ),
4303 Err(EnsureWorkflowError::RootActivationDesiredConflict { .. })
4304 ));
4305 assert_eq!(state, before);
4306 assert!(
4307 reconcile_retained_planning_state::<std::io::Error>(
4308 &mut state,
4309 Some(&plan),
4310 Some(&journal),
4311 "fleet",
4312 "local",
4313 "desired"
4314 )
4315 .unwrap()
4316 .is_some()
4317 );
4318 }
4319
4320 fn terminal_component_entry(canister: &str, parent: &str) -> crate::registry::RegistryEntry {
4321 crate::registry::RegistryEntry {
4322 pid: canister.to_string(),
4323 role: Some("managed_component".to_string()),
4324 parent_pid: Some(parent.to_string()),
4325 module_hash: Some("11".repeat(32)),
4326 protocol_binding: Some(crate::protocol_binding::RegistryProtocolBinding {
4327 release_identity: "0.110.test".to_string(),
4328 role: canic_core::ids::CanisterRole::from("managed_component"),
4329 capabilities: BTreeSet::new(),
4330 candid_sha256: [1; 32],
4331 protocol_profile_digest:
4332 canic_core::role_contract::ProtocolProfileDigest::from_bytes([2; 32]),
4333 }),
4334 }
4335 }
4336
4337 pub(super) fn retained_evidence() -> (FleetEnsureStateRecord, FleetEnsureJournalRecord) {
4338 let state = FleetEnsureStateRecord {
4339 active_registry: None,
4340 completed_reinstall_action_sha256: BTreeMap::from([(
4341 "root".to_string(),
4342 "action".to_string(),
4343 )]),
4344 completed_reinstall_operation_id: Some("operation".to_string()),
4345 completed_reinstalls: BTreeMap::from([("root".to_string(), 7)]),
4346 fleet: "fleet".to_string(),
4347 pending_principals: BTreeMap::new(),
4348 principals: BTreeMap::new(),
4349 retained_cycles_by_principal: BTreeMap::new(),
4350 schema_version: crate::fleet_ensure::model::FLEET_ENSURE_SCHEMA_VERSION,
4351 topology: BTreeMap::new(),
4352 };
4353 let journal = FleetEnsureJournalRecord {
4354 funding_reviews: Vec::new(),
4355 successor_phases: Vec::new(),
4356 completion: FleetEnsureCompletion::ReplanRequired,
4357 estate_funding_required: None,
4358 effects: vec![crate::fleet_ensure::model::EffectRecord {
4359 maintenance_attempts: 0,
4360 action_sha256: "action".to_string(),
4361 created_principal: None,
4362 destination_post_cycles: None,
4363 destination_pre_cycles: None,
4364 post_cycles: None,
4365 pre_cycles: None,
4366 pre_canister_version: Some(7),
4367 progress_identity: None,
4368 receipt: None,
4369 state: EffectState::Applied,
4370 }],
4371 fleet: "fleet".to_string(),
4372 initial_controlled_cycles: 0,
4373 initial_estate_funding_cycles_by_root: BTreeMap::new(),
4374 initial_operator_cycles: 0,
4375 operation_id: "operation".to_string(),
4376 plan_sha256: "plan".to_string(),
4377 schema_version: crate::fleet_ensure::model::FLEET_ENSURE_SCHEMA_VERSION,
4378 stalled_observations: 0,
4379 };
4380 (state, journal)
4381 }
4382
4383 pub(super) fn estate_funding_plan() -> FleetEnsurePlan {
4384 FleetEnsurePlan {
4385 continuation: None,
4386 canisters: Vec::new(),
4387 conservation: CycleConservation {
4388 estate_funding_domains: vec![crate::fleet_ensure::model::EstateFundingDomainPlan {
4389 allocated_workloads: 0,
4390 available_cycles: Some(40),
4391 available_pool_slots: 2,
4392 creation_amount_cycles: 45,
4393 creation_execution_margin_cycles: 5,
4394 readiness_floor_cycles: 35,
4395 cycles_ledger: "ledger".to_string(),
4396 eligible_ready_pool_assets: 0,
4397 initial_pool_assets: Vec::new(),
4398 ledger_fee_cycles: 5,
4399 management_creation_fee_cycles: 5,
4400 maximum_creation_debit_cycles: 100,
4401 maximum_creation_fee_cycles: 20,
4402 maximum_funding_cycles: 60,
4403 occupied_pool_assets: 0,
4404 pending_creation_count: 0,
4405 pending_creation: None,
4406 planned_initial_workloads: 2,
4407 pool_maximum_size: 2,
4408 required_creation_count: 2,
4409 root: "root".to_string(),
4410 root_principal: Some("rrkah-fqaaa-aaaaa-aaaaq-cai".to_string()),
4411 shortfall_cycles: 60,
4412 }],
4413 expected_post_operation_cycles: 80,
4414 maximum_execution_burn_cycles: 0,
4415 maximum_new_funding_cycles: 0,
4416 maximum_operator_debit_cycles: 0,
4417 maximum_unavoidable_fee_cycles: 0,
4418 observed_controlled_cycles: 40,
4419 retained_in_reused_canisters_cycles: 0,
4420 scheduled_transfer_cycles: 0,
4421 },
4422 desired_sha256: "desired".to_string(),
4423 environment: "local".to_string(),
4424 fleet: "fleet".to_string(),
4425 operation_id: "operation".to_string(),
4426 plan_sha256: "plan".to_string(),
4427 planned_at_time: 1,
4428 protocol_actions: Vec::new(),
4429 recovery_review: None,
4430 reinstall: None,
4431 root_reinstall_bindings: Vec::new(),
4432 root_start_authority: None,
4433 reviewed_desired: None,
4434 schema_version: FLEET_ENSURE_SCHEMA_VERSION,
4435 scope: FleetEnsurePlanScope::Full,
4436 terminal_inventory_operation_id: None,
4437 }
4438 }
4439
4440 pub(super) fn estate_funding_observation(balance_cycles: Option<u128>) -> FleetObservation {
4441 FleetObservation {
4442 additional_controlled_cycles: BTreeMap::new(),
4443 canisters: BTreeMap::new(),
4444 estate_funding_domains: BTreeMap::from([(
4445 "root".to_string(),
4446 crate::fleet_ensure::model::EstateFundingDomainObservation {
4447 balance_cycles,
4448 cycles_ledger: "ledger".to_string(),
4449 pool: None,
4450 root_principal: Some("rrkah-fqaaa-aaaaa-aaaaq-cai".to_string()),
4451 },
4452 )]),
4453 ledger_fee_cycles: 0,
4454 operator_cycles: 0,
4455 protocol_ready: BTreeMap::new(),
4456 }
4457 }
4458
4459 fn terminal_estate_creation_observation() -> FleetObservation {
4460 let mut observation = estate_funding_observation(Some(0));
4461 let receipt =
4462 |byte, block_index| crate::fleet_ensure::model::EstatePoolCreationReceiptObservation {
4463 block_index,
4464 operation_id: format!("{byte:02x}").repeat(32),
4465 cycles_ledger: "ledger".to_string(),
4466 ledger_amount_cycles: 45,
4467 ledger_fee_cycles: 5,
4468 readiness_floor_cycles: 35,
4469 creation_execution_margin_cycles: 5,
4470 management_creation_fee_cycles: 5,
4471 first_observed_cycles: Some(39),
4472 };
4473 observation
4474 .estate_funding_domains
4475 .get_mut("root")
4476 .expect("Root funding domain")
4477 .pool = Some(crate::fleet_ensure::model::EstatePoolInventoryObservation {
4478 assets: vec![
4479 crate::fleet_ensure::model::EstatePoolAssetObservation {
4480 creation_receipt: Some(receipt(1, 10)),
4481 cycles: 35,
4482 lifecycle: crate::fleet_ensure::model::EstatePoolAssetLifecycle::Ready,
4483 origin: crate::fleet_ensure::model::EstatePoolAssetOrigin::Created,
4484 principal: "created-one".to_string(),
4485 },
4486 crate::fleet_ensure::model::EstatePoolAssetObservation {
4487 creation_receipt: Some(receipt(2, 11)),
4488 cycles: 35,
4489 lifecycle: crate::fleet_ensure::model::EstatePoolAssetLifecycle::Workload,
4490 origin: crate::fleet_ensure::model::EstatePoolAssetOrigin::Created,
4491 principal: "created-two".to_string(),
4492 },
4493 ],
4494 maximum_size: 2,
4495 minimum_size: 0,
4496 pending_creation: None,
4497 readiness_floor_cycles: 35,
4498 creation_execution_margin_cycles: 5,
4499 });
4500 observation
4501 }
4502
4503 fn estate_creation_receipt(
4504 observation: &mut FleetObservation,
4505 asset_index: usize,
4506 ) -> &mut crate::fleet_ensure::model::EstatePoolCreationReceiptObservation {
4507 observation
4508 .estate_funding_domains
4509 .get_mut("root")
4510 .expect("Root domain")
4511 .pool
4512 .as_mut()
4513 .expect("pool")
4514 .assets[asset_index]
4515 .creation_receipt
4516 .as_mut()
4517 .expect("receipt")
4518 }
4519
4520 #[test]
4521 fn autonomous_reserve_creation_reconciles_without_host_provisioning() {
4522 let mut plan = estate_funding_plan();
4523 let domain = &mut plan.conservation.estate_funding_domains[0];
4524 domain.available_cycles = Some(100);
4525 domain.maximum_funding_cycles = 0;
4526 domain.shortfall_cycles = 0;
4527 let (state, mut journal) = retained_evidence();
4528 journal.effects.clear();
4529 journal
4530 .initial_estate_funding_cycles_by_root
4531 .insert("root".to_string(), 100);
4532 let terminal = terminal_estate_creation_observation();
4533 assert_eq!(
4534 reconcile_estate_funding::<std::io::Error>(&plan, &journal, &state, &terminal)
4535 .expect("reconcile exact Root-owned creation against the reviewed reserve"),
4536 (0, 20)
4537 );
4538 }
4539
4540 #[test]
4541 fn terminal_estate_creation_uses_exact_protected_receipts() {
4542 let plan = estate_funding_plan();
4543 let domain = &plan.conservation.estate_funding_domains[0];
4544 let (state, _) = retained_evidence();
4545 let exact = terminal_estate_creation_observation();
4546 assert_eq!(
4547 exact_estate_creation_costs::<std::io::Error>(&state, &exact, domain)
4548 .expect("reconcile exact creation receipts"),
4549 (100, 20)
4550 );
4551 for required_creation_count in [0, 1] {
4552 let mut insufficient = domain.clone();
4553 insufficient.required_creation_count = required_creation_count;
4554 assert!(matches!(
4555 exact_estate_creation_costs::<std::io::Error>(&state, &exact, &insufficient),
4556 Err(EnsureWorkflowError::JournalIntegrity)
4557 ));
4558 }
4559
4560 let mut drifted = Vec::new();
4561 let mut changed = exact.clone();
4562 estate_creation_receipt(&mut changed, 0).ledger_amount_cycles = 44;
4563 drifted.push(changed);
4564 let mut changed = exact.clone();
4565 estate_creation_receipt(&mut changed, 1).operation_id = "01".repeat(32);
4566 drifted.push(changed);
4567 let mut changed = exact.clone();
4568 estate_creation_receipt(&mut changed, 1).block_index = 10;
4569 drifted.push(changed);
4570 let mut changed = exact.clone();
4571 estate_creation_receipt(&mut changed, 0).first_observed_cycles = None;
4572 drifted.push(changed);
4573 let mut changed = exact.clone();
4574 estate_creation_receipt(&mut changed, 0).first_observed_cycles = Some(34);
4575 drifted.push(changed);
4576 let mut changed = exact.clone();
4577 estate_creation_receipt(&mut changed, 0).first_observed_cycles = Some(41);
4578 drifted.push(changed);
4579 let mut changed = exact;
4580 changed
4581 .estate_funding_domains
4582 .get_mut("root")
4583 .expect("Root domain")
4584 .pool
4585 .as_mut()
4586 .expect("pool")
4587 .assets[0]
4588 .origin = crate::fleet_ensure::model::EstatePoolAssetOrigin::Imported;
4589 drifted.push(changed);
4590 for observation in drifted {
4591 assert!(matches!(
4592 exact_estate_creation_costs::<std::io::Error>(&state, &observation, domain,),
4593 Err(EnsureWorkflowError::JournalIntegrity)
4594 ));
4595 }
4596 }
4597
4598 #[test]
4599 fn estate_funding_pause_is_exact_and_clears_only_after_sufficient_balance() {
4600 let plan = estate_funding_plan();
4601 let (state, mut journal) = retained_evidence();
4602 let required = estate_funding_requirement::<std::io::Error>(
4603 &plan,
4604 &state,
4605 &estate_funding_observation(Some(40)),
4606 )
4607 .expect("validate funding observation")
4608 .expect("underfunded estate pauses");
4609 assert_eq!(required.available_cycles, 40);
4610 assert_eq!(required.maximum_creation_debit_cycles, 100);
4611 assert_eq!(required.shortfall_cycles, 60);
4612 assert!(
4613 estate_funding_requirement::<std::io::Error>(
4614 &plan,
4615 &state,
4616 &estate_funding_observation(Some(100)),
4617 )
4618 .expect("validate sufficient funding observation")
4619 .is_none()
4620 );
4621
4622 journal.completion = FleetEnsureCompletion::InProgress;
4623 journal.effects.clear();
4624 journal.estate_funding_required = Some(required);
4625 journal.initial_estate_funding_cycles_by_root = BTreeMap::from([("root".to_string(), 40)]);
4626 assert!(initial_estate_funding_is_exact(&journal, &plan));
4627 assert!(estate_funding_pause_is_exact(&journal, &plan, &state));
4628
4629 journal
4630 .estate_funding_required
4631 .as_mut()
4632 .expect("funding pause")
4633 .shortfall_cycles = 59;
4634 assert!(!estate_funding_pause_is_exact(&journal, &plan, &state));
4635
4636 *journal
4637 .initial_estate_funding_cycles_by_root
4638 .get_mut("root")
4639 .expect("initial Root funding balance") = 41;
4640 assert!(!initial_estate_funding_is_exact(&journal, &plan));
4641 }
4642
4643 #[test]
4644 fn current_creation_funding_pause_is_bound_to_exact_runtime_evidence() {
4645 let mut plan = estate_funding_plan();
4646 plan.conservation.estate_funding_domains[0].cycles_ledger = "aaaaa-aa".to_string();
4647 let (state, mut journal) = retained_evidence();
4648 let funding = canic_core::dto::component_provisioning::RootEstateFundingRequired {
4649 available: Cycles::new(40),
4650 attempt_count: 2,
4651 creation_amount: Cycles::new(45),
4652 cycles_ledger: candid::Principal::from_text("aaaaa-aa").expect("Ledger Principal"),
4653 execution_margin: Cycles::new(5),
4654 last_attempt_at_ns: Some(10),
4655 ledger_fee: Cycles::new(5),
4656 management_creation_fee: Cycles::new(5),
4657 operation_id: [17; 32],
4658 readiness_floor: Cycles::new(35),
4659 required: Cycles::new(50),
4660 retry_at_ns: 20,
4661 root: candid::Principal::from_text("rrkah-fqaaa-aaaaa-aaaaq-cai")
4662 .expect("Root Principal"),
4663 shortfall: Cycles::new(10),
4664 };
4665 let required =
4666 protocol_estate_funding_requirement::<std::io::Error>(&plan, &state, &funding)
4667 .expect("bind runtime funding evidence");
4668 journal.completion = FleetEnsureCompletion::InProgress;
4669 journal.effects.clear();
4670 journal.estate_funding_required = Some(required);
4671 assert!(estate_funding_pause_is_exact(&journal, &plan, &state));
4672
4673 journal
4674 .estate_funding_required
4675 .as_mut()
4676 .expect("funding pause")
4677 .pending_creation_operation_id = Some("AA".repeat(32));
4678 assert!(!estate_funding_pause_is_exact(&journal, &plan, &state));
4679
4680 let mut drifted = funding;
4681 drifted.execution_margin = Cycles::new(6);
4682 assert!(matches!(
4683 protocol_estate_funding_requirement::<std::io::Error>(&plan, &state, &drifted),
4684 Err(EnsureWorkflowError::PlanIntegrity)
4685 ));
4686 }
4687
4688 #[test]
4689 fn estate_funding_observation_rejects_missing_or_mismatched_authority() {
4690 let plan = estate_funding_plan();
4691 let (state, _) = retained_evidence();
4692 let mut missing_balance = estate_funding_observation(None);
4693 assert!(matches!(
4694 estate_funding_requirement::<std::io::Error>(&plan, &state, &missing_balance),
4695 Err(EnsureWorkflowError::PlanIntegrity)
4696 ));
4697
4698 let observed = missing_balance
4699 .estate_funding_domains
4700 .get_mut("root")
4701 .expect("Root funding domain");
4702 observed.balance_cycles = Some(40);
4703 observed.root_principal = Some("aaaaa-aa".to_string());
4704 assert!(matches!(
4705 estate_funding_requirement::<std::io::Error>(&plan, &state, &missing_balance),
4706 Err(EnsureWorkflowError::PlanIntegrity)
4707 ));
4708
4709 for changed_balance in [39, 101] {
4710 assert!(matches!(
4711 estate_funding_requirement::<std::io::Error>(
4712 &plan,
4713 &state,
4714 &estate_funding_observation(Some(changed_balance)),
4715 ),
4716 Err(EnsureWorkflowError::DriftedBeforeApply)
4717 ));
4718 }
4719 }
4720
4721 #[test]
4722 fn non_creation_protocol_completion_does_not_consume_estate_creation_authority() {
4723 let mut plan = estate_funding_plan();
4724 plan.conservation.estate_funding_domains[0].maximum_creation_debit_cycles = 0;
4725 plan.conservation.estate_funding_domains[0].maximum_creation_fee_cycles = 0;
4726 plan.conservation.estate_funding_domains[0].maximum_funding_cycles = 0;
4727 plan.conservation.estate_funding_domains[0].required_creation_count = 0;
4728 plan.conservation.estate_funding_domains[0].shortfall_cycles = 0;
4729 let action = EnsureAction::FleetProtocol {
4730 action: Box::new(
4731 crate::fleet_ensure::model::CurrentFleetProtocolAction::PrepareStoreChunkSet {
4732 request: canic_control_plane::dto::template::TemplateChunkSetPrepareInput {
4733 template_id: canic_control_plane::ids::TemplateId::from("root"),
4734 version: canic_control_plane::ids::TemplateVersion::from("current"),
4735 payload_hash: vec![1; 32],
4736 payload_size_bytes: 1,
4737 chunk_hashes: vec![vec![2; 32]],
4738 },
4739 },
4740 ),
4741 candid: "store.did".to_string(),
4742 candid_sha256: "11".repeat(32),
4743 maximum_execution_burn_cycles: 0,
4744 name: "prepare-store-chunks".to_string(),
4745 principal: "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(),
4746 };
4747 let action_hash = action_sha256(&action);
4748 plan.protocol_actions.push(action);
4749 let (state, mut journal) = retained_evidence();
4750 journal.initial_estate_funding_cycles_by_root = BTreeMap::from([("root".to_string(), 40)]);
4751 journal.effects = vec![EffectRecord {
4752 maintenance_attempts: 0,
4753 action_sha256: action_hash,
4754 created_principal: None,
4755 destination_post_cycles: None,
4756 destination_pre_cycles: None,
4757 post_cycles: None,
4758 pre_cycles: None,
4759 pre_canister_version: None,
4760 progress_identity: Some("store chunks prepared".to_string()),
4761 receipt: Some("receipt".to_string()),
4762 state: EffectState::Applied,
4763 }];
4764
4765 assert_eq!(
4766 reconcile_estate_funding::<std::io::Error>(
4767 &plan,
4768 &journal,
4769 &state,
4770 &estate_funding_observation(Some(40)),
4771 )
4772 .expect("non-creation protocol preserves the Root Ledger balance"),
4773 (0, 0),
4774 );
4775 }
4776
4777 #[test]
4778 fn fresh_estate_funding_binds_the_created_root_from_durable_state() {
4779 let mut plan = estate_funding_plan();
4780 plan.conservation.estate_funding_domains[0].available_cycles = None;
4781 plan.conservation.estate_funding_domains[0].maximum_funding_cycles = 100;
4782 plan.conservation.estate_funding_domains[0].root_principal = None;
4783 let (mut state, _) = retained_evidence();
4784 state.pending_principals.insert(
4785 "root".to_string(),
4786 "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(),
4787 );
4788
4789 let required = estate_funding_requirement::<std::io::Error>(
4790 &plan,
4791 &state,
4792 &estate_funding_observation(Some(0)),
4793 )
4794 .expect("validate created Root funding authority")
4795 .expect("fresh underfunded estate pauses");
4796 assert_eq!(required.shortfall_cycles, 100);
4797 assert_eq!(required.root_principal, "rrkah-fqaaa-aaaaa-aaaaq-cai");
4798
4799 state
4800 .pending_principals
4801 .insert("root".to_string(), "aaaaa-aa".to_string());
4802 assert!(matches!(
4803 estate_funding_requirement::<std::io::Error>(
4804 &plan,
4805 &state,
4806 &estate_funding_observation(Some(0)),
4807 ),
4808 Err(EnsureWorkflowError::PlanIntegrity)
4809 ));
4810 }
4811
4812 #[test]
4813 fn unchanged_estate_funding_pause_does_not_rewrite_its_journal() {
4814 let root = crate::test_support::temp_dir("estate-funding-pause");
4815 let paths = EnsurePaths::under(&root, "local", "fleet");
4816 let plan = estate_funding_plan();
4817 let (state, mut journal) = retained_evidence();
4818 journal.completion = FleetEnsureCompletion::InProgress;
4819 journal.effects.clear();
4820 journal.initial_estate_funding_cycles_by_root = BTreeMap::from([("root".to_string(), 40)]);
4821 let required = estate_funding_requirement::<std::io::Error>(
4822 &plan,
4823 &state,
4824 &estate_funding_observation(Some(40)),
4825 )
4826 .expect("validate funding observation")
4827 .expect("underfunded estate pauses");
4828
4829 assert!(
4830 retain_estate_funding_pause::<std::io::Error>(&paths, &mut journal, Some(&required),)
4831 .expect("retain first funding pause")
4832 );
4833 let retained = std::fs::read(&paths.journal).expect("read first retained pause");
4834 assert!(
4835 !retain_estate_funding_pause::<std::io::Error>(&paths, &mut journal, Some(&required),)
4836 .expect("replay unchanged funding pause")
4837 );
4838 assert_eq!(
4839 std::fs::read(&paths.journal).expect("reread unchanged retained pause"),
4840 retained
4841 );
4842 assert!(
4843 retain_estate_funding_pause::<std::io::Error>(&paths, &mut journal, None)
4844 .expect("clear funded pause")
4845 );
4846 assert!(journal.estate_funding_required.is_none());
4847
4848 std::fs::remove_dir_all(root).expect("remove funding-pause fixture");
4849 }
4850
4851 #[test]
4852 fn estate_funding_is_rechecked_until_the_first_protocol_effect_starts() {
4853 let protocol_action = || EnsureAction::FleetProtocol {
4854 action: Box::new(
4855 crate::fleet_ensure::model::CurrentFleetProtocolAction::PrepareStoreChunkSet {
4856 request: canic_control_plane::dto::template::TemplateChunkSetPrepareInput {
4857 template_id: canic_control_plane::ids::TemplateId::from("root"),
4858 version: canic_control_plane::ids::TemplateVersion::from("current"),
4859 payload_hash: vec![1; 32],
4860 payload_size_bytes: 1,
4861 chunk_hashes: vec![vec![2; 32]],
4862 },
4863 },
4864 ),
4865 candid: "store.did".to_string(),
4866 candid_sha256: "11".repeat(32),
4867 maximum_execution_burn_cycles: 1,
4868 name: "prepare".to_string(),
4869 principal: "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(),
4870 };
4871 let first = protocol_action();
4872 let second = protocol_action();
4873 let actions = [&first, &second];
4874 let (_, mut journal) = retained_evidence();
4875 journal.effects = vec![EffectRecord {
4876 maintenance_attempts: 0,
4877 action_sha256: action_sha256(&first),
4878 created_principal: None,
4879 destination_post_cycles: None,
4880 destination_pre_cycles: None,
4881 post_cycles: None,
4882 pre_cycles: None,
4883 pre_canister_version: None,
4884 progress_identity: None,
4885 receipt: None,
4886 state: EffectState::Intent,
4887 }];
4888 assert!(!prior_fleet_protocol_effect_started(&actions, &journal, 1));
4889
4890 journal.effects[0].state = EffectState::Issued;
4891 assert!(prior_fleet_protocol_effect_started(&actions, &journal, 1));
4892 }
4893
4894 #[test]
4895 fn terminal_inventory_allows_only_verified_pool_to_component_parent_transition() {
4896 let asset = "rrkah-fqaaa-aaaaa-aaaaq-cai";
4897 let old_root = "r7inp-6aaaa-aaaaa-aaabq-cai";
4898 let hub = "ryjl3-tyaaa-aaaaa-aaaba-cai";
4899 let state = || {
4900 let (mut state, _) = retained_evidence();
4901 state.principals = BTreeMap::from([
4902 ("asset".to_string(), asset.to_string()),
4903 ("hub".to_string(), hub.to_string()),
4904 ("root".to_string(), old_root.to_string()),
4905 ]);
4906 state.topology.insert(
4907 "asset".to_string(),
4908 crate::fleet_ensure::model::FleetEnsureTopologyRecord {
4909 kind: crate::fleet_ensure::model::DesiredCanisterKind::Pool,
4910 module_hash: None,
4911 parent: Some("root".to_string()),
4912 protocol_binding: None,
4913 role: Some("canister_pool_asset".to_string()),
4914 },
4915 );
4916 state
4917 };
4918
4919 let mut promoted = state();
4920 merge_terminal_inventory::<std::io::Error>(
4921 &mut promoted,
4922 crate::fleet_ensure::ops::TerminalFleetInventory {
4923 active_registry: None,
4924 controlled_cycles_by_principal: BTreeMap::new(),
4925 entries: vec![terminal_component_entry(asset, hub)],
4926 },
4927 )
4928 .expect("verified pool workload assumes its terminal Component parent");
4929 let promoted = promoted.topology.get("asset").expect("promoted asset");
4930 assert_eq!(
4931 promoted.kind,
4932 crate::fleet_ensure::model::DesiredCanisterKind::Component
4933 );
4934 assert_eq!(promoted.parent.as_deref(), Some("hub"));
4935
4936 let mut already_component = state();
4937 already_component
4938 .topology
4939 .get_mut("asset")
4940 .expect("retained asset")
4941 .kind = crate::fleet_ensure::model::DesiredCanisterKind::Component;
4942 assert!(matches!(
4943 merge_terminal_inventory::<std::io::Error>(
4944 &mut already_component,
4945 crate::fleet_ensure::ops::TerminalFleetInventory {
4946 active_registry: None,
4947 controlled_cycles_by_principal: BTreeMap::new(),
4948 entries: vec![terminal_component_entry(asset, hub)],
4949 },
4950 ),
4951 Err(EnsureWorkflowError::TerminalInventory(
4952 TerminalInventoryError::RetainedParentConflict {
4953 canister,
4954 retained_parent,
4955 terminal_parent,
4956 }
4957 )) if canister == asset
4958 && retained_parent == "root"
4959 && terminal_parent == "hub"
4960 ));
4961
4962 for incomplete_entry in [
4963 crate::registry::RegistryEntry {
4964 module_hash: None,
4965 ..terminal_component_entry(asset, hub)
4966 },
4967 crate::registry::RegistryEntry {
4968 protocol_binding: None,
4969 ..terminal_component_entry(asset, hub)
4970 },
4971 ] {
4972 assert!(matches!(
4973 merge_terminal_inventory::<std::io::Error>(
4974 &mut state(),
4975 crate::fleet_ensure::ops::TerminalFleetInventory {
4976 active_registry: None,
4977 controlled_cycles_by_principal: BTreeMap::new(),
4978 entries: vec![incomplete_entry],
4979 },
4980 ),
4981 Err(EnsureWorkflowError::TerminalInventory(
4982 TerminalInventoryError::RetainedParentConflict {
4983 canister,
4984 retained_parent,
4985 terminal_parent,
4986 }
4987 )) if canister == asset
4988 && retained_parent == "root"
4989 && terminal_parent == "hub"
4990 ));
4991 }
4992 }
4993
4994 #[test]
4995 fn completed_reinstall_evidence_requires_exact_operation_action_and_version() {
4996 let (state, journal) = retained_evidence();
4997 assert!(retained_reinstall_operation_matches(
4998 &state,
4999 "fleet",
5000 "operation",
5001 &journal,
5002 ));
5003
5004 let mut wrong_action = journal.clone();
5005 wrong_action.effects[0].action_sha256 = "other".to_string();
5006 assert!(!retained_reinstall_operation_matches(
5007 &state,
5008 "fleet",
5009 "operation",
5010 &wrong_action,
5011 ));
5012
5013 let mut wrong_version = journal.clone();
5014 wrong_version.effects[0].pre_canister_version = Some(8);
5015 assert!(!retained_reinstall_operation_matches(
5016 &state,
5017 "fleet",
5018 "operation",
5019 &wrong_version,
5020 ));
5021
5022 for (fleet, operation) in [("other", "operation"), ("fleet", "other")] {
5023 assert!(!retained_reinstall_operation_matches(
5024 &state, fleet, operation, &journal,
5025 ));
5026 }
5027
5028 let mut incomplete = journal;
5029 incomplete.effects[0].state = EffectState::Issued;
5030 assert!(!retained_reinstall_operation_matches(
5031 &state,
5032 "fleet",
5033 "operation",
5034 &incomplete,
5035 ));
5036 }
5037
5038 #[test]
5039 fn fresh_fleet_alternate_desired_cannot_clear_completed_reinstall_evidence() {
5040 let (state, journal) = retained_evidence();
5041 assert!(retained_reinstall_desired_conflict(
5042 &state,
5043 "fleet",
5044 "operation",
5045 "reviewed-desired",
5046 "alternate-desired",
5047 &journal,
5048 ));
5049 assert!(!retained_reinstall_desired_conflict(
5050 &state,
5051 "fleet",
5052 "operation",
5053 "reviewed-desired",
5054 "reviewed-desired",
5055 &journal,
5056 ));
5057
5058 let mut terminal = journal;
5059 terminal.completion = FleetEnsureCompletion::Converged;
5060 assert!(!retained_reinstall_desired_conflict(
5061 &state,
5062 "fleet",
5063 "operation",
5064 "reviewed-desired",
5065 "alternate-desired",
5066 &terminal,
5067 ));
5068 }
5069
5070 fn cycle_observation(
5071 lifecycle: crate::fleet_ensure::model::RootOwnedCanisterLifecycle,
5072 cycles: u128,
5073 ) -> FleetObservation {
5074 FleetObservation {
5075 additional_controlled_cycles: BTreeMap::new(),
5076 canisters: BTreeMap::from([(
5077 "asset".to_string(),
5078 Some(crate::fleet_ensure::model::LiveCanister {
5079 canister_version: Some(1),
5080 controllers: Vec::new(),
5081 cycles,
5082 module_sha256: None,
5083 principal: "controlled-principal".to_string(),
5084 reinstall_required: false,
5085 root_owned_lifecycle: Some(lifecycle),
5086 status: crate::fleet_ensure::model::CanisterRuntimeStatus::Stopped,
5087 }),
5088 )]),
5089 estate_funding_domains: BTreeMap::new(),
5090 ledger_fee_cycles: 0,
5091 operator_cycles: 0,
5092 protocol_ready: BTreeMap::new(),
5093 }
5094 }
5095
5096 #[test]
5097 fn fresh_fleet_idle_cycle_duplicate_requires_exact_balance() {
5098 use crate::fleet_ensure::model::RootOwnedCanisterLifecycle;
5099
5100 let additional = BTreeMap::from([("controlled-principal".to_string(), 100)]);
5101 let mut idle = cycle_observation(RootOwnedCanisterLifecycle::Idle, 100);
5102 attach_terminal_cycles::<std::io::Error>(&mut idle, additional.clone())
5103 .expect("merge exact Idle observation");
5104 assert!(idle.additional_controlled_cycles.is_empty());
5105
5106 let mut conflicting_idle = cycle_observation(RootOwnedCanisterLifecycle::Idle, 99);
5107 assert!(matches!(
5108 attach_terminal_cycles::<std::io::Error>(&mut conflicting_idle, additional.clone()),
5109 Err(EnsureWorkflowError::TerminalInventory(
5110 TerminalInventoryError::IdleCycleObservationConflict {
5111 canister,
5112 expected: 99,
5113 observed: 100,
5114 }
5115 )) if canister == "controlled-principal"
5116 ));
5117
5118 let mut workload = cycle_observation(RootOwnedCanisterLifecycle::Workload, 99);
5119 attach_terminal_cycles::<std::io::Error>(&mut workload, additional)
5120 .expect("merge conservative workload observation");
5121 assert_eq!(
5122 workload
5123 .canisters
5124 .get("asset")
5125 .and_then(Option::as_ref)
5126 .map(|live| live.cycles),
5127 Some(99),
5128 );
5129 }
5130}