1pub mod attempt_recovery;
8#[cfg(test)]
9mod balance_observation;
10pub mod capacity_import;
11pub mod clean_reinstall;
12mod continuation;
13mod funding;
14pub mod funding_observation;
15#[cfg(test)]
16mod funding_tests;
17mod independent_effects;
18pub mod infrastructure_bootstrap;
19pub mod operator_mint;
20pub mod readiness;
21mod reinstall;
22pub mod release;
23mod retained_plan;
24mod root_reinstall;
25
26use crate::fleet_ensure::{
27 dto::{FleetEnsurePhase, FleetEnsureProgress, FleetEnsureProgressState},
28 inventory::project_current_fleet_inventory,
29 model::{
30 ActualCycleConservation, CanisterDisposition, CurrentFleetProtocolAction,
31 CycleConservation, EffectRecord, EffectState, EnsureAction, EstateFundingDomainPlan,
32 EstateFundingRequiredRecord, FLEET_ENSURE_SCHEMA_VERSION, FleetEnsureCompletion,
33 FleetEnsureJournalRecord, FleetEnsurePlan, FleetEnsurePlanScope, FleetEnsureReport,
34 FleetEnsureStateRecord, FleetObservation, RootManagementObservation,
35 create_balance_is_terminal,
36 },
37 ops::{
38 EffectRetry, EnsurePaths, EnsurePlatform, EnsureStateError, action_sha256,
39 continuation::verify_release_transition, effect_preparation::prepare_effect,
40 lock_operation, read_root_start_authority, read_state, reserve_fixture_publication_attempt,
41 resolve_desired_artifacts, retain_configured_principal_bindings, write_journal, write_plan,
42 write_state,
43 },
44 policy::{
45 EnsurePolicyError, RootStartPlanInput, compile_plan, compile_root_start_prerequisite_plan,
46 effect_observation_policy, expected_plan_sha256, operation_id,
47 recompile_root_start_prerequisite_plan, validate_creation_fee_scope,
48 validate_path_identity, validate_path_labels,
49 },
50};
51use canic_contracts::cycles::Cycles;
52use std::{
53 collections::{BTreeMap, BTreeSet},
54 path::Path,
55 time::Instant,
56};
57use thiserror::Error as ThisError;
58
59pub use reinstall::plan_reinstall;
60
61#[derive(Debug, ThisError)]
64#[error(
65 "fresh pool canister {canister} requested {requested_creation_funding_cycles} creation cycles for readiness floor {readiness_floor_cycles}; its first live balance is {live_balance_cycles} cycles (readiness shortfall {readiness_shortfall_cycles}) and must retain {remaining_controller_burn_cycles} cycles for controller finalization (required pre-finalization balance {required_pre_finalization_balance_cycles}, pre-finalization shortfall {pre_finalization_shortfall_cycles}; maximum first-observation burn {maximum_observation_burn_cycles}); no controller or protocol action followed"
66)]
67pub struct FreshPoolCreationUnderfundedError {
68 pub canister: String,
69 pub live_balance_cycles: u128,
70 pub maximum_observation_burn_cycles: u128,
71 pub pre_finalization_shortfall_cycles: u128,
72 pub readiness_floor_cycles: u128,
73 pub readiness_shortfall_cycles: u128,
74 pub remaining_controller_burn_cycles: u128,
75 pub requested_creation_funding_cycles: u128,
76 pub required_pre_finalization_balance_cycles: u128,
77}
78
79#[derive(Debug, ThisError)]
81#[error(
82 "Root {root} Cycles Ledger account {root_principal} on {cycles_ledger} has {available_cycles} cycles; {required_creation_count} autonomous creations require at most {maximum_creation_debit_cycles} cycles ({creation_amount_cycles} gross amount containing {readiness_floor_cycles} readiness floor, {management_creation_fee_cycles} management creation fee and {creation_execution_margin_cycles} execution margin, plus {ledger_fee_cycles} Ledger fee each; total reviewed fees {maximum_creation_fee_cycles}), shortfall {shortfall_cycles}; pending_creation={pending_creation_operation_id:?} attempts={attempt_count:?} last_attempt_at_ns={last_attempt_at_ns:?} retry_at_ns={retry_at_ns:?}; preserve this operation, repeat Fleet ensure without --apply and review funding_review, then apply its review_sha256 for the exact shortfall; do not fund the account out of band"
83)]
84pub struct EstateFundingRequiredError {
85 pub attempt_count: Option<u32>,
86 pub available_cycles: u128,
87 pub creation_amount_cycles: u128,
88 pub cycles_ledger: String,
89 pub creation_execution_margin_cycles: u128,
90 pub readiness_floor_cycles: u128,
91 pub ledger_fee_cycles: u128,
92 pub management_creation_fee_cycles: u128,
93 pub maximum_creation_debit_cycles: u128,
94 pub maximum_creation_fee_cycles: u128,
95 pub pending_creation_operation_id: Option<String>,
96 pub required_creation_count: u32,
97 pub root: String,
98 pub root_principal: String,
99 pub last_attempt_at_ns: Option<u64>,
100 pub retry_at_ns: Option<u64>,
101 pub shortfall_cycles: u128,
102}
103
104#[derive(Debug, ThisError)]
106pub enum TerminalInventoryError {
107 #[error("one Principal is retained under more than one desired role")]
108 DuplicateRetainedPrincipal,
109
110 #[error("verified canister {canister} lost its Principal binding")]
111 LostPrincipalBinding { canister: String },
112
113 #[error("canister {canister} names unknown parent {parent}")]
114 UnknownParent { canister: String, parent: String },
115
116 #[error(
117 "canister {canister} retained parent {retained_parent} differs from terminal parent {terminal_parent}"
118 )]
119 RetainedParentConflict {
120 canister: String,
121 retained_parent: String,
122 terminal_parent: String,
123 },
124
125 #[error(
126 "terminal Root-owned Idle canister {canister} cycle observations differ: expected {expected}, observed {observed}"
127 )]
128 IdleCycleObservationConflict {
129 canister: String,
130 expected: u128,
131 observed: u128,
132 },
133
134 #[error(
135 "terminal cycle observation duplicates canister {canister} outside its exact Root-owned Idle or Workload lifecycle"
136 )]
137 CycleObservationLifecycleConflict { canister: String },
138}
139
140#[derive(Debug, ThisError)]
143pub enum EnsureWorkflowError<E>
144where
145 E: std::error::Error + 'static,
146{
147 #[error(transparent)]
148 InfrastructureBootstrap(
149 #[from] crate::fleet_ensure::ops::infrastructure_bootstrap::InfrastructureBootstrapError,
150 ),
151 #[error(transparent)]
152 RetainedContract(#[from] crate::fleet_ensure::ops::retained_contract::RetainedContractError),
153
154 #[error(transparent)]
155 FundingObservation(
156 #[from] crate::fleet_ensure::model::funding_observation::FundingObservationError,
157 ),
158
159 #[error(
160 "operator conversion must be resolved or its unapproved review cancelled before Fleet effects continue: {review_sha256}; inspect with fleet ensure --operator-mint"
161 )]
162 OperatorMintPending { review_sha256: String },
163
164 #[error(
165 "operator conversion required for retained funding review {review_sha256}; preserve the original withdrawal intent"
166 )]
167 OperatorFundingRequired { review_sha256: String },
168
169 #[error("operator conversion review, approval digest or current authority differs")]
170 OperatorMintReviewConflict,
171
172 #[error("operator conversion reply conflicts with a retained transaction outcome")]
173 OperatorMintReplyConflict,
174
175 #[error("operator conversion wire data is invalid: {0}")]
176 OperatorMintWire(#[from] crate::fleet_ensure::ops::operator_mint::OperatorMintWireError),
177
178 #[error(
179 "retained Fleet operation cannot be resumed: {source}; local source evidence identifies operation {operation_id}, journal plan reference {plan_sha256}, source document {source_document_sha256}; preserve plan, journal, state and artifacts; request an explicit --reinstall review without --apply using the selected corrected release; live authority, complete estate and cycle checks must pass before any reset; see docs/features/operations/fleet-ensure.md#unreadable-retained-plan"
180 )]
181 RetainedActivationReviewRequired {
182 operation_id: String,
183 plan_sha256: String,
184 source_document_sha256: String,
185 #[source]
186 source: Box<EnsureStateError>,
187 },
188
189 #[error(
190 "retained Fleet operation is unreadable: {source}; preserve plan, journal, state, artifacts and paid-effect receipts; local evidence does not establish a supported separate recovery review; resolve issued effects under their exact authority before a release transition; do not edit fields, replace digests or discard the operation; see docs/features/operations/fleet-ensure.md#unreadable-retained-plan"
191 )]
192 RetainedPlanUnreadable {
193 #[source]
194 source: Box<EnsureStateError>,
195 },
196
197 #[error(
198 "activation {operation_id} has an Issued provisioning action after its Applied host prefix (source document {source_document_sha256}); preserve the plan and journal: required live evidence for the bounded partial-activation recovery is unavailable"
199 )]
200 PartialActivationResetUnavailable {
201 operation_id: String,
202 source_document_sha256: String,
203 },
204
205 #[error(
206 "reinstall requires a converged Fleet with retained physical authority or an admitted single-Root partial-activation recovery source; the supplied source or target does not satisfy either contract; preserve the retained plan and journal"
207 )]
208 ReinstallConflict,
209
210 #[error(
211 "retained Fleet operation {operation_id} must recover before a new reinstall; preserve original plan {plan_sha256}, journal and receipts; run ensure without --reinstall to review retained funding, use --operator-mint if conversion is required, and resume the original reviewed operation before requesting a selected-release reinstall"
212 )]
213 RetainedOperationRecoveryRequired {
214 operation_id: String,
215 plan_sha256: String,
216 },
217
218 #[error("reviewed Fleet plan digest changed before its first effect")]
219 DriftedBeforeApply,
220
221 #[error(
222 "replan-required Fleet operation owns completed reinstall evidence for desired input {expected}; refusing alternate desired input {actual}"
223 )]
224 RetainedReinstallDesiredConflict { actual: String, expected: String },
225
226 #[error(
227 "Root prerequisite retains activation authority for desired input {expected}; refreshed input {actual} is refused before Store effects; preserve the retained plan and journal for recovery review"
228 )]
229 RootActivationDesiredConflict { actual: String, expected: String },
230
231 #[error("reviewed Fleet plan {expected} does not match retained plan {actual}")]
232 PlanDigestMismatch { actual: String, expected: String },
233
234 #[error("reviewed Fleet plan is missing; run `canic fleet ensure <fleet>` first")]
235 PlanMissing,
236
237 #[error("retained Fleet plan failed its content digest")]
238 PlanIntegrity,
239
240 #[error("retained Fleet journal does not match the reviewed Fleet operation")]
241 JournalIntegrity,
242
243 #[error(
244 "Fleet ensure action {action} made no progress for {observations} consecutive observations (last progress: {progress_identity}); operation remains resumable"
245 )]
246 Stalled {
247 action: String,
248 observations: u32,
249 progress_identity: String,
250 },
251
252 #[error("Root {root} provisioning has made no progress for {observations} observations: stage={stage:?}, target={target}, operation={operation}, diagnostic={diagnostic_code}, retry=Backoff, failed_at_ns={failed_at_ns}; the issued operation remains retained; resume the same reviewed apply after the dependency recovers", operation = canic_core::cdk::utils::hash::hex_bytes(.operation_id))]
253 ProvisioningRetryPending {
254 root: candid::Principal,
255 stage: canic_contracts::dto::component_provisioning::ProvisioningFailureStage,
256 target: candid::Principal,
257 operation_id: [u8; 32],
258 diagnostic_code: u16,
259 observations: u32,
260 failed_at_ns: u64,
261 },
262
263 #[error(
264 "Root-owned canister {target} remained {last_lifecycle} for {observations} consecutive passive observations; no command was reissued and the operation remains resumable"
265 )]
266 RootOwnedObservationStalled {
267 target: String,
268 observations: u32,
269 last_lifecycle: String,
270 },
271
272 #[error(
273 "selected Cycles Ledger account has {actual} cycles, below reviewed maximum debit {required}; shortfall {} cycles; no new execution journal or funding intent was retained; fund the selected operator account before retrying this reviewed plan; repeat fleet ensure with --operator-mint and without --apply for a read-only ICP quote",
274 required.saturating_sub(*actual)
275 )]
276 InsufficientOperatorCycles { actual: u128, required: u128 },
277
278 #[error("terminal Fleet cycle conservation failed: {0}")]
279 Conservation(String),
280
281 #[error(
282 "completed Fleet operation {operation_id} cannot replay accounting for plan {plan_sha256}: current operator balance {current_operator_cycles} is outside the reviewed range {minimum_operator_cycles}..={operator_source_cycles}; subsequent account activity cannot be attributed to this Fleet operation; preserve the completed plan, journal and receipts, run `canic fleet ensure {fleet}` without --apply to review a fresh plan, then apply only its newly reviewed digest and debit; do not repeat the completed deployment or edit its accounting"
283 )]
284 TerminalReplayBalanceChanged {
285 fleet: String,
286 operation_id: String,
287 plan_sha256: String,
288 current_operator_cycles: u128,
289 minimum_operator_cycles: u128,
290 operator_source_cycles: u128,
291 },
292
293 #[error("terminal Fleet inventory failed exact current-authority validation: {0}")]
294 TerminalInventory(#[source] TerminalInventoryError),
295
296 #[error(
297 "reviewed effects completed, but live Fleet state still differs from desired state; review a new plan"
298 )]
299 ConvergenceDrift,
300
301 #[error(
302 "fixture publication action {action} exhausted its {maximum_attempts} reviewed paid attempts; preserve the operation and reconcile its exact source status; no additional update was issued"
303 )]
304 FixturePublicationBound {
305 action: String,
306 maximum_attempts: u32,
307 },
308
309 #[error(
310 "Root pool maintenance {action} exhausted its reviewed limit of {maximum_updates} updates; retained receipts remain authoritative"
311 )]
312 PoolMaintenanceBound {
313 action: String,
314 maximum_updates: u32,
315 },
316
317 #[error(
318 "created canister {canister} retained one exact Ledger creation receipt, but its first live balance is {actual_cycles} cycles instead of requested target {requested_cycles} (deficit {deficit_cycles}; configured management creation fee {configured_fee_cycles}); no controller or protocol action followed, and the immutable operation is now replan-required"
319 )]
320 ReplanRequiredAfterCreateBalanceDrift {
321 actual_cycles: u128,
322 canister: String,
323 configured_fee_cycles: u128,
324 deficit_cycles: u128,
325 requested_cycles: u128,
326 },
327
328 #[error("Fleet successor requires a new review: {reason:?}; {}", .review.as_deref().map(ToString::to_string).unwrap_or_default())]
329 SuccessorReviewRequired {
330 reason: crate::fleet_ensure::model::FleetEnsureSuccessorReviewReason,
331 review: Option<Box<crate::fleet_ensure::model::FleetSuccessorReview>>,
332 },
333
334 #[error(transparent)]
335 FreshPoolCreationUnderfunded(Box<FreshPoolCreationUnderfundedError>),
336
337 #[error(transparent)]
338 EstateFundingRequired(Box<EstateFundingRequiredError>),
339
340 #[error("native Root funding needs explicit approval of review {review_sha256}")]
341 NativeFundingRequired { review_sha256: String },
342
343 #[error(transparent)]
344 Policy(#[from] EnsurePolicyError),
345
346 #[error(transparent)]
347 Inventory(#[from] crate::fleet_ensure::CurrentFleetInventoryError),
348
349 #[error("Fleet platform operation failed: {0}")]
350 Platform(#[source] E),
351
352 #[error(transparent)]
353 State(#[from] EnsureStateError),
354}
355
356pub fn retained_reinstall_apply_plan<E: std::error::Error + 'static>(
358 root: &Path,
359 environment: &str,
360 requested_fleet: &str,
361 reviewed_plan_sha256: &str,
362) -> Result<Option<FleetEnsurePlan>, EnsureWorkflowError<E>> {
363 validate_path_labels(environment, requested_fleet)?;
364 let paths = EnsurePaths::under(root, environment, requested_fleet);
365 let _lock = lock_operation(&paths)?;
366 let plan = if let Some(review) = crate::fleet_ensure::ops::reinstall::adoption::review(&paths)?
368 {
369 Some(review)
370 } else {
371 if crate::fleet_ensure::ops::operation_selection::completed(
372 &paths,
373 environment,
374 requested_fleet,
375 )?
376 .is_some_and(|completed| completed.plan_sha256 != reviewed_plan_sha256)
377 {
378 return Ok(None);
379 }
380 retained_plan::read(&paths, environment, requested_fleet)?
381 };
382 let Some(plan) = plan else {
383 return Ok(None);
384 };
385 let plan = verified_plan(plan)?;
386 if plan.environment != environment || plan.fleet != requested_fleet {
387 return Err(EnsureWorkflowError::PlanIntegrity);
388 }
389 if plan.reinstall.is_some() && plan.plan_sha256 == reviewed_plan_sha256 {
390 Ok(Some(plan))
391 } else {
392 Ok(None)
393 }
394}
395
396pub fn retained_in_progress_plan<E>(
398 root: &Path,
399 environment: &str,
400 requested_fleet: &str,
401) -> Result<Option<FleetEnsurePlan>, EnsureWorkflowError<E>>
402where
403 E: std::error::Error + 'static,
404{
405 validate_path_labels(environment, requested_fleet)?;
406 let paths = EnsurePaths::under(root, environment, requested_fleet);
407 let _lock = lock_operation(&paths)?;
408 if let Some(review) = crate::fleet_ensure::ops::reinstall::adoption::review(&paths)? {
409 if review.environment != environment || review.fleet != requested_fleet {
410 return Err(EnsureWorkflowError::PlanIntegrity);
411 }
412 return Ok(Some(review));
413 }
414 if crate::fleet_ensure::ops::operation_selection::completed(
415 &paths,
416 environment,
417 requested_fleet,
418 )?
419 .is_some()
420 {
421 return Ok(None);
422 }
423 let Some(journal) = retained_plan::journal(&paths, environment, requested_fleet)? else {
424 return Ok(None);
425 };
426 if !matches!(
427 journal.completion,
428 FleetEnsureCompletion::InProgress | FleetEnsureCompletion::Prepared
429 ) {
430 return Ok(None);
431 }
432 let retained = verified_plan(
433 retained_plan::read(&paths, environment, requested_fleet)?
434 .ok_or(EnsureWorkflowError::PlanMissing)?,
435 )?;
436 if retained.environment != environment || retained.fleet != requested_fleet {
437 return Err(EnsureWorkflowError::PlanIntegrity);
438 }
439 let state = read_state(&paths, requested_fleet)?;
440 let prepared_full = journal.completion == FleetEnsureCompletion::Prepared
441 && (retained.scope == FleetEnsurePlanScope::Full
442 || (retained.scope == FleetEnsurePlanScope::RootReinstallPrerequisite
443 && retained.reinstall.as_ref().is_some_and(|intent| {
444 intent
445 .activation_reset
446 .as_ref()
447 .is_some_and(|activation| activation.preparation.is_some())
448 })))
449 && retained.reinstall.as_ref().is_some_and(|intent| {
450 intent.operation_id == journal.operation_id
451 && intent.operation_id == retained.operation_id
452 });
453 if !prepared_full {
454 verify_journal_integrity(&journal, &retained, requested_fleet, &state)?;
455 }
456 Ok(Some(retained))
457}
458
459pub fn plan<P>(
461 root: &Path,
462 desired: &crate::fleet_ensure::model::DesiredFleet,
463 desired_sha256: &str,
464 requested_fleet: &str,
465 created_at_time: u64,
466 platform: &mut P,
467) -> Result<FleetEnsureReport, EnsureWorkflowError<P::Error>>
468where
469 P: EnsurePlatform,
470{
471 platform.with_planning_observations(|platform| {
472 plan_with_observations(
473 root,
474 desired,
475 desired_sha256,
476 requested_fleet,
477 created_at_time,
478 platform,
479 )
480 })
481}
482
483#[expect(
484 clippy::too_many_lines,
485 reason = "one locked read-only transaction reconciles retained authority, observation and the immutable plan"
486)]
487fn plan_with_observations<P>(
488 root: &Path,
489 desired: &crate::fleet_ensure::model::DesiredFleet,
490 desired_sha256: &str,
491 requested_fleet: &str,
492 created_at_time: u64,
493 platform: &mut P,
494) -> Result<FleetEnsureReport, EnsureWorkflowError<P::Error>>
495where
496 P: EnsurePlatform,
497{
498 validate_path_identity(desired, requested_fleet)?;
499 let paths = EnsurePaths::under(root, &desired.environment, requested_fleet);
500 let _lock = lock_operation(&paths)?;
501 if let Some(review) = crate::fleet_ensure::ops::reinstall::adoption::review(&paths)? {
502 if review.desired_sha256 != desired_sha256 {
503 return Err(EnsureWorkflowError::DriftedBeforeApply);
504 }
505 verify_release_transition(root, desired, review.scope)?;
506 return Ok(FleetEnsureReport {
507 funding_review: None,
508 actual_conservation: None,
509 effects_applied: 0,
510 plan: review,
511 terminal: false,
512 });
513 }
514 if let Some(mut journal) =
515 retained_plan::journal(&paths, &desired.environment, requested_fleet)?
516 && journal.completion == FleetEnsureCompletion::InProgress
517 {
518 let retained = verified_plan(
519 retained_plan::read(&paths, &desired.environment, requested_fleet)?
520 .ok_or(EnsureWorkflowError::PlanMissing)?,
521 )?;
522 let state = read_state(&paths, requested_fleet)?;
523 verify_journal(&journal, &retained, requested_fleet, &state)?;
524 let operation_desired = retained
525 .reviewed_desired
526 .as_deref()
527 .ok_or(EnsureWorkflowError::PlanIntegrity)?
528 .desired();
529 verify_release_transition(root, operation_desired, retained.scope)?;
530 if journal.estate_funding_required.is_some()
531 || funding::native_review_applicable(&retained, &journal)
532 {
533 platform
534 .bind_reviewed_desired(operation_desired)
535 .map_err(EnsureWorkflowError::Platform)?;
536 }
537 let funding_review = funding::prepare(
538 &paths,
539 &retained,
540 &mut journal,
541 &state,
542 created_at_time,
543 platform,
544 )?;
545 return Ok(FleetEnsureReport {
546 funding_review,
547 actual_conservation: None,
548 effects_applied: applied_count(&journal),
549 plan: retained,
550 terminal: false,
551 });
552 }
553 verify_release_transition(root, desired, FleetEnsurePlanScope::RootStartPrerequisite)?;
554 let mut state = read_state(&paths, requested_fleet)?;
555 let prior_plan = retained_plan::read(&paths, &desired.environment, requested_fleet)?
556 .map(verified_plan)
557 .transpose()?;
558 let prior_journal = retained_plan::journal(&paths, &desired.environment, requested_fleet)?;
559 if let Some(prior) = prior_plan
560 .as_ref()
561 .filter(|prior| prior.scope == FleetEnsurePlanScope::InfrastructureBootstrap)
562 {
563 crate::fleet_ensure::ops::infrastructure_bootstrap::convergence::prepare(
564 &paths,
565 prior,
566 prior_journal
567 .as_ref()
568 .ok_or(EnsureWorkflowError::JournalIntegrity)?,
569 &state,
570 desired,
571 )?;
572 }
573 if let Some(prior) = &prior_plan {
574 let recovery_ready = prior_journal
575 .as_ref()
576 .map(|journal| reinstall_recovery_ready(prior, journal, &state))
577 .transpose()?
578 .unwrap_or(false);
579 if recovery_ready && let Some(journal) = &prior_journal {
580 if prior.desired_sha256 != desired_sha256 {
581 return Err(EnsureWorkflowError::ReinstallConflict);
582 }
583 retain_completed_reinstalls(&mut state, prior, journal);
584 }
585 let activation_reset = prior
586 .reinstall
587 .as_ref()
588 .is_some_and(|intent| intent.activation_reset.is_some());
589 if activation_reset
590 && prior.scope == FleetEnsurePlanScope::RootReinstallPrerequisite
591 && let Some(journal) = prior_journal
592 .as_ref()
593 .filter(|journal| journal.completion == FleetEnsureCompletion::Converged)
594 {
595 verify_journal(journal, prior, requested_fleet, &state)?;
596 retain_completed_reinstalls(&mut state, prior, journal);
597 write_state(&paths, &state)?;
598 }
599 let unfinished_reset = prior.reinstall.is_some()
600 && !recovery_ready
601 && !prior_journal.as_ref().is_some_and(|journal| {
602 journal.plan_sha256 == prior.plan_sha256
603 && journal.completion == FleetEnsureCompletion::Converged
604 });
605 if prior.scope == FleetEnsurePlanScope::ReinstallPreparation
606 || unfinished_reset
607 || (activation_reset && prior.scope == FleetEnsurePlanScope::RootReinstallPrerequisite)
608 {
609 if prior.desired_sha256 != desired_sha256 {
610 return Err(EnsureWorkflowError::ReinstallConflict);
611 }
612 return reinstall::continue_plan(
613 root,
614 &paths,
615 desired,
616 prior,
617 prior_journal.as_ref(),
618 &state,
619 platform,
620 );
621 }
622 }
623 let matching_prior = reconcile_retained_planning_state(
624 &mut state,
625 prior_plan.as_ref(),
626 prior_journal.as_ref(),
627 requested_fleet,
628 &desired.environment,
629 desired_sha256,
630 )?;
631 if let Some(prior) = matching_prior {
632 retain_plan_cycles(&mut state, prior);
633 if let Some(journal) = &prior_journal {
634 retain_completed_reinstalls(&mut state, prior, journal);
635 }
636 }
637 let operation_id = prior_plan
640 .as_ref()
641 .filter(|prior| prior.scope == FleetEnsurePlanScope::InfrastructureBootstrap)
642 .or_else(|| matching_prior.filter(|prior| prior.scope == FleetEnsurePlanScope::Full))
643 .map_or_else(
644 || operation_id(desired_sha256, &desired.environment, requested_fleet),
645 |prior| prior.operation_id.clone(),
646 );
647 if let Some(management) = platform
648 .observe_root_management(&state, &BTreeSet::new())
649 .map_err(EnsureWorkflowError::Platform)?
650 {
651 let root_start_authority = read_root_start_authority(&paths)?;
652 if let Some(plan) = compile_root_start_prerequisite_plan(RootStartPlanInput {
653 state: &state,
654 authority: root_start_authority.as_ref(),
655 created_at_time,
656 desired,
657 desired_sha256,
658 observation: &management,
659 requested_fleet,
660 })? {
661 write_plan(&paths, &plan)?;
662 return Ok(FleetEnsureReport {
663 funding_review: None,
664 actual_conservation: None,
665 effects_applied: 0,
666 plan,
667 terminal: false,
668 });
669 }
670 verify_release_transition(root, desired, FleetEnsurePlanScope::Full)?;
671 if let Some(plan) = crate::fleet_ensure::policy::root_reinstall::compile(
672 RootStartPlanInput {
673 state: &state,
674 authority: None,
675 created_at_time,
676 desired,
677 desired_sha256,
678 observation: &management,
679 requested_fleet,
680 },
681 &resolve_desired_artifacts(root, desired)?,
682 )? {
683 write_plan(&paths, &plan)?;
684 return Ok(FleetEnsureReport {
685 funding_review: None,
686 actual_conservation: None,
687 effects_applied: 0,
688 plan,
689 terminal: false,
690 });
691 }
692 }
693 verify_release_transition(root, desired, FleetEnsurePlanScope::Full)?;
694 let artifacts = resolve_desired_artifacts(root, desired)?;
695 let mut observation = platform
696 .observe(&operation_id, &state)
697 .map_err(EnsureWorkflowError::Platform)?;
698 retain_observed_cycles(&mut state, &observation);
699 write_state(&paths, &state)?;
702 let terminal_inventory_operation_id = current_terminal_inventory_operation(
703 &state,
704 prior_plan.as_ref(),
705 prior_journal.as_ref(),
706 requested_fleet,
707 )?;
708 if let Some(operation_id) = terminal_inventory_operation_id.as_deref() {
709 attach_terminal_inventory_cycles(operation_id, &state, platform, &mut observation)?;
710 }
711 let mut protocol_actions = platform
712 .protocol_actions(
713 terminal_inventory_operation_id
714 .as_deref()
715 .unwrap_or(&operation_id),
716 &state,
717 )
718 .map_err(EnsureWorkflowError::Platform)?;
719 let mut retained_observations = 0_u32;
720 let mut plan = loop {
721 match compile_plan(
722 desired,
723 &artifacts,
724 &protocol_actions,
725 desired_sha256,
726 requested_fleet,
727 &observation,
728 created_at_time,
729 &operation_id,
730 None,
731 ) {
732 Ok(plan) => break plan,
733 Err(EnsurePolicyError::PendingRootOwnedBalance { name }) => {
734 retained_observations = retained_observations.saturating_add(1);
735 if retained_observations >= desired.maximum_stalled_observations {
736 return Err(EnsureWorkflowError::RootOwnedObservationStalled {
737 last_lifecycle: root_owned_lifecycle_label(&observation, &name).to_string(),
738 observations: retained_observations,
739 target: name,
740 });
741 }
742 platform.pace_root_owned_observation(&name, retained_observations);
743 observation = platform
744 .observe(&operation_id, &state)
745 .map_err(EnsureWorkflowError::Platform)?;
746 retain_observed_cycles(&mut state, &observation);
747 write_state(&paths, &state)?;
748 if let Some(operation_id) = terminal_inventory_operation_id.as_deref() {
749 attach_terminal_inventory_cycles(
750 operation_id,
751 &state,
752 platform,
753 &mut observation,
754 )?;
755 }
756 protocol_actions = platform
757 .protocol_actions(
758 terminal_inventory_operation_id
759 .as_deref()
760 .unwrap_or(&operation_id),
761 &state,
762 )
763 .map_err(EnsureWorkflowError::Platform)?;
764 }
765 Err(error) => return Err(error.into()),
766 }
767 };
768 for action in ordered_actions(&plan) {
769 verify_store_activation_binding(&plan, action, &state, platform)?;
770 }
771 bind_terminal_inventory_operation(&mut plan, terminal_inventory_operation_id);
772 write_plan(&paths, &plan)?;
773 Ok(FleetEnsureReport {
774 funding_review: None,
775 actual_conservation: None,
776 effects_applied: 0,
777 plan,
778 terminal: false,
779 })
780}
781
782fn current_terminal_inventory_operation<E>(
783 state: &FleetEnsureStateRecord,
784 prior_plan: Option<&FleetEnsurePlan>,
785 prior_journal: Option<&FleetEnsureJournalRecord>,
786 requested_fleet: &str,
787) -> Result<Option<String>, EnsureWorkflowError<E>>
788where
789 E: std::error::Error + 'static,
790{
791 if state.active_registry.is_none() {
792 return Ok(None);
793 }
794 let prior_plan = prior_plan.ok_or(EnsureWorkflowError::PlanMissing)?;
795 let prior_journal = prior_journal.ok_or(EnsureWorkflowError::JournalIntegrity)?;
796 verify_journal(prior_journal, prior_plan, requested_fleet, state)?;
797 if prior_plan.scope == FleetEnsurePlanScope::InfrastructureBootstrap
800 && prior_journal.completion == FleetEnsureCompletion::Converged
801 {
802 return Ok(None);
803 }
804 if prior_journal.completion != FleetEnsureCompletion::Converged
805 || prior_plan.scope != FleetEnsurePlanScope::Full
806 {
807 return Err(EnsureWorkflowError::JournalIntegrity);
808 }
809 let operation_id = prior_plan
810 .terminal_inventory_operation_id
811 .as_ref()
812 .unwrap_or(&prior_plan.operation_id);
813 if operation_id.is_empty() {
814 return Err(EnsureWorkflowError::JournalIntegrity);
815 }
816 Ok(Some(operation_id.clone()))
817}
818
819fn bind_terminal_inventory_operation(plan: &mut FleetEnsurePlan, operation_id: Option<String>) {
820 plan.terminal_inventory_operation_id = operation_id;
821 plan.plan_sha256 = expected_plan_sha256(plan);
822}
823
824fn reconcile_retained_planning_state<'a, E>(
825 state: &mut FleetEnsureStateRecord,
826 prior_plan: Option<&'a FleetEnsurePlan>,
827 prior_journal: Option<&FleetEnsureJournalRecord>,
828 requested_fleet: &str,
829 desired_environment: &str,
830 desired_sha256: &str,
831) -> Result<Option<&'a FleetEnsurePlan>, EnsureWorkflowError<E>>
832where
833 E: std::error::Error + 'static,
834{
835 if let Some((prior, journal)) = prior_plan.zip(prior_journal)
836 && prior.fleet == requested_fleet
837 && prior.environment == desired_environment
838 && retained_reinstall_desired_conflict(
839 state,
840 &prior.fleet,
841 &prior.operation_id,
842 &prior.desired_sha256,
843 desired_sha256,
844 journal,
845 )
846 {
847 return Err(EnsureWorkflowError::RetainedReinstallDesiredConflict {
848 actual: desired_sha256.to_string(),
849 expected: prior.desired_sha256.clone(),
850 });
851 }
852 if let Some((prior, journal)) = prior_plan.zip(prior_journal)
853 && prior.scope == FleetEnsurePlanScope::RootReinstallPrerequisite
854 && prior.desired_sha256 != desired_sha256
855 {
856 verify_journal(journal, prior, requested_fleet, state)?;
857 return Err(EnsureWorkflowError::RootActivationDesiredConflict {
858 actual: desired_sha256.to_string(),
859 expected: prior.desired_sha256.clone(),
860 });
861 }
862 let matching_prior = prior_plan.filter(|prior| {
863 prior.fleet == requested_fleet
864 && prior.environment == desired_environment
865 && prior.desired_sha256 == desired_sha256
866 });
867 let retained_operation_matches =
868 matching_prior
869 .zip(prior_journal)
870 .is_some_and(|(prior, journal)| {
871 retained_reinstall_operation_matches(
872 state,
873 &prior.fleet,
874 &prior.operation_id,
875 journal,
876 )
877 });
878 if !retained_operation_matches {
879 state.completed_reinstall_action_sha256.clear();
880 state.completed_reinstall_operation_id = None;
881 state.completed_reinstalls.clear();
882 }
883 Ok(matching_prior)
884}
885
886fn retain_completed_reinstalls(
887 state: &mut FleetEnsureStateRecord,
888 plan: &FleetEnsurePlan,
889 journal: &FleetEnsureJournalRecord,
890) {
891 if journal.fleet != plan.fleet || journal.operation_id != plan.operation_id {
892 return;
893 }
894 for action in ordered_actions(plan) {
895 let EnsureAction::Install {
896 mode: crate::fleet_ensure::model::InstallMode::Reinstall,
897 name,
898 principal,
899 wasm_sha256,
900 ..
901 } = action
902 else {
903 continue;
904 };
905 let action_sha256 = action_sha256(action);
906 let Some(effect) = journal
907 .effects
908 .iter()
909 .find(|effect| effect.action_sha256 == action_sha256)
910 else {
911 continue;
912 };
913 if effect.state == EffectState::Applied
914 && let Some(pre_canister_version) = effect.pre_canister_version
915 && state
916 .principals
917 .get(name)
918 .is_some_and(|retained| retained == principal)
919 {
920 state.completed_reinstall_operation_id = Some(journal.operation_id.clone());
921 state
922 .completed_reinstall_action_sha256
923 .insert(name.clone(), action_sha256);
924 state
925 .completed_reinstalls
926 .insert(name.clone(), pre_canister_version);
927 if let Some(topology) = state.topology.get_mut(name) {
928 topology.module_hash = Some(wasm_sha256.clone());
929 }
930 }
931 }
932}
933
934fn completed_reinstall_evidence_matches(
935 state: &FleetEnsureStateRecord,
936 journal: &FleetEnsureJournalRecord,
937) -> bool {
938 !state.completed_reinstalls.is_empty()
939 && state
940 .completed_reinstalls
941 .iter()
942 .all(|(name, pre_version)| {
943 state
944 .completed_reinstall_action_sha256
945 .get(name)
946 .and_then(|action_sha256| {
947 journal
948 .effects
949 .iter()
950 .find(|effect| effect.action_sha256 == *action_sha256)
951 })
952 .is_some_and(|effect| {
953 effect.state == EffectState::Applied
954 && effect.pre_canister_version == Some(*pre_version)
955 })
956 })
957 && state.completed_reinstall_action_sha256.len() == state.completed_reinstalls.len()
958}
959
960fn reinstall_recovery_ready<E: std::error::Error + 'static>(
962 plan: &FleetEnsurePlan,
963 journal: &FleetEnsureJournalRecord,
964 state: &FleetEnsureStateRecord,
965) -> Result<bool, EnsureWorkflowError<E>> {
966 if plan.scope != FleetEnsurePlanScope::Full
967 || plan.reinstall.is_none()
968 || journal.completion != FleetEnsureCompletion::ReplanRequired
969 || journal.plan_sha256 != plan.plan_sha256
970 {
971 return Ok(false);
972 }
973 verify_journal(journal, plan, &plan.fleet, state)?;
974 let actions = continuation::actions(plan, journal);
975 let complete = actions.len() == journal.effects.len()
976 && journal
977 .effects
978 .iter()
979 .all(|effect| effect.state == EffectState::Applied);
980 let installs_recorded = actions
981 .iter()
982 .zip(&journal.effects)
983 .all(|(action, effect)| {
984 !matches!(action, EnsureAction::Install { .. }) || effect.pre_canister_version.is_some()
985 });
986 if !complete || !installs_recorded {
987 return Err(EnsureWorkflowError::JournalIntegrity);
988 }
989 Ok(true)
990}
991
992fn retained_reinstall_operation_matches(
993 state: &FleetEnsureStateRecord,
994 prior_fleet: &str,
995 prior_operation_id: &str,
996 journal: &FleetEnsureJournalRecord,
997) -> bool {
998 journal.completion == FleetEnsureCompletion::ReplanRequired
999 && journal.fleet == prior_fleet
1000 && journal.operation_id == prior_operation_id
1001 && state.completed_reinstall_operation_id.as_deref() == Some(journal.operation_id.as_str())
1002 && completed_reinstall_evidence_matches(state, journal)
1003}
1004
1005fn retained_reinstall_desired_conflict(
1006 state: &FleetEnsureStateRecord,
1007 prior_fleet: &str,
1008 prior_operation_id: &str,
1009 prior_desired_sha256: &str,
1010 supplied_desired_sha256: &str,
1011 journal: &FleetEnsureJournalRecord,
1012) -> bool {
1013 prior_desired_sha256 != supplied_desired_sha256
1014 && retained_reinstall_operation_matches(state, prior_fleet, prior_operation_id, journal)
1015}
1016
1017fn retain_plan_cycles(state: &mut FleetEnsureStateRecord, plan: &FleetEnsurePlan) {
1018 for canister in &plan.canisters {
1019 if let Some(principal) = &canister.principal {
1020 state
1021 .retained_cycles_by_principal
1022 .entry(principal.clone())
1023 .or_insert(canister.observed_cycles);
1024 }
1025 }
1026}
1027
1028fn retain_observed_cycles(state: &mut FleetEnsureStateRecord, observation: &FleetObservation) {
1029 for live in observation.canisters.values().filter_map(Option::as_ref) {
1030 state
1031 .retained_cycles_by_principal
1032 .insert(live.principal.clone(), live.cycles);
1033 }
1034 state
1035 .retained_cycles_by_principal
1036 .extend(observation.additional_controlled_cycles.clone());
1037}
1038
1039fn root_owned_lifecycle_label(observation: &FleetObservation, target: &str) -> &'static str {
1040 observation
1041 .canisters
1042 .get(target)
1043 .and_then(Option::as_ref)
1044 .and_then(|live| live.root_owned_lifecycle)
1045 .map_or("unavailable", |lifecycle| lifecycle.label())
1046}
1047
1048#[expect(
1050 clippy::too_many_lines,
1051 reason = "the sole effect driver keeps intent, reconciliation and terminal publication visibly ordered"
1052)]
1053pub fn apply<P>(
1054 root: &Path,
1055 desired: &crate::fleet_ensure::model::DesiredFleet,
1056 desired_sha256: &str,
1057 requested_fleet: &str,
1058 reviewed_plan_sha256: &str,
1059 platform: &mut P,
1060) -> Result<FleetEnsureReport, EnsureWorkflowError<P::Error>>
1061where
1062 P: EnsurePlatform,
1063{
1064 validate_path_identity(desired, requested_fleet)?;
1065 let paths = EnsurePaths::under(root, &desired.environment, requested_fleet);
1066 let _lock = lock_operation(&paths)?;
1067 let pending = crate::fleet_ensure::ops::reinstall::adoption::review(&paths)?;
1068 let adopting = pending.is_some();
1069 let retained_plan = verified_plan(match pending {
1070 Some(plan) => plan,
1071 None => retained_plan::read(&paths, &desired.environment, requested_fleet)?
1072 .ok_or(EnsureWorkflowError::PlanMissing)?,
1073 })?;
1074 let retained_journal = if adopting {
1075 None
1076 } else {
1077 retained_plan::journal(&paths, &desired.environment, requested_fleet)?
1078 };
1079 if retained_plan.plan_sha256 != reviewed_plan_sha256
1080 && !retained_journal
1081 .as_ref()
1082 .is_some_and(|journal| funding::accepts(&retained_plan, journal, reviewed_plan_sha256))
1083 {
1084 return Err(EnsureWorkflowError::PlanDigestMismatch {
1085 actual: retained_plan.plan_sha256,
1086 expected: reviewed_plan_sha256.to_string(),
1087 });
1088 }
1089 let mut state = read_state(&paths, requested_fleet)?;
1090 let in_progress = retained_journal
1091 .as_ref()
1092 .is_some_and(|journal| journal.completion == FleetEnsureCompletion::InProgress);
1093 if let Some(journal) = retained_journal.as_ref().filter(|_| in_progress) {
1094 verify_journal(journal, &retained_plan, requested_fleet, &state)?;
1095 }
1096 if !in_progress
1097 && retained_plan.reinstall.is_none()
1098 && retained_plan.desired_sha256 != desired_sha256
1099 {
1100 return Err(EnsureWorkflowError::DriftedBeforeApply);
1101 }
1102 let operation_desired = if in_progress || retained_plan.reinstall.is_some() {
1103 let reviewed = retained_plan
1104 .reviewed_desired
1105 .as_deref()
1106 .ok_or(EnsureWorkflowError::PlanIntegrity)?
1107 .desired();
1108 validate_path_identity(reviewed, requested_fleet)?;
1109 if reviewed.environment != retained_plan.environment {
1110 return Err(EnsureWorkflowError::PlanIntegrity);
1111 }
1112 reviewed
1113 } else {
1114 if retained_plan.desired_sha256 != desired_sha256 {
1115 return Err(EnsureWorkflowError::DriftedBeforeApply);
1116 }
1117 desired
1118 };
1119 if let Some(journal) = &retained_journal
1120 && let Some(actual) = crate::fleet_ensure::ops::terminal::read(
1121 &paths,
1122 &retained_plan,
1123 journal,
1124 &state,
1125 &funding_plan::<P::Error>(&retained_plan, journal)?.conservation,
1126 )?
1127 {
1128 verify_journal(journal, &retained_plan, requested_fleet, &state)?;
1129 if journal.completion != FleetEnsureCompletion::Converged {
1130 let mut completed = journal.clone();
1131 completed.completion = FleetEnsureCompletion::Converged;
1132 completed.stalled_observations = 0;
1133 write_journal(&paths, &completed)?;
1134 }
1135 return Ok(FleetEnsureReport {
1136 funding_review: None,
1137 actual_conservation: Some(actual),
1138 effects_applied: 0,
1139 plan: retained_plan,
1140 terminal: true,
1141 });
1142 }
1143 if let Some(journal) = &retained_journal
1144 && let Some(actual) = crate::fleet_ensure::ops::infrastructure_bootstrap::terminal::read(
1145 &paths,
1146 &retained_plan,
1147 journal,
1148 &state,
1149 )?
1150 {
1151 verify_journal(journal, &retained_plan, requested_fleet, &state)?;
1152 if journal.completion != FleetEnsureCompletion::Converged {
1153 let mut completed = journal.clone();
1154 completed.completion = FleetEnsureCompletion::Converged;
1155 completed.stalled_observations = 0;
1156 write_journal(&paths, &completed)?;
1157 }
1158 return Ok(FleetEnsureReport {
1159 funding_review: None,
1160 actual_conservation: Some(actual),
1161 effects_applied: 0,
1162 plan: retained_plan,
1163 terminal: true,
1164 });
1165 }
1166 if retained_plan.scope == FleetEnsurePlanScope::InfrastructureBootstrap {
1167 crate::fleet_ensure::ops::infrastructure_bootstrap::verify_plan(root, &retained_plan)?;
1168 }
1169 verify_release_transition(root, operation_desired, retained_plan.scope)?;
1170 if !retained_journal.as_ref().is_some_and(|journal| {
1171 journal.completion == FleetEnsureCompletion::Converged
1172 && journal.plan_sha256 == retained_plan.plan_sha256
1173 }) {
1174 validate_creation_fee_scope(
1175 operation_desired,
1176 &retained_plan.canisters,
1177 &retained_plan.conservation.estate_funding_domains,
1178 )?;
1179 }
1180 platform
1181 .bind_reviewed_desired(operation_desired)
1182 .map_err(EnsureWorkflowError::Platform)?;
1183 if let Some(source) = &retained_plan.infrastructure_bootstrap {
1184 platform
1185 .bind_infrastructure_bootstrap(source)
1186 .map_err(EnsureWorkflowError::Platform)?;
1187 }
1188 let operation_desired_sha256 = retained_plan.desired_sha256.as_str();
1189 if let Some(review) = retained_journal.as_ref().and_then(
1190 crate::fleet_ensure::ops::infrastructure_bootstrap::registration_recovery::approved,
1191 ) && !platform
1192 .bind_bootstrap_registration_recovery(review)
1193 .map_err(EnsureWorkflowError::Platform)?
1194 {
1195 return Err(EnsureWorkflowError::PlanIntegrity);
1196 }
1197 continuation::verify_inputs(root, operation_desired, &retained_plan)?;
1198 if let Some(journal) = &retained_journal {
1199 continuation::verify_canonical(&retained_plan, journal, &state, platform)?;
1200 if reinstall_recovery_ready(&retained_plan, journal, &state)? {
1201 return Err(EnsureWorkflowError::SuccessorReviewRequired {
1202 reason:
1203 crate::fleet_ensure::model::FleetEnsureSuccessorReviewReason::AdditionalEffect,
1204 review: None,
1205 });
1206 }
1207 }
1208 let completed_journal = retained_journal.as_ref().filter(|journal| {
1209 (journal.completion == FleetEnsureCompletion::Converged
1210 || (journal.completion == FleetEnsureCompletion::Prepared
1211 && retained_plan.scope == FleetEnsurePlanScope::ReinstallPreparation))
1212 && journal.plan_sha256 == retained_plan.plan_sha256
1213 });
1214 if retained_plan.scope == FleetEnsurePlanScope::ReinstallPreparation
1215 && let Some(journal) = completed_journal
1216 {
1217 let actual = reinstall::complete(&retained_plan, journal, &state, platform)?;
1218 return Ok(FleetEnsureReport {
1219 funding_review: None,
1220 actual_conservation: Some(actual),
1221 effects_applied: 0,
1222 plan: retained_plan,
1223 terminal: true,
1224 });
1225 }
1226 if retained_plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite
1227 && let Some(journal) = completed_journal
1228 {
1229 verify_journal(journal, &retained_plan, requested_fleet, &state)?;
1230 let actual = root_reinstall::complete(&retained_plan, journal, &state, platform)?;
1231 report_progress_state(
1232 platform,
1233 &retained_plan,
1234 journal,
1235 FleetEnsurePhase::Infrastructure,
1236 FleetEnsureProgressState::PrerequisiteComplete,
1237 );
1238 return Ok(FleetEnsureReport {
1239 funding_review: None,
1240 actual_conservation: Some(actual),
1241 effects_applied: 0,
1242 plan: retained_plan,
1243 terminal: true,
1244 });
1245 }
1246 if retained_plan.scope == FleetEnsurePlanScope::RootStartPrerequisite
1247 && let Some(journal) = completed_journal
1248 {
1249 verify_terminal_root_start_replay(
1250 operation_desired,
1251 &retained_plan,
1252 journal,
1253 requested_fleet,
1254 platform,
1255 &state,
1256 )?;
1257 report_progress_state(
1258 platform,
1259 &retained_plan,
1260 journal,
1261 FleetEnsurePhase::Infrastructure,
1262 FleetEnsureProgressState::PrerequisiteComplete,
1263 );
1264 return Ok(FleetEnsureReport {
1265 funding_review: None,
1266 actual_conservation: None,
1267 effects_applied: applied_count(journal),
1268 plan: retained_plan,
1269 terminal: true,
1270 });
1271 }
1272 let mut journal = match retained_journal {
1273 Some(journal) if journal.completion == FleetEnsureCompletion::InProgress => {
1274 verify_journal(&journal, &retained_plan, requested_fleet, &state)?;
1275 journal
1276 }
1277 _ => {
1278 let (observation, initial_controlled_cycles) = verify_fresh_plan(
1279 root,
1280 operation_desired,
1281 operation_desired_sha256,
1282 requested_fleet,
1283 &retained_plan,
1284 platform,
1285 &state,
1286 )?;
1287 let required = retained_plan.conservation.maximum_operator_debit_cycles;
1290 if observation.operator_cycles < required {
1291 return Err(EnsureWorkflowError::InsufficientOperatorCycles {
1292 actual: observation.operator_cycles,
1293 required,
1294 });
1295 }
1296 root_reinstall::verify_initial_funding(
1297 &retained_plan,
1298 observation.operator_cycles,
1299 platform,
1300 )?;
1301 let journal = FleetEnsureJournalRecord {
1302 release: None,
1303 bootstrap_registration_recovery: None,
1304 funding_observations: BTreeMap::new(),
1305 funding_reviews: Vec::new(),
1306 successor_phases: Vec::new(),
1307 completion: FleetEnsureCompletion::InProgress,
1308 estate_funding_required: None,
1309 effects: Vec::new(),
1310 fleet: requested_fleet.to_string(),
1311 initial_controlled_cycles,
1312 initial_estate_funding_cycles_by_root: retained_plan
1313 .conservation
1314 .estate_funding_domains
1315 .iter()
1316 .map(|domain| {
1317 (
1318 domain.root.clone(),
1319 domain.available_cycles.unwrap_or_default(),
1320 )
1321 })
1322 .collect(),
1323 initial_operator_cycles: observation.operator_cycles,
1324 operation_id: retained_plan.operation_id.clone(),
1325 plan_sha256: retained_plan.plan_sha256.clone(),
1326 schema_version: FLEET_ENSURE_SCHEMA_VERSION,
1327 stalled_observations: 0,
1328 };
1329 if adopting {
1331 crate::fleet_ensure::ops::reinstall::adoption::adopt(
1332 &paths,
1333 &retained_plan,
1334 &journal,
1335 )?;
1336 } else {
1337 write_journal(&paths, &journal)?;
1338 }
1339 journal
1340 }
1341 };
1342
1343 funding::resume(
1344 &paths,
1345 &retained_plan,
1346 &mut journal,
1347 &state,
1348 reviewed_plan_sha256,
1349 platform,
1350 )?;
1351 if journal
1352 .estate_funding_required
1353 .as_ref()
1354 .is_some_and(|pause| funding::covered(&journal, pause))
1355 {
1356 retain_estate_funding_pause(&paths, &mut journal, None)?;
1357 }
1358 let mut terminal_state;
1359 let mut bootstrap_coordinator_verified = false;
1360 let mut continuation_observation: Option<ContinuationObservation> = None;
1363 loop {
1364 let execution_actions = continuation::actions(&retained_plan, &journal)
1365 .into_iter()
1366 .cloned()
1367 .collect::<Vec<_>>();
1368 let actions = execution_actions.iter().collect::<Vec<_>>();
1369 let mut replayed_issued_commands = BTreeSet::new();
1370 let mut reported_phase = None;
1371 loop {
1372 let mut deferred_controller_observation = false;
1373 for (index, action) in actions.iter().enumerate() {
1374 let observation_policy = effect_observation_policy(operation_desired, action)?;
1375 let action_hash = action_sha256(action);
1376 let retained_effect = journal.effects.get(index);
1377 let continued_observation = retained_effect
1378 .is_none_or(|effect| effect.state != EffectState::Applied)
1379 .then(|| continuation_observation.take())
1380 .flatten()
1381 .filter(|evidence| evidence.first_action_sha256 == action_hash);
1382 if retained_effect.is_none_or(|effect| effect.state != EffectState::Applied) {
1383 if retained_plan.infrastructure_bootstrap.is_some()
1384 && index < ordered_actions(&retained_plan).len()
1385 && !bootstrap_coordinator_verified
1386 && operation_desired
1387 .bootstrap
1388 .as_ref()
1389 .is_some_and(|bootstrap| action.name() != bootstrap.coordinator)
1390 {
1391 if !platform
1392 .verify_bootstrap_coordinator(&state)
1393 .map_err(EnsureWorkflowError::Platform)?
1394 {
1395 return Err(EnsureWorkflowError::ConvergenceDrift);
1396 }
1397 bootstrap_coordinator_verified = true;
1398 }
1399 let phase = action_progress_phase(action);
1400 if reported_phase != Some(phase) {
1401 report_progress(platform, &retained_plan, &journal, phase);
1402 reported_phase = Some(phase);
1403 }
1404 }
1405 if retained_plan.infrastructure_bootstrap.is_none()
1406 && matches!(action, EnsureAction::FleetProtocol { .. })
1407 && retained_effect.is_none_or(|effect| effect.state == EffectState::Intent)
1408 && !prior_fleet_protocol_effect_started(&actions, &journal, index)
1409 {
1410 let funding_observation = match continued_observation {
1411 Some(evidence) => evidence.observation,
1412 None => platform
1413 .observe(&journal.operation_id, &state)
1414 .map_err(EnsureWorkflowError::Platform)?,
1415 };
1416 let funded_plan = funding_plan(&retained_plan, &journal)?;
1417 let required =
1418 estate_funding_requirement(&funded_plan, &state, &funding_observation)?;
1419 retain_estate_funding_pause(&paths, &mut journal, required.as_ref())?;
1420 if let Some(required) = required {
1421 report_progress_state(
1422 platform,
1423 &retained_plan,
1424 &journal,
1425 action_progress_phase(action),
1426 FleetEnsureProgressState::FundingRequired,
1427 );
1428 return Err(EnsureWorkflowError::EstateFundingRequired(Box::new(
1429 estate_funding_error(&required),
1430 )));
1431 }
1432 }
1433 independent_effects::apply_batch(
1434 &paths,
1435 &retained_plan,
1436 &actions,
1437 index,
1438 &mut journal,
1439 &state,
1440 platform,
1441 observation_policy.maximum_stalled_observations,
1442 )?;
1443 let mut initial_observation = if journal.effects.len() <= index {
1444 crate::fleet_ensure::ops::infrastructure_bootstrap::inspection::reserve_effect(
1445 &paths,
1446 &retained_plan,
1447 action,
1448 )?;
1449 let prepared = prepare_effect(platform, &journal.operation_id, action, &state)
1450 .map_err(EnsureWorkflowError::Platform)?;
1451 journal.effects.push(prepared.record);
1452 write_journal(&paths, &journal)?;
1453 prepared.observation
1454 } else {
1455 None
1456 };
1457 let observation_started = Instant::now();
1458 loop {
1459 let record = journal
1460 .effects
1461 .get(index)
1462 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1463 if record.action_sha256 != action_hash {
1464 return Err(EnsureWorkflowError::DriftedBeforeApply);
1465 }
1466 if !create_record_is_exact(action, record, &state) {
1467 return Err(EnsureWorkflowError::JournalIntegrity);
1468 }
1469 if matches!(record.state, EffectState::Applied) {
1470 let retained_create = journal
1471 .effects
1472 .get(index)
1473 .ok_or(EnsureWorkflowError::JournalIntegrity)
1474 .and_then(|record| {
1475 retain_applied_create_authority(
1476 operation_desired,
1477 action,
1478 record,
1479 &mut state,
1480 )
1481 })?;
1482 let retained_funding = journal.effects.get(index).is_some_and(|record| {
1483 retain_applied_funding_cycles(&mut state, action, record)
1484 });
1485 if retained_create || retained_funding {
1486 write_state(&paths, &state)?;
1487 }
1488 break;
1489 }
1490
1491 let observed = if let Some(observed) = initial_observation.take() {
1492 observed
1493 } else {
1494 crate::fleet_ensure::ops::infrastructure_bootstrap::inspection::reserve_effect(
1495 &paths, &retained_plan, action,
1496 )?;
1497 platform
1498 .observe_effect(&journal.operation_id, action, record, &state)
1499 .map_err(EnsureWorkflowError::Platform)?
1500 };
1501 let source_cycles = if observed.post_cycles.is_some() {
1502 observed.post_cycles
1503 } else {
1504 platform
1505 .action_cycles(action, &state)
1506 .map_err(EnsureWorkflowError::Platform)?
1507 };
1508 let destination_cycles = platform
1509 .action_destination_cycles(action, &state)
1510 .map_err(EnsureWorkflowError::Platform)?;
1511 let replan_after_create_balance =
1512 observed.retry == EffectRetry::ReplanRequiredAfterCreateBalanceDrift;
1513 let retained_create_balance_is_exact = !replan_after_create_balance
1514 || (journal.effects.len() == index + 1
1515 && journal.effects[..index]
1516 .iter()
1517 .all(|effect| effect.state == EffectState::Applied)
1518 && record.state == EffectState::Issued
1519 && create_identity_is_exact(action.name(), record, &state)
1520 && observed.post_cycles.is_some()
1521 && observed.post_cycles == source_cycles);
1522 let protocol_funding_required = if record.state == EffectState::Issued {
1523 observed
1524 .estate_funding_required
1525 .as_ref()
1526 .map(|funding| {
1527 protocol_estate_funding_requirement(&retained_plan, &state, funding)
1528 })
1529 .transpose()?
1530 } else {
1531 None
1532 };
1533 if let Some(required) = protocol_funding_required.as_ref()
1534 && !funding::covered(&journal, required)
1535 {
1536 retain_estate_funding_pause(&paths, &mut journal, Some(required))?;
1537 report_progress_state(
1538 platform,
1539 &retained_plan,
1540 &journal,
1541 action_progress_phase(action),
1542 FleetEnsureProgressState::FundingRequired,
1543 );
1544 return Err(EnsureWorkflowError::EstateFundingRequired(Box::new(
1545 estate_funding_error(required),
1546 )));
1547 }
1548 if journal
1549 .estate_funding_required
1550 .as_ref()
1551 .is_some_and(|required| required.pending_creation_operation_id.is_some())
1552 {
1553 retain_estate_funding_pause(&paths, &mut journal, None)?;
1554 }
1555 let record = journal
1556 .effects
1557 .get_mut(index)
1558 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1559 merge_observed_cycles(record, source_cycles, destination_cycles);
1560 if replan_after_create_balance {
1561 if !retained_create_balance_is_exact {
1562 return Err(EnsureWorkflowError::JournalIntegrity);
1563 }
1564 let EnsureAction::Create {
1565 name,
1566 requested_initial_cycles,
1567 ..
1568 } = action
1569 else {
1570 return Err(EnsureWorkflowError::JournalIntegrity);
1571 };
1572 let actual_cycles = record
1573 .post_cycles
1574 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1575 record.progress_identity = Some(observed.progress_identity);
1576 record.state = EffectState::Applied;
1577 retain_created_canister_for_replan(
1578 operation_desired,
1579 name,
1580 record,
1581 &mut state,
1582 )?;
1583 journal.completion = FleetEnsureCompletion::ReplanRequired;
1584 journal.stalled_observations = 0;
1585 write_journal(&paths, &journal)?;
1586 write_state(&paths, &state)?;
1587 let configured_fee_cycles = operation_desired
1588 .management_creation_fee_cycles
1589 .parse::<Cycles>()
1590 .map(|cycles| cycles.to_u128())
1591 .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
1592 return Err(EnsureWorkflowError::ReplanRequiredAfterCreateBalanceDrift {
1593 actual_cycles,
1594 canister: name.clone(),
1595 configured_fee_cycles,
1596 deficit_cycles: requested_initial_cycles.saturating_sub(actual_cycles),
1597 requested_cycles: *requested_initial_cycles,
1598 });
1599 }
1600 if observed.applied {
1601 record.progress_identity = Some(observed.progress_identity);
1602 record.state = EffectState::Applied;
1603 journal.stalled_observations = 0;
1604 write_journal(&paths, &journal)?;
1605 let record = journal
1606 .effects
1607 .get(index)
1608 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1609 let retained_create = retain_applied_create_authority(
1610 operation_desired,
1611 action,
1612 record,
1613 &mut state,
1614 )?;
1615 let retained_funding =
1616 retain_applied_funding_cycles(&mut state, action, record);
1617 if retained_create || retained_funding {
1618 write_state(&paths, &state)?;
1619 }
1620 report_progress(
1622 platform,
1623 &retained_plan,
1624 &journal,
1625 action_progress_phase(action),
1626 );
1627 break;
1628 }
1629
1630 if observed.retry == EffectRetry::DeferUntilControllerObservation {
1631 if !deferred_create_observation_is_exact(
1632 operation_desired,
1633 &actions,
1634 index,
1635 action,
1636 record,
1637 &state,
1638 ) {
1639 return Err(EnsureWorkflowError::JournalIntegrity);
1640 }
1641 let progress_identity = observed.progress_identity.clone();
1642 if record.progress_identity.as_deref() == Some(&observed.progress_identity)
1643 {
1644 journal.stalled_observations =
1645 journal.stalled_observations.saturating_add(1);
1646 } else {
1647 record.progress_identity = Some(observed.progress_identity);
1648 journal.stalled_observations = 0;
1649 }
1650 write_journal(&paths, &journal)?;
1651 if journal.stalled_observations
1652 >= observation_policy.maximum_stalled_observations
1653 {
1654 return Err(EnsureWorkflowError::Stalled {
1655 action: action.name().to_string(),
1656 observations: journal.stalled_observations,
1657 progress_identity,
1658 });
1659 }
1660 deferred_controller_observation = true;
1661 break;
1662 }
1663
1664 let maintenance_continuation =
1665 observed.retry == EffectRetry::ContinuePoolMaintenance;
1666 if matches!(record.state, EffectState::Intent) || maintenance_continuation {
1667 crate::fleet_ensure::policy::infrastructure_bootstrap::validate_effect_headroom(
1668 &retained_plan,
1669 action,
1670 source_cycles,
1671 )?;
1672 if let Some(maximum_attempts) = action.fixture_publication_attempt_limit() {
1673 if !reserve_fixture_publication_attempt(record, maximum_attempts) {
1674 return Err(EnsureWorkflowError::FixturePublicationBound {
1675 action: action.name().to_string(),
1676 maximum_attempts,
1677 });
1678 }
1679 write_journal(&paths, &journal)?;
1681 }
1682 let record = journal
1683 .effects
1684 .get_mut(index)
1685 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1686 if let EnsureAction::FleetProtocol {
1687 action: current, ..
1688 } = action
1689 && let CurrentFleetProtocolAction::MaintainPoolReadiness {
1690 maximum_updates,
1691 ..
1692 } = current.as_ref()
1693 {
1694 if record.maintenance_attempts >= *maximum_updates {
1695 return Err(EnsureWorkflowError::PoolMaintenanceBound {
1696 action: action.name().to_string(),
1697 maximum_updates: *maximum_updates,
1698 });
1699 }
1700 if record.progress_identity.as_deref()
1701 == Some(&observed.progress_identity)
1702 {
1703 journal.stalled_observations =
1704 journal.stalled_observations.saturating_add(1);
1705 } else {
1706 journal.stalled_observations = 0;
1707 }
1708 record.maintenance_attempts += 1;
1709 let attempts = record.maintenance_attempts;
1710 write_journal(&paths, &journal)?;
1712 if attempts > 1 {
1713 platform
1714 .pace_effect_observation(action, journal.stalled_observations);
1715 }
1716 } else if maintenance_continuation {
1717 return Err(EnsureWorkflowError::JournalIntegrity);
1718 }
1719 let record = journal
1720 .effects
1721 .get(index)
1722 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1723 if retained_plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite {
1724 root_reinstall::verify_effect_authority(
1725 &retained_plan,
1726 action,
1727 record.pre_cycles,
1728 &state,
1729 platform,
1730 )?;
1731 }
1732 verify_store_activation_binding(&retained_plan, action, &state, platform)?;
1733 reinstall::verify_effect_authority(
1734 &retained_plan,
1735 action,
1736 &state,
1737 platform,
1738 )?;
1739 let outcome =
1740 match platform.apply(&journal.operation_id, action, record, &state) {
1741 Ok(outcome) => outcome,
1742 Err(source) => {
1743 let progress_identity = record
1744 .progress_identity
1745 .clone()
1746 .unwrap_or_else(|| "effect-call-failed".to_string());
1747 journal.stalled_observations =
1748 journal.stalled_observations.saturating_add(1);
1749 write_journal(&paths, &journal)?;
1750 if journal.stalled_observations
1751 >= observation_policy.maximum_stalled_observations
1752 {
1753 return Err(EnsureWorkflowError::Stalled {
1754 action: action.name().to_string(),
1755 observations: journal.stalled_observations,
1756 progress_identity,
1757 });
1758 }
1759 return Err(EnsureWorkflowError::Platform(source));
1760 }
1761 };
1762 if !create_outcome_is_exact(action, record, &state, &outcome) {
1763 return Err(EnsureWorkflowError::JournalIntegrity);
1764 }
1765 if let Some(created) = &outcome.created_principal {
1766 state
1767 .pending_principals
1768 .insert(action.name().to_string(), created.clone());
1769 write_state(&paths, &state)?;
1770 }
1771 let record = journal
1772 .effects
1773 .get_mut(index)
1774 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1775 record.created_principal = outcome.created_principal;
1776 record.receipt = outcome.receipt;
1777 record.post_cycles = outcome.post_cycles;
1778 record.progress_identity = Some(observed.progress_identity);
1779 record.state = EffectState::Issued;
1780 if !maintenance_continuation {
1782 journal.stalled_observations = 0;
1783 }
1784 write_journal(&paths, &journal)?;
1785 continue;
1786 }
1787
1788 if observed.retry == EffectRetry::ReplayExactIssuedCommand
1789 && !replayed_issued_commands.contains(&index)
1790 {
1791 if !matches!(
1792 action,
1793 EnsureAction::FleetProtocol { action, .. }
1794 if matches!(
1795 action.as_ref(),
1796 crate::fleet_ensure::model::CurrentFleetProtocolAction::ProvisionComponents { .. }
1797 )
1798 ) {
1799 return Err(EnsureWorkflowError::JournalIntegrity);
1800 }
1801 let retained_receipt = record.receipt.clone();
1802 let outcome = platform
1803 .apply(&journal.operation_id, action, record, &state)
1804 .map_err(EnsureWorkflowError::Platform)?;
1805 if outcome.created_principal.is_some()
1806 || outcome.receipt != retained_receipt
1807 || outcome.post_cycles.is_some()
1808 {
1809 return Err(EnsureWorkflowError::JournalIntegrity);
1810 }
1811 let record = journal
1812 .effects
1813 .get_mut(index)
1814 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1815 record.progress_identity = Some(observed.progress_identity);
1816 journal.stalled_observations = 0;
1817 write_journal(&paths, &journal)?;
1818 replayed_issued_commands.insert(index);
1819 continue;
1820 }
1821
1822 if record.progress_identity.as_deref() == Some(&observed.progress_identity) {
1823 journal.stalled_observations =
1824 journal.stalled_observations.saturating_add(1);
1825 } else {
1826 record.progress_identity = Some(observed.progress_identity);
1827 journal.stalled_observations = 0;
1828 }
1829 let progress_identity = record
1830 .progress_identity
1831 .clone()
1832 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1833 write_journal(&paths, &journal)?;
1834 if journal.stalled_observations
1835 >= observation_policy.maximum_stalled_observations
1836 {
1837 if let Some((failure, origin)) = observed
1838 .provisioning_failure
1839 .and_then(|failure| failure.origin.map(|origin| (failure, origin)))
1840 {
1841 return Err(EnsureWorkflowError::ProvisioningRetryPending {
1842 root: failure.fleet_subnet_root,
1843 stage: origin.stage,
1844 target: origin.target,
1845 operation_id: origin.operation_id,
1846 diagnostic_code: origin.diagnostic_code,
1847 observations: journal.stalled_observations,
1848 failed_at_ns: origin.failed_at_ns,
1849 });
1850 }
1851 return Err(EnsureWorkflowError::Stalled {
1852 action: action.name().to_string(),
1853 observations: journal.stalled_observations,
1854 progress_identity,
1855 });
1856 }
1857 if observation_policy.paced {
1858 report_progress_state(
1859 platform,
1860 &retained_plan,
1861 &journal,
1862 action_progress_phase(action),
1863 FleetEnsureProgressState::AwaitingProgress {
1864 elapsed_seconds: observation_started.elapsed().as_secs(),
1865 provisioning: observed.provisioning_progress,
1866 },
1867 );
1868 platform.pace_effect_observation(action, journal.stalled_observations);
1869 }
1870 }
1871 }
1872 if journal.effects.len() == actions.len()
1873 && journal
1874 .effects
1875 .iter()
1876 .all(|record| record.state == EffectState::Applied)
1877 {
1878 break;
1879 }
1880 if !deferred_controller_observation {
1881 return Err(EnsureWorkflowError::JournalIntegrity);
1882 }
1883 }
1884
1885 report_progress(
1886 platform,
1887 &retained_plan,
1888 &journal,
1889 FleetEnsurePhase::TerminalVerification,
1890 );
1891 if retained_plan.scope == FleetEnsurePlanScope::InfrastructureBootstrap {
1892 if infrastructure_bootstrap::advance(
1893 root,
1894 &retained_plan,
1895 &mut journal,
1896 &mut state,
1897 platform,
1898 )? {
1899 continue;
1900 }
1901 let phase = journal
1902 .successor_phases
1903 .first()
1904 .and_then(|phase| phase.plan.as_ref())
1905 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1906 let expected =
1907 crate::fleet_ensure::ops::infrastructure_bootstrap::registration::registry(phase)?;
1908 if !platform
1909 .verify_bootstrap_registry(&expected)
1910 .map_err(EnsureWorkflowError::Platform)?
1911 {
1912 return Err(EnsureWorkflowError::ConvergenceDrift);
1913 }
1914 let (actual, observed) = infrastructure_bootstrap::complete(
1915 root,
1916 &retained_plan,
1917 &journal,
1918 &state,
1919 platform,
1920 )?;
1921 if !platform
1922 .verify_bootstrap_registry(&expected)
1923 .map_err(EnsureWorkflowError::Platform)?
1924 {
1925 return Err(EnsureWorkflowError::ConvergenceDrift);
1926 }
1927 publish_terminal_state(operation_desired, &retained_plan, &journal, &mut state)
1928 .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
1929 crate::fleet_ensure::ops::infrastructure_bootstrap::registration::bind_state(
1930 root, &mut state, phase,
1931 )?;
1932 write_state(&paths, &state)?;
1933 crate::fleet_ensure::ops::infrastructure_bootstrap::terminal::retain(
1934 &paths,
1935 &retained_plan,
1936 &journal,
1937 &state,
1938 actual.clone(),
1939 &observed,
1940 )?;
1941 journal.completion = FleetEnsureCompletion::Converged;
1942 journal.stalled_observations = 0;
1943 write_journal(&paths, &journal)?;
1944 report_progress_state(
1945 platform,
1946 &retained_plan,
1947 &journal,
1948 FleetEnsurePhase::Infrastructure,
1949 FleetEnsureProgressState::PrerequisiteComplete,
1950 );
1951 return Ok(FleetEnsureReport {
1952 funding_review: None,
1953 actual_conservation: Some(actual),
1954 effects_applied: applied_count(&journal),
1955 plan: retained_plan,
1956 terminal: true,
1957 });
1958 }
1959 if retained_plan.scope == FleetEnsurePlanScope::ReinstallPreparation {
1960 let actual = reinstall::complete(&retained_plan, &journal, &state, platform)?;
1961 journal.completion = FleetEnsureCompletion::Prepared;
1962 journal.stalled_observations = 0;
1963 write_journal(&paths, &journal)?;
1964 report_progress_state(
1965 platform,
1966 &retained_plan,
1967 &journal,
1968 FleetEnsurePhase::Infrastructure,
1969 FleetEnsureProgressState::PrerequisiteComplete,
1970 );
1971 return Ok(FleetEnsureReport {
1972 funding_review: None,
1973 actual_conservation: Some(actual),
1974 effects_applied: applied_count(&journal),
1975 plan: retained_plan,
1976 terminal: true,
1977 });
1978 }
1979 if retained_plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite {
1980 let actual = root_reinstall::complete(&retained_plan, &journal, &state, platform)?;
1981 publish_terminal_state(operation_desired, &retained_plan, &journal, &mut state)
1982 .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
1983 state.active_registry = None;
1984 write_state(&paths, &state)?;
1985 journal.completion = FleetEnsureCompletion::Converged;
1986 journal.stalled_observations = 0;
1987 write_journal(&paths, &journal)?;
1988 report_progress_state(
1989 platform,
1990 &retained_plan,
1991 &journal,
1992 FleetEnsurePhase::Infrastructure,
1993 FleetEnsureProgressState::PrerequisiteComplete,
1994 );
1995 return Ok(FleetEnsureReport {
1996 funding_review: None,
1997 actual_conservation: Some(actual),
1998 effects_applied: applied_count(&journal),
1999 plan: retained_plan,
2000 terminal: true,
2001 });
2002 }
2003 if retained_plan.scope == FleetEnsurePlanScope::RootStartPrerequisite {
2004 let actual_conservation = complete_root_start_prerequisite(
2005 operation_desired,
2006 &retained_plan,
2007 platform,
2008 &state,
2009 &journal,
2010 )?;
2011 journal.completion = FleetEnsureCompletion::Converged;
2012 journal.stalled_observations = 0;
2013 write_journal(&paths, &journal)?;
2014 report_progress_state(
2015 platform,
2016 &retained_plan,
2017 &journal,
2018 FleetEnsurePhase::Infrastructure,
2019 FleetEnsureProgressState::PrerequisiteComplete,
2020 );
2021 return Ok(FleetEnsureReport {
2022 funding_review: None,
2023 actual_conservation: Some(actual_conservation),
2024 effects_applied: applied_count(&journal),
2025 plan: retained_plan,
2026 terminal: true,
2027 });
2028 }
2029
2030 terminal_state = state.clone();
2031 publish_terminal_state(
2032 operation_desired,
2033 &retained_plan,
2034 &journal,
2035 &mut terminal_state,
2036 )
2037 .map_err(|TerminalStatePublicationError::JournalIntegrity| {
2038 EnsureWorkflowError::JournalIntegrity
2039 })?;
2040 if completed_infrastructure_reinstall(operation_desired, &retained_plan, &journal) {
2041 terminal_state.active_registry = None;
2042 }
2043 project_current_fleet_inventory(&terminal_state)?;
2044 {
2045 let (terminal_observation, converged) =
2049 platform.with_planning_observations(|platform| {
2050 let mut terminal_observation = platform
2051 .observe(&retained_plan.operation_id, &terminal_state)
2052 .map_err(EnsureWorkflowError::Platform)?;
2053 let artifacts = resolve_desired_artifacts(root, operation_desired)?;
2054 let observation_started = Instant::now();
2055 let converged = loop {
2056 let protocol_actions = platform
2057 .protocol_actions(protocol_operation(&retained_plan), &terminal_state)
2058 .map_err(EnsureWorkflowError::Platform)?;
2059 match compile_plan(
2060 operation_desired,
2061 &artifacts,
2062 &protocol_actions,
2063 operation_desired_sha256,
2064 requested_fleet,
2065 &terminal_observation,
2066 retained_plan.planned_at_time,
2067 &retained_plan.operation_id,
2068 None,
2069 ) {
2070 Ok(plan) => {
2071 if journal.stalled_observations != 0 {
2072 journal.stalled_observations = 0;
2073 write_journal(&paths, &journal)?;
2074 }
2075 break plan;
2076 }
2077 Err(EnsurePolicyError::PendingRootOwnedBalance { name }) => {
2078 journal.stalled_observations =
2079 journal.stalled_observations.saturating_add(1);
2080 retain_observed_cycles(&mut state, &terminal_observation);
2081 write_state(&paths, &state)?;
2082 write_journal(&paths, &journal)?;
2083 if journal.stalled_observations
2084 >= operation_desired.maximum_stalled_observations
2085 {
2086 return Err(EnsureWorkflowError::RootOwnedObservationStalled {
2087 last_lifecycle: root_owned_lifecycle_label(
2088 &terminal_observation,
2089 &name,
2090 )
2091 .to_string(),
2092 observations: journal.stalled_observations,
2093 target: name,
2094 });
2095 }
2096 report_progress_state(
2097 platform,
2098 &retained_plan,
2099 &journal,
2100 FleetEnsurePhase::TerminalVerification,
2101 FleetEnsureProgressState::AwaitingProgress {
2102 elapsed_seconds: observation_started.elapsed().as_secs(),
2103 provisioning: None,
2104 },
2105 );
2106 platform.pace_root_owned_observation(
2107 &name,
2108 journal.stalled_observations,
2109 );
2110 terminal_observation = platform
2111 .observe(&retained_plan.operation_id, &terminal_state)
2112 .map_err(EnsureWorkflowError::Platform)?;
2113 }
2114 Err(error @ EnsurePolicyError::EstatePoolCapacity { .. }) => {
2115 retain_observed_cycles(&mut terminal_state, &terminal_observation);
2116 retain_completed_reinstalls(
2117 &mut terminal_state,
2118 &retained_plan,
2119 &journal,
2120 );
2121 write_state(&paths, &terminal_state)?;
2122 journal.completion = FleetEnsureCompletion::ReplanRequired;
2123 journal.stalled_observations = 0;
2124 write_journal(&paths, &journal)?;
2125 return Err(error.into());
2126 }
2127 Err(error) => return Err(error.into()),
2128 }
2129 };
2130 Ok::<_, EnsureWorkflowError<P::Error>>((terminal_observation, converged))
2131 })?;
2132 if converged
2133 .canisters
2134 .iter()
2135 .any(|canister| !canister.actions.is_empty())
2136 || !converged.protocol_actions.is_empty()
2137 {
2138 if retained_plan.continuation.is_some() {
2139 let appended = continuation::append(
2140 &paths,
2141 operation_desired,
2142 &retained_plan,
2143 &mut journal,
2144 &terminal_state,
2145 &terminal_observation,
2146 converged,
2147 platform,
2148 );
2149 match appended {
2150 Ok(()) => {
2151 write_state(&paths, &terminal_state)?;
2152 state = terminal_state.clone();
2153 continuation_observation = journal
2154 .successor_phases
2155 .last()
2156 .and_then(|phase| phase.plan.as_ref())
2157 .and_then(|phase| phase.protocol_actions.first())
2158 .map(|first| ContinuationObservation {
2159 first_action_sha256: action_sha256(first),
2160 observation: terminal_observation,
2161 });
2162 report_progress_state(
2163 platform,
2164 &retained_plan,
2165 &journal,
2166 FleetEnsurePhase::TerminalVerification,
2167 FleetEnsureProgressState::PrerequisiteComplete,
2168 );
2169 continue;
2170 }
2171 Err(EnsureWorkflowError::SuccessorReviewRequired { reason, review }) => {
2172 retain_observed_cycles(&mut terminal_state, &terminal_observation);
2173 retain_completed_reinstalls(
2174 &mut terminal_state,
2175 &retained_plan,
2176 &journal,
2177 );
2178 write_state(&paths, &terminal_state)?;
2179 journal.completion = FleetEnsureCompletion::ReplanRequired;
2180 write_journal(&paths, &journal)?;
2181 report_progress_state(
2182 platform,
2183 &retained_plan,
2184 &journal,
2185 FleetEnsurePhase::TerminalVerification,
2186 FleetEnsureProgressState::ReviewRequired {
2187 reason,
2188 review: review.clone(),
2189 },
2190 );
2191 return Err(EnsureWorkflowError::SuccessorReviewRequired {
2192 reason,
2193 review,
2194 });
2195 }
2196 Err(error) => return Err(error),
2197 }
2198 }
2199 retain_observed_cycles(&mut terminal_state, &terminal_observation);
2200 retain_completed_reinstalls(&mut terminal_state, &retained_plan, &journal);
2201 write_state(&paths, &terminal_state)?;
2202 journal.completion = FleetEnsureCompletion::ReplanRequired;
2203 journal.stalled_observations = 0;
2204 write_journal(&paths, &journal)?;
2205 let reason =
2206 crate::fleet_ensure::model::FleetEnsureSuccessorReviewReason::AdditionalEffect;
2207 let review = Some(Box::new(
2208 crate::fleet_ensure::ops::recovery::review_details(&converged),
2209 ));
2210 report_progress_state(
2211 platform,
2212 &retained_plan,
2213 &journal,
2214 FleetEnsurePhase::TerminalVerification,
2215 FleetEnsureProgressState::ReviewRequired {
2216 reason,
2217 review: review.clone(),
2218 },
2219 );
2220 return Err(EnsureWorkflowError::SuccessorReviewRequired { reason, review });
2221 }
2222 }
2223 break;
2224 }
2225 let terminal_inventory_operation_id =
2226 completed_inventory_operation(&retained_plan, &terminal_state)?;
2227 let terminal_inventory = platform
2228 .terminal_inventory(terminal_inventory_operation_id, &terminal_state)
2229 .map_err(EnsureWorkflowError::Platform)?;
2230 let terminal_cycles = terminal_inventory.controlled_cycles_by_principal.clone();
2231 merge_terminal_inventory(&mut terminal_state, terminal_inventory)?;
2232 project_current_fleet_inventory(&terminal_state)?;
2233 let mut final_observation = platform
2234 .observe(&retained_plan.operation_id, &terminal_state)
2235 .map_err(EnsureWorkflowError::Platform)?;
2236 attach_terminal_cycles(&mut final_observation, terminal_cycles)?;
2237 reinstall::verify_terminal_estate(&retained_plan, &final_observation)?;
2238 reinstall::verify_terminal_authority(&retained_plan, &terminal_state, platform)?;
2239 let actual_conservation = verify_terminal_conservation(
2240 &retained_plan,
2241 &journal,
2242 &terminal_state,
2243 &final_observation,
2244 )?;
2245 terminal_state.completed_reinstall_action_sha256.clear();
2246 terminal_state.completed_reinstall_operation_id = None;
2247 terminal_state.completed_reinstalls.clear();
2248 write_state(&paths, &terminal_state)?;
2250 journal.completion = FleetEnsureCompletion::Converged;
2251 journal.stalled_observations = 0;
2252 crate::fleet_ensure::ops::terminal::retain(
2253 &paths,
2254 &retained_plan,
2255 &journal,
2256 &terminal_state,
2257 &actual_conservation,
2258 &funding_plan::<P::Error>(&retained_plan, &journal)?.conservation,
2259 )?;
2260 write_journal(&paths, &journal)?;
2261 report_progress(
2262 platform,
2263 &retained_plan,
2264 &journal,
2265 FleetEnsurePhase::Complete,
2266 );
2267 Ok(FleetEnsureReport {
2268 funding_review: None,
2269 actual_conservation: Some(actual_conservation),
2270 effects_applied: applied_count(&journal),
2271 plan: retained_plan,
2272 terminal: true,
2273 })
2274}
2275
2276fn action_progress_phase(action: &EnsureAction) -> FleetEnsurePhase {
2277 match action {
2278 EnsureAction::FleetProtocol { action, .. } => match action.as_ref() {
2279 CurrentFleetProtocolAction::ReconcilePoolAsset { .. } => {
2280 FleetEnsurePhase::ImportReconciliation
2281 }
2282 CurrentFleetProtocolAction::ProvisionComponents { .. } => {
2283 FleetEnsurePhase::WorkloadProvisioning
2284 }
2285 CurrentFleetProtocolAction::MaintainPoolReadiness { .. }
2286 | CurrentFleetProtocolAction::ObservePoolReadiness { .. } => {
2287 FleetEnsurePhase::PoolReadiness
2288 }
2289 _ => FleetEnsurePhase::ControlPlane,
2290 },
2291 _ => FleetEnsurePhase::Infrastructure,
2292 }
2293}
2294
2295fn report_progress<P: EnsurePlatform>(
2296 platform: &mut P,
2297 plan: &FleetEnsurePlan,
2298 journal: &FleetEnsureJournalRecord,
2299 phase: FleetEnsurePhase,
2300) {
2301 let state = if phase == FleetEnsurePhase::Complete {
2302 FleetEnsureProgressState::Complete
2303 } else {
2304 FleetEnsureProgressState::Advancing
2305 };
2306 report_progress_state(platform, plan, journal, phase, state);
2307}
2308
2309fn report_progress_state<P: EnsurePlatform>(
2310 platform: &mut P,
2311 plan: &FleetEnsurePlan,
2312 journal: &FleetEnsureJournalRecord,
2313 phase: FleetEnsurePhase,
2314 state: FleetEnsureProgressState,
2315) {
2316 platform.report_progress(FleetEnsureProgress {
2317 next_action: if matches!(
2318 state,
2319 FleetEnsureProgressState::Advancing
2320 | FleetEnsureProgressState::AwaitingProgress { .. }
2321 | FleetEnsureProgressState::FundingRequired
2322 ) && phase != FleetEnsurePhase::TerminalVerification
2323 {
2324 crate::fleet_ensure::ops::progress::next_action(
2325 &continuation::actions(plan, journal),
2326 journal,
2327 )
2328 } else {
2329 None
2330 },
2331 state,
2332 operation_id: plan.operation_id.clone(),
2333 plan_sha256: plan.plan_sha256.clone(),
2334 phase,
2335 applied_effects: applied_count(journal),
2336 reviewed_effects: continuation::actions(plan, journal).len()
2337 + journal
2338 .funding_reviews
2339 .iter()
2340 .filter(|review| review.effect.is_some())
2341 .count(),
2342 });
2343}
2344
2345const fn merge_observed_cycles(
2346 record: &mut EffectRecord,
2347 source_cycles: Option<u128>,
2348 destination_cycles: Option<u128>,
2349) {
2350 if let Some(cycles) = source_cycles {
2351 record.post_cycles = Some(cycles);
2352 }
2353 if let Some(cycles) = destination_cycles {
2354 record.destination_post_cycles = Some(cycles);
2355 }
2356}
2357
2358pub(super) fn retain_applied_create_authority<E>(
2359 desired: &crate::fleet_ensure::model::DesiredFleet,
2360 action: &EnsureAction,
2361 record: &EffectRecord,
2362 state: &mut FleetEnsureStateRecord,
2363) -> Result<bool, EnsureWorkflowError<E>>
2364where
2365 E: std::error::Error + 'static,
2366{
2367 let EnsureAction::Create {
2368 name,
2369 requested_initial_cycles,
2370 ..
2371 } = action
2372 else {
2373 return Ok(false);
2374 };
2375 if record.state != EffectState::Applied {
2376 return Err(EnsureWorkflowError::JournalIntegrity);
2377 }
2378 let configured = desired
2379 .canisters
2380 .iter()
2381 .find(|canister| {
2382 canister.name == *name
2383 && canister.presence == crate::fleet_ensure::model::DesiredPresence::Present
2384 })
2385 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2386 let principal = record
2387 .created_principal
2388 .as_deref()
2389 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2390 let cycles = record
2391 .post_cycles
2392 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2393 let maximum_observation_burn_cycles =
2394 crate::fleet_ensure::ops::maximum_creation_observation_burn(desired)
2395 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2396 if !create_identity_is_exact(name, record, state)
2397 || !create_balance_is_terminal(
2398 Some(cycles),
2399 *requested_initial_cycles,
2400 maximum_observation_burn_cycles,
2401 )
2402 || state
2403 .retained_cycles_by_principal
2404 .get(principal)
2405 .is_some_and(|retained| *retained != cycles)
2406 {
2407 return Err(EnsureWorkflowError::JournalIntegrity);
2408 }
2409 validate_fresh_pool_creation_balance(
2410 desired,
2411 configured,
2412 name,
2413 cycles,
2414 maximum_observation_burn_cycles,
2415 *requested_initial_cycles,
2416 )?;
2417
2418 let prior_topology = state.topology.get(name);
2419 let topology_identity_matches = prior_topology.is_some_and(|topology| {
2420 topology.kind == configured.kind
2421 && topology.parent == configured.parent
2422 && topology.protocol_binding == configured.protocol_binding
2423 && topology.role
2424 == configured
2425 .protocol_binding
2426 .as_ref()
2427 .map(|binding| binding.role.to_string())
2428 });
2429 let topology = crate::fleet_ensure::model::FleetEnsureTopologyRecord {
2430 kind: configured.kind,
2431 module_hash: if topology_identity_matches {
2432 prior_topology.and_then(|topology| topology.module_hash.clone())
2433 } else {
2434 None
2435 },
2436 parent: configured.parent.clone(),
2437 protocol_binding: configured.protocol_binding.clone(),
2438 role: configured
2439 .protocol_binding
2440 .as_ref()
2441 .map(|binding| binding.role.to_string()),
2442 };
2443 let changed = state.retained_cycles_by_principal.get(principal) != Some(&cycles)
2444 || state.topology.get(name) != Some(&topology);
2445 state
2446 .retained_cycles_by_principal
2447 .insert(principal.to_string(), cycles);
2448 state.topology.insert(name.clone(), topology);
2449 Ok(changed)
2450}
2451
2452fn validate_fresh_pool_creation_balance<E>(
2453 desired: &crate::fleet_ensure::model::DesiredFleet,
2454 configured: &crate::fleet_ensure::model::DesiredCanister,
2455 name: &str,
2456 cycles: u128,
2457 maximum_observation_burn_cycles: u128,
2458 requested_initial_cycles: u128,
2459) -> Result<(), EnsureWorkflowError<E>>
2460where
2461 E: std::error::Error + 'static,
2462{
2463 if configured.kind == crate::fleet_ensure::model::DesiredCanisterKind::Pool
2464 && desired
2465 .bootstrap
2466 .as_ref()
2467 .is_some_and(|bootstrap| bootstrap.fresh_estate)
2468 && configured.principal.is_none()
2469 && configured.controllers.is_empty()
2470 && configured.parent.as_ref().is_some_and(|parent| {
2471 configured.controller_canisters.as_slice() == std::slice::from_ref(parent)
2472 })
2473 {
2474 let readiness_floor_cycles = configured
2475 .minimum_cycles
2476 .parse::<Cycles>()
2477 .map(|cycles| cycles.to_u128())
2478 .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
2479 let remaining_controller_burn_cycles = desired
2480 .maximum_update_burn_cycles
2481 .parse::<Cycles>()
2482 .map(|cycles| cycles.to_u128())
2483 .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
2484 let required_pre_finalization_balance_cycles = readiness_floor_cycles
2485 .checked_add(remaining_controller_burn_cycles)
2486 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2487 if cycles < required_pre_finalization_balance_cycles {
2488 return Err(EnsureWorkflowError::FreshPoolCreationUnderfunded(Box::new(
2489 FreshPoolCreationUnderfundedError {
2490 canister: name.to_string(),
2491 live_balance_cycles: cycles,
2492 maximum_observation_burn_cycles,
2493 pre_finalization_shortfall_cycles: required_pre_finalization_balance_cycles
2494 - cycles,
2495 readiness_floor_cycles,
2496 readiness_shortfall_cycles: readiness_floor_cycles.saturating_sub(cycles),
2497 remaining_controller_burn_cycles,
2498 requested_creation_funding_cycles: requested_initial_cycles,
2499 required_pre_finalization_balance_cycles,
2500 },
2501 )));
2502 }
2503 }
2504 Ok(())
2505}
2506
2507fn retain_applied_funding_cycles(
2508 state: &mut FleetEnsureStateRecord,
2509 action: &EnsureAction,
2510 record: &EffectRecord,
2511) -> bool {
2512 let EnsureAction::Fund { principal, .. } = action else {
2513 return false;
2514 };
2515 if record.state != EffectState::Applied || record.receipt.is_none() {
2516 return false;
2517 }
2518 let Some(cycles) = record.post_cycles else {
2519 return false;
2520 };
2521 let principal = principal.strip_prefix("created:").map_or_else(
2522 || Some(principal.as_str()),
2523 |name| state.pending_principals.get(name).map(String::as_str),
2524 );
2525 let Some(principal) = principal else {
2526 return false;
2527 };
2528 if state.retained_cycles_by_principal.get(principal) == Some(&cycles) {
2529 return false;
2530 }
2531 state
2532 .retained_cycles_by_principal
2533 .insert(principal.to_string(), cycles);
2534 true
2535}
2536
2537fn retain_created_canister_for_replan<E>(
2538 desired: &crate::fleet_ensure::model::DesiredFleet,
2539 name: &str,
2540 record: &EffectRecord,
2541 state: &mut FleetEnsureStateRecord,
2542) -> Result<(), EnsureWorkflowError<E>>
2543where
2544 E: std::error::Error + 'static,
2545{
2546 let configured = desired
2547 .canisters
2548 .iter()
2549 .find(|canister| {
2550 canister.name == name
2551 && canister.presence == crate::fleet_ensure::model::DesiredPresence::Present
2552 })
2553 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2554 let principal = record
2555 .created_principal
2556 .as_deref()
2557 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2558 let cycles = record
2559 .post_cycles
2560 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2561 if state.pending_principals.get(name).map(String::as_str) != Some(principal) {
2562 return Err(EnsureWorkflowError::JournalIntegrity);
2563 }
2564 state.pending_principals.remove(name);
2565 state
2566 .principals
2567 .insert(name.to_string(), principal.to_string());
2568 state
2569 .retained_cycles_by_principal
2570 .insert(principal.to_string(), cycles);
2571 state.topology.insert(
2572 name.to_string(),
2573 crate::fleet_ensure::model::FleetEnsureTopologyRecord {
2574 kind: configured.kind,
2575 module_hash: None,
2576 parent: configured.parent.clone(),
2577 protocol_binding: configured.protocol_binding.clone(),
2578 role: configured
2579 .protocol_binding
2580 .as_ref()
2581 .map(|binding| binding.role.to_string()),
2582 },
2583 );
2584 Ok(())
2585}
2586
2587fn create_record_is_exact(
2588 action: &EnsureAction,
2589 record: &EffectRecord,
2590 state: &FleetEnsureStateRecord,
2591) -> bool {
2592 let EnsureAction::Create {
2593 name,
2594 requested_initial_cycles,
2595 ..
2596 } = action
2597 else {
2598 return true;
2599 };
2600 match record.state {
2601 EffectState::Intent => {
2602 record.created_principal.is_none()
2603 && record.receipt.is_none()
2604 && record.post_cycles.is_none()
2605 }
2606 EffectState::Issued => {
2607 create_identity_is_exact(name, record, state)
2608 && record.post_cycles == Some(*requested_initial_cycles)
2609 }
2610 EffectState::Applied => create_identity_is_exact(name, record, state),
2611 }
2612}
2613
2614fn create_identity_is_exact(
2615 name: &str,
2616 record: &EffectRecord,
2617 state: &FleetEnsureStateRecord,
2618) -> bool {
2619 let Some(created_principal) = record
2620 .created_principal
2621 .as_deref()
2622 .filter(|value| !value.is_empty())
2623 else {
2624 return false;
2625 };
2626 if record.receipt.as_deref().is_none_or(str::is_empty) {
2627 return false;
2628 }
2629 state_create_principal_matches(name, created_principal, state, true)
2630}
2631
2632fn state_create_principal_matches(
2633 name: &str,
2634 created_principal: &str,
2635 state: &FleetEnsureStateRecord,
2636 require_retained: bool,
2637) -> bool {
2638 let pending = state.pending_principals.get(name).map(String::as_str);
2639 let terminal = state.principals.get(name).map(String::as_str);
2640 let retained = pending.is_some() || terminal.is_some();
2641 let pending_matches = pending.is_none_or(|principal| principal == created_principal);
2642 let terminal_matches = terminal.is_none_or(|principal| principal == created_principal);
2643 (!require_retained || retained) && pending_matches && terminal_matches
2644}
2645
2646fn create_outcome_is_exact(
2647 action: &EnsureAction,
2648 record: &EffectRecord,
2649 state: &FleetEnsureStateRecord,
2650 outcome: &crate::fleet_ensure::ops::EffectOutcome,
2651) -> bool {
2652 if matches!(action, EnsureAction::FundEstate { .. }) {
2653 return outcome.created_principal.is_none()
2654 && outcome.post_cycles.is_none()
2655 && outcome
2656 .receipt
2657 .as_deref()
2658 .is_some_and(|receipt| !receipt.is_empty());
2659 }
2660 let EnsureAction::Create {
2661 name,
2662 requested_initial_cycles,
2663 ..
2664 } = action
2665 else {
2666 return true;
2667 };
2668 let Some(created_principal) = outcome
2669 .created_principal
2670 .as_deref()
2671 .filter(|value| !value.is_empty())
2672 else {
2673 return false;
2674 };
2675 let Some(receipt) = outcome.receipt.as_deref().filter(|value| !value.is_empty()) else {
2676 return false;
2677 };
2678 if outcome.post_cycles != Some(*requested_initial_cycles) {
2679 return false;
2680 }
2681 let principal_matches = record
2682 .created_principal
2683 .as_deref()
2684 .is_none_or(|retained| retained == created_principal);
2685 let receipt_matches = record
2686 .receipt
2687 .as_deref()
2688 .is_none_or(|retained| retained == receipt);
2689 let state_matches = state_create_principal_matches(name, created_principal, state, false);
2690 principal_matches && receipt_matches && state_matches
2691}
2692
2693fn verify_fresh_plan<P>(
2694 root: &Path,
2695 desired: &crate::fleet_ensure::model::DesiredFleet,
2696 desired_sha256: &str,
2697 requested_fleet: &str,
2698 retained_plan: &FleetEnsurePlan,
2699 platform: &mut P,
2700 state: &FleetEnsureStateRecord,
2701) -> Result<(FleetObservation, u128), EnsureWorkflowError<P::Error>>
2702where
2703 P: EnsurePlatform,
2704{
2705 if retained_plan.scope == FleetEnsurePlanScope::InfrastructureBootstrap {
2706 return infrastructure_bootstrap::verify_before_apply(root, retained_plan, state, platform);
2707 }
2708 if retained_plan.reinstall.is_some() {
2709 return reinstall::verify_before_apply(root, desired, retained_plan, platform, state);
2710 }
2711 if retained_plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite {
2712 return root_reinstall::verify_before_apply(root, desired, retained_plan, platform, state);
2713 }
2714 if retained_plan.scope == FleetEnsurePlanScope::RootStartPrerequisite {
2715 let targets = reviewed_root_start_targets(retained_plan)?;
2716 let management = platform
2717 .observe_root_management(state, &targets)
2718 .map_err(EnsureWorkflowError::Platform)?
2719 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2720 let current = recompile_root_start_prerequisite_plan(
2721 RootStartPlanInput {
2722 state,
2723 authority: retained_plan.root_start_authority.as_deref(),
2724 created_at_time: retained_plan.planned_at_time,
2725 desired,
2726 desired_sha256,
2727 observation: &management,
2728 requested_fleet,
2729 },
2730 &targets,
2731 )?;
2732 if !compatible_root_start_prerequisite(retained_plan, ¤t, desired) {
2733 return Err(EnsureWorkflowError::DriftedBeforeApply);
2734 }
2735 let observation = root_management_fleet_observation(&management, &targets)?;
2736 return Ok((observation, current.conservation.observed_controlled_cycles));
2737 }
2738 let mut observation = platform
2739 .observe(&retained_plan.operation_id, state)
2740 .map_err(EnsureWorkflowError::Platform)?;
2741 let terminal_inventory_operation_id =
2742 fresh_plan_inventory_operation(root, retained_plan, state)?;
2743 if let Some(operation_id) = terminal_inventory_operation_id {
2744 attach_terminal_inventory_cycles(operation_id, state, platform, &mut observation)?;
2745 }
2746 let artifacts = resolve_desired_artifacts(root, desired)?;
2747 let mut retained_observations = 0_u32;
2748 let mut current = loop {
2749 let protocol_actions = platform
2750 .protocol_actions(protocol_operation(retained_plan), state)
2751 .map_err(EnsureWorkflowError::Platform)?;
2752 match compile_plan(
2753 desired,
2754 &artifacts,
2755 &protocol_actions,
2756 desired_sha256,
2757 requested_fleet,
2758 &observation,
2759 retained_plan.planned_at_time,
2760 &retained_plan.operation_id,
2761 None,
2762 ) {
2763 Ok(plan) => break plan,
2764 Err(EnsurePolicyError::PendingRootOwnedBalance { name }) => {
2765 retained_observations = retained_observations.saturating_add(1);
2766 if retained_observations >= desired.maximum_stalled_observations {
2767 return Err(EnsureWorkflowError::RootOwnedObservationStalled {
2768 last_lifecycle: root_owned_lifecycle_label(&observation, &name).to_string(),
2769 observations: retained_observations,
2770 target: name,
2771 });
2772 }
2773 platform.pace_root_owned_observation(&name, retained_observations);
2774 observation = platform
2775 .observe(&retained_plan.operation_id, state)
2776 .map_err(EnsureWorkflowError::Platform)?;
2777 if let Some(operation_id) = terminal_inventory_operation_id {
2778 attach_terminal_inventory_cycles(
2779 operation_id,
2780 state,
2781 platform,
2782 &mut observation,
2783 )?;
2784 }
2785 }
2786 Err(error) => return Err(error.into()),
2787 }
2788 };
2789 bind_terminal_inventory_operation(
2790 &mut current,
2791 retained_plan.terminal_inventory_operation_id.clone(),
2792 );
2793 if current.plan_sha256 != retained_plan.plan_sha256
2794 && !compatible_after_bounded_observation(retained_plan, ¤t, desired, &observation)
2795 {
2796 return Err(EnsureWorkflowError::DriftedBeforeApply);
2797 }
2798 Ok((observation, current.conservation.observed_controlled_cycles))
2799}
2800
2801fn estate_funding_requirement<E>(
2802 plan: &FleetEnsurePlan,
2803 state: &FleetEnsureStateRecord,
2804 observation: &FleetObservation,
2805) -> Result<Option<EstateFundingRequiredRecord>, EnsureWorkflowError<E>>
2806where
2807 E: std::error::Error + 'static,
2808{
2809 for reviewed in &plan.conservation.estate_funding_domains {
2810 let observed = observation
2811 .estate_funding_domains
2812 .get(&reviewed.root)
2813 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2814 if observed.cycles_ledger != reviewed.cycles_ledger {
2815 return Err(EnsureWorkflowError::PlanIntegrity);
2816 }
2817 let root_principal = observed
2818 .root_principal
2819 .as_deref()
2820 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2821 if reviewed_estate_root_principal(reviewed, state) != Some(root_principal) {
2822 return Err(EnsureWorkflowError::PlanIntegrity);
2823 }
2824 let available_cycles = observed
2825 .balance_cycles
2826 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2827 let reviewed_available = reviewed.available_cycles.unwrap_or_default();
2828 if available_cycles < reviewed_available {
2831 return Err(EnsureWorkflowError::DriftedBeforeApply);
2832 }
2833 if reviewed.required_creation_count == 0 {
2834 continue;
2835 }
2836 if available_cycles < reviewed.maximum_creation_debit_cycles {
2837 return Ok(Some(EstateFundingRequiredRecord {
2838 attempt_count: None,
2839 available_cycles,
2840 creation_amount_cycles: reviewed.creation_amount_cycles,
2841 creation_execution_margin_cycles: reviewed.creation_execution_margin_cycles,
2842 readiness_floor_cycles: reviewed.readiness_floor_cycles,
2843 cycles_ledger: reviewed.cycles_ledger.clone(),
2844 ledger_fee_cycles: reviewed.ledger_fee_cycles,
2845 management_creation_fee_cycles: reviewed.management_creation_fee_cycles,
2846 maximum_creation_debit_cycles: reviewed.maximum_creation_debit_cycles,
2847 maximum_creation_fee_cycles: reviewed.maximum_creation_fee_cycles,
2848 operation_id: plan.operation_id.clone(),
2849 pending_creation_operation_id: None,
2850 plan_sha256: plan.plan_sha256.clone(),
2851 required_creation_count: reviewed.required_creation_count,
2852 root: reviewed.root.clone(),
2853 root_principal: root_principal.to_string(),
2854 last_attempt_at_ns: None,
2855 retry_at_ns: None,
2856 shortfall_cycles: reviewed
2857 .maximum_creation_debit_cycles
2858 .saturating_sub(available_cycles),
2859 }));
2860 }
2861 }
2862 Ok(None)
2863}
2864
2865fn protocol_estate_funding_requirement<E>(
2866 plan: &FleetEnsurePlan,
2867 state: &FleetEnsureStateRecord,
2868 funding: &canic_contracts::dto::component_provisioning::RootEstateFundingRequired,
2869) -> Result<EstateFundingRequiredRecord, EnsureWorkflowError<E>>
2870where
2871 E: std::error::Error + 'static,
2872{
2873 let root_principal = funding.root.to_text();
2874 let reviewed = plan
2875 .conservation
2876 .estate_funding_domains
2877 .iter()
2878 .find(|domain| reviewed_estate_root_principal(domain, state) == Some(&root_principal))
2879 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2880 let creation_amount_cycles = funding.creation_amount.to_u128();
2881 let ledger_fee_cycles = funding.ledger_fee.to_u128();
2882 let management_creation_fee_cycles = funding.management_creation_fee.to_u128();
2883 let required_cycles = creation_amount_cycles
2884 .checked_add(ledger_fee_cycles)
2885 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2886 let maximum_creation_fee_cycles = management_creation_fee_cycles
2887 .checked_add(ledger_fee_cycles)
2888 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2889 let exact_reviewed_authority = reviewed.required_creation_count > 0
2890 && reviewed.cycles_ledger == funding.cycles_ledger.to_text()
2891 && reviewed.creation_amount_cycles == creation_amount_cycles
2892 && reviewed.creation_execution_margin_cycles == funding.execution_margin.to_u128()
2893 && reviewed.readiness_floor_cycles == funding.readiness_floor.to_u128()
2894 && reviewed.ledger_fee_cycles == ledger_fee_cycles
2895 && reviewed.management_creation_fee_cycles == management_creation_fee_cycles;
2896 let exact_runtime_arithmetic = funding.required.to_u128() == required_cycles
2897 && funding.available < funding.required
2898 && funding.shortfall.to_u128()
2899 == required_cycles.saturating_sub(funding.available.to_u128());
2900 if !exact_reviewed_authority || !exact_runtime_arithmetic {
2901 return Err(EnsureWorkflowError::PlanIntegrity);
2902 }
2903 Ok(EstateFundingRequiredRecord {
2904 attempt_count: Some(funding.attempt_count),
2905 available_cycles: funding.available.to_u128(),
2906 creation_amount_cycles,
2907 cycles_ledger: funding.cycles_ledger.to_text(),
2908 creation_execution_margin_cycles: funding.execution_margin.to_u128(),
2909 readiness_floor_cycles: funding.readiness_floor.to_u128(),
2910 ledger_fee_cycles,
2911 management_creation_fee_cycles,
2912 maximum_creation_debit_cycles: required_cycles,
2913 maximum_creation_fee_cycles,
2914 operation_id: plan.operation_id.clone(),
2915 pending_creation_operation_id: Some(canic_core::cdk::utils::hash::hex_bytes(
2916 funding.operation_id,
2917 )),
2918 plan_sha256: plan.plan_sha256.clone(),
2919 required_creation_count: 1,
2920 root: reviewed.root.clone(),
2921 root_principal,
2922 last_attempt_at_ns: funding.last_attempt_at_ns,
2923 retry_at_ns: Some(funding.retry_at_ns),
2924 shortfall_cycles: funding.shortfall.to_u128(),
2925 })
2926}
2927
2928fn estate_funding_error(required: &EstateFundingRequiredRecord) -> EstateFundingRequiredError {
2929 EstateFundingRequiredError {
2930 attempt_count: required.attempt_count,
2931 available_cycles: required.available_cycles,
2932 creation_amount_cycles: required.creation_amount_cycles,
2933 creation_execution_margin_cycles: required.creation_execution_margin_cycles,
2934 readiness_floor_cycles: required.readiness_floor_cycles,
2935 cycles_ledger: required.cycles_ledger.clone(),
2936 ledger_fee_cycles: required.ledger_fee_cycles,
2937 management_creation_fee_cycles: required.management_creation_fee_cycles,
2938 maximum_creation_debit_cycles: required.maximum_creation_debit_cycles,
2939 maximum_creation_fee_cycles: required.maximum_creation_fee_cycles,
2940 pending_creation_operation_id: required.pending_creation_operation_id.clone(),
2941 required_creation_count: required.required_creation_count,
2942 root: required.root.clone(),
2943 root_principal: required.root_principal.clone(),
2944 last_attempt_at_ns: required.last_attempt_at_ns,
2945 retry_at_ns: required.retry_at_ns,
2946 shortfall_cycles: required.shortfall_cycles,
2947 }
2948}
2949
2950fn retain_estate_funding_pause<E>(
2951 paths: &EnsurePaths,
2952 journal: &mut FleetEnsureJournalRecord,
2953 required: Option<&EstateFundingRequiredRecord>,
2954) -> Result<bool, EnsureWorkflowError<E>>
2955where
2956 E: std::error::Error + 'static,
2957{
2958 if journal.estate_funding_required.as_ref() == required {
2959 return Ok(false);
2960 }
2961 journal.estate_funding_required = required.cloned();
2962 write_journal(paths, journal)?;
2963 Ok(true)
2964}
2965
2966fn reviewed_estate_root_principal<'a>(
2967 domain: &'a crate::fleet_ensure::model::EstateFundingDomainPlan,
2968 state: &'a FleetEnsureStateRecord,
2969) -> Option<&'a str> {
2970 domain
2971 .root_principal
2972 .as_deref()
2973 .or_else(|| {
2974 state
2975 .pending_principals
2976 .get(&domain.root)
2977 .map(String::as_str)
2978 })
2979 .or_else(|| state.principals.get(&domain.root).map(String::as_str))
2980}
2981
2982struct ContinuationObservation {
2984 first_action_sha256: String,
2985 observation: FleetObservation,
2986}
2987
2988fn prior_fleet_protocol_effect_started(
2989 actions: &[&EnsureAction],
2990 journal: &FleetEnsureJournalRecord,
2991 current_index: usize,
2992) -> bool {
2993 actions
2994 .iter()
2995 .take(current_index)
2996 .zip(&journal.effects)
2997 .any(|(action, effect)| {
2998 matches!(action, EnsureAction::FleetProtocol { .. })
2999 && matches!(effect.state, EffectState::Issued | EffectState::Applied)
3000 })
3001}
3002
3003fn protocol_operation(plan: &FleetEnsurePlan) -> &str {
3006 plan.terminal_inventory_operation_id
3007 .as_deref()
3008 .unwrap_or(&plan.operation_id)
3009}
3010
3011fn completed_inventory_operation<'a, E>(
3012 plan: &'a FleetEnsurePlan,
3013 state: &FleetEnsureStateRecord,
3014) -> Result<&'a str, EnsureWorkflowError<E>>
3015where
3016 E: std::error::Error + 'static,
3017{
3018 if plan.terminal_inventory_operation_id.is_some() {
3019 reviewed_terminal_inventory_operation(plan, state)?
3020 .ok_or(EnsureWorkflowError::PlanIntegrity)
3021 } else {
3022 Ok(&plan.operation_id)
3023 }
3024}
3025
3026fn fresh_plan_inventory_operation<'a, E>(
3027 root: &Path,
3028 plan: &'a FleetEnsurePlan,
3029 state: &FleetEnsureStateRecord,
3030) -> Result<Option<&'a str>, EnsureWorkflowError<E>>
3031where
3032 E: std::error::Error + 'static,
3033{
3034 if state.active_registry.is_some() && plan.terminal_inventory_operation_id.is_none() {
3035 crate::fleet_ensure::ops::infrastructure_bootstrap::convergence::verify_origin(
3036 &EnsurePaths::under(root, &plan.environment, &plan.fleet),
3037 plan,
3038 state,
3039 )?;
3040 Ok(None)
3041 } else {
3042 reviewed_terminal_inventory_operation(plan, state)
3043 }
3044}
3045
3046fn reviewed_terminal_inventory_operation<'a, E>(
3047 plan: &'a FleetEnsurePlan,
3048 state: &FleetEnsureStateRecord,
3049) -> Result<Option<&'a str>, EnsureWorkflowError<E>>
3050where
3051 E: std::error::Error + 'static,
3052{
3053 match (
3054 state.active_registry.is_some(),
3055 plan.terminal_inventory_operation_id.as_deref(),
3056 ) {
3057 (false, None) => Ok(None),
3058 (true, Some(operation_id)) if !operation_id.is_empty() => Ok(Some(operation_id)),
3059 _ => Err(EnsureWorkflowError::PlanIntegrity),
3060 }
3061}
3062
3063fn complete_root_start_prerequisite<P>(
3064 desired: &crate::fleet_ensure::model::DesiredFleet,
3065 retained_plan: &FleetEnsurePlan,
3066 platform: &mut P,
3067 state: &FleetEnsureStateRecord,
3068 journal: &FleetEnsureJournalRecord,
3069) -> Result<ActualCycleConservation, EnsureWorkflowError<P::Error>>
3070where
3071 P: EnsurePlatform,
3072{
3073 let targets = reviewed_root_start_targets(retained_plan)?;
3074 let management = platform
3075 .observe_root_management(state, &targets)
3076 .map_err(EnsureWorkflowError::Platform)?
3077 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
3078 let current = recompile_root_start_prerequisite_plan(
3079 RootStartPlanInput {
3080 state,
3081 authority: retained_plan.root_start_authority.as_deref(),
3082 created_at_time: retained_plan.planned_at_time,
3083 desired,
3084 desired_sha256: &retained_plan.desired_sha256,
3085 observation: &management,
3086 requested_fleet: &retained_plan.fleet,
3087 },
3088 &targets,
3089 )?;
3090 if !compatible_root_start_prerequisite(retained_plan, ¤t, desired) {
3091 return Err(EnsureWorkflowError::ConvergenceDrift);
3092 }
3093 if management.roots.iter().any(|(name, observed)| {
3094 targets.contains(name)
3095 && observed.live.status != crate::fleet_ensure::model::CanisterRuntimeStatus::Running
3096 }) {
3097 return Err(EnsureWorkflowError::ConvergenceDrift);
3098 }
3099 let terminal = root_management_fleet_observation(&management, &targets)?;
3100 verify_terminal_conservation(retained_plan, journal, state, &terminal)
3101}
3102
3103fn verify_terminal_root_start_replay<P>(
3104 desired: &crate::fleet_ensure::model::DesiredFleet,
3105 plan: &FleetEnsurePlan,
3106 journal: &FleetEnsureJournalRecord,
3107 requested_fleet: &str,
3108 platform: &mut P,
3109 state: &FleetEnsureStateRecord,
3110) -> Result<(), EnsureWorkflowError<P::Error>>
3111where
3112 P: EnsurePlatform,
3113{
3114 verify_journal(journal, plan, requested_fleet, state)?;
3115 let actions = ordered_actions(plan);
3116 if actions.len() != journal.effects.len()
3117 || journal
3118 .effects
3119 .iter()
3120 .any(|effect| effect.state != EffectState::Applied)
3121 {
3122 return Err(EnsureWorkflowError::JournalIntegrity);
3123 }
3124 for (action, record) in actions.into_iter().zip(&journal.effects) {
3125 if action_sha256(action) != record.action_sha256
3126 || !platform
3127 .observe_effect(&journal.operation_id, action, record, state)
3128 .map_err(EnsureWorkflowError::Platform)?
3129 .applied
3130 {
3131 return Err(EnsureWorkflowError::ConvergenceDrift);
3132 }
3133 }
3134 let targets = reviewed_root_start_targets(plan)?;
3135 let management = platform
3136 .observe_root_management(state, &targets)
3137 .map_err(EnsureWorkflowError::Platform)?
3138 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
3139 let current = recompile_root_start_prerequisite_plan(
3140 RootStartPlanInput {
3141 state,
3142 authority: plan.root_start_authority.as_deref(),
3143 created_at_time: plan.planned_at_time,
3144 desired,
3145 desired_sha256: &plan.desired_sha256,
3146 observation: &management,
3147 requested_fleet: &plan.fleet,
3148 },
3149 &targets,
3150 )?;
3151 if normalized_plan(plan) != normalized_plan(¤t)
3152 || management.roots.iter().any(|(name, observed)| {
3153 targets.contains(name)
3154 && observed.live.status
3155 != crate::fleet_ensure::model::CanisterRuntimeStatus::Running
3156 })
3157 {
3158 return Err(EnsureWorkflowError::ConvergenceDrift);
3159 }
3160 Ok(())
3161}
3162
3163fn reviewed_root_start_targets<E>(
3164 plan: &FleetEnsurePlan,
3165) -> Result<BTreeSet<String>, EnsureWorkflowError<E>>
3166where
3167 E: std::error::Error + 'static,
3168{
3169 if plan.scope != FleetEnsurePlanScope::RootStartPrerequisite
3170 || !plan.protocol_actions.is_empty()
3171 || plan.canisters.is_empty()
3172 || plan.conservation.maximum_new_funding_cycles != 0
3173 || plan.conservation.maximum_operator_debit_cycles != 0
3174 || plan.conservation.maximum_unavoidable_fee_cycles != 0
3175 || !plan.conservation.estate_funding_domains.is_empty()
3176 || plan.conservation.scheduled_transfer_cycles != 0
3177 || plan.conservation.observed_controlled_cycles
3178 != plan.conservation.retained_in_reused_canisters_cycles
3179 || plan
3180 .conservation
3181 .observed_controlled_cycles
3182 .checked_sub(plan.conservation.maximum_execution_burn_cycles)
3183 != Some(plan.conservation.expected_post_operation_cycles)
3184 {
3185 return Err(EnsureWorkflowError::PlanIntegrity);
3186 }
3187 let mut targets = BTreeSet::new();
3188 for canister in &plan.canisters {
3189 if canister.disposition != CanisterDisposition::Reuse
3190 || canister.principal.is_none()
3191 || !matches!(
3192 canister.actions.as_slice(),
3193 [EnsureAction::Start { name, principal }]
3194 if name == &canister.name
3195 && Some(principal) == canister.principal.as_ref()
3196 )
3197 || !targets.insert(canister.name.clone())
3198 {
3199 return Err(EnsureWorkflowError::PlanIntegrity);
3200 }
3201 }
3202 Ok(targets)
3203}
3204
3205fn root_management_fleet_observation<E>(
3206 management: &RootManagementObservation,
3207 targets: &BTreeSet<String>,
3208) -> Result<FleetObservation, EnsureWorkflowError<E>>
3209where
3210 E: std::error::Error + 'static,
3211{
3212 let canisters = targets
3213 .iter()
3214 .map(|name| {
3215 management
3216 .roots
3217 .get(name)
3218 .map(|observed| (name.clone(), Some(observed.live.clone())))
3219 .ok_or(EnsureWorkflowError::PlanIntegrity)
3220 })
3221 .collect::<Result<BTreeMap<_, _>, _>>()?;
3222 Ok(FleetObservation {
3223 additional_controlled_cycles: BTreeMap::new(),
3224 canisters,
3225 estate_funding_domains: BTreeMap::new(),
3226 ledger_fee_cycles: 0,
3227 operator_cycles: management.operator_cycles,
3228 protocol_ready: BTreeMap::new(),
3229 })
3230}
3231
3232fn compatible_root_start_prerequisite(
3233 retained: &FleetEnsurePlan,
3234 current: &FleetEnsurePlan,
3235 desired: &crate::fleet_ensure::model::DesiredFleet,
3236) -> bool {
3237 let Ok(maximum_movement) = desired
3238 .maximum_update_burn_cycles
3239 .parse::<Cycles>()
3240 .map(|cycles| cycles.to_u128())
3241 else {
3242 return false;
3243 };
3244 retained.canisters.len() == current.canisters.len()
3245 && retained
3246 .canisters
3247 .iter()
3248 .zip(¤t.canisters)
3249 .all(|(retained, current)| {
3250 retained
3251 .observed_cycles
3252 .saturating_sub(current.observed_cycles)
3253 <= maximum_movement
3254 })
3255 && normalized_plan(retained) == normalized_plan(current)
3256}
3257
3258fn attach_terminal_inventory_cycles<P>(
3259 operation_id: &str,
3260 state: &FleetEnsureStateRecord,
3261 platform: &mut P,
3262 observation: &mut FleetObservation,
3263) -> Result<(), EnsureWorkflowError<P::Error>>
3264where
3265 P: EnsurePlatform,
3266{
3267 let inventory = platform
3268 .terminal_inventory(operation_id, state)
3269 .map_err(EnsureWorkflowError::Platform)?;
3270 attach_terminal_cycles(observation, inventory.controlled_cycles_by_principal)
3271}
3272
3273pub(super) fn completed_infrastructure_reinstall(
3274 desired: &crate::fleet_ensure::model::DesiredFleet,
3275 plan: &FleetEnsurePlan,
3276 journal: &FleetEnsureJournalRecord,
3277) -> bool {
3278 if desired.protocol.is_none() {
3279 return false;
3280 }
3281 let infrastructure = desired.canisters.iter().filter(|canister| {
3282 canister.presence == crate::fleet_ensure::model::DesiredPresence::Present
3283 && matches!(
3284 canister.kind,
3285 crate::fleet_ensure::model::DesiredCanisterKind::Coordinator
3286 | crate::fleet_ensure::model::DesiredCanisterKind::Root
3287 | crate::fleet_ensure::model::DesiredCanisterKind::Store
3288 )
3289 });
3290 let mut count = 0_usize;
3291 for configured in infrastructure {
3292 count += 1;
3293 let Some(action) = plan
3294 .canisters
3295 .iter()
3296 .find(|canister| {
3297 canister.name == configured.name
3298 && canister.disposition == CanisterDisposition::Reinstall
3299 })
3300 .and_then(|canister| {
3301 canister.actions.iter().find(|action| {
3302 matches!(
3303 action,
3304 EnsureAction::Install {
3305 mode: crate::fleet_ensure::model::InstallMode::Reinstall,
3306 name,
3307 ..
3308 } if name == &configured.name
3309 )
3310 })
3311 })
3312 else {
3313 return false;
3314 };
3315 let action_sha256 = action_sha256(action);
3316 if !journal.effects.iter().any(|effect| {
3317 effect.action_sha256 == action_sha256 && effect.state == EffectState::Applied
3318 }) {
3319 return false;
3320 }
3321 }
3322 count > 0
3323}
3324
3325fn compatible_after_bounded_observation(
3326 retained: &FleetEnsurePlan,
3327 current: &FleetEnsurePlan,
3328 desired: &crate::fleet_ensure::model::DesiredFleet,
3329 observation: &FleetObservation,
3330) -> bool {
3331 let Ok(maximum_observation_movement) = desired
3332 .maximum_observation_burn_cycles
3333 .parse::<Cycles>()
3334 .map(|cycles| cycles.to_u128())
3335 else {
3336 return false;
3337 };
3338 if retained.canisters.len() != current.canisters.len() {
3339 return false;
3340 }
3341 for (retained_canister, current_canister) in retained.canisters.iter().zip(¤t.canisters) {
3342 let movement = retained_canister
3343 .observed_cycles
3344 .saturating_sub(current_canister.observed_cycles);
3345 if movement > maximum_observation_movement {
3346 return false;
3347 }
3348 }
3349 if !retained_funding_remains_sufficient(retained, desired, observation) {
3350 return false;
3351 }
3352 normalized_plan(retained) == normalized_plan(current)
3353}
3354
3355fn retained_funding_remains_sufficient(
3356 retained: &FleetEnsurePlan,
3357 desired: &crate::fleet_ensure::model::DesiredFleet,
3358 observation: &FleetObservation,
3359) -> bool {
3360 retained
3361 .canisters
3362 .iter()
3363 .flat_map(|canister| &canister.actions)
3364 .filter_map(|action| match action {
3365 EnsureAction::Fund {
3366 amount,
3367 name,
3368 pool_funding,
3369 principal,
3370 ..
3371 } => Some((amount, name, pool_funding, principal)),
3372 _ => None,
3373 })
3374 .all(|(amount, name, pool_funding, principal)| {
3375 if let Some(root) = pool_funding {
3376 return observation
3377 .estate_funding_domains
3378 .values()
3379 .find(|domain| domain.root_principal.as_ref() == Some(&root.root))
3380 .and_then(|domain| domain.pool.as_ref())
3381 .is_some_and(|pool| {
3382 pool.assets.iter().any(|asset| {
3383 asset.principal == *principal
3384 && asset.lifecycle == root.lifecycle
3385 && asset
3386 .cycles
3387 .checked_add(*amount)
3388 .is_some_and(|funded| funded >= pool.readiness_floor_cycles)
3389 })
3390 });
3391 }
3392 let Some(live) = observation.canisters.get(name).and_then(Option::as_ref) else {
3393 return false;
3394 };
3395 let Some(configured) = desired
3396 .canisters
3397 .iter()
3398 .find(|configured| configured.name == *name)
3399 else {
3400 return false;
3401 };
3402 let Ok(minimum) = configured
3403 .minimum_cycles
3404 .parse::<Cycles>()
3405 .map(|cycles| cycles.to_u128())
3406 else {
3407 return false;
3408 };
3409 live.cycles
3410 .checked_add(*amount)
3411 .is_some_and(|funded| funded >= minimum)
3412 })
3413}
3414
3415fn normalized_plan(plan: &FleetEnsurePlan) -> FleetEnsurePlan {
3416 let mut normalized = plan.clone();
3417 normalized.plan_sha256.clear();
3418 normalized.reviewed_desired = None;
3419 if let Some(review) = &mut normalized.recovery_review {
3420 review.continuation_reserve_cycles = 0;
3422 for funding in &mut review.known_pool_funding {
3423 funding.amount_cycles = 0;
3424 funding.funding_deficit_cycles = 0;
3425 }
3426 }
3427 normalized.conservation = CycleConservation {
3428 estate_funding_domains: plan
3429 .conservation
3430 .estate_funding_domains
3431 .iter()
3432 .cloned()
3433 .map(|mut domain| {
3434 domain.available_cycles = None;
3435 domain.maximum_funding_cycles = 0;
3436 domain.root_principal = None;
3437 domain.shortfall_cycles = 0;
3438 domain
3439 })
3440 .collect(),
3441 expected_post_operation_cycles: 0,
3442 maximum_execution_burn_cycles: 0,
3443 maximum_new_funding_cycles: 0,
3444 maximum_operator_debit_cycles: 0,
3445 maximum_unavoidable_fee_cycles: 0,
3446 observed_controlled_cycles: 0,
3447 retained_in_reused_canisters_cycles: 0,
3448 scheduled_transfer_cycles: 0,
3449 };
3450 for canister in &mut normalized.canisters {
3451 canister.observed_cycles = 0;
3452 for action in &mut canister.actions {
3453 if let EnsureAction::Fund {
3454 amount,
3455 funding_deficit_cycles,
3456 ..
3457 } = action
3458 {
3459 *amount = 0;
3460 *funding_deficit_cycles = 0;
3461 }
3462 }
3463 }
3464 normalized
3465}
3466
3467fn attach_terminal_cycles<E>(
3468 observation: &mut FleetObservation,
3469 additional: BTreeMap<String, u128>,
3470) -> Result<(), EnsureWorkflowError<E>>
3471where
3472 E: std::error::Error + 'static,
3473{
3474 let mut retained = BTreeMap::new();
3475 for (principal, cycles) in additional {
3476 let Some(configured) = observation
3477 .canisters
3478 .values_mut()
3479 .filter_map(Option::as_mut)
3480 .find(|canister| canister.principal == principal)
3481 else {
3482 retained.insert(principal, cycles);
3483 continue;
3484 };
3485 match configured.root_owned_lifecycle {
3486 Some(crate::fleet_ensure::model::RootOwnedCanisterLifecycle::Workload) => {
3487 configured.cycles = configured.cycles.min(cycles);
3490 }
3491 Some(crate::fleet_ensure::model::RootOwnedCanisterLifecycle::Idle)
3492 if cycles >= configured.cycles =>
3493 {
3494 configured.cycles = cycles;
3497 }
3498 Some(crate::fleet_ensure::model::RootOwnedCanisterLifecycle::Idle) => {
3499 return Err(EnsureWorkflowError::TerminalInventory(
3500 TerminalInventoryError::IdleCycleObservationConflict {
3501 canister: principal,
3502 expected: configured.cycles,
3503 observed: cycles,
3504 },
3505 ));
3506 }
3507 _ => {
3508 return Err(EnsureWorkflowError::TerminalInventory(
3509 TerminalInventoryError::CycleObservationLifecycleConflict {
3510 canister: principal,
3511 },
3512 ));
3513 }
3514 }
3515 }
3516 observation.additional_controlled_cycles = retained;
3517 Ok(())
3518}
3519
3520fn funding_plan<'a, E: std::error::Error + 'static>(
3521 plan: &'a FleetEnsurePlan,
3522 journal: &FleetEnsureJournalRecord,
3523) -> Result<std::borrow::Cow<'a, FleetEnsurePlan>, EnsureWorkflowError<E>> {
3524 if journal.funding_reviews.is_empty()
3525 && journal.funding_observations.is_empty()
3526 && journal.bootstrap_registration_recovery.is_none()
3527 {
3528 return Ok(std::borrow::Cow::Borrowed(plan));
3529 }
3530 funding_observation::verify(plan, journal)?;
3531 let mut funded = plan.clone();
3532 funded.conservation =
3533 crate::fleet_ensure::ops::infrastructure_bootstrap::registration_recovery::conservation(
3534 plan, journal,
3535 )?;
3536 let (amount, fee) = funding::totals::<E>(journal, None)?;
3537 let conservation = &mut funded.conservation;
3538 conservation.maximum_execution_burn_cycles = conservation
3539 .maximum_execution_burn_cycles
3540 .checked_add(funding_observation::total(journal)?)
3541 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3542 conservation.maximum_operator_debit_cycles = conservation
3543 .maximum_operator_debit_cycles
3544 .checked_add(amount)
3545 .and_then(|n| n.checked_add(fee))
3546 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3547 conservation.maximum_new_funding_cycles = conservation
3548 .maximum_new_funding_cycles
3549 .checked_add(amount)
3550 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3551 conservation.maximum_unavoidable_fee_cycles = conservation
3552 .maximum_unavoidable_fee_cycles
3553 .checked_add(fee)
3554 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3555 for domain in &mut conservation.estate_funding_domains {
3556 domain.maximum_funding_cycles = domain
3557 .maximum_funding_cycles
3558 .checked_add(funding::totals::<E>(journal, Some(&domain.root))?.0)
3559 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3560 }
3561 Ok(std::borrow::Cow::Owned(funded))
3562}
3563
3564fn verify_terminal_conservation<E>(
3565 plan: &FleetEnsurePlan,
3566 journal: &FleetEnsureJournalRecord,
3567 state: &FleetEnsureStateRecord,
3568 terminal: &FleetObservation,
3569) -> Result<ActualCycleConservation, EnsureWorkflowError<E>>
3570where
3571 E: std::error::Error + 'static,
3572{
3573 verify_terminal_conservation_with_total(
3574 plan,
3575 journal,
3576 state,
3577 terminal,
3578 controlled_cycles(terminal)?,
3579 )
3580}
3581
3582fn verify_terminal_conservation_with_total<E: std::error::Error + 'static>(
3583 plan: &FleetEnsurePlan,
3584 journal: &FleetEnsureJournalRecord,
3585 state: &FleetEnsureStateRecord,
3586 terminal: &FleetObservation,
3587 final_controlled_cycles: u128,
3588) -> Result<ActualCycleConservation, EnsureWorkflowError<E>> {
3589 let funded = funding_plan(plan, journal)?;
3590 let conservation = &funded.conservation;
3591 let operator_source_cycles =
3592 crate::fleet_ensure::policy::operator_mint::operator_source(journal)
3593 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3594 let minimum_operator_cycles =
3595 operator_source_cycles.saturating_sub(conservation.maximum_operator_debit_cycles);
3596 if journal.completion == FleetEnsureCompletion::Converged
3597 && !(minimum_operator_cycles..=operator_source_cycles).contains(&terminal.operator_cycles)
3598 {
3599 return Err(EnsureWorkflowError::TerminalReplayBalanceChanged {
3600 fleet: plan.fleet.clone(),
3601 operation_id: plan.operation_id.clone(),
3602 plan_sha256: plan.plan_sha256.clone(),
3603 current_operator_cycles: terminal.operator_cycles,
3604 minimum_operator_cycles,
3605 operator_source_cycles,
3606 });
3607 }
3608 let operator_debit_cycles = operator_source_cycles
3609 .checked_sub(terminal.operator_cycles)
3610 .ok_or_else(|| {
3611 EnsureWorkflowError::Conservation(
3612 "operator balance increased during apply; review a new plan".to_string(),
3613 )
3614 })?;
3615 if operator_debit_cycles > conservation.maximum_operator_debit_cycles {
3616 return Err(EnsureWorkflowError::Conservation(format!(
3617 "operator debit {operator_debit_cycles} exceeded reviewed maximum {}",
3618 conservation.maximum_operator_debit_cycles
3619 )));
3620 }
3621 let received_new_funding_cycles = operator_debit_cycles
3622 .checked_sub(conservation.maximum_unavoidable_fee_cycles)
3623 .ok_or_else(|| {
3624 EnsureWorkflowError::Conservation(
3625 "operator debit was below the exact reviewed fee total".to_string(),
3626 )
3627 })?;
3628 if received_new_funding_cycles > conservation.maximum_new_funding_cycles {
3629 return Err(EnsureWorkflowError::Conservation(format!(
3630 "received funding {received_new_funding_cycles} exceeded reviewed maximum {}",
3631 conservation.maximum_new_funding_cycles
3632 )));
3633 }
3634 let (estate_funding_cycles, exact_estate_creation_fee_cycles) =
3635 reconcile_estate_funding(plan, journal, state, terminal)?;
3636 let available = journal
3637 .initial_controlled_cycles
3638 .checked_add(received_new_funding_cycles)
3639 .ok_or_else(|| {
3640 EnsureWorkflowError::Conservation(
3641 "terminal controlled-cycle arithmetic overflowed".to_string(),
3642 )
3643 })?;
3644 let after_estate_fees = available
3645 .checked_sub(exact_estate_creation_fee_cycles)
3646 .ok_or_else(|| {
3647 EnsureWorkflowError::Conservation(
3648 "terminal controlled cycles cannot cover reviewed estate creation fees".to_string(),
3649 )
3650 })?;
3651 let observed_net_cycle_debit_cycles = after_estate_fees.saturating_sub(final_controlled_cycles);
3654 let observed_net_cycle_credit_cycles =
3655 final_controlled_cycles.saturating_sub(after_estate_fees);
3656 if observed_net_cycle_debit_cycles > conservation.maximum_execution_burn_cycles {
3657 return Err(EnsureWorkflowError::Conservation(format!(
3658 "observed net cycle debit {observed_net_cycle_debit_cycles} exceeded reviewed maximum {}",
3659 conservation.maximum_execution_burn_cycles
3660 )));
3661 }
3662 Ok(ActualCycleConservation {
3663 estate_funding_cycles,
3664 exact_estate_creation_fee_cycles,
3665 exact_unavoidable_fee_cycles: conservation.maximum_unavoidable_fee_cycles,
3666 final_controlled_cycles,
3667 observed_net_cycle_debit_cycles,
3668 observed_starting_cycles: journal.initial_controlled_cycles,
3669 observed_net_cycle_credit_cycles,
3670 operator_debit_cycles,
3671 received_new_funding_cycles,
3672 })
3673}
3674
3675fn resolved_initial_pool_assets<E>(
3676 domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3677 state: &FleetEnsureStateRecord,
3678) -> Result<BTreeSet<String>, EnsureWorkflowError<E>>
3679where
3680 E: std::error::Error + 'static,
3681{
3682 domain
3683 .initial_pool_assets
3684 .iter()
3685 .map(|identity| {
3686 identity.strip_prefix("created:").map_or_else(
3687 || Ok(identity.clone()),
3688 |name| {
3689 state
3690 .pending_principals
3691 .get(name)
3692 .or_else(|| state.principals.get(name))
3693 .cloned()
3694 .ok_or(EnsureWorkflowError::JournalIntegrity)
3695 },
3696 )
3697 })
3698 .collect()
3699}
3700
3701fn exact_estate_creation_costs<E>(
3702 state: &FleetEnsureStateRecord,
3703 terminal: &FleetObservation,
3704 domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3705) -> Result<(u128, u128), EnsureWorkflowError<E>>
3706where
3707 E: std::error::Error + 'static,
3708{
3709 let observed = terminal
3710 .estate_funding_domains
3711 .get(&domain.root)
3712 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3713 let Some(pool) = observed.pool.as_ref() else {
3714 return if domain.required_creation_count == 0 {
3715 Ok((0, 0))
3716 } else {
3717 Err(EnsureWorkflowError::JournalIntegrity)
3718 };
3719 };
3720 if pool.pending_creation.is_some() {
3721 return Err(EnsureWorkflowError::JournalIntegrity);
3722 }
3723
3724 estate_creation_costs(state, terminal, domain)
3725}
3726
3727fn estate_creation_costs<E: std::error::Error + 'static>(
3728 state: &FleetEnsureStateRecord,
3729 observation: &FleetObservation,
3730 domain: &EstateFundingDomainPlan,
3731) -> Result<(u128, u128), EnsureWorkflowError<E>> {
3732 let pool = observation
3733 .estate_funding_domains
3734 .get(&domain.root)
3735 .and_then(|domain| domain.pool.as_ref())
3736 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3737 let initial = resolved_initial_pool_assets(domain, state)?;
3738 let terminal_principals = pool
3739 .assets
3740 .iter()
3741 .map(|asset| asset.principal.clone())
3742 .collect::<BTreeSet<_>>();
3743 if terminal_principals.len() != pool.assets.len() || !initial.is_subset(&terminal_principals) {
3744 return Err(EnsureWorkflowError::JournalIntegrity);
3745 }
3746 let created = pool
3747 .assets
3748 .iter()
3749 .filter(|asset| !initial.contains(&asset.principal))
3750 .collect::<Vec<_>>();
3751 let creation_count_is_bounded = created.len()
3752 <= usize::try_from(domain.required_creation_count)
3753 .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
3754 let pool_policy_is_exact = created.is_empty() || terminal_pool_policy_is_exact(pool, domain);
3757 if !creation_count_is_bounded || !pool_policy_is_exact {
3758 return Err(EnsureWorkflowError::JournalIntegrity);
3759 }
3760
3761 let mut operation_ids = BTreeSet::new();
3762 let mut block_indexes = BTreeSet::new();
3763 let mut creation_debit = 0_u128;
3764 let mut creation_fee = 0_u128;
3765 for asset in created {
3766 let (asset_debit, asset_fee) = exact_estate_creation_receipt_costs(
3767 asset,
3768 domain,
3769 &mut operation_ids,
3770 &mut block_indexes,
3771 )?;
3772 creation_debit = creation_debit.checked_add(asset_debit).ok_or_else(|| {
3773 EnsureWorkflowError::Conservation(
3774 "estate creation-debit arithmetic overflowed".to_string(),
3775 )
3776 })?;
3777 creation_fee = creation_fee.checked_add(asset_fee).ok_or_else(|| {
3778 EnsureWorkflowError::Conservation(
3779 "estate creation-fee arithmetic overflowed".to_string(),
3780 )
3781 })?;
3782 }
3783 if creation_debit > domain.maximum_creation_debit_cycles
3784 || creation_fee > domain.maximum_creation_fee_cycles
3785 {
3786 return Err(EnsureWorkflowError::JournalIntegrity);
3787 }
3788 Ok((creation_debit, creation_fee))
3789}
3790
3791const fn terminal_pool_policy_is_exact(
3792 pool: &crate::fleet_ensure::model::EstatePoolInventoryObservation,
3793 domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3794) -> bool {
3795 let expected_maximum_size = domain.pool_maximum_size;
3796 let expected_readiness_floor = domain.readiness_floor_cycles;
3797 let expected_execution_margin = domain.creation_execution_margin_cycles;
3798 pool.maximum_size == expected_maximum_size
3799 && pool.readiness_floor_cycles == expected_readiness_floor
3800 && pool.creation_execution_margin_cycles == expected_execution_margin
3801}
3802
3803#[derive(Eq, PartialEq)]
3804struct EstateCreationReceiptAuthority<'a> {
3805 creation_execution_margin_cycles: u128,
3806 cycles_ledger: &'a str,
3807 ledger_amount_cycles: u128,
3808 ledger_fee_cycles: u128,
3809 management_creation_fee_cycles: u128,
3810 readiness_floor_cycles: u128,
3811}
3812
3813fn observed_estate_creation_authority(
3814 receipt: &crate::fleet_ensure::model::EstatePoolCreationReceiptObservation,
3815) -> EstateCreationReceiptAuthority<'_> {
3816 EstateCreationReceiptAuthority {
3817 creation_execution_margin_cycles: receipt.creation_execution_margin_cycles,
3818 cycles_ledger: &receipt.cycles_ledger,
3819 ledger_amount_cycles: receipt.ledger_amount_cycles,
3820 ledger_fee_cycles: receipt.ledger_fee_cycles,
3821 management_creation_fee_cycles: receipt.management_creation_fee_cycles,
3822 readiness_floor_cycles: receipt.readiness_floor_cycles,
3823 }
3824}
3825
3826fn reviewed_estate_creation_authority(
3827 domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3828) -> EstateCreationReceiptAuthority<'_> {
3829 EstateCreationReceiptAuthority {
3830 creation_execution_margin_cycles: domain.creation_execution_margin_cycles,
3831 cycles_ledger: &domain.cycles_ledger,
3832 ledger_amount_cycles: domain.creation_amount_cycles,
3833 ledger_fee_cycles: domain.ledger_fee_cycles,
3834 management_creation_fee_cycles: domain.management_creation_fee_cycles,
3835 readiness_floor_cycles: domain.readiness_floor_cycles,
3836 }
3837}
3838
3839fn first_estate_creation_observation_is_exact(
3840 receipt: &crate::fleet_ensure::model::EstatePoolCreationReceiptObservation,
3841) -> bool {
3842 receipt
3843 .ledger_amount_cycles
3844 .checked_sub(receipt.management_creation_fee_cycles)
3845 .zip(receipt.first_observed_cycles)
3846 .is_some_and(|(funded_native_cycles, observed_cycles)| {
3847 observed_cycles >= receipt.readiness_floor_cycles
3848 && funded_native_cycles.saturating_sub(observed_cycles)
3849 <= receipt.creation_execution_margin_cycles
3850 })
3851}
3852
3853fn exact_estate_creation_receipt_costs<E>(
3854 asset: &crate::fleet_ensure::model::EstatePoolAssetObservation,
3855 domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3856 operation_ids: &mut BTreeSet<String>,
3857 block_indexes: &mut BTreeSet<u64>,
3858) -> Result<(u128, u128), EnsureWorkflowError<E>>
3859where
3860 E: std::error::Error + 'static,
3861{
3862 let receipt = asset
3863 .creation_receipt
3864 .as_ref()
3865 .filter(|_| asset.origin == crate::fleet_ensure::model::EstatePoolAssetOrigin::Created)
3866 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3867 let authority_is_exact =
3868 observed_estate_creation_authority(receipt) == reviewed_estate_creation_authority(domain);
3869 if !is_sha256_hex(&receipt.operation_id)
3870 || !authority_is_exact
3871 || !first_estate_creation_observation_is_exact(receipt)
3872 || !operation_ids.insert(receipt.operation_id.clone())
3873 || !block_indexes.insert(receipt.block_index)
3874 {
3875 return Err(EnsureWorkflowError::JournalIntegrity);
3876 }
3877 let debit = receipt
3878 .ledger_amount_cycles
3879 .checked_add(receipt.ledger_fee_cycles)
3880 .ok_or_else(|| {
3881 EnsureWorkflowError::Conservation(
3882 "estate creation-debit arithmetic overflowed".to_string(),
3883 )
3884 })?;
3885 let fee = receipt
3886 .management_creation_fee_cycles
3887 .checked_add(receipt.ledger_fee_cycles)
3888 .ok_or_else(|| {
3889 EnsureWorkflowError::Conservation(
3890 "estate creation-fee arithmetic overflowed".to_string(),
3891 )
3892 })?;
3893 Ok((debit, fee))
3894}
3895
3896fn reconcile_estate_funding<E>(
3897 plan: &FleetEnsurePlan,
3898 journal: &FleetEnsureJournalRecord,
3899 state: &FleetEnsureStateRecord,
3900 terminal: &FleetObservation,
3901) -> Result<(u128, u128), EnsureWorkflowError<E>>
3902where
3903 E: std::error::Error + 'static,
3904{
3905 plan.conservation
3906 .estate_funding_domains
3907 .iter()
3908 .try_fold((0_u128, 0_u128), |(funding_total, fee_total), domain| {
3909 let initial = journal
3910 .initial_estate_funding_cycles_by_root
3911 .get(&domain.root)
3912 .copied()
3913 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3914 let observed = terminal
3915 .estate_funding_domains
3916 .get(&domain.root)
3917 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3918 if observed.cycles_ledger != domain.cycles_ledger
3919 || reviewed_estate_root_principal(domain, state)
3920 != observed.root_principal.as_deref()
3921 {
3922 return Err(EnsureWorkflowError::JournalIntegrity);
3923 }
3924 let terminal_balance = observed
3925 .balance_cycles
3926 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3927 let (creation_debit, creation_fee) = exact_estate_creation_costs(
3928 state,
3929 terminal,
3930 domain,
3931 )?;
3932 let funding = applied_estate_funding_for_domain(plan, journal, state, domain)?
3933 .checked_add(funding::totals::<E>(journal, Some(&domain.root))?.0)
3934 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3935 let expected_terminal_balance = initial
3936 .checked_add(funding)
3937 .and_then(|available| available.checked_sub(creation_debit))
3938 .ok_or_else(|| {
3939 EnsureWorkflowError::Conservation(format!(
3940 "Root {} funding-account balance cannot reconcile its reviewed funding and creation debit",
3941 domain.root
3942 ))
3943 })?;
3944 if terminal_balance < expected_terminal_balance {
3947 return Err(EnsureWorkflowError::Conservation(format!(
3948 "Root {} funding-account balance {terminal_balance} differs from reviewed terminal balance {expected_terminal_balance}",
3949 domain.root
3950 )));
3951 }
3952 Ok((
3953 funding_total.checked_add(funding).ok_or_else(|| {
3954 EnsureWorkflowError::Conservation(
3955 "estate funding arithmetic overflowed".to_string(),
3956 )
3957 })?,
3958 fee_total
3959 .checked_add(creation_fee)
3960 .ok_or_else(|| {
3961 EnsureWorkflowError::Conservation(
3962 "estate creation-fee arithmetic overflowed".to_string(),
3963 )
3964 })?,
3965 ))
3966 })
3967}
3968
3969fn applied_estate_funding_for_domain<E>(
3970 plan: &FleetEnsurePlan,
3971 journal: &FleetEnsureJournalRecord,
3972 state: &FleetEnsureStateRecord,
3973 domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3974) -> Result<u128, EnsureWorkflowError<E>>
3975where
3976 E: std::error::Error + 'static,
3977{
3978 let action = plan
3979 .canisters
3980 .iter()
3981 .find(|canister| canister.name == domain.root)
3982 .into_iter()
3983 .flat_map(|canister| &canister.actions)
3984 .find(|action| matches!(action, EnsureAction::FundEstate { .. }));
3985 let Some(action) = action else {
3986 return if domain.maximum_funding_cycles == 0 {
3987 Ok(0)
3988 } else {
3989 Err(EnsureWorkflowError::JournalIntegrity)
3990 };
3991 };
3992 let EnsureAction::FundEstate {
3993 amount,
3994 expected_post_cycles,
3995 ledger,
3996 ledger_fee_cycles,
3997 principal,
3998 ..
3999 } = action
4000 else {
4001 unreachable!("filtered estate-funding action")
4002 };
4003 let root_principal = principal.strip_prefix("created:").map_or_else(
4004 || Some(principal.as_str()),
4005 |name| {
4006 state
4007 .pending_principals
4008 .get(name)
4009 .or_else(|| state.principals.get(name))
4010 .map(String::as_str)
4011 },
4012 );
4013 let expected_destination = domain
4014 .available_cycles
4015 .unwrap_or_default()
4016 .checked_add(domain.shortfall_cycles)
4017 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
4018 let action_authority_is_exact = *amount == domain.shortfall_cycles
4019 && *expected_post_cycles == expected_destination
4020 && ledger == &domain.cycles_ledger
4021 && *ledger_fee_cycles == domain.ledger_fee_cycles
4022 && root_principal == reviewed_estate_root_principal(domain, state);
4023 let effect = journal
4024 .effects
4025 .iter()
4026 .find(|effect| effect.action_sha256 == action_sha256(action));
4027 let Some(effect) = effect.filter(|effect| {
4028 effect.state == EffectState::Applied
4029 && effect
4030 .receipt
4031 .as_deref()
4032 .is_some_and(|receipt| !receipt.is_empty())
4033 }) else {
4034 return Err(EnsureWorkflowError::JournalIntegrity);
4035 };
4036 let balance_evidence_is_exact = crate::fleet_ensure::ops::estate_funding_applied(
4037 crate::fleet_ensure::ops::EstateFundingObservation {
4038 amount: *amount,
4039 destination_after: effect.destination_post_cycles.unwrap_or_default(),
4040 destination_before: effect.destination_pre_cycles,
4041 expected_destination_after: *expected_post_cycles,
4042 ledger_fee_cycles: *ledger_fee_cycles,
4043 source_after: effect.post_cycles.unwrap_or_default(),
4044 source_before: effect.pre_cycles,
4045 },
4046 );
4047 if !action_authority_is_exact || !balance_evidence_is_exact {
4048 return Err(EnsureWorkflowError::JournalIntegrity);
4049 }
4050 Ok(*amount)
4051}
4052
4053fn controlled_cycles<E>(observation: &FleetObservation) -> Result<u128, EnsureWorkflowError<E>>
4054where
4055 E: std::error::Error + 'static,
4056{
4057 observation
4058 .canisters
4059 .values()
4060 .filter_map(Option::as_ref)
4061 .try_fold(0_u128, |total, live| total.checked_add(live.cycles))
4062 .and_then(|total| {
4063 observation
4064 .additional_controlled_cycles
4065 .values()
4066 .try_fold(total, |subtotal, cycles| subtotal.checked_add(*cycles))
4067 })
4068 .and_then(|total| {
4069 observation
4070 .estate_funding_domains
4071 .values()
4072 .filter_map(|domain| domain.balance_cycles)
4073 .try_fold(total, u128::checked_add)
4074 })
4075 .ok_or_else(|| {
4076 EnsureWorkflowError::Conservation("controlled-cycle total overflowed".to_string())
4077 })
4078}
4079
4080fn verified_plan<E>(plan: FleetEnsurePlan) -> Result<FleetEnsurePlan, EnsureWorkflowError<E>>
4081where
4082 E: std::error::Error + 'static,
4083{
4084 if plan.reviewed_desired.is_none()
4085 || expected_plan_sha256(&plan) != plan.plan_sha256
4086 || (plan.scope == FleetEnsurePlanScope::InfrastructureBootstrap)
4087 != plan.infrastructure_bootstrap.is_some()
4088 {
4089 return Err(EnsureWorkflowError::PlanIntegrity);
4090 }
4091 Ok(plan)
4092}
4093
4094fn verify_journal<E>(
4095 journal: &FleetEnsureJournalRecord,
4096 plan: &FleetEnsurePlan,
4097 requested_fleet: &str,
4098 state: &FleetEnsureStateRecord,
4099) -> Result<(), EnsureWorkflowError<E>>
4100where
4101 E: std::error::Error + 'static,
4102{
4103 verify_journal_integrity(journal, plan, requested_fleet, state)?;
4104 if let Some(review) = journal
4105 .funding_reviews
4106 .iter()
4107 .filter_map(|r| r.operator_mint.as_ref())
4108 .find(|review| review.receipt.is_none())
4109 {
4110 return Err(EnsureWorkflowError::OperatorMintPending {
4111 review_sha256: review.review_sha256.clone(),
4112 });
4113 }
4114 Ok(())
4115}
4116
4117fn verify_journal_integrity<E>(
4118 journal: &FleetEnsureJournalRecord,
4119 plan: &FleetEnsurePlan,
4120 requested_fleet: &str,
4121 state: &FleetEnsureStateRecord,
4122) -> Result<(), EnsureWorkflowError<E>>
4123where
4124 E: std::error::Error + 'static,
4125{
4126 if journal.fleet != requested_fleet
4127 || journal.operation_id != plan.operation_id
4128 || journal.plan_sha256 != plan.plan_sha256
4129 || !initial_estate_funding_is_exact(journal, plan)
4130 || !estate_funding_pause_is_exact(journal, plan, state)
4131 {
4132 return Err(EnsureWorkflowError::JournalIntegrity);
4133 }
4134 crate::fleet_ensure::ops::infrastructure_bootstrap::registration_recovery::verify(
4135 plan, journal, state,
4136 )?;
4137 funding::verify(plan, journal, state)?;
4138 funding_observation::verify(plan, journal)?;
4139 operator_mint::verify(plan, journal)?;
4140 continuation::verify_records(plan, journal)?;
4141 let actions = continuation::actions(plan, journal);
4142 let effect_count_matches = journal.effects.len() <= actions.len();
4143 let action_hashes_match = journal.effects.iter().zip(actions).all(|(effect, action)| {
4144 effect.action_sha256 == action_sha256(action) && effect.attempts_match(action)
4145 });
4146 if !(effect_count_matches && action_hashes_match) {
4147 return Err(EnsureWorkflowError::JournalIntegrity);
4148 }
4149 Ok(())
4150}
4151
4152fn initial_estate_funding_is_exact(
4153 journal: &FleetEnsureJournalRecord,
4154 plan: &FleetEnsurePlan,
4155) -> bool {
4156 journal.initial_estate_funding_cycles_by_root.len()
4157 == plan.conservation.estate_funding_domains.len()
4158 && plan
4159 .conservation
4160 .estate_funding_domains
4161 .iter()
4162 .all(|domain| {
4163 journal
4164 .initial_estate_funding_cycles_by_root
4165 .get(&domain.root)
4166 .copied()
4167 == Some(domain.available_cycles.unwrap_or_default())
4168 })
4169}
4170
4171fn estate_funding_pause_is_exact(
4172 journal: &FleetEnsureJournalRecord,
4173 plan: &FleetEnsurePlan,
4174 state: &FleetEnsureStateRecord,
4175) -> bool {
4176 let Some(required) = journal.estate_funding_required.as_ref() else {
4177 return true;
4178 };
4179 journal.completion == FleetEnsureCompletion::InProgress
4180 && estate_funding_record_is_exact(required, plan, state)
4181}
4182
4183fn estate_funding_record_is_exact(
4184 required: &EstateFundingRequiredRecord,
4185 plan: &FleetEnsurePlan,
4186 state: &FleetEnsureStateRecord,
4187) -> bool {
4188 let Some(reviewed) = plan
4189 .conservation
4190 .estate_funding_domains
4191 .iter()
4192 .find(|domain| domain.root == required.root)
4193 else {
4194 return false;
4195 };
4196 let operation_authority_is_exact =
4197 required.operation_id == plan.operation_id && required.plan_sha256 == plan.plan_sha256;
4198 let funding_authority_is_exact =
4199 EstateFundingPauseAuthority::from(required) == EstateFundingPauseAuthority::from(reviewed);
4200 let root_authority_is_exact =
4201 reviewed_estate_root_principal(reviewed, state) == Some(required.root_principal.as_str());
4202 let shortfall_is_exact = required.available_cycles < required.maximum_creation_debit_cycles
4203 && required.shortfall_cycles
4204 == required
4205 .maximum_creation_debit_cycles
4206 .saturating_sub(required.available_cycles);
4207 if !(operation_authority_is_exact
4208 && funding_authority_is_exact
4209 && root_authority_is_exact
4210 && shortfall_is_exact)
4211 {
4212 return false;
4213 }
4214 if required.pending_creation_operation_id.is_some() {
4215 current_creation_funding_pause_is_exact(required, reviewed)
4216 } else {
4217 planned_estate_funding_pause_is_exact(required, reviewed)
4218 }
4219}
4220
4221#[derive(Eq, PartialEq)]
4222struct EstateFundingPauseAuthority<'a> {
4223 creation_amount_cycles: u128,
4224 creation_execution_margin_cycles: u128,
4225 cycles_ledger: &'a str,
4226 ledger_fee_cycles: u128,
4227 management_creation_fee_cycles: u128,
4228 readiness_floor_cycles: u128,
4229 root: &'a str,
4230}
4231
4232impl<'a> From<&'a EstateFundingRequiredRecord> for EstateFundingPauseAuthority<'a> {
4233 fn from(required: &'a EstateFundingRequiredRecord) -> Self {
4234 Self {
4235 creation_amount_cycles: required.creation_amount_cycles,
4236 creation_execution_margin_cycles: required.creation_execution_margin_cycles,
4237 cycles_ledger: &required.cycles_ledger,
4238 ledger_fee_cycles: required.ledger_fee_cycles,
4239 management_creation_fee_cycles: required.management_creation_fee_cycles,
4240 readiness_floor_cycles: required.readiness_floor_cycles,
4241 root: &required.root,
4242 }
4243 }
4244}
4245
4246impl<'a> From<&'a EstateFundingDomainPlan> for EstateFundingPauseAuthority<'a> {
4247 fn from(reviewed: &'a EstateFundingDomainPlan) -> Self {
4248 Self {
4249 creation_amount_cycles: reviewed.creation_amount_cycles,
4250 creation_execution_margin_cycles: reviewed.creation_execution_margin_cycles,
4251 cycles_ledger: &reviewed.cycles_ledger,
4252 ledger_fee_cycles: reviewed.ledger_fee_cycles,
4253 management_creation_fee_cycles: reviewed.management_creation_fee_cycles,
4254 readiness_floor_cycles: reviewed.readiness_floor_cycles,
4255 root: &reviewed.root,
4256 }
4257 }
4258}
4259
4260fn current_creation_funding_pause_is_exact(
4261 required: &EstateFundingRequiredRecord,
4262 reviewed: &EstateFundingDomainPlan,
4263) -> bool {
4264 let operation_id_is_exact = required
4265 .pending_creation_operation_id
4266 .as_deref()
4267 .is_some_and(is_sha256_hex);
4268 let attempt_evidence_is_exact = required.attempt_count.is_some()
4269 && required
4270 .retry_at_ns
4271 .is_some_and(|retry_at_ns| retry_at_ns > 0)
4272 && required
4273 .last_attempt_at_ns
4274 .is_none_or(|attempted_at_ns| attempted_at_ns > 0);
4275 let single_creation_debit = required
4276 .creation_amount_cycles
4277 .checked_add(required.ledger_fee_cycles);
4278 let single_creation_fees = required
4279 .management_creation_fee_cycles
4280 .checked_add(required.ledger_fee_cycles);
4281 let debit_authority_is_exact = Some(required.maximum_creation_debit_cycles)
4282 == single_creation_debit
4283 && Some(required.maximum_creation_fee_cycles) == single_creation_fees;
4284 reviewed.required_creation_count > 0
4285 && required.required_creation_count == 1
4286 && operation_id_is_exact
4287 && attempt_evidence_is_exact
4288 && debit_authority_is_exact
4289}
4290
4291const fn planned_estate_funding_pause_is_exact(
4292 required: &EstateFundingRequiredRecord,
4293 reviewed: &EstateFundingDomainPlan,
4294) -> bool {
4295 let runtime_attempt_is_absent = required.attempt_count.is_none()
4296 && required.last_attempt_at_ns.is_none()
4297 && required.retry_at_ns.is_none();
4298 let reviewed_debit_is_exact = required.maximum_creation_debit_cycles
4299 == reviewed.maximum_creation_debit_cycles
4300 && required.maximum_creation_fee_cycles == reviewed.maximum_creation_fee_cycles
4301 && required.required_creation_count == reviewed.required_creation_count;
4302 runtime_attempt_is_absent && reviewed_debit_is_exact
4303}
4304
4305fn is_sha256_hex(value: &str) -> bool {
4306 value.len() == 64
4307 && value
4308 .bytes()
4309 .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
4310}
4311
4312#[derive(Clone, Copy, Debug, Eq, PartialEq)]
4313enum TerminalStatePublicationError {
4314 JournalIntegrity,
4315}
4316
4317fn publish_terminal_state(
4318 desired: &crate::fleet_ensure::model::DesiredFleet,
4319 plan: &FleetEnsurePlan,
4320 journal: &FleetEnsureJournalRecord,
4321 state: &mut FleetEnsureStateRecord,
4322) -> Result<(), TerminalStatePublicationError> {
4323 let maximum_observation_burn_cycles =
4324 crate::fleet_ensure::ops::maximum_creation_observation_burn(desired)
4325 .ok_or(TerminalStatePublicationError::JournalIntegrity)?;
4326 let actions = continuation::actions(plan, journal);
4327 if actions.len() != journal.effects.len() {
4328 return Err(TerminalStatePublicationError::JournalIntegrity);
4329 }
4330 for (action, record) in actions.into_iter().zip(&journal.effects) {
4331 if action_sha256(action) != record.action_sha256 || record.state != EffectState::Applied {
4332 return Err(TerminalStatePublicationError::JournalIntegrity);
4333 }
4334 let EnsureAction::Create {
4335 name,
4336 requested_initial_cycles,
4337 ..
4338 } = action
4339 else {
4340 continue;
4341 };
4342 let Some(created_principal) = record.created_principal.as_deref() else {
4343 return Err(TerminalStatePublicationError::JournalIntegrity);
4344 };
4345 let Some(post_cycles) = record.post_cycles else {
4346 return Err(TerminalStatePublicationError::JournalIntegrity);
4347 };
4348 if !create_balance_is_terminal(
4349 Some(post_cycles),
4350 *requested_initial_cycles,
4351 maximum_observation_burn_cycles,
4352 ) || !create_identity_is_exact(name, record, state)
4353 || state
4354 .retained_cycles_by_principal
4355 .get(created_principal)
4356 .is_some_and(|retained| *retained != post_cycles)
4357 {
4358 return Err(TerminalStatePublicationError::JournalIntegrity);
4359 }
4360 state
4361 .retained_cycles_by_principal
4362 .insert(created_principal.to_string(), post_cycles);
4363 }
4364
4365 let prior_topology = state.topology.clone();
4366 for canister in &plan.canisters {
4367 if !desired
4368 .canisters
4369 .iter()
4370 .any(|configured| configured.name == canister.name)
4371 {
4372 if !is_reviewed_dynamic_pool_funding(plan, canister) {
4375 return Err(TerminalStatePublicationError::JournalIntegrity);
4376 }
4377 continue;
4378 }
4379
4380 match canister.disposition {
4381 CanisterDisposition::Create | CanisterDisposition::Replace => {
4382 if let Some(principal) = state.pending_principals.remove(&canister.name) {
4383 state.principals.insert(canister.name.clone(), principal);
4384 }
4385 }
4386 CanisterDisposition::Reinstall | CanisterDisposition::Reuse => {
4387 if let Some(principal) = state
4388 .pending_principals
4389 .remove(&canister.name)
4390 .or_else(|| canister.principal.clone())
4391 {
4392 state.principals.insert(canister.name.clone(), principal);
4393 }
4394 }
4395 CanisterDisposition::Delete => {
4396 state.principals.remove(&canister.name);
4397 state.pending_principals.remove(&canister.name);
4398 }
4399 }
4400 }
4401 publish_topology(desired, plan, &prior_topology, state);
4402 retain_configured_principal_bindings(state);
4403 Ok(())
4404}
4405
4406fn publish_topology(
4407 desired: &crate::fleet_ensure::model::DesiredFleet,
4408 plan: &FleetEnsurePlan,
4409 prior_topology: &BTreeMap<String, crate::fleet_ensure::model::FleetEnsureTopologyRecord>,
4410 state: &mut FleetEnsureStateRecord,
4411) {
4412 state.topology = desired
4413 .canisters
4414 .iter()
4415 .filter(|canister| {
4416 canister.presence == crate::fleet_ensure::model::DesiredPresence::Present
4417 && (plan.scope != FleetEnsurePlanScope::InfrastructureBootstrap
4418 || plan
4419 .canisters
4420 .iter()
4421 .any(|target| target.name == canister.name))
4422 })
4423 .map(|canister| {
4424 (
4425 canister.name.clone(),
4426 crate::fleet_ensure::model::FleetEnsureTopologyRecord {
4427 kind: canister.kind,
4428 module_hash: projected_module_hash(plan, prior_topology, canister),
4429 parent: canister.parent.clone(),
4430 protocol_binding: canister.protocol_binding.clone(),
4431 role: canister
4432 .protocol_binding
4433 .as_ref()
4434 .map(|binding| binding.role.to_string()),
4435 },
4436 )
4437 })
4438 .collect();
4439}
4440
4441fn is_reviewed_dynamic_pool_funding(
4442 plan: &FleetEnsurePlan,
4443 canister: &crate::fleet_ensure::model::CanisterPlan,
4444) -> bool {
4445 canister.disposition == CanisterDisposition::Reuse
4446 && !canister.actions.is_empty()
4447 && canister
4448 .actions
4449 .iter()
4450 .all(|action| is_reviewed_pool_funding(action, plan, canister.principal.as_deref()))
4451}
4452
4453fn is_reviewed_pool_funding(
4454 action: &EnsureAction,
4455 plan: &FleetEnsurePlan,
4456 expected_principal: Option<&str>,
4457) -> bool {
4458 let EnsureAction::Fund {
4459 pool_funding: Some(root),
4460 principal,
4461 ..
4462 } = action
4463 else {
4464 return false;
4465 };
4466 if Some(principal.as_str()) != expected_principal {
4467 return false;
4468 }
4469 plan.protocol_actions.iter().any(|candidate| {
4470 let EnsureAction::FleetProtocol {
4471 principal: target,
4472 action,
4473 ..
4474 } = candidate
4475 else {
4476 return false;
4477 };
4478 let CurrentFleetProtocolAction::ReconcilePoolAsset { request, .. } = action.as_ref() else {
4479 return false;
4480 };
4481 target == &root.root && request.canister_id.to_text() == *principal
4482 })
4483}
4484
4485fn projected_module_hash(
4486 plan: &FleetEnsurePlan,
4487 prior_topology: &BTreeMap<String, crate::fleet_ensure::model::FleetEnsureTopologyRecord>,
4488 canister: &crate::fleet_ensure::model::DesiredCanister,
4489) -> Option<String> {
4490 let installed = plan
4491 .canisters
4492 .iter()
4493 .find(|planned| planned.name == canister.name)
4494 .and_then(|planned| {
4495 planned
4496 .actions
4497 .iter()
4498 .rev()
4499 .find_map(|action| match action {
4500 EnsureAction::Install { wasm_sha256, .. } => Some(wasm_sha256.clone()),
4501 _ => None,
4502 })
4503 });
4504 installed.or_else(|| {
4505 prior_topology
4506 .get(&canister.name)
4507 .filter(|retained| retained.kind == canister.kind && retained.parent == canister.parent)
4508 .and_then(|retained| retained.module_hash.clone())
4509 })
4510}
4511
4512fn merge_terminal_inventory<E>(
4513 state: &mut FleetEnsureStateRecord,
4514 inventory: crate::fleet_ensure::ops::TerminalFleetInventory,
4515) -> Result<(), EnsureWorkflowError<E>>
4516where
4517 E: std::error::Error + 'static,
4518{
4519 let entries = inventory.entries;
4520 let mut names_by_principal = state
4521 .principals
4522 .iter()
4523 .map(|(name, principal)| (principal.clone(), name.clone()))
4524 .collect::<BTreeMap<_, _>>();
4525 if names_by_principal.len() != state.principals.len() {
4526 return Err(EnsureWorkflowError::TerminalInventory(
4527 TerminalInventoryError::DuplicateRetainedPrincipal,
4528 ));
4529 }
4530 for entry in &entries {
4531 names_by_principal
4532 .entry(entry.pid.clone())
4533 .or_insert_with(|| format!("observed:{}", entry.pid));
4534 }
4535 for entry in entries {
4536 let name = names_by_principal.get(&entry.pid).cloned().ok_or_else(|| {
4537 EnsureWorkflowError::TerminalInventory(TerminalInventoryError::LostPrincipalBinding {
4538 canister: entry.pid.clone(),
4539 })
4540 })?;
4541 let parent = entry
4542 .parent_pid
4543 .as_ref()
4544 .map(|principal| {
4545 names_by_principal.get(principal).cloned().ok_or_else(|| {
4546 EnsureWorkflowError::TerminalInventory(TerminalInventoryError::UnknownParent {
4547 canister: entry.pid.clone(),
4548 parent: principal.clone(),
4549 })
4550 })
4551 })
4552 .transpose()?;
4553 let existing = state.topology.get(&name);
4554 let kind = terminal_inventory_kind(existing, &entry);
4555 let parent_conflict = existing
4556 .and_then(|topology| topology.parent.as_ref())
4557 .zip(parent.as_ref())
4558 .filter(|(expected, actual)| expected != actual);
4559 if let Some((retained_parent, terminal_parent)) = parent_conflict
4560 && !verified_pool_to_component_parent_transition(existing, kind, &entry)
4561 {
4562 return Err(EnsureWorkflowError::TerminalInventory(
4563 TerminalInventoryError::RetainedParentConflict {
4564 canister: entry.pid,
4565 retained_parent: retained_parent.clone(),
4566 terminal_parent: terminal_parent.clone(),
4567 },
4568 ));
4569 }
4570 state.principals.insert(name.clone(), entry.pid);
4571 state.topology.insert(
4572 name,
4573 crate::fleet_ensure::model::FleetEnsureTopologyRecord {
4574 kind,
4575 module_hash: entry.module_hash,
4576 parent,
4577 protocol_binding: entry.protocol_binding,
4578 role: entry.role,
4579 },
4580 );
4581 }
4582 state.active_registry = inventory.active_registry;
4583 Ok(())
4584}
4585
4586fn terminal_inventory_kind(
4587 existing: Option<&crate::fleet_ensure::model::FleetEnsureTopologyRecord>,
4588 entry: &crate::registry::RegistryEntry,
4589) -> crate::fleet_ensure::model::DesiredCanisterKind {
4590 existing.map_or_else(
4591 || {
4592 if entry.module_hash.is_some() {
4593 crate::fleet_ensure::model::DesiredCanisterKind::Component
4594 } else {
4595 crate::fleet_ensure::model::DesiredCanisterKind::Auxiliary
4596 }
4597 },
4598 |topology| {
4599 if topology.kind == crate::fleet_ensure::model::DesiredCanisterKind::Pool
4600 && entry.module_hash.is_some()
4601 && entry.protocol_binding.is_some()
4602 {
4603 crate::fleet_ensure::model::DesiredCanisterKind::Component
4604 } else {
4605 topology.kind
4606 }
4607 },
4608 )
4609}
4610
4611fn verified_pool_to_component_parent_transition(
4612 existing: Option<&crate::fleet_ensure::model::FleetEnsureTopologyRecord>,
4613 terminal_kind: crate::fleet_ensure::model::DesiredCanisterKind,
4614 entry: &crate::registry::RegistryEntry,
4615) -> bool {
4616 existing.is_some_and(|topology| {
4617 topology.kind == crate::fleet_ensure::model::DesiredCanisterKind::Pool
4618 && terminal_kind == crate::fleet_ensure::model::DesiredCanisterKind::Component
4619 && entry.module_hash.is_some()
4620 && entry.protocol_binding.is_some()
4621 })
4622}
4623
4624fn applied_count(journal: &FleetEnsureJournalRecord) -> u32 {
4625 u32::try_from(
4626 journal
4627 .effects
4628 .iter()
4629 .chain(
4630 journal
4631 .funding_reviews
4632 .iter()
4633 .filter_map(|review| review.effect.as_ref()),
4634 )
4635 .filter(|effect| matches!(effect.state, EffectState::Applied))
4636 .count(),
4637 )
4638 .unwrap_or(u32::MAX)
4639}
4640
4641fn verify_store_activation_binding<P: EnsurePlatform>(
4642 plan: &FleetEnsurePlan,
4643 action: &EnsureAction,
4644 state: &FleetEnsureStateRecord,
4645 platform: &mut P,
4646) -> Result<(), EnsureWorkflowError<P::Error>> {
4647 use crate::fleet_ensure::model::DesiredCanisterInit;
4648 let EnsureAction::Install {
4649 canic_init: Some(DesiredCanisterInit::Store { root }),
4650 ..
4651 } = action
4652 else {
4653 return Ok(());
4654 };
4655 let root_will_be_installed = plan.canisters.iter().flat_map(|canister| &canister.actions)
4656 .any(|action| matches!(action,
4657 EnsureAction::Install { name, canic_init: Some(DesiredCanisterInit::Root { root: installed_root }), .. }
4658 if name == root && installed_root == root));
4659 if !root_will_be_installed {
4660 platform
4661 .require_retained_root_activation(&plan.operation_id, root, state)
4662 .map_err(EnsureWorkflowError::Platform)?;
4663 }
4664 Ok(())
4665}
4666
4667pub(super) fn ordered_actions(plan: &FleetEnsurePlan) -> Vec<&EnsureAction> {
4668 let mut actions = plan
4669 .canisters
4670 .iter()
4671 .flat_map(|canister| canister.actions.iter())
4672 .chain(plan.protocol_actions.iter())
4673 .collect::<Vec<_>>();
4674 if plan.scope == FleetEnsurePlanScope::ReinstallPreparation
4675 && plan
4676 .reinstall
4677 .as_ref()
4678 .is_some_and(|intent| intent.activation_reset.is_some())
4679 {
4680 actions.sort_by_key(|action| matches!(action, EnsureAction::Start { .. }));
4682 return actions;
4683 }
4684 if matches!(
4685 plan.scope,
4686 FleetEnsurePlanScope::RootReinstallPrerequisite
4687 | FleetEnsurePlanScope::InfrastructureBootstrap
4688 ) {
4689 return actions;
4690 }
4691 let temporary_pool_observation_finalizations = plan
4692 .reviewed_desired
4693 .as_deref()
4694 .map(|reviewed| {
4695 let desired = reviewed.desired();
4696 plan.canisters
4697 .iter()
4698 .filter_map(|canister_plan| {
4699 let configured = desired.canisters.iter().find(|configured| {
4700 configured.name == canister_plan.name
4701 && configured.kind
4702 == crate::fleet_ensure::model::DesiredCanisterKind::Pool
4703 })?;
4704 let temporary_create = desired
4705 .bootstrap
4706 .as_ref()
4707 .is_some_and(|bootstrap| bootstrap.fresh_estate)
4708 && configured.principal.is_none()
4709 && desired.bootstrap.as_ref().is_some_and(|bootstrap| {
4710 configured.controllers
4711 == bootstrap
4712 .recovery_controllers
4713 .iter()
4714 .map(candid::Principal::to_text)
4715 .collect::<Vec<_>>()
4716 })
4717 && configured.controller_canisters.len() == 1
4718 && canister_plan.actions.iter().any(|action| {
4719 matches!(
4720 action,
4721 EnsureAction::Create {
4722 controller_canisters,
4723 controllers,
4724 ..
4725 } if controller_canisters == &configured.controller_canisters
4726 && controllers == &{
4727 let mut expected = configured.controllers.clone();
4728 expected.push(desired.operator.clone());
4729 expected.sort();
4730 expected
4731 }
4732 )
4733 });
4734 temporary_create.then_some(canister_plan.name.as_str())
4735 })
4736 .collect::<BTreeSet<_>>()
4737 })
4738 .unwrap_or_default();
4739 actions.sort_by_key(|action| {
4740 if matches!(action, EnsureAction::SetControllers { name, .. } if temporary_pool_observation_finalizations.contains(name.as_str()))
4741 {
4742 6
4743 } else {
4744 action_order(action)
4745 }
4746 });
4747 actions
4748}
4749
4750fn deferred_create_observation_is_exact(
4751 desired: &crate::fleet_ensure::model::DesiredFleet,
4752 actions: &[&EnsureAction],
4753 index: usize,
4754 action: &EnsureAction,
4755 record: &EffectRecord,
4756 state: &FleetEnsureStateRecord,
4757) -> bool {
4758 let EnsureAction::Create {
4759 controller_canisters,
4760 controllers,
4761 name,
4762 ..
4763 } = action
4764 else {
4765 return false;
4766 };
4767 let Some(configured) = desired.canisters.iter().find(|canister| {
4768 canister.name == *name
4769 && canister.kind == crate::fleet_ensure::model::DesiredCanisterKind::Pool
4770 }) else {
4771 return false;
4772 };
4773 let Some(parent) = configured.parent.as_deref() else {
4774 return false;
4775 };
4776 let fresh_root_only_create = desired
4777 .bootstrap
4778 .as_ref()
4779 .is_some_and(|bootstrap| bootstrap.fresh_estate)
4780 && configured.principal.is_none()
4781 && configured.controllers.is_empty()
4782 && configured.controller_canisters.as_slice() == [parent]
4783 && controller_canisters == &configured.controller_canisters
4784 && controllers.is_empty();
4785 let exact_issued_create = record.state == EffectState::Issued
4786 && create_identity_is_exact(name, record, state)
4787 && record.post_cycles.is_some();
4788 let later_root_install = actions[index.saturating_add(1)..].iter().any(|candidate| {
4789 matches!(
4790 candidate,
4791 EnsureAction::Install {
4792 canic_init: Some(crate::fleet_ensure::model::DesiredCanisterInit::Root { root }),
4793 name: install_name,
4794 ..
4795 } if root == parent && install_name == parent
4796 )
4797 });
4798 let infrastructure_only = actions.iter().all(|candidate| {
4799 matches!(
4800 candidate,
4801 EnsureAction::Create { .. }
4802 | EnsureAction::FundEstate { .. }
4803 | EnsureAction::Install { .. }
4804 | EnsureAction::SetControllers { .. }
4805 )
4806 });
4807 fresh_root_only_create && exact_issued_create && later_root_install && infrastructure_only
4808}
4809
4810pub(super) const fn action_order(action: &EnsureAction) -> u8 {
4811 match action {
4812 EnsureAction::Create { .. } => 0,
4813 EnsureAction::Fund { .. } | EnsureAction::FundEstate { .. } => 1,
4814 EnsureAction::Install {
4815 canic_init: Some(crate::fleet_ensure::model::DesiredCanisterInit::Coordinator),
4816 ..
4817 } => 2,
4818 EnsureAction::SetControllers { .. } => 3,
4819 EnsureAction::Install {
4820 canic_init: Some(crate::fleet_ensure::model::DesiredCanisterInit::Store { .. }),
4821 ..
4822 }
4823 | EnsureAction::DeleteSnapshot { .. } => 4,
4824 EnsureAction::Uninstall { .. } | EnsureAction::Install { .. } => 5,
4825 EnsureAction::Start { .. } => 6,
4826 EnsureAction::FleetProtocol { .. } | EnsureAction::Protocol { .. } => 7,
4827 EnsureAction::Transfer { .. } => 8,
4828 EnsureAction::Stop { .. } => 9,
4829 EnsureAction::Delete { .. } => 10,
4830 }
4831}
4832
4833#[cfg(test)]
4834mod tests {
4835 use super::*;
4836
4837 #[test]
4838 fn store_binding_requires_exact_retained_root_or_reviewed_root_install() {
4839 use crate::fleet_ensure::{
4840 model::{CanisterDisposition, CanisterPlan, DesiredCanisterInit, InstallMode},
4841 tests::MockPlatform,
4842 };
4843 let (state, _) = retained_evidence();
4844 let mut plan = estate_funding_plan();
4845 let desired = serde_json::from_value(serde_json::json!({
4846 "canisters": [], "cycles_ledger": "um5iw-rqaaa-aaaaq-qaaba-cai",
4847 "environment": "local", "fleet": "fleet", "ledger_fee_cycles": "0",
4848 "management_creation_fee_cycles": "0", "material_cycle_threshold": "1",
4849 "maximum_observation_burn_cycles": "1", "maximum_stalled_observations": 2,
4850 "maximum_update_burn_cycles": "2", "operator": "rrkah-fqaaa-aaaaa-aaaaq-cai",
4851 "schema_version": FLEET_ENSURE_SCHEMA_VERSION, "treasury": "root"
4852 }))
4853 .unwrap();
4854 let mut platform = MockPlatform::new(desired, []);
4855 platform.rejected_activation_root = Some("root".into());
4856 let store = EnsureAction::Install {
4857 canic_init: Some(DesiredCanisterInit::Store {
4858 root: "root".into(),
4859 }),
4860 reinstall_witness: None,
4861 init_arg: None,
4862 init_arg_sha256: None,
4863 init_candid: None,
4864 init_candid_sha256: None,
4865 mode: InstallMode::Install,
4866 name: "store".into(),
4867 principal: "store-principal".into(),
4868 wasm: "store.wasm".into(),
4869 wasm_sha256: "store-hash".into(),
4870 };
4871 assert!(matches!(
4872 verify_store_activation_binding(&plan, &store, &state, &mut platform),
4873 Err(EnsureWorkflowError::Platform(_))
4874 ));
4875 assert_eq!(
4876 platform.retained_activation_checks,
4877 [(plan.operation_id.clone(), "root".into())]
4878 );
4879 let mut root_install = store.clone();
4880 if let EnsureAction::Install {
4881 name, canic_init, ..
4882 } = &mut root_install
4883 {
4884 *name = "other-root".into();
4885 *canic_init = Some(DesiredCanisterInit::Root {
4886 root: "other-root".into(),
4887 });
4888 }
4889 plan.canisters.push(CanisterPlan {
4890 actions: vec![root_install],
4891 disposition: CanisterDisposition::Create,
4892 name: "root".into(),
4893 observed_cycles: 0,
4894 principal: None,
4895 });
4896 assert!(verify_store_activation_binding(&plan, &store, &state, &mut platform).is_err());
4897 if let EnsureAction::Install {
4898 name, canic_init, ..
4899 } = &mut plan.canisters.last_mut().unwrap().actions[0]
4900 {
4901 *name = "root".into();
4902 *canic_init = Some(DesiredCanisterInit::Root {
4903 root: "root".into(),
4904 });
4905 }
4906 let reads = platform.retained_activation_checks.len();
4907 verify_store_activation_binding(&plan, &store, &state, &mut platform).unwrap();
4908 assert_eq!(platform.retained_activation_checks.len(), reads);
4909 plan.canisters.pop();
4910 platform.rejected_activation_root = None;
4911 verify_store_activation_binding(&plan, &store, &state, &mut platform).unwrap();
4912 assert_eq!(platform.retained_activation_checks.len(), reads + 1);
4913 }
4914
4915 #[test]
4916 fn root_prerequisite_retains_activation_authority_across_desired_refresh() {
4917 let (mut state, mut journal) = retained_evidence();
4918 let mut plan = estate_funding_plan();
4919 plan.scope = FleetEnsurePlanScope::RootReinstallPrerequisite;
4920 plan.conservation.estate_funding_domains.clear();
4921 journal.effects.clear();
4922 journal.completion = FleetEnsureCompletion::Converged;
4923 let before = state.clone();
4924 assert!(matches!(
4925 reconcile_retained_planning_state::<std::io::Error>(
4926 &mut state,
4927 Some(&plan),
4928 Some(&journal),
4929 "fleet",
4930 "local",
4931 "refreshed-imports"
4932 ),
4933 Err(EnsureWorkflowError::RootActivationDesiredConflict { .. })
4934 ));
4935 assert_eq!(state, before);
4936 assert!(
4937 reconcile_retained_planning_state::<std::io::Error>(
4938 &mut state,
4939 Some(&plan),
4940 Some(&journal),
4941 "fleet",
4942 "local",
4943 "desired"
4944 )
4945 .unwrap()
4946 .is_some()
4947 );
4948 }
4949
4950 fn terminal_component_entry(canister: &str, parent: &str) -> crate::registry::RegistryEntry {
4951 crate::registry::RegistryEntry {
4952 pid: canister.to_string(),
4953 role: Some("managed_component".to_string()),
4954 parent_pid: Some(parent.to_string()),
4955 module_hash: Some("11".repeat(32)),
4956 protocol_binding: Some(crate::protocol_binding::RegistryProtocolBinding {
4957 release_identity: "0.110.test".to_string(),
4958 role: canic_contracts::ids::CanisterRole::from("managed_component"),
4959 capabilities: BTreeSet::new(),
4960 candid_sha256: [1; 32],
4961 protocol_profile_digest:
4962 canic_core::role_contract::ProtocolProfileDigest::from_bytes([2; 32]),
4963 }),
4964 }
4965 }
4966
4967 pub(super) fn retained_evidence() -> (FleetEnsureStateRecord, FleetEnsureJournalRecord) {
4968 let state = FleetEnsureStateRecord {
4969 active_registry: None,
4970 completed_reinstall_action_sha256: BTreeMap::from([(
4971 "root".to_string(),
4972 "action".to_string(),
4973 )]),
4974 completed_reinstall_operation_id: Some("operation".to_string()),
4975 completed_reinstalls: BTreeMap::from([("root".to_string(), 7)]),
4976 fleet: "fleet".to_string(),
4977 pending_principals: BTreeMap::new(),
4978 principals: BTreeMap::new(),
4979 retained_cycles_by_principal: BTreeMap::new(),
4980 schema_version: crate::fleet_ensure::model::FLEET_ENSURE_SCHEMA_VERSION,
4981 topology: BTreeMap::new(),
4982 };
4983 let journal = FleetEnsureJournalRecord {
4984 release: None,
4985 bootstrap_registration_recovery: None,
4986 funding_observations: BTreeMap::new(),
4987 funding_reviews: Vec::new(),
4988 successor_phases: Vec::new(),
4989 completion: FleetEnsureCompletion::ReplanRequired,
4990 estate_funding_required: None,
4991 effects: vec![crate::fleet_ensure::model::EffectRecord {
4992 publication_attempts: 0,
4993 maintenance_attempts: 0,
4994 action_sha256: "action".to_string(),
4995 created_principal: None,
4996 destination_post_cycles: None,
4997 destination_pre_cycles: None,
4998 post_cycles: None,
4999 pre_cycles: None,
5000 pre_canister_version: Some(7),
5001 progress_identity: None,
5002 receipt: None,
5003 state: EffectState::Applied,
5004 }],
5005 fleet: "fleet".to_string(),
5006 initial_controlled_cycles: 0,
5007 initial_estate_funding_cycles_by_root: BTreeMap::new(),
5008 initial_operator_cycles: 0,
5009 operation_id: "operation".to_string(),
5010 plan_sha256: "plan".to_string(),
5011 schema_version: crate::fleet_ensure::model::FLEET_ENSURE_SCHEMA_VERSION,
5012 stalled_observations: 0,
5013 };
5014 (state, journal)
5015 }
5016
5017 pub(super) fn estate_funding_plan() -> FleetEnsurePlan {
5018 FleetEnsurePlan {
5019 continuation: None,
5020 canisters: Vec::new(),
5021 conservation: CycleConservation {
5022 estate_funding_domains: vec![crate::fleet_ensure::model::EstateFundingDomainPlan {
5023 allocated_workloads: 0,
5024 available_cycles: Some(40),
5025 available_pool_slots: 2,
5026 creation_amount_cycles: 45,
5027 creation_execution_margin_cycles: 5,
5028 readiness_floor_cycles: 35,
5029 cycles_ledger: "ledger".to_string(),
5030 eligible_ready_pool_assets: 0,
5031 initial_pool_assets: Vec::new(),
5032 ledger_fee_cycles: 5,
5033 management_creation_fee_cycles: 5,
5034 maximum_creation_debit_cycles: 100,
5035 maximum_creation_fee_cycles: 20,
5036 maximum_funding_cycles: 60,
5037 occupied_pool_assets: 0,
5038 pending_creation_count: 0,
5039 pending_creation: None,
5040 planned_initial_workloads: 2,
5041 pool_maximum_size: 2,
5042 required_creation_count: 2,
5043 root: "root".to_string(),
5044 root_principal: Some("rrkah-fqaaa-aaaaa-aaaaq-cai".to_string()),
5045 shortfall_cycles: 60,
5046 }],
5047 expected_post_operation_cycles: 80,
5048 maximum_execution_burn_cycles: 0,
5049 maximum_new_funding_cycles: 0,
5050 maximum_operator_debit_cycles: 0,
5051 maximum_unavoidable_fee_cycles: 0,
5052 observed_controlled_cycles: 40,
5053 retained_in_reused_canisters_cycles: 0,
5054 scheduled_transfer_cycles: 0,
5055 },
5056 desired_sha256: "desired".to_string(),
5057 environment: "local".to_string(),
5058 fleet: "fleet".to_string(),
5059 operation_id: "operation".to_string(),
5060 plan_sha256: "plan".to_string(),
5061 planned_at_time: 1,
5062 protocol_actions: Vec::new(),
5063 recovery_review: None,
5064 reinstall: None,
5065 infrastructure_bootstrap: None,
5066 root_reinstall_bindings: Vec::new(),
5067 root_start_authority: None,
5068 reviewed_desired: None,
5069 schema_version: FLEET_ENSURE_SCHEMA_VERSION,
5070 scope: FleetEnsurePlanScope::Full,
5071 terminal_inventory_operation_id: None,
5072 }
5073 }
5074
5075 pub(super) fn estate_funding_observation(balance_cycles: Option<u128>) -> FleetObservation {
5076 FleetObservation {
5077 additional_controlled_cycles: BTreeMap::new(),
5078 canisters: BTreeMap::new(),
5079 estate_funding_domains: BTreeMap::from([(
5080 "root".to_string(),
5081 crate::fleet_ensure::model::EstateFundingDomainObservation {
5082 balance_cycles,
5083 cycles_ledger: "ledger".to_string(),
5084 pool: None,
5085 root_principal: Some("rrkah-fqaaa-aaaaa-aaaaq-cai".to_string()),
5086 },
5087 )]),
5088 ledger_fee_cycles: 0,
5089 operator_cycles: 0,
5090 protocol_ready: BTreeMap::new(),
5091 }
5092 }
5093
5094 fn terminal_estate_creation_observation() -> FleetObservation {
5095 let mut observation = estate_funding_observation(Some(0));
5096 let receipt =
5097 |byte, block_index| crate::fleet_ensure::model::EstatePoolCreationReceiptObservation {
5098 block_index,
5099 operation_id: format!("{byte:02x}").repeat(32),
5100 cycles_ledger: "ledger".to_string(),
5101 ledger_amount_cycles: 45,
5102 ledger_fee_cycles: 5,
5103 readiness_floor_cycles: 35,
5104 creation_execution_margin_cycles: 5,
5105 management_creation_fee_cycles: 5,
5106 first_observed_cycles: Some(39),
5107 };
5108 observation
5109 .estate_funding_domains
5110 .get_mut("root")
5111 .expect("Root funding domain")
5112 .pool = Some(crate::fleet_ensure::model::EstatePoolInventoryObservation {
5113 assets: vec![
5114 crate::fleet_ensure::model::EstatePoolAssetObservation {
5115 creation_receipt: Some(receipt(1, 10)),
5116 cycles: 35,
5117 lifecycle: crate::fleet_ensure::model::EstatePoolAssetLifecycle::Ready,
5118 origin: crate::fleet_ensure::model::EstatePoolAssetOrigin::Created,
5119 principal: "created-one".to_string(),
5120 },
5121 crate::fleet_ensure::model::EstatePoolAssetObservation {
5122 creation_receipt: Some(receipt(2, 11)),
5123 cycles: 35,
5124 lifecycle: crate::fleet_ensure::model::EstatePoolAssetLifecycle::Workload,
5125 origin: crate::fleet_ensure::model::EstatePoolAssetOrigin::Created,
5126 principal: "created-two".to_string(),
5127 },
5128 ],
5129 maximum_size: 2,
5130 minimum_size: 0,
5131 pending_creation: None,
5132 readiness_floor_cycles: 35,
5133 creation_execution_margin_cycles: 5,
5134 });
5135 observation
5136 }
5137
5138 fn estate_creation_receipt(
5139 observation: &mut FleetObservation,
5140 asset_index: usize,
5141 ) -> &mut crate::fleet_ensure::model::EstatePoolCreationReceiptObservation {
5142 observation
5143 .estate_funding_domains
5144 .get_mut("root")
5145 .expect("Root domain")
5146 .pool
5147 .as_mut()
5148 .expect("pool")
5149 .assets[asset_index]
5150 .creation_receipt
5151 .as_mut()
5152 .expect("receipt")
5153 }
5154
5155 #[test]
5156 fn autonomous_reserve_creation_reconciles_without_host_provisioning() {
5157 let mut plan = estate_funding_plan();
5158 let domain = &mut plan.conservation.estate_funding_domains[0];
5159 domain.available_cycles = Some(100);
5160 domain.maximum_funding_cycles = 0;
5161 domain.shortfall_cycles = 0;
5162 let (state, mut journal) = retained_evidence();
5163 journal.effects.clear();
5164 journal
5165 .initial_estate_funding_cycles_by_root
5166 .insert("root".to_string(), 100);
5167 let terminal = terminal_estate_creation_observation();
5168 assert_eq!(
5169 reconcile_estate_funding::<std::io::Error>(&plan, &journal, &state, &terminal)
5170 .expect("reconcile exact Root-owned creation against the reviewed reserve"),
5171 (0, 20)
5172 );
5173 }
5174
5175 #[test]
5176 fn terminal_estate_creation_uses_exact_protected_receipts() {
5177 let plan = estate_funding_plan();
5178 let domain = &plan.conservation.estate_funding_domains[0];
5179 let (state, _) = retained_evidence();
5180 let exact = terminal_estate_creation_observation();
5181 assert_eq!(
5182 exact_estate_creation_costs::<std::io::Error>(&state, &exact, domain)
5183 .expect("reconcile exact creation receipts"),
5184 (100, 20)
5185 );
5186 for required_creation_count in [0, 1] {
5187 let mut insufficient = domain.clone();
5188 insufficient.required_creation_count = required_creation_count;
5189 assert!(matches!(
5190 exact_estate_creation_costs::<std::io::Error>(&state, &exact, &insufficient),
5191 Err(EnsureWorkflowError::JournalIntegrity)
5192 ));
5193 }
5194
5195 let mut drifted = Vec::new();
5196 let mut changed = exact.clone();
5197 estate_creation_receipt(&mut changed, 0).ledger_amount_cycles = 44;
5198 drifted.push(changed);
5199 let mut changed = exact.clone();
5200 estate_creation_receipt(&mut changed, 1).operation_id = "01".repeat(32);
5201 drifted.push(changed);
5202 let mut changed = exact.clone();
5203 estate_creation_receipt(&mut changed, 1).block_index = 10;
5204 drifted.push(changed);
5205 let mut changed = exact.clone();
5206 estate_creation_receipt(&mut changed, 0).first_observed_cycles = None;
5207 drifted.push(changed);
5208 let mut changed = exact.clone();
5209 estate_creation_receipt(&mut changed, 0).first_observed_cycles = Some(34);
5210 drifted.push(changed);
5211 let mut donated = exact.clone();
5212 estate_creation_receipt(&mut donated, 0).first_observed_cycles = Some(41);
5213 assert_eq!(
5214 exact_estate_creation_costs::<std::io::Error>(&state, &donated, domain)
5215 .expect("external donation leaves exact creation cost unchanged"),
5216 (100, 20)
5217 );
5218 let mut changed = exact;
5219 changed
5220 .estate_funding_domains
5221 .get_mut("root")
5222 .expect("Root domain")
5223 .pool
5224 .as_mut()
5225 .expect("pool")
5226 .assets[0]
5227 .origin = crate::fleet_ensure::model::EstatePoolAssetOrigin::Imported;
5228 drifted.push(changed);
5229 for observation in drifted {
5230 assert!(matches!(
5231 exact_estate_creation_costs::<std::io::Error>(&state, &observation, domain,),
5232 Err(EnsureWorkflowError::JournalIntegrity)
5233 ));
5234 }
5235 }
5236
5237 #[test]
5238 fn estate_funding_pause_is_exact_and_clears_only_after_sufficient_balance() {
5239 let plan = estate_funding_plan();
5240 let (state, mut journal) = retained_evidence();
5241 let required = estate_funding_requirement::<std::io::Error>(
5242 &plan,
5243 &state,
5244 &estate_funding_observation(Some(40)),
5245 )
5246 .expect("validate funding observation")
5247 .expect("underfunded estate pauses");
5248 assert_eq!(required.available_cycles, 40);
5249 assert_eq!(required.maximum_creation_debit_cycles, 100);
5250 assert_eq!(required.shortfall_cycles, 60);
5251 assert!(
5252 estate_funding_requirement::<std::io::Error>(
5253 &plan,
5254 &state,
5255 &estate_funding_observation(Some(100)),
5256 )
5257 .expect("validate sufficient funding observation")
5258 .is_none()
5259 );
5260
5261 journal.completion = FleetEnsureCompletion::InProgress;
5262 journal.effects.clear();
5263 journal.estate_funding_required = Some(required);
5264 journal.initial_estate_funding_cycles_by_root = BTreeMap::from([("root".to_string(), 40)]);
5265 assert!(initial_estate_funding_is_exact(&journal, &plan));
5266 assert!(estate_funding_pause_is_exact(&journal, &plan, &state));
5267
5268 journal
5269 .estate_funding_required
5270 .as_mut()
5271 .expect("funding pause")
5272 .shortfall_cycles = 59;
5273 assert!(!estate_funding_pause_is_exact(&journal, &plan, &state));
5274
5275 *journal
5276 .initial_estate_funding_cycles_by_root
5277 .get_mut("root")
5278 .expect("initial Root funding balance") = 41;
5279 assert!(!initial_estate_funding_is_exact(&journal, &plan));
5280 }
5281
5282 #[test]
5283 fn current_creation_funding_pause_is_bound_to_exact_runtime_evidence() {
5284 let mut plan = estate_funding_plan();
5285 plan.conservation.estate_funding_domains[0].cycles_ledger = "aaaaa-aa".to_string();
5286 let (state, mut journal) = retained_evidence();
5287 let funding = canic_contracts::dto::component_provisioning::RootEstateFundingRequired {
5288 available: Cycles::new(40),
5289 attempt_count: 2,
5290 creation_amount: Cycles::new(45),
5291 cycles_ledger: candid::Principal::from_text("aaaaa-aa").expect("Ledger Principal"),
5292 execution_margin: Cycles::new(5),
5293 last_attempt_at_ns: Some(10),
5294 ledger_fee: Cycles::new(5),
5295 management_creation_fee: Cycles::new(5),
5296 operation_id: [17; 32],
5297 readiness_floor: Cycles::new(35),
5298 required: Cycles::new(50),
5299 retry_at_ns: 20,
5300 root: candid::Principal::from_text("rrkah-fqaaa-aaaaa-aaaaq-cai")
5301 .expect("Root Principal"),
5302 shortfall: Cycles::new(10),
5303 };
5304 let required =
5305 protocol_estate_funding_requirement::<std::io::Error>(&plan, &state, &funding)
5306 .expect("bind runtime funding evidence");
5307 journal.completion = FleetEnsureCompletion::InProgress;
5308 journal.effects.clear();
5309 journal.estate_funding_required = Some(required);
5310 assert!(estate_funding_pause_is_exact(&journal, &plan, &state));
5311
5312 journal
5313 .estate_funding_required
5314 .as_mut()
5315 .expect("funding pause")
5316 .pending_creation_operation_id = Some("AA".repeat(32));
5317 assert!(!estate_funding_pause_is_exact(&journal, &plan, &state));
5318
5319 let mut drifted = funding;
5320 drifted.execution_margin = Cycles::new(6);
5321 assert!(matches!(
5322 protocol_estate_funding_requirement::<std::io::Error>(&plan, &state, &drifted),
5323 Err(EnsureWorkflowError::PlanIntegrity)
5324 ));
5325 }
5326
5327 #[test]
5328 fn estate_funding_observation_rejects_missing_or_mismatched_authority() {
5329 let plan = estate_funding_plan();
5330 let (state, _) = retained_evidence();
5331 let mut missing_balance = estate_funding_observation(None);
5332 assert!(matches!(
5333 estate_funding_requirement::<std::io::Error>(&plan, &state, &missing_balance),
5334 Err(EnsureWorkflowError::PlanIntegrity)
5335 ));
5336
5337 let observed = missing_balance
5338 .estate_funding_domains
5339 .get_mut("root")
5340 .expect("Root funding domain");
5341 observed.balance_cycles = Some(40);
5342 observed.root_principal = Some("aaaaa-aa".to_string());
5343 assert!(matches!(
5344 estate_funding_requirement::<std::io::Error>(&plan, &state, &missing_balance),
5345 Err(EnsureWorkflowError::PlanIntegrity)
5346 ));
5347
5348 for changed_balance in [0, 39] {
5349 assert!(matches!(
5350 estate_funding_requirement::<std::io::Error>(
5351 &plan,
5352 &state,
5353 &estate_funding_observation(Some(changed_balance)),
5354 ),
5355 Err(EnsureWorkflowError::DriftedBeforeApply)
5356 ));
5357 }
5358 }
5359
5360 #[test]
5361 fn estate_funding_surplus_preserves_reviewed_creation_limits() {
5362 let plan = estate_funding_plan();
5363 let original = plan.clone();
5364 let (state, _) = retained_evidence();
5365 assert!(
5366 estate_funding_requirement::<std::io::Error>(
5367 &plan,
5368 &state,
5369 &estate_funding_observation(Some(150)),
5370 )
5371 .unwrap()
5372 .is_none()
5373 );
5374 assert_eq!(plan, original);
5375 let mut extra_creation = terminal_estate_creation_observation();
5376 let pool = extra_creation
5377 .estate_funding_domains
5378 .get_mut("root")
5379 .unwrap()
5380 .pool
5381 .as_mut()
5382 .unwrap();
5383 let mut extra = pool.assets[0].clone();
5384 extra.principal = "unreviewed-third-creation".into();
5385 pool.assets.push(extra);
5386 assert!(matches!(
5387 exact_estate_creation_costs::<std::io::Error>(
5388 &state,
5389 &extra_creation,
5390 &plan.conservation.estate_funding_domains[0]
5391 ),
5392 Err(EnsureWorkflowError::JournalIntegrity)
5393 ));
5394 }
5395
5396 #[test]
5397 fn non_creation_protocol_completion_does_not_consume_estate_creation_authority() {
5398 let mut plan = estate_funding_plan();
5399 plan.conservation.estate_funding_domains[0].maximum_creation_debit_cycles = 0;
5400 plan.conservation.estate_funding_domains[0].maximum_creation_fee_cycles = 0;
5401 plan.conservation.estate_funding_domains[0].maximum_funding_cycles = 0;
5402 plan.conservation.estate_funding_domains[0].required_creation_count = 0;
5403 plan.conservation.estate_funding_domains[0].shortfall_cycles = 0;
5404 let action = EnsureAction::FleetProtocol {
5405 action: Box::new(
5406 crate::fleet_ensure::model::CurrentFleetProtocolAction::PublishStoreChunk {
5407 request: canic_contracts::dto::template::TemplateChunkInput {
5408 preparation: None,
5409 template_id: canic_contracts::ids::TemplateId::from("root"),
5410 version: canic_contracts::ids::TemplateVersion::from("current"),
5411 chunk_index: 0,
5412 bytes: vec![2],
5413 },
5414 },
5415 ),
5416 candid: "store.did".to_string(),
5417 candid_sha256: "11".repeat(32),
5418 maximum_execution_burn_cycles: 0,
5419 name: "publish-store-chunk".to_string(),
5420 principal: "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(),
5421 };
5422 let action_hash = action_sha256(&action);
5423 plan.protocol_actions.push(action);
5424 let (state, mut journal) = retained_evidence();
5425 journal.initial_estate_funding_cycles_by_root = BTreeMap::from([("root".to_string(), 40)]);
5426 journal.effects = vec![EffectRecord {
5427 publication_attempts: 0,
5428 maintenance_attempts: 0,
5429 action_sha256: action_hash,
5430 created_principal: None,
5431 destination_post_cycles: None,
5432 destination_pre_cycles: None,
5433 post_cycles: None,
5434 pre_cycles: None,
5435 pre_canister_version: None,
5436 progress_identity: Some("store chunks prepared".to_string()),
5437 receipt: Some("receipt".to_string()),
5438 state: EffectState::Applied,
5439 }];
5440
5441 assert_eq!(
5442 reconcile_estate_funding::<std::io::Error>(
5443 &plan,
5444 &journal,
5445 &state,
5446 &estate_funding_observation(Some(40)),
5447 )
5448 .expect("non-creation protocol preserves the Root Ledger balance"),
5449 (0, 0),
5450 );
5451 journal.initial_controlled_cycles = 40;
5452 journal.initial_operator_cycles = 0;
5453 let original = plan.clone();
5454 for balance in [40, 41, 150] {
5455 let observed = estate_funding_observation(Some(balance));
5456 assert!(
5457 estate_funding_requirement::<std::io::Error>(&plan, &state, &observed)
5458 .unwrap()
5459 .is_none()
5460 );
5461 let actual =
5462 verify_terminal_conservation::<std::io::Error>(&plan, &journal, &state, &observed)
5463 .unwrap();
5464 assert_eq!(actual.final_controlled_cycles, balance);
5465 assert_eq!(actual.observed_net_cycle_credit_cycles, balance - 40);
5466 assert_eq!(actual.estate_funding_cycles, 0);
5467 assert_eq!(actual.operator_debit_cycles, 0);
5468 assert_eq!(actual.exact_estate_creation_fee_cycles, 0);
5469 }
5470 assert_eq!(plan, original);
5471 assert!(matches!(
5472 reconcile_estate_funding::<std::io::Error>(
5473 &plan,
5474 &journal,
5475 &state,
5476 &estate_funding_observation(Some(39))
5477 ),
5478 Err(EnsureWorkflowError::Conservation(_))
5479 ));
5480 }
5481
5482 #[test]
5483 fn fresh_estate_funding_binds_the_created_root_from_durable_state() {
5484 let mut plan = estate_funding_plan();
5485 plan.conservation.estate_funding_domains[0].available_cycles = None;
5486 plan.conservation.estate_funding_domains[0].maximum_funding_cycles = 100;
5487 plan.conservation.estate_funding_domains[0].root_principal = None;
5488 let (mut state, _) = retained_evidence();
5489 state.pending_principals.insert(
5490 "root".to_string(),
5491 "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(),
5492 );
5493
5494 let required = estate_funding_requirement::<std::io::Error>(
5495 &plan,
5496 &state,
5497 &estate_funding_observation(Some(0)),
5498 )
5499 .expect("validate created Root funding authority")
5500 .expect("fresh underfunded estate pauses");
5501 assert_eq!(required.shortfall_cycles, 100);
5502 assert_eq!(required.root_principal, "rrkah-fqaaa-aaaaa-aaaaq-cai");
5503
5504 state
5505 .pending_principals
5506 .insert("root".to_string(), "aaaaa-aa".to_string());
5507 assert!(matches!(
5508 estate_funding_requirement::<std::io::Error>(
5509 &plan,
5510 &state,
5511 &estate_funding_observation(Some(0)),
5512 ),
5513 Err(EnsureWorkflowError::PlanIntegrity)
5514 ));
5515 }
5516
5517 #[test]
5518 fn unchanged_estate_funding_pause_does_not_rewrite_its_journal() {
5519 let root = crate::test_support::temp_dir("estate-funding-pause");
5520 let paths = EnsurePaths::under(&root, "local", "fleet");
5521 let plan = estate_funding_plan();
5522 let (state, mut journal) = retained_evidence();
5523 journal.completion = FleetEnsureCompletion::InProgress;
5524 journal.effects.clear();
5525 journal.initial_estate_funding_cycles_by_root = BTreeMap::from([("root".to_string(), 40)]);
5526 let required = estate_funding_requirement::<std::io::Error>(
5527 &plan,
5528 &state,
5529 &estate_funding_observation(Some(40)),
5530 )
5531 .expect("validate funding observation")
5532 .expect("underfunded estate pauses");
5533
5534 assert!(
5535 retain_estate_funding_pause::<std::io::Error>(&paths, &mut journal, Some(&required),)
5536 .expect("retain first funding pause")
5537 );
5538 let retained = std::fs::read(&paths.journal).expect("read first retained pause");
5539 assert!(
5540 !retain_estate_funding_pause::<std::io::Error>(&paths, &mut journal, Some(&required),)
5541 .expect("replay unchanged funding pause")
5542 );
5543 assert_eq!(
5544 std::fs::read(&paths.journal).expect("reread unchanged retained pause"),
5545 retained
5546 );
5547 assert!(
5548 retain_estate_funding_pause::<std::io::Error>(&paths, &mut journal, None)
5549 .expect("clear funded pause")
5550 );
5551 assert!(journal.estate_funding_required.is_none());
5552
5553 std::fs::remove_dir_all(root).expect("remove funding-pause fixture");
5554 }
5555
5556 #[test]
5557 fn estate_funding_is_rechecked_until_the_first_protocol_effect_starts() {
5558 let protocol_action = || EnsureAction::FleetProtocol {
5559 action: Box::new(
5560 crate::fleet_ensure::model::CurrentFleetProtocolAction::PublishStoreChunk {
5561 request: canic_contracts::dto::template::TemplateChunkInput {
5562 preparation: None,
5563 template_id: canic_contracts::ids::TemplateId::from("root"),
5564 version: canic_contracts::ids::TemplateVersion::from("current"),
5565 chunk_index: 0,
5566 bytes: vec![2],
5567 },
5568 },
5569 ),
5570 candid: "store.did".to_string(),
5571 candid_sha256: "11".repeat(32),
5572 maximum_execution_burn_cycles: 1,
5573 name: "prepare".to_string(),
5574 principal: "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(),
5575 };
5576 let first = protocol_action();
5577 let second = protocol_action();
5578 let actions = [&first, &second];
5579 let (_, mut journal) = retained_evidence();
5580 journal.effects = vec![EffectRecord {
5581 publication_attempts: 0,
5582 maintenance_attempts: 0,
5583 action_sha256: action_sha256(&first),
5584 created_principal: None,
5585 destination_post_cycles: None,
5586 destination_pre_cycles: None,
5587 post_cycles: None,
5588 pre_cycles: None,
5589 pre_canister_version: None,
5590 progress_identity: None,
5591 receipt: None,
5592 state: EffectState::Intent,
5593 }];
5594 assert!(!prior_fleet_protocol_effect_started(&actions, &journal, 1));
5595
5596 journal.effects[0].state = EffectState::Issued;
5597 assert!(prior_fleet_protocol_effect_started(&actions, &journal, 1));
5598 }
5599
5600 #[test]
5601 fn configured_projection_publishes_unique_bindings_without_losing_dynamic_cycles() {
5602 let asset = "rrkah-fqaaa-aaaaa-aaaaq-cai";
5603 let dynamic = "ryjl3-tyaaa-aaaaa-aaaba-cai";
5604 for kind in ["pool", "component"] {
5605 let desired = serde_json::from_value(serde_json::json!({
5606 "canisters": [{
5607 "controllers": [], "initial_cycles": "0", "kind": kind,
5608 "minimum_cycles": "0", "name": "configured", "presence": "present",
5609 "principal": asset, "replace": false, "subnet": "subnet"
5610 }],
5611 "cycles_ledger": "um5iw-rqaaa-aaaaq-qaaba-cai",
5612 "environment": "local", "fleet": "fleet", "ledger_fee_cycles": "0",
5613 "management_creation_fee_cycles": "0", "material_cycle_threshold": "1",
5614 "maximum_observation_burn_cycles": "1", "maximum_stalled_observations": 2,
5615 "maximum_update_burn_cycles": "2", "operator": asset,
5616 "schema_version": FLEET_ENSURE_SCHEMA_VERSION, "treasury": "configured"
5617 }))
5618 .expect("current desired projection");
5619 let (mut state, mut journal) = retained_evidence();
5620 journal.effects.clear();
5621 state.principals = BTreeMap::from([
5622 ("configured".to_string(), asset.to_string()),
5623 (format!("observed:{asset}"), asset.to_string()),
5624 (format!("observed:{dynamic}"), dynamic.to_string()),
5625 ]);
5626 state.retained_cycles_by_principal =
5627 BTreeMap::from([(asset.to_string(), 100), (dynamic.to_string(), 60)]);
5628 let cycles = state.retained_cycles_by_principal.clone();
5629 let journal_before = serde_json::to_vec(&journal).expect("journal before projection");
5630 let plan = estate_funding_plan();
5631
5632 publish_terminal_state(&desired, &plan, &journal, &mut state)
5633 .expect("publish current configured topology");
5634 assert_eq!(
5635 state.principals,
5636 BTreeMap::from([
5637 ("configured".to_string(), asset.to_string()),
5638 (format!("observed:{dynamic}"), dynamic.to_string()),
5639 ])
5640 );
5641 assert_eq!(state.retained_cycles_by_principal, cycles);
5642 let published = state.clone();
5643 publish_terminal_state(&desired, &plan, &journal, &mut state)
5644 .expect("repeat projection");
5645 assert_eq!(state, published);
5646 assert_eq!(
5647 serde_json::to_vec(&journal).expect("journal after projection"),
5648 journal_before
5649 );
5650 }
5651 }
5652
5653 #[test]
5654 fn configured_projection_preserves_duplicate_authority_rejection() {
5655 let asset = "rrkah-fqaaa-aaaaa-aaaaq-cai";
5656 for configured in [false, true] {
5657 let (mut state, _) = retained_evidence();
5658 state.principals = BTreeMap::from([
5659 ("first".to_string(), asset.to_string()),
5660 ("second".to_string(), asset.to_string()),
5661 ]);
5662 if configured {
5663 let topology = crate::fleet_ensure::model::FleetEnsureTopologyRecord {
5664 kind: crate::fleet_ensure::model::DesiredCanisterKind::Pool,
5665 module_hash: None,
5666 parent: None,
5667 protocol_binding: None,
5668 role: None,
5669 };
5670 state.topology = BTreeMap::from([
5671 ("first".to_string(), topology.clone()),
5672 ("second".to_string(), topology),
5673 ]);
5674 }
5675 let before = state.clone();
5676 retain_configured_principal_bindings(&mut state);
5677 assert_eq!(state, before);
5678 for entries in [Vec::new(), vec![terminal_component_entry(asset, "parent")]] {
5679 assert!(matches!(
5680 merge_terminal_inventory::<std::io::Error>(
5681 &mut state,
5682 crate::fleet_ensure::ops::TerminalFleetInventory {
5683 entries,
5684 ..Default::default()
5685 }
5686 ),
5687 Err(EnsureWorkflowError::TerminalInventory(
5688 TerminalInventoryError::DuplicateRetainedPrincipal
5689 ))
5690 ));
5691 assert_eq!(state, before);
5692 }
5693 }
5694 }
5695
5696 #[test]
5697 fn terminal_inventory_allows_only_verified_pool_to_component_parent_transition() {
5698 let asset = "rrkah-fqaaa-aaaaa-aaaaq-cai";
5699 let old_root = "r7inp-6aaaa-aaaaa-aaabq-cai";
5700 let hub = "ryjl3-tyaaa-aaaaa-aaaba-cai";
5701 let state = || {
5702 let (mut state, _) = retained_evidence();
5703 state.principals = BTreeMap::from([
5704 ("asset".to_string(), asset.to_string()),
5705 ("hub".to_string(), hub.to_string()),
5706 ("root".to_string(), old_root.to_string()),
5707 ]);
5708 state.topology.insert(
5709 "asset".to_string(),
5710 crate::fleet_ensure::model::FleetEnsureTopologyRecord {
5711 kind: crate::fleet_ensure::model::DesiredCanisterKind::Pool,
5712 module_hash: None,
5713 parent: Some("root".to_string()),
5714 protocol_binding: None,
5715 role: Some("canister_pool_asset".to_string()),
5716 },
5717 );
5718 state
5719 };
5720
5721 let mut promoted = state();
5722 merge_terminal_inventory::<std::io::Error>(
5723 &mut promoted,
5724 crate::fleet_ensure::ops::TerminalFleetInventory {
5725 active_registry: None,
5726 controlled_cycles_by_principal: BTreeMap::new(),
5727 entries: vec![terminal_component_entry(asset, hub)],
5728 },
5729 )
5730 .expect("verified pool workload assumes its terminal Component parent");
5731 let promoted = promoted.topology.get("asset").expect("promoted asset");
5732 assert_eq!(
5733 promoted.kind,
5734 crate::fleet_ensure::model::DesiredCanisterKind::Component
5735 );
5736 assert_eq!(promoted.parent.as_deref(), Some("hub"));
5737
5738 let mut already_component = state();
5739 already_component
5740 .topology
5741 .get_mut("asset")
5742 .expect("retained asset")
5743 .kind = crate::fleet_ensure::model::DesiredCanisterKind::Component;
5744 assert!(matches!(
5745 merge_terminal_inventory::<std::io::Error>(
5746 &mut already_component,
5747 crate::fleet_ensure::ops::TerminalFleetInventory {
5748 active_registry: None,
5749 controlled_cycles_by_principal: BTreeMap::new(),
5750 entries: vec![terminal_component_entry(asset, hub)],
5751 },
5752 ),
5753 Err(EnsureWorkflowError::TerminalInventory(
5754 TerminalInventoryError::RetainedParentConflict {
5755 canister,
5756 retained_parent,
5757 terminal_parent,
5758 }
5759 )) if canister == asset
5760 && retained_parent == "root"
5761 && terminal_parent == "hub"
5762 ));
5763
5764 for incomplete_entry in [
5765 crate::registry::RegistryEntry {
5766 module_hash: None,
5767 ..terminal_component_entry(asset, hub)
5768 },
5769 crate::registry::RegistryEntry {
5770 protocol_binding: None,
5771 ..terminal_component_entry(asset, hub)
5772 },
5773 ] {
5774 assert!(matches!(
5775 merge_terminal_inventory::<std::io::Error>(
5776 &mut state(),
5777 crate::fleet_ensure::ops::TerminalFleetInventory {
5778 active_registry: None,
5779 controlled_cycles_by_principal: BTreeMap::new(),
5780 entries: vec![incomplete_entry],
5781 },
5782 ),
5783 Err(EnsureWorkflowError::TerminalInventory(
5784 TerminalInventoryError::RetainedParentConflict {
5785 canister,
5786 retained_parent,
5787 terminal_parent,
5788 }
5789 )) if canister == asset
5790 && retained_parent == "root"
5791 && terminal_parent == "hub"
5792 ));
5793 }
5794 }
5795
5796 #[test]
5797 fn reinstall_recovery_review_requires_complete_exact_install_evidence() {
5798 use crate::fleet_ensure::model::{CanisterPlan, FleetReinstallRecord, InstallMode};
5799 let (state, mut journal) = retained_evidence();
5800 let mut plan = estate_funding_plan();
5801 plan.conservation.estate_funding_domains.clear();
5802 plan.reinstall = Some(Box::new(FleetReinstallRecord {
5803 activation_reset: None,
5804 operation_id: plan.operation_id.clone(),
5805 source_operation_id: "source-operation".into(),
5806 authorities: Vec::new(),
5807 assets: Vec::new(),
5808 }));
5809 let action = EnsureAction::Install {
5810 canic_init: None,
5811 reinstall_witness: None,
5812 init_arg: None,
5813 init_arg_sha256: None,
5814 init_candid: None,
5815 init_candid_sha256: None,
5816 mode: InstallMode::Reinstall,
5817 name: "root".into(),
5818 principal: "rrkah-fqaaa-aaaaa-aaaaq-cai".into(),
5819 wasm: "root.wasm".into(),
5820 wasm_sha256: "22".repeat(32),
5821 };
5822 journal.effects[0].action_sha256 = action_sha256(&action);
5823 plan.canisters.push(CanisterPlan {
5824 actions: vec![action],
5825 disposition: CanisterDisposition::Reuse,
5826 name: "root".into(),
5827 observed_cycles: 0,
5828 principal: Some("rrkah-fqaaa-aaaaa-aaaaq-cai".into()),
5829 });
5830 let admit = |journal: &FleetEnsureJournalRecord| {
5831 reinstall_recovery_ready::<std::io::Error>(&plan, journal, &state)
5832 };
5833 assert!(admit(&journal).unwrap());
5834 let mut issued = journal.clone();
5835 issued.effects[0].state = EffectState::Issued;
5836 let mut missing = journal.clone();
5837 missing.effects.clear();
5838 let mut wrong_action = journal.clone();
5839 wrong_action.effects[0].action_sha256 = "other".into();
5840 let mut no_version = journal.clone();
5841 no_version.effects[0].pre_canister_version = None;
5842 let mut wrong_operation = journal.clone();
5843 wrong_operation.operation_id = "other".into();
5844 for invalid in [issued, missing, wrong_action, no_version, wrong_operation] {
5845 assert!(matches!(
5846 admit(&invalid),
5847 Err(EnsureWorkflowError::JournalIntegrity)
5848 ));
5849 }
5850 for completion in [
5851 FleetEnsureCompletion::InProgress,
5852 FleetEnsureCompletion::Converged,
5853 ] {
5854 let mut other = journal.clone();
5855 other.completion = completion;
5856 assert!(!admit(&other).unwrap());
5857 }
5858 }
5859
5860 #[test]
5861 fn completed_reinstall_evidence_requires_exact_operation_action_and_version() {
5862 let (state, journal) = retained_evidence();
5863 assert!(retained_reinstall_operation_matches(
5864 &state,
5865 "fleet",
5866 "operation",
5867 &journal,
5868 ));
5869
5870 let mut wrong_action = journal.clone();
5871 wrong_action.effects[0].action_sha256 = "other".to_string();
5872 assert!(!retained_reinstall_operation_matches(
5873 &state,
5874 "fleet",
5875 "operation",
5876 &wrong_action,
5877 ));
5878
5879 let mut wrong_version = journal.clone();
5880 wrong_version.effects[0].pre_canister_version = Some(8);
5881 assert!(!retained_reinstall_operation_matches(
5882 &state,
5883 "fleet",
5884 "operation",
5885 &wrong_version,
5886 ));
5887
5888 for (fleet, operation) in [("other", "operation"), ("fleet", "other")] {
5889 assert!(!retained_reinstall_operation_matches(
5890 &state, fleet, operation, &journal,
5891 ));
5892 }
5893
5894 let mut incomplete = journal;
5895 incomplete.effects[0].state = EffectState::Issued;
5896 assert!(!retained_reinstall_operation_matches(
5897 &state,
5898 "fleet",
5899 "operation",
5900 &incomplete,
5901 ));
5902 }
5903
5904 #[test]
5905 fn fresh_fleet_alternate_desired_cannot_clear_completed_reinstall_evidence() {
5906 let (state, journal) = retained_evidence();
5907 assert!(retained_reinstall_desired_conflict(
5908 &state,
5909 "fleet",
5910 "operation",
5911 "reviewed-desired",
5912 "alternate-desired",
5913 &journal,
5914 ));
5915 assert!(!retained_reinstall_desired_conflict(
5916 &state,
5917 "fleet",
5918 "operation",
5919 "reviewed-desired",
5920 "reviewed-desired",
5921 &journal,
5922 ));
5923
5924 let mut terminal = journal;
5925 terminal.completion = FleetEnsureCompletion::Converged;
5926 assert!(!retained_reinstall_desired_conflict(
5927 &state,
5928 "fleet",
5929 "operation",
5930 "reviewed-desired",
5931 "alternate-desired",
5932 &terminal,
5933 ));
5934 }
5935
5936 fn cycle_observation(
5937 lifecycle: crate::fleet_ensure::model::RootOwnedCanisterLifecycle,
5938 cycles: u128,
5939 ) -> FleetObservation {
5940 FleetObservation {
5941 additional_controlled_cycles: BTreeMap::new(),
5942 canisters: BTreeMap::from([(
5943 "asset".to_string(),
5944 Some(crate::fleet_ensure::model::LiveCanister {
5945 canister_version: Some(1),
5946 controllers: Vec::new(),
5947 cycles,
5948 module_sha256: None,
5949 principal: "controlled-principal".to_string(),
5950 reinstall_required: false,
5951 root_owned_lifecycle: Some(lifecycle),
5952 status: crate::fleet_ensure::model::CanisterRuntimeStatus::Stopped,
5953 }),
5954 )]),
5955 estate_funding_domains: BTreeMap::new(),
5956 ledger_fee_cycles: 0,
5957 operator_cycles: 0,
5958 protocol_ready: BTreeMap::new(),
5959 }
5960 }
5961
5962 #[test]
5963 fn fresh_fleet_idle_observation_accepts_donations_without_double_counting() {
5964 use crate::fleet_ensure::model::RootOwnedCanisterLifecycle;
5965
5966 let additional = BTreeMap::from([("controlled-principal".to_string(), 100)]);
5967 let mut idle = cycle_observation(RootOwnedCanisterLifecycle::Idle, 100);
5968 attach_terminal_cycles::<std::io::Error>(&mut idle, additional.clone())
5969 .expect("merge exact Idle observation");
5970 assert!(idle.additional_controlled_cycles.is_empty());
5971
5972 let mut donated_idle = cycle_observation(RootOwnedCanisterLifecycle::Idle, 99);
5973 attach_terminal_cycles::<std::io::Error>(&mut donated_idle, additional.clone()).unwrap();
5974 assert_eq!(
5975 controlled_cycles::<std::io::Error>(&donated_idle).unwrap(),
5976 100
5977 );
5978 assert!(donated_idle.additional_controlled_cycles.is_empty());
5979
5980 let mut conflicting_idle = cycle_observation(RootOwnedCanisterLifecycle::Idle, 101);
5981 assert!(matches!(
5982 attach_terminal_cycles::<std::io::Error>(&mut conflicting_idle, additional.clone()),
5983 Err(EnsureWorkflowError::TerminalInventory(
5984 TerminalInventoryError::IdleCycleObservationConflict {
5985 canister,
5986 expected: 101,
5987 observed: 100,
5988 }
5989 )) if canister == "controlled-principal"
5990 ));
5991
5992 let mut workload = cycle_observation(RootOwnedCanisterLifecycle::Workload, 99);
5993 attach_terminal_cycles::<std::io::Error>(&mut workload, additional)
5994 .expect("merge conservative workload observation");
5995 assert_eq!(
5996 workload
5997 .canisters
5998 .get("asset")
5999 .and_then(Option::as_ref)
6000 .map(|live| live.cycles),
6001 Some(99),
6002 );
6003 }
6004}