1mod continuation;
8mod funding;
9#[cfg(test)]
10mod funding_tests;
11mod reinstall;
12mod retained_plan;
13mod root_reinstall;
14mod settlement;
15
16use crate::fleet_ensure::{
17 dto::{FleetEnsurePhase, FleetEnsureProgress, FleetEnsureProgressState},
18 inventory::project_current_fleet_inventory,
19 model::{
20 ActualCycleConservation, CanisterDisposition, CurrentFleetProtocolAction,
21 CycleConservation, EffectRecord, EffectState, EnsureAction, EstateFundingDomainPlan,
22 EstateFundingRequiredRecord, FLEET_ENSURE_SCHEMA_VERSION, FleetEnsureCompletion,
23 FleetEnsureJournalRecord, FleetEnsurePlan, FleetEnsurePlanScope, FleetEnsureReport,
24 FleetEnsureStateRecord, FleetObservation, RootManagementObservation,
25 create_balance_is_terminal,
26 },
27 ops::{
28 EffectRetry, EnsurePaths, EnsurePlatform, EnsureStateError, action_sha256,
29 compact_inline_plan, effect_preparation::prepare_effect, lock_operation, read_plan,
30 read_root_start_authority, read_state, reserve_fixture_publication_attempt,
31 resolve_desired_artifacts, retain_configured_principal_bindings, write_journal, write_plan,
32 write_state,
33 },
34 policy::{
35 EnsurePolicyError, RootStartPlanInput, compile_plan, compile_root_start_prerequisite_plan,
36 effect_observation_policy, expected_plan_sha256, operation_id,
37 recompile_root_start_prerequisite_plan, validate_creation_fee_scope,
38 validate_path_identity, validate_path_labels,
39 },
40};
41use canic_core::cdk::types::Cycles;
42use std::{
43 collections::{BTreeMap, BTreeSet},
44 path::Path,
45 time::Instant,
46};
47use thiserror::Error as ThisError;
48
49pub use reinstall::plan_reinstall;
50
51#[derive(Debug, ThisError)]
54#[error(
55 "fresh pool canister {canister} requested {requested_creation_funding_cycles} creation cycles for readiness floor {readiness_floor_cycles}; its first live balance is {live_balance_cycles} cycles (readiness shortfall {readiness_shortfall_cycles}) and must retain {remaining_controller_burn_cycles} cycles for controller finalization (required pre-finalization balance {required_pre_finalization_balance_cycles}, pre-finalization shortfall {pre_finalization_shortfall_cycles}; maximum first-observation burn {maximum_observation_burn_cycles}); no controller or protocol action followed"
56)]
57pub struct FreshPoolCreationUnderfundedError {
58 pub canister: String,
59 pub live_balance_cycles: u128,
60 pub maximum_observation_burn_cycles: u128,
61 pub pre_finalization_shortfall_cycles: u128,
62 pub readiness_floor_cycles: u128,
63 pub readiness_shortfall_cycles: u128,
64 pub remaining_controller_burn_cycles: u128,
65 pub requested_creation_funding_cycles: u128,
66 pub required_pre_finalization_balance_cycles: u128,
67}
68
69#[derive(Debug, ThisError)]
71#[error(
72 "Root {root} Cycles Ledger account {root_principal} on {cycles_ledger} has {available_cycles} cycles; {required_creation_count} autonomous creations require at most {maximum_creation_debit_cycles} cycles ({creation_amount_cycles} gross amount containing {readiness_floor_cycles} readiness floor, {management_creation_fee_cycles} management creation fee and {creation_execution_margin_cycles} execution margin, plus {ledger_fee_cycles} Ledger fee each; total reviewed fees {maximum_creation_fee_cycles}), shortfall {shortfall_cycles}; pending_creation={pending_creation_operation_id:?} attempts={attempt_count:?} last_attempt_at_ns={last_attempt_at_ns:?} retry_at_ns={retry_at_ns:?}; preserve this operation, repeat Fleet ensure without --apply and review funding_review, then apply its review_sha256 for the exact shortfall; do not fund the account out of band"
73)]
74pub struct EstateFundingRequiredError {
75 pub attempt_count: Option<u32>,
76 pub available_cycles: u128,
77 pub creation_amount_cycles: u128,
78 pub cycles_ledger: String,
79 pub creation_execution_margin_cycles: u128,
80 pub readiness_floor_cycles: u128,
81 pub ledger_fee_cycles: u128,
82 pub management_creation_fee_cycles: u128,
83 pub maximum_creation_debit_cycles: u128,
84 pub maximum_creation_fee_cycles: u128,
85 pub pending_creation_operation_id: Option<String>,
86 pub required_creation_count: u32,
87 pub root: String,
88 pub root_principal: String,
89 pub last_attempt_at_ns: Option<u64>,
90 pub retry_at_ns: Option<u64>,
91 pub shortfall_cycles: u128,
92}
93
94#[derive(Debug, ThisError)]
96pub enum TerminalInventoryError {
97 #[error("one Principal is retained under more than one desired role")]
98 DuplicateRetainedPrincipal,
99
100 #[error("verified canister {canister} lost its Principal binding")]
101 LostPrincipalBinding { canister: String },
102
103 #[error("canister {canister} names unknown parent {parent}")]
104 UnknownParent { canister: String, parent: String },
105
106 #[error(
107 "canister {canister} retained parent {retained_parent} differs from terminal parent {terminal_parent}"
108 )]
109 RetainedParentConflict {
110 canister: String,
111 retained_parent: String,
112 terminal_parent: String,
113 },
114
115 #[error(
116 "terminal Root-owned Idle canister {canister} cycle observations differ: expected {expected}, observed {observed}"
117 )]
118 IdleCycleObservationConflict {
119 canister: String,
120 expected: u128,
121 observed: u128,
122 },
123
124 #[error(
125 "terminal cycle observation duplicates canister {canister} outside its exact Root-owned Idle or Workload lifecycle"
126 )]
127 CycleObservationLifecycleConflict { canister: String },
128}
129
130#[derive(Debug, ThisError)]
133pub enum EnsureWorkflowError<E>
134where
135 E: std::error::Error + 'static,
136{
137 #[error(
138 "retained Fleet operation cannot be resumed: {source}; local source evidence identifies operation {operation_id}, journal plan reference {plan_sha256}, source document {source_document_sha256}; preserve plan, journal, state and artifacts; request an explicit --reinstall review without --apply using the selected corrected release; live authority, complete estate and cycle checks must pass before any reset; see docs/features/operations/fleet-ensure.md#unreadable-retained-plan"
139 )]
140 RetainedActivationReviewRequired {
141 operation_id: String,
142 plan_sha256: String,
143 source_document_sha256: String,
144 #[source]
145 source: Box<EnsureStateError>,
146 },
147
148 #[error(
149 "retained Fleet operation is unreadable: {source}; preserve plan, journal, state, artifacts and paid-effect receipts; local evidence does not establish the supported partial-activation review path; resolve issued effects under their exact authority before a release transition; do not edit fields, replace digests or discard the operation; see docs/features/operations/fleet-ensure.md#unreadable-retained-plan"
150 )]
151 RetainedPlanUnreadable {
152 #[source]
153 source: Box<EnsureStateError>,
154 },
155
156 #[error(
157 "activation {operation_id} has an Issued provisioning action after its Applied host prefix (source document {source_document_sha256}); preserve the plan and journal: required live evidence for the bounded partial-activation recovery is unavailable"
158 )]
159 PartialActivationResetUnavailable {
160 operation_id: String,
161 source_document_sha256: String,
162 },
163
164 #[error(
165 "reinstall requires a converged Fleet with retained physical authority or an admitted single-Root partial-activation recovery source; the supplied source or target does not satisfy either contract; preserve the retained plan and journal"
166 )]
167 ReinstallConflict,
168
169 #[error("reviewed Fleet plan digest changed before its first effect")]
170 DriftedBeforeApply,
171
172 #[error(
173 "in-progress Fleet plan requires exact desired input {expected}; supplied input is {actual}"
174 )]
175 RetainedDesiredUnavailable { actual: String, expected: String },
176
177 #[error(
178 "replan-required Fleet operation owns completed reinstall evidence for desired input {expected}; refusing alternate desired input {actual}"
179 )]
180 RetainedReinstallDesiredConflict { actual: String, expected: String },
181
182 #[error(
183 "Root prerequisite retains activation authority for desired input {expected}; refreshed input {actual} is refused before Store effects; preserve the retained plan and journal for recovery review"
184 )]
185 RootActivationDesiredConflict { actual: String, expected: String },
186
187 #[error("reviewed Fleet plan {expected} does not match retained plan {actual}")]
188 PlanDigestMismatch { actual: String, expected: String },
189
190 #[error("reviewed Fleet plan is missing; run `canic fleet ensure <fleet>` first")]
191 PlanMissing,
192
193 #[error("retained Fleet plan failed its content digest")]
194 PlanIntegrity,
195
196 #[error("retained Fleet journal does not match the reviewed Fleet operation")]
197 JournalIntegrity,
198
199 #[error(
200 "Fleet ensure action {action} made no progress for {observations} consecutive observations (last progress: {progress_identity}); operation remains resumable"
201 )]
202 Stalled {
203 action: String,
204 observations: u32,
205 progress_identity: String,
206 },
207
208 #[error("Root {root} provisioning has made no progress for {observations} observations: stage={stage:?}, target={target}, operation={operation}, diagnostic={diagnostic_code}, retry=Backoff, failed_at_ns={failed_at_ns}; the issued operation remains retained; resume the same reviewed apply after the dependency recovers", operation = canic_core::cdk::utils::hash::hex_bytes(.operation_id))]
209 ProvisioningRetryPending {
210 root: candid::Principal,
211 stage: canic_core::dto::component_provisioning::ProvisioningFailureStage,
212 target: candid::Principal,
213 operation_id: [u8; 32],
214 diagnostic_code: u16,
215 observations: u32,
216 failed_at_ns: u64,
217 },
218
219 #[error(
220 "Root-owned canister {target} remained {last_lifecycle} for {observations} consecutive passive observations; no command was reissued and the operation remains resumable"
221 )]
222 RootOwnedObservationStalled {
223 target: String,
224 observations: u32,
225 last_lifecycle: String,
226 },
227
228 #[error(
229 "selected Cycles Ledger account has {actual} cycles, below reviewed maximum debit {required}"
230 )]
231 InsufficientOperatorCycles { actual: u128, required: u128 },
232
233 #[error("terminal Fleet cycle conservation failed: {0}")]
234 Conservation(String),
235
236 #[error("terminal Fleet inventory failed exact current-authority validation: {0}")]
237 TerminalInventory(#[source] TerminalInventoryError),
238
239 #[error(
240 "reviewed effects completed, but live Fleet state still differs from desired state; review a new plan"
241 )]
242 ConvergenceDrift,
243
244 #[error(
245 "fixture publication action {action} exhausted its {maximum_attempts} reviewed paid attempts; preserve the operation and reconcile its exact source status; no additional update was issued"
246 )]
247 FixturePublicationBound {
248 action: String,
249 maximum_attempts: u32,
250 },
251
252 #[error(
253 "Root pool maintenance {action} exhausted its reviewed limit of {maximum_updates} updates; retained receipts remain authoritative"
254 )]
255 PoolMaintenanceBound {
256 action: String,
257 maximum_updates: u32,
258 },
259
260 #[error(
261 "created canister {canister} retained one exact Ledger creation receipt, but its first live balance is {actual_cycles} cycles instead of requested target {requested_cycles} (deficit {deficit_cycles}; configured management creation fee {configured_fee_cycles}); no controller or protocol action followed, and the immutable operation is now replan-required"
262 )]
263 ReplanRequiredAfterCreateBalanceDrift {
264 actual_cycles: u128,
265 canister: String,
266 configured_fee_cycles: u128,
267 deficit_cycles: u128,
268 requested_cycles: u128,
269 },
270
271 #[error("Fleet successor requires a new review: {reason:?}; {}", .review.as_deref().map(ToString::to_string).unwrap_or_default())]
272 SuccessorReviewRequired {
273 reason: crate::fleet_ensure::model::FleetEnsureSuccessorReviewReason,
274 review: Option<Box<crate::fleet_ensure::model::FleetSuccessorReview>>,
275 },
276
277 #[error(transparent)]
278 FreshPoolCreationUnderfunded(Box<FreshPoolCreationUnderfundedError>),
279
280 #[error(transparent)]
281 EstateFundingRequired(Box<EstateFundingRequiredError>),
282
283 #[error("native Root funding needs explicit approval of review {review_sha256}")]
284 NativeFundingRequired { review_sha256: String },
285
286 #[error(transparent)]
287 Policy(#[from] EnsurePolicyError),
288
289 #[error(transparent)]
290 Inventory(#[from] crate::fleet_ensure::CurrentFleetInventoryError),
291
292 #[error("Fleet platform operation failed: {0}")]
293 Platform(#[source] E),
294
295 #[error(transparent)]
296 State(#[from] EnsureStateError),
297}
298
299pub fn retained_reinstall_apply_plan<E: std::error::Error + 'static>(
301 root: &Path,
302 environment: &str,
303 requested_fleet: &str,
304 reviewed_plan_sha256: &str,
305) -> Result<Option<FleetEnsurePlan>, EnsureWorkflowError<E>> {
306 validate_path_labels(environment, requested_fleet)?;
307 let paths = EnsurePaths::under(root, environment, requested_fleet);
308 let _lock = lock_operation(&paths)?;
309 let plan = match crate::fleet_ensure::ops::reinstall::adoption::review(&paths)? {
311 Some(review) => Some(review),
312 None => retained_plan::read(&paths, environment, requested_fleet)?,
313 };
314 let Some(plan) = plan else {
315 return Ok(None);
316 };
317 let plan = verified_plan(plan)?;
318 if plan.environment != environment || plan.fleet != requested_fleet {
319 return Err(EnsureWorkflowError::PlanIntegrity);
320 }
321 if plan.reinstall.is_some() && plan.plan_sha256 == reviewed_plan_sha256 {
322 Ok(Some(plan))
323 } else {
324 Ok(None)
325 }
326}
327
328pub fn retained_in_progress_plan<E>(
330 root: &Path,
331 environment: &str,
332 requested_fleet: &str,
333) -> Result<Option<FleetEnsurePlan>, EnsureWorkflowError<E>>
334where
335 E: std::error::Error + 'static,
336{
337 validate_path_labels(environment, requested_fleet)?;
338 let paths = EnsurePaths::under(root, environment, requested_fleet);
339 let _lock = lock_operation(&paths)?;
340 if let Some(review) = crate::fleet_ensure::ops::reinstall::adoption::review(&paths)? {
341 if review.environment != environment || review.fleet != requested_fleet {
342 return Err(EnsureWorkflowError::PlanIntegrity);
343 }
344 return Ok(Some(review));
345 }
346 let Some(journal) = retained_plan::journal(&paths, environment, requested_fleet)? else {
347 return Ok(None);
348 };
349 if !matches!(
350 journal.completion,
351 FleetEnsureCompletion::InProgress | FleetEnsureCompletion::Prepared
352 ) {
353 return Ok(None);
354 }
355 let retained = verified_plan(
356 retained_plan::read(&paths, environment, requested_fleet)?
357 .ok_or(EnsureWorkflowError::PlanMissing)?,
358 )?;
359 if retained.environment != environment || retained.fleet != requested_fleet {
360 return Err(EnsureWorkflowError::PlanIntegrity);
361 }
362 let state = read_state(&paths, requested_fleet)?;
363 let prepared_full = journal.completion == FleetEnsureCompletion::Prepared
364 && (retained.scope == FleetEnsurePlanScope::Full
365 || (retained.scope == FleetEnsurePlanScope::RootReinstallPrerequisite
366 && retained.reinstall.as_ref().is_some_and(|intent| {
367 intent
368 .activation_reset
369 .as_ref()
370 .is_some_and(|activation| activation.preparation.is_some())
371 })))
372 && retained.reinstall.as_ref().is_some_and(|intent| {
373 intent.operation_id == journal.operation_id
374 && intent.operation_id == retained.operation_id
375 });
376 if !prepared_full {
377 verify_journal(&journal, &retained, requested_fleet, &state)?;
378 }
379 Ok(Some(retained))
380}
381
382#[expect(
384 clippy::too_many_lines,
385 reason = "one locked read-only transaction reconciles retained authority, observation and the immutable plan"
386)]
387pub fn plan<P>(
388 root: &Path,
389 desired: &crate::fleet_ensure::model::DesiredFleet,
390 desired_sha256: &str,
391 requested_fleet: &str,
392 created_at_time: u64,
393 platform: &mut P,
394) -> Result<FleetEnsureReport, EnsureWorkflowError<P::Error>>
395where
396 P: EnsurePlatform,
397{
398 validate_path_identity(desired, requested_fleet)?;
399 let paths = EnsurePaths::under(root, &desired.environment, requested_fleet);
400 let _lock = lock_operation(&paths)?;
401 if let Some(review) = crate::fleet_ensure::ops::reinstall::adoption::review(&paths)? {
402 if review.desired_sha256 != desired_sha256 {
403 return Err(EnsureWorkflowError::DriftedBeforeApply);
404 }
405 return Ok(FleetEnsureReport {
406 funding_review: None,
407 actual_conservation: None,
408 effects_applied: 0,
409 plan: review,
410 terminal: false,
411 });
412 }
413 if let Some(mut journal) =
414 retained_plan::journal(&paths, &desired.environment, requested_fleet)?
415 && journal.completion == FleetEnsureCompletion::InProgress
416 {
417 let retained = verified_plan(
418 retained_plan::read(&paths, &desired.environment, requested_fleet)?
419 .ok_or(EnsureWorkflowError::PlanMissing)?,
420 )?;
421 let state = read_state(&paths, requested_fleet)?;
422 verify_journal(&journal, &retained, requested_fleet, &state)?;
423 if journal.estate_funding_required.is_some()
424 || funding::native_review_applicable(&retained, &journal)
425 {
426 if let Some(reviewed) = retained.reviewed_desired.as_deref() {
427 platform
428 .bind_reviewed_desired(reviewed.desired())
429 .map_err(EnsureWorkflowError::Platform)?;
430 } else if retained.desired_sha256 != desired_sha256 {
431 return Err(EnsureWorkflowError::RetainedDesiredUnavailable {
432 actual: desired_sha256.to_string(),
433 expected: retained.desired_sha256.clone(),
434 });
435 }
436 }
437 let funding_review = funding::prepare(
438 &paths,
439 &retained,
440 &mut journal,
441 &state,
442 created_at_time,
443 platform,
444 )?;
445 return Ok(FleetEnsureReport {
446 funding_review,
447 actual_conservation: None,
448 effects_applied: applied_count(&journal),
449 plan: retained,
450 terminal: false,
451 });
452 }
453 let mut state = read_state(&paths, requested_fleet)?;
454 let prior_plan = retained_plan::read(&paths, &desired.environment, requested_fleet)?
455 .map(verified_plan)
456 .transpose()?;
457 let prior_journal = retained_plan::journal(&paths, &desired.environment, requested_fleet)?;
458 if let Some(prior) = &prior_plan {
459 let recovery_ready = prior_journal
460 .as_ref()
461 .map(|journal| reinstall_recovery_ready(prior, journal, &state))
462 .transpose()?
463 .unwrap_or(false);
464 if recovery_ready && let Some(journal) = &prior_journal {
465 if prior.desired_sha256 != desired_sha256 {
466 return Err(EnsureWorkflowError::ReinstallConflict);
467 }
468 retain_completed_reinstalls(&mut state, prior, journal);
469 }
470 let activation_reset = prior
471 .reinstall
472 .as_ref()
473 .is_some_and(|intent| intent.activation_reset.is_some());
474 if activation_reset
475 && prior.scope == FleetEnsurePlanScope::RootReinstallPrerequisite
476 && let Some(journal) = prior_journal
477 .as_ref()
478 .filter(|journal| journal.completion == FleetEnsureCompletion::Converged)
479 {
480 verify_journal(journal, prior, requested_fleet, &state)?;
481 retain_completed_reinstalls(&mut state, prior, journal);
482 write_state(&paths, &state)?;
483 }
484 let unfinished_reset = prior.reinstall.is_some()
485 && !recovery_ready
486 && !prior_journal.as_ref().is_some_and(|journal| {
487 journal.plan_sha256 == prior.plan_sha256
488 && journal.completion == FleetEnsureCompletion::Converged
489 });
490 if prior.scope == FleetEnsurePlanScope::ReinstallPreparation
491 || unfinished_reset
492 || (activation_reset && prior.scope == FleetEnsurePlanScope::RootReinstallPrerequisite)
493 {
494 if prior.desired_sha256 != desired_sha256 {
495 return Err(EnsureWorkflowError::ReinstallConflict);
496 }
497 return reinstall::continue_plan(
498 root,
499 &paths,
500 desired,
501 prior,
502 prior_journal.as_ref(),
503 &state,
504 platform,
505 );
506 }
507 }
508 let matching_prior = reconcile_retained_planning_state(
509 &mut state,
510 prior_plan.as_ref(),
511 prior_journal.as_ref(),
512 requested_fleet,
513 &desired.environment,
514 desired_sha256,
515 )?;
516 if let Some(prior) = matching_prior {
517 retain_plan_cycles(&mut state, prior);
518 if let Some(journal) = &prior_journal {
519 retain_completed_reinstalls(&mut state, prior, journal);
520 }
521 }
522 let operation_id = matching_prior
523 .filter(|prior| prior.scope == FleetEnsurePlanScope::Full)
524 .map_or_else(
525 || operation_id(desired_sha256, &desired.environment, requested_fleet),
526 |prior| prior.operation_id.clone(),
527 );
528 if let Some(management) = platform
529 .observe_root_management(&state, &BTreeSet::new())
530 .map_err(EnsureWorkflowError::Platform)?
531 {
532 let root_start_authority = read_root_start_authority(&paths)?;
533 if let Some(plan) = compile_root_start_prerequisite_plan(RootStartPlanInput {
534 state: &state,
535 authority: root_start_authority.as_ref(),
536 created_at_time,
537 desired,
538 desired_sha256,
539 observation: &management,
540 requested_fleet,
541 })? {
542 write_plan(&paths, &plan)?;
543 return Ok(FleetEnsureReport {
544 funding_review: None,
545 actual_conservation: None,
546 effects_applied: 0,
547 plan,
548 terminal: false,
549 });
550 }
551 if let Some(plan) = crate::fleet_ensure::policy::root_reinstall::compile(
552 RootStartPlanInput {
553 state: &state,
554 authority: None,
555 created_at_time,
556 desired,
557 desired_sha256,
558 observation: &management,
559 requested_fleet,
560 },
561 &resolve_desired_artifacts(root, desired)?,
562 )? {
563 write_plan(&paths, &plan)?;
564 return Ok(FleetEnsureReport {
565 funding_review: None,
566 actual_conservation: None,
567 effects_applied: 0,
568 plan,
569 terminal: false,
570 });
571 }
572 }
573 let artifacts = resolve_desired_artifacts(root, desired)?;
574 let mut observation = platform
575 .observe(&operation_id, &state)
576 .map_err(EnsureWorkflowError::Platform)?;
577 retain_observed_cycles(&mut state, &observation);
578 write_state(&paths, &state)?;
581 let terminal_inventory_operation_id = current_terminal_inventory_operation(
582 &state,
583 prior_plan.as_ref(),
584 prior_journal.as_ref(),
585 requested_fleet,
586 )?;
587 if let Some(operation_id) = terminal_inventory_operation_id.as_deref() {
588 attach_terminal_inventory_cycles(operation_id, &state, platform, &mut observation)?;
589 }
590 let mut protocol_actions = platform
591 .protocol_actions(&operation_id, &state)
592 .map_err(EnsureWorkflowError::Platform)?;
593 let mut retained_observations = 0_u32;
594 let mut plan = loop {
595 match compile_plan(
596 desired,
597 &artifacts,
598 &protocol_actions,
599 desired_sha256,
600 requested_fleet,
601 &observation,
602 created_at_time,
603 &operation_id,
604 None,
605 ) {
606 Ok(plan) => break plan,
607 Err(EnsurePolicyError::PendingRootOwnedBalance { name }) => {
608 retained_observations = retained_observations.saturating_add(1);
609 if retained_observations >= desired.maximum_stalled_observations {
610 return Err(EnsureWorkflowError::RootOwnedObservationStalled {
611 last_lifecycle: root_owned_lifecycle_label(&observation, &name).to_string(),
612 observations: retained_observations,
613 target: name,
614 });
615 }
616 platform.pace_root_owned_observation(&name, retained_observations);
617 observation = platform
618 .observe(&operation_id, &state)
619 .map_err(EnsureWorkflowError::Platform)?;
620 retain_observed_cycles(&mut state, &observation);
621 write_state(&paths, &state)?;
622 if let Some(operation_id) = terminal_inventory_operation_id.as_deref() {
623 attach_terminal_inventory_cycles(
624 operation_id,
625 &state,
626 platform,
627 &mut observation,
628 )?;
629 }
630 protocol_actions = platform
631 .protocol_actions(&operation_id, &state)
632 .map_err(EnsureWorkflowError::Platform)?;
633 }
634 Err(error) => return Err(error.into()),
635 }
636 };
637 for action in ordered_actions(&plan) {
638 verify_store_activation_binding(&plan, action, &state, platform)?;
639 }
640 bind_terminal_inventory_operation(&mut plan, terminal_inventory_operation_id);
641 write_plan(&paths, &plan)?;
642 Ok(FleetEnsureReport {
643 funding_review: None,
644 actual_conservation: None,
645 effects_applied: 0,
646 plan,
647 terminal: false,
648 })
649}
650
651fn current_terminal_inventory_operation<E>(
652 state: &FleetEnsureStateRecord,
653 prior_plan: Option<&FleetEnsurePlan>,
654 prior_journal: Option<&FleetEnsureJournalRecord>,
655 requested_fleet: &str,
656) -> Result<Option<String>, EnsureWorkflowError<E>>
657where
658 E: std::error::Error + 'static,
659{
660 if state.active_registry.is_none() {
661 return Ok(None);
662 }
663 let prior_plan = prior_plan.ok_or(EnsureWorkflowError::PlanMissing)?;
664 let prior_journal = prior_journal.ok_or(EnsureWorkflowError::JournalIntegrity)?;
665 verify_journal(prior_journal, prior_plan, requested_fleet, state)?;
666 if prior_journal.completion != FleetEnsureCompletion::Converged
667 || prior_plan.scope != FleetEnsurePlanScope::Full
668 {
669 return Err(EnsureWorkflowError::JournalIntegrity);
670 }
671 let operation_id = prior_plan
672 .terminal_inventory_operation_id
673 .as_ref()
674 .unwrap_or(&prior_plan.operation_id);
675 if operation_id.is_empty() {
676 return Err(EnsureWorkflowError::JournalIntegrity);
677 }
678 Ok(Some(operation_id.clone()))
679}
680
681fn bind_terminal_inventory_operation(plan: &mut FleetEnsurePlan, operation_id: Option<String>) {
682 plan.terminal_inventory_operation_id = operation_id;
683 plan.plan_sha256 = expected_plan_sha256(plan);
684}
685
686fn reconcile_retained_planning_state<'a, E>(
687 state: &mut FleetEnsureStateRecord,
688 prior_plan: Option<&'a FleetEnsurePlan>,
689 prior_journal: Option<&FleetEnsureJournalRecord>,
690 requested_fleet: &str,
691 desired_environment: &str,
692 desired_sha256: &str,
693) -> Result<Option<&'a FleetEnsurePlan>, EnsureWorkflowError<E>>
694where
695 E: std::error::Error + 'static,
696{
697 if let Some((prior, journal)) = prior_plan.zip(prior_journal)
698 && prior.fleet == requested_fleet
699 && prior.environment == desired_environment
700 && retained_reinstall_desired_conflict(
701 state,
702 &prior.fleet,
703 &prior.operation_id,
704 &prior.desired_sha256,
705 desired_sha256,
706 journal,
707 )
708 {
709 return Err(EnsureWorkflowError::RetainedReinstallDesiredConflict {
710 actual: desired_sha256.to_string(),
711 expected: prior.desired_sha256.clone(),
712 });
713 }
714 if let Some((prior, journal)) = prior_plan.zip(prior_journal)
715 && prior.scope == FleetEnsurePlanScope::RootReinstallPrerequisite
716 && prior.desired_sha256 != desired_sha256
717 {
718 verify_journal(journal, prior, requested_fleet, state)?;
719 return Err(EnsureWorkflowError::RootActivationDesiredConflict {
720 actual: desired_sha256.to_string(),
721 expected: prior.desired_sha256.clone(),
722 });
723 }
724 let matching_prior = prior_plan.filter(|prior| {
725 prior.fleet == requested_fleet
726 && prior.environment == desired_environment
727 && prior.desired_sha256 == desired_sha256
728 });
729 let retained_operation_matches =
730 matching_prior
731 .zip(prior_journal)
732 .is_some_and(|(prior, journal)| {
733 retained_reinstall_operation_matches(
734 state,
735 &prior.fleet,
736 &prior.operation_id,
737 journal,
738 )
739 });
740 if !retained_operation_matches {
741 state.completed_reinstall_action_sha256.clear();
742 state.completed_reinstall_operation_id = None;
743 state.completed_reinstalls.clear();
744 }
745 Ok(matching_prior)
746}
747
748fn retain_completed_reinstalls(
749 state: &mut FleetEnsureStateRecord,
750 plan: &FleetEnsurePlan,
751 journal: &FleetEnsureJournalRecord,
752) {
753 if journal.fleet != plan.fleet || journal.operation_id != plan.operation_id {
754 return;
755 }
756 for action in ordered_actions(plan) {
757 let EnsureAction::Install {
758 mode: crate::fleet_ensure::model::InstallMode::Reinstall,
759 name,
760 principal,
761 wasm_sha256,
762 ..
763 } = action
764 else {
765 continue;
766 };
767 let action_sha256 = action_sha256(action);
768 let Some(effect) = journal
769 .effects
770 .iter()
771 .find(|effect| effect.action_sha256 == action_sha256)
772 else {
773 continue;
774 };
775 if effect.state == EffectState::Applied
776 && let Some(pre_canister_version) = effect.pre_canister_version
777 && state
778 .principals
779 .get(name)
780 .is_some_and(|retained| retained == principal)
781 {
782 state.completed_reinstall_operation_id = Some(journal.operation_id.clone());
783 state
784 .completed_reinstall_action_sha256
785 .insert(name.clone(), action_sha256);
786 state
787 .completed_reinstalls
788 .insert(name.clone(), pre_canister_version);
789 if let Some(topology) = state.topology.get_mut(name) {
790 topology.module_hash = Some(wasm_sha256.clone());
791 }
792 }
793 }
794}
795
796fn completed_reinstall_evidence_matches(
797 state: &FleetEnsureStateRecord,
798 journal: &FleetEnsureJournalRecord,
799) -> bool {
800 !state.completed_reinstalls.is_empty()
801 && state
802 .completed_reinstalls
803 .iter()
804 .all(|(name, pre_version)| {
805 state
806 .completed_reinstall_action_sha256
807 .get(name)
808 .and_then(|action_sha256| {
809 journal
810 .effects
811 .iter()
812 .find(|effect| effect.action_sha256 == *action_sha256)
813 })
814 .is_some_and(|effect| {
815 effect.state == EffectState::Applied
816 && effect.pre_canister_version == Some(*pre_version)
817 })
818 })
819 && state.completed_reinstall_action_sha256.len() == state.completed_reinstalls.len()
820}
821
822fn reinstall_recovery_ready<E: std::error::Error + 'static>(
824 plan: &FleetEnsurePlan,
825 journal: &FleetEnsureJournalRecord,
826 state: &FleetEnsureStateRecord,
827) -> Result<bool, EnsureWorkflowError<E>> {
828 if plan.scope != FleetEnsurePlanScope::Full
829 || plan.reinstall.is_none()
830 || journal.completion != FleetEnsureCompletion::ReplanRequired
831 || journal.plan_sha256 != plan.plan_sha256
832 {
833 return Ok(false);
834 }
835 verify_journal(journal, plan, &plan.fleet, state)?;
836 let actions = continuation::actions(plan, journal);
837 let complete = actions.len() == journal.effects.len()
838 && journal
839 .effects
840 .iter()
841 .all(|effect| effect.state == EffectState::Applied);
842 let installs_recorded = actions
843 .iter()
844 .zip(&journal.effects)
845 .all(|(action, effect)| {
846 !matches!(action, EnsureAction::Install { .. }) || effect.pre_canister_version.is_some()
847 });
848 if !complete || !installs_recorded {
849 return Err(EnsureWorkflowError::JournalIntegrity);
850 }
851 Ok(true)
852}
853
854fn retained_reinstall_operation_matches(
855 state: &FleetEnsureStateRecord,
856 prior_fleet: &str,
857 prior_operation_id: &str,
858 journal: &FleetEnsureJournalRecord,
859) -> bool {
860 journal.completion == FleetEnsureCompletion::ReplanRequired
861 && journal.fleet == prior_fleet
862 && journal.operation_id == prior_operation_id
863 && state.completed_reinstall_operation_id.as_deref() == Some(journal.operation_id.as_str())
864 && completed_reinstall_evidence_matches(state, journal)
865}
866
867fn retained_reinstall_desired_conflict(
868 state: &FleetEnsureStateRecord,
869 prior_fleet: &str,
870 prior_operation_id: &str,
871 prior_desired_sha256: &str,
872 supplied_desired_sha256: &str,
873 journal: &FleetEnsureJournalRecord,
874) -> bool {
875 prior_desired_sha256 != supplied_desired_sha256
876 && retained_reinstall_operation_matches(state, prior_fleet, prior_operation_id, journal)
877}
878
879fn retain_plan_cycles(state: &mut FleetEnsureStateRecord, plan: &FleetEnsurePlan) {
880 for canister in &plan.canisters {
881 if let Some(principal) = &canister.principal {
882 state
883 .retained_cycles_by_principal
884 .entry(principal.clone())
885 .or_insert(canister.observed_cycles);
886 }
887 }
888}
889
890fn retain_observed_cycles(state: &mut FleetEnsureStateRecord, observation: &FleetObservation) {
891 for live in observation.canisters.values().filter_map(Option::as_ref) {
892 state
893 .retained_cycles_by_principal
894 .insert(live.principal.clone(), live.cycles);
895 }
896 state
897 .retained_cycles_by_principal
898 .extend(observation.additional_controlled_cycles.clone());
899}
900
901fn root_owned_lifecycle_label(observation: &FleetObservation, target: &str) -> &'static str {
902 observation
903 .canisters
904 .get(target)
905 .and_then(Option::as_ref)
906 .and_then(|live| live.root_owned_lifecycle)
907 .map_or("unavailable", |lifecycle| lifecycle.label())
908}
909
910#[expect(
912 clippy::too_many_lines,
913 reason = "the sole effect driver keeps intent, reconciliation and terminal publication visibly ordered"
914)]
915pub fn apply<P>(
916 root: &Path,
917 desired: &crate::fleet_ensure::model::DesiredFleet,
918 desired_sha256: &str,
919 requested_fleet: &str,
920 reviewed_plan_sha256: &str,
921 platform: &mut P,
922) -> Result<FleetEnsureReport, EnsureWorkflowError<P::Error>>
923where
924 P: EnsurePlatform,
925{
926 validate_path_identity(desired, requested_fleet)?;
927 let paths = EnsurePaths::under(root, &desired.environment, requested_fleet);
928 let _lock = lock_operation(&paths)?;
929 let pending = crate::fleet_ensure::ops::reinstall::adoption::review(&paths)?;
930 let adopting = pending.is_some();
931 let retained_plan = verified_plan(match pending {
932 Some(plan) => plan,
933 None => retained_plan::read(&paths, &desired.environment, requested_fleet)?
934 .ok_or(EnsureWorkflowError::PlanMissing)?,
935 })?;
936 let retained_journal = if adopting {
937 None
938 } else {
939 retained_plan::journal(&paths, &desired.environment, requested_fleet)?
940 };
941 if retained_plan.plan_sha256 != reviewed_plan_sha256
942 && !retained_journal
943 .as_ref()
944 .is_some_and(|journal| funding::accepts(&retained_plan, journal, reviewed_plan_sha256))
945 {
946 return Err(EnsureWorkflowError::PlanDigestMismatch {
947 actual: retained_plan.plan_sha256,
948 expected: reviewed_plan_sha256.to_string(),
949 });
950 }
951 let mut state = read_state(&paths, requested_fleet)?;
952 let in_progress = retained_journal
953 .as_ref()
954 .is_some_and(|journal| journal.completion == FleetEnsureCompletion::InProgress);
955 if let Some(journal) = retained_journal.as_ref().filter(|_| in_progress) {
956 verify_journal(journal, &retained_plan, requested_fleet, &state)?;
957 compact_inline_plan(&paths, &retained_plan)?;
958 }
959 if !in_progress
960 && retained_plan.reinstall.is_none()
961 && retained_plan.desired_sha256 != desired_sha256
962 {
963 return Err(EnsureWorkflowError::DriftedBeforeApply);
964 }
965 let operation_desired = if in_progress || retained_plan.reinstall.is_some() {
966 if let Some(reviewed) = retained_plan.reviewed_desired.as_deref() {
967 let reviewed = reviewed.desired();
968 validate_path_identity(reviewed, requested_fleet)?;
969 if reviewed.environment != retained_plan.environment {
970 return Err(EnsureWorkflowError::PlanIntegrity);
971 }
972 reviewed
973 } else if retained_plan.desired_sha256 == desired_sha256 {
974 desired
975 } else {
976 return Err(EnsureWorkflowError::RetainedDesiredUnavailable {
977 actual: desired_sha256.to_string(),
978 expected: retained_plan.desired_sha256.clone(),
979 });
980 }
981 } else {
982 if retained_plan.desired_sha256 != desired_sha256 {
983 return Err(EnsureWorkflowError::DriftedBeforeApply);
984 }
985 desired
986 };
987 if !retained_journal.as_ref().is_some_and(|journal| {
988 journal.completion == FleetEnsureCompletion::Converged
989 && journal.plan_sha256 == retained_plan.plan_sha256
990 }) {
991 validate_creation_fee_scope(
992 operation_desired,
993 &retained_plan.canisters,
994 &retained_plan.conservation.estate_funding_domains,
995 )?;
996 }
997 platform
998 .bind_reviewed_desired(operation_desired)
999 .map_err(EnsureWorkflowError::Platform)?;
1000 let operation_desired_sha256 = retained_plan.desired_sha256.as_str();
1001 continuation::verify_inputs(root, operation_desired, &retained_plan)?;
1002 if let Some(journal) = &retained_journal {
1003 continuation::verify_canonical(&retained_plan, journal, &state, platform)?;
1004 if reinstall_recovery_ready(&retained_plan, journal, &state)? {
1005 return Err(EnsureWorkflowError::SuccessorReviewRequired {
1006 reason:
1007 crate::fleet_ensure::model::FleetEnsureSuccessorReviewReason::AdditionalEffect,
1008 review: None,
1009 });
1010 }
1011 }
1012 let completed_journal = retained_journal.as_ref().filter(|journal| {
1013 (journal.completion == FleetEnsureCompletion::Converged
1014 || (journal.completion == FleetEnsureCompletion::Prepared
1015 && retained_plan.scope == FleetEnsurePlanScope::ReinstallPreparation))
1016 && journal.plan_sha256 == retained_plan.plan_sha256
1017 });
1018 if retained_plan.scope == FleetEnsurePlanScope::ReinstallPreparation
1019 && let Some(journal) = completed_journal
1020 {
1021 let actual = reinstall::complete(&retained_plan, journal, &state, platform)?;
1022 return Ok(FleetEnsureReport {
1023 funding_review: None,
1024 actual_conservation: Some(actual),
1025 effects_applied: 0,
1026 plan: retained_plan,
1027 terminal: true,
1028 });
1029 }
1030 if retained_plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite
1031 && let Some(journal) = completed_journal
1032 {
1033 verify_journal(journal, &retained_plan, requested_fleet, &state)?;
1034 let actual = root_reinstall::complete(&retained_plan, journal, &state, platform)?;
1035 report_progress_state(
1036 platform,
1037 &retained_plan,
1038 journal,
1039 FleetEnsurePhase::Infrastructure,
1040 FleetEnsureProgressState::PrerequisiteComplete,
1041 );
1042 return Ok(FleetEnsureReport {
1043 funding_review: None,
1044 actual_conservation: Some(actual),
1045 effects_applied: 0,
1046 plan: retained_plan,
1047 terminal: true,
1048 });
1049 }
1050 if retained_plan.scope == FleetEnsurePlanScope::Full
1051 && let Some(journal) = completed_journal
1052 {
1053 verify_journal(journal, &retained_plan, requested_fleet, &state)?;
1054 let actual = continuation::replay(
1055 root,
1056 operation_desired,
1057 &retained_plan,
1058 journal,
1059 &state,
1060 platform,
1061 )?;
1062 report_progress(
1063 platform,
1064 &retained_plan,
1065 journal,
1066 FleetEnsurePhase::Complete,
1067 );
1068 return Ok(FleetEnsureReport {
1069 funding_review: None,
1070 actual_conservation: Some(actual),
1071 effects_applied: 0,
1072 plan: retained_plan,
1073 terminal: true,
1074 });
1075 }
1076 if retained_plan.scope == FleetEnsurePlanScope::RootStartPrerequisite
1077 && let Some(journal) = completed_journal
1078 {
1079 verify_terminal_root_start_replay(
1080 operation_desired,
1081 &retained_plan,
1082 journal,
1083 requested_fleet,
1084 platform,
1085 &state,
1086 )?;
1087 report_progress_state(
1088 platform,
1089 &retained_plan,
1090 journal,
1091 FleetEnsurePhase::Infrastructure,
1092 FleetEnsureProgressState::PrerequisiteComplete,
1093 );
1094 return Ok(FleetEnsureReport {
1095 funding_review: None,
1096 actual_conservation: None,
1097 effects_applied: applied_count(journal),
1098 plan: retained_plan,
1099 terminal: true,
1100 });
1101 }
1102 let mut journal = match retained_journal {
1103 Some(journal) if journal.completion == FleetEnsureCompletion::InProgress => {
1104 verify_journal(&journal, &retained_plan, requested_fleet, &state)?;
1105 journal
1106 }
1107 _ => {
1108 let (observation, initial_controlled_cycles) = verify_fresh_plan(
1109 root,
1110 operation_desired,
1111 operation_desired_sha256,
1112 requested_fleet,
1113 &retained_plan,
1114 platform,
1115 &state,
1116 )?;
1117 root_reinstall::verify_initial_funding(
1118 &retained_plan,
1119 observation.operator_cycles,
1120 platform,
1121 )?;
1122 let journal = FleetEnsureJournalRecord {
1123 funding_reviews: Vec::new(),
1124 successor_phases: Vec::new(),
1125 completion: FleetEnsureCompletion::InProgress,
1126 estate_funding_required: None,
1127 effects: Vec::new(),
1128 fleet: requested_fleet.to_string(),
1129 initial_controlled_cycles,
1130 initial_estate_funding_cycles_by_root: retained_plan
1131 .conservation
1132 .estate_funding_domains
1133 .iter()
1134 .map(|domain| {
1135 (
1136 domain.root.clone(),
1137 domain.available_cycles.unwrap_or_default(),
1138 )
1139 })
1140 .collect(),
1141 initial_operator_cycles: observation.operator_cycles,
1142 operation_id: retained_plan.operation_id.clone(),
1143 plan_sha256: retained_plan.plan_sha256.clone(),
1144 schema_version: FLEET_ENSURE_SCHEMA_VERSION,
1145 stalled_observations: 0,
1146 };
1147 if adopting {
1149 crate::fleet_ensure::ops::reinstall::adoption::adopt(
1150 &paths,
1151 &retained_plan,
1152 &journal,
1153 )?;
1154 } else {
1155 write_journal(&paths, &journal)?;
1156 }
1157 journal
1158 }
1159 };
1160
1161 funding::resume(
1162 &paths,
1163 &retained_plan,
1164 &mut journal,
1165 &state,
1166 reviewed_plan_sha256,
1167 platform,
1168 )?;
1169 if journal
1170 .estate_funding_required
1171 .as_ref()
1172 .is_some_and(|pause| funding::covered(&journal, pause))
1173 {
1174 retain_estate_funding_pause(&paths, &mut journal, None)?;
1175 }
1176 let mut terminal_state;
1177 loop {
1178 let execution_actions = continuation::actions(&retained_plan, &journal)
1179 .into_iter()
1180 .cloned()
1181 .collect::<Vec<_>>();
1182 let actions = execution_actions.iter().collect::<Vec<_>>();
1183 let mut replayed_issued_commands = BTreeSet::new();
1184 let mut reported_phase = None;
1185 loop {
1186 let mut deferred_controller_observation = false;
1187 for (index, action) in actions.iter().enumerate() {
1188 let action_hash = action_sha256(action);
1189 let observation_policy = effect_observation_policy(operation_desired, action)?;
1190 let retained_effect = journal.effects.get(index);
1191 if retained_effect.is_none_or(|effect| effect.state != EffectState::Applied) {
1192 let phase = action_progress_phase(action);
1193 if reported_phase != Some(phase) {
1194 report_progress(platform, &retained_plan, &journal, phase);
1195 reported_phase = Some(phase);
1196 }
1197 }
1198 if matches!(action, EnsureAction::FleetProtocol { .. })
1199 && retained_effect.is_none_or(|effect| effect.state == EffectState::Intent)
1200 && !prior_fleet_protocol_effect_started(&actions, &journal, index)
1201 {
1202 let funding_observation = platform
1203 .observe(&journal.operation_id, &state)
1204 .map_err(EnsureWorkflowError::Platform)?;
1205 let funded_plan = funding_plan(&retained_plan, &journal)?;
1206 let required =
1207 estate_funding_requirement(&funded_plan, &state, &funding_observation)?;
1208 retain_estate_funding_pause(&paths, &mut journal, required.as_ref())?;
1209 if let Some(required) = required {
1210 report_progress_state(
1211 platform,
1212 &retained_plan,
1213 &journal,
1214 action_progress_phase(action),
1215 FleetEnsureProgressState::FundingRequired,
1216 );
1217 return Err(EnsureWorkflowError::EstateFundingRequired(Box::new(
1218 estate_funding_error(&required),
1219 )));
1220 }
1221 }
1222 let mut initial_observation = if journal.effects.len() <= index {
1223 let prepared = prepare_effect(platform, &journal.operation_id, action, &state)
1224 .map_err(EnsureWorkflowError::Platform)?;
1225 journal.effects.push(prepared.record);
1226 write_journal(&paths, &journal)?;
1227 prepared.observation
1228 } else {
1229 None
1230 };
1231 let observation_started = Instant::now();
1232 loop {
1233 let record = journal
1234 .effects
1235 .get(index)
1236 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1237 if record.action_sha256 != action_hash {
1238 return Err(EnsureWorkflowError::DriftedBeforeApply);
1239 }
1240 if !create_record_is_exact(action, record, &state) {
1241 return Err(EnsureWorkflowError::JournalIntegrity);
1242 }
1243 if matches!(record.state, EffectState::Applied) {
1244 let retained_create = journal
1245 .effects
1246 .get(index)
1247 .ok_or(EnsureWorkflowError::JournalIntegrity)
1248 .and_then(|record| {
1249 retain_applied_create_authority(
1250 operation_desired,
1251 action,
1252 record,
1253 &mut state,
1254 )
1255 })?;
1256 let retained_funding = journal.effects.get(index).is_some_and(|record| {
1257 retain_applied_funding_cycles(&mut state, action, record)
1258 });
1259 if retained_create || retained_funding {
1260 write_state(&paths, &state)?;
1261 }
1262 break;
1263 }
1264
1265 let observed = match initial_observation.take() {
1266 Some(observed) => observed,
1267 None => platform
1268 .observe_effect(&journal.operation_id, action, record, &state)
1269 .map_err(EnsureWorkflowError::Platform)?,
1270 };
1271 let source_cycles = if observed.post_cycles.is_some() {
1272 observed.post_cycles
1273 } else {
1274 platform
1275 .action_cycles(action, &state)
1276 .map_err(EnsureWorkflowError::Platform)?
1277 };
1278 let destination_cycles = platform
1279 .action_destination_cycles(action, &state)
1280 .map_err(EnsureWorkflowError::Platform)?;
1281 let replan_after_create_balance =
1282 observed.retry == EffectRetry::ReplanRequiredAfterCreateBalanceDrift;
1283 let retained_create_balance_is_exact = !replan_after_create_balance
1284 || (journal.effects.len() == index + 1
1285 && journal.effects[..index]
1286 .iter()
1287 .all(|effect| effect.state == EffectState::Applied)
1288 && record.state == EffectState::Issued
1289 && create_identity_is_exact(action.name(), record, &state)
1290 && observed.post_cycles.is_some()
1291 && observed.post_cycles == source_cycles);
1292 let protocol_funding_required = if record.state == EffectState::Issued {
1293 observed
1294 .estate_funding_required
1295 .as_ref()
1296 .map(|funding| {
1297 protocol_estate_funding_requirement(&retained_plan, &state, funding)
1298 })
1299 .transpose()?
1300 } else {
1301 None
1302 };
1303 if let Some(required) = protocol_funding_required.as_ref()
1304 && !funding::covered(&journal, required)
1305 {
1306 retain_estate_funding_pause(&paths, &mut journal, Some(required))?;
1307 report_progress_state(
1308 platform,
1309 &retained_plan,
1310 &journal,
1311 action_progress_phase(action),
1312 FleetEnsureProgressState::FundingRequired,
1313 );
1314 return Err(EnsureWorkflowError::EstateFundingRequired(Box::new(
1315 estate_funding_error(required),
1316 )));
1317 }
1318 if journal
1319 .estate_funding_required
1320 .as_ref()
1321 .is_some_and(|required| required.pending_creation_operation_id.is_some())
1322 {
1323 retain_estate_funding_pause(&paths, &mut journal, None)?;
1324 }
1325 let record = journal
1326 .effects
1327 .get_mut(index)
1328 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1329 merge_observed_cycles(record, source_cycles, destination_cycles);
1330 if replan_after_create_balance {
1331 if !retained_create_balance_is_exact {
1332 return Err(EnsureWorkflowError::JournalIntegrity);
1333 }
1334 let EnsureAction::Create {
1335 name,
1336 requested_initial_cycles,
1337 ..
1338 } = action
1339 else {
1340 return Err(EnsureWorkflowError::JournalIntegrity);
1341 };
1342 let actual_cycles = record
1343 .post_cycles
1344 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1345 record.progress_identity = Some(observed.progress_identity);
1346 record.state = EffectState::Applied;
1347 retain_created_canister_for_replan(
1348 operation_desired,
1349 name,
1350 record,
1351 &mut state,
1352 )?;
1353 journal.completion = FleetEnsureCompletion::ReplanRequired;
1354 journal.stalled_observations = 0;
1355 write_journal(&paths, &journal)?;
1356 write_state(&paths, &state)?;
1357 let configured_fee_cycles = operation_desired
1358 .management_creation_fee_cycles
1359 .parse::<Cycles>()
1360 .map(|cycles| cycles.to_u128())
1361 .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
1362 return Err(EnsureWorkflowError::ReplanRequiredAfterCreateBalanceDrift {
1363 actual_cycles,
1364 canister: name.clone(),
1365 configured_fee_cycles,
1366 deficit_cycles: requested_initial_cycles.saturating_sub(actual_cycles),
1367 requested_cycles: *requested_initial_cycles,
1368 });
1369 }
1370 if observed.applied {
1371 record.progress_identity = Some(observed.progress_identity);
1372 record.state = EffectState::Applied;
1373 journal.stalled_observations = 0;
1374 write_journal(&paths, &journal)?;
1375 let record = journal
1376 .effects
1377 .get(index)
1378 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1379 let retained_create = retain_applied_create_authority(
1380 operation_desired,
1381 action,
1382 record,
1383 &mut state,
1384 )?;
1385 let retained_funding =
1386 retain_applied_funding_cycles(&mut state, action, record);
1387 if retained_create || retained_funding {
1388 write_state(&paths, &state)?;
1389 }
1390 report_progress(
1392 platform,
1393 &retained_plan,
1394 &journal,
1395 action_progress_phase(action),
1396 );
1397 break;
1398 }
1399
1400 if observed.retry == EffectRetry::DeferUntilControllerObservation {
1401 if !deferred_create_observation_is_exact(
1402 operation_desired,
1403 &actions,
1404 index,
1405 action,
1406 record,
1407 &state,
1408 ) {
1409 return Err(EnsureWorkflowError::JournalIntegrity);
1410 }
1411 let progress_identity = observed.progress_identity.clone();
1412 if record.progress_identity.as_deref() == Some(&observed.progress_identity)
1413 {
1414 journal.stalled_observations =
1415 journal.stalled_observations.saturating_add(1);
1416 } else {
1417 record.progress_identity = Some(observed.progress_identity);
1418 journal.stalled_observations = 0;
1419 }
1420 write_journal(&paths, &journal)?;
1421 if journal.stalled_observations
1422 >= observation_policy.maximum_stalled_observations
1423 {
1424 return Err(EnsureWorkflowError::Stalled {
1425 action: action.name().to_string(),
1426 observations: journal.stalled_observations,
1427 progress_identity,
1428 });
1429 }
1430 deferred_controller_observation = true;
1431 break;
1432 }
1433
1434 let maintenance_continuation =
1435 observed.retry == EffectRetry::ContinuePoolMaintenance;
1436 if matches!(record.state, EffectState::Intent) || maintenance_continuation {
1437 if let Some(maximum_attempts) = action.fixture_publication_attempt_limit() {
1438 if !reserve_fixture_publication_attempt(record, maximum_attempts) {
1439 return Err(EnsureWorkflowError::FixturePublicationBound {
1440 action: action.name().to_string(),
1441 maximum_attempts,
1442 });
1443 }
1444 write_journal(&paths, &journal)?;
1446 }
1447 let record = journal
1448 .effects
1449 .get_mut(index)
1450 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1451 if let EnsureAction::FleetProtocol {
1452 action: current, ..
1453 } = action
1454 && let CurrentFleetProtocolAction::MaintainPoolReadiness {
1455 maximum_updates,
1456 ..
1457 } = current.as_ref()
1458 {
1459 if record.maintenance_attempts >= *maximum_updates {
1460 return Err(EnsureWorkflowError::PoolMaintenanceBound {
1461 action: action.name().to_string(),
1462 maximum_updates: *maximum_updates,
1463 });
1464 }
1465 if record.progress_identity.as_deref()
1466 == Some(&observed.progress_identity)
1467 {
1468 journal.stalled_observations =
1469 journal.stalled_observations.saturating_add(1);
1470 } else {
1471 journal.stalled_observations = 0;
1472 }
1473 record.maintenance_attempts += 1;
1474 let attempts = record.maintenance_attempts;
1475 write_journal(&paths, &journal)?;
1477 if attempts > 1 {
1478 platform
1479 .pace_effect_observation(action, journal.stalled_observations);
1480 }
1481 } else if maintenance_continuation {
1482 return Err(EnsureWorkflowError::JournalIntegrity);
1483 }
1484 let record = journal
1485 .effects
1486 .get(index)
1487 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1488 if retained_plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite {
1489 root_reinstall::verify_effect_authority(
1490 &retained_plan,
1491 action,
1492 record.pre_cycles,
1493 &state,
1494 platform,
1495 )?;
1496 }
1497 verify_store_activation_binding(&retained_plan, action, &state, platform)?;
1498 reinstall::verify_effect_authority(
1499 &retained_plan,
1500 action,
1501 &state,
1502 platform,
1503 )?;
1504 let outcome =
1505 match platform.apply(&journal.operation_id, action, record, &state) {
1506 Ok(outcome) => outcome,
1507 Err(source) => {
1508 let progress_identity = record
1509 .progress_identity
1510 .clone()
1511 .unwrap_or_else(|| "effect-call-failed".to_string());
1512 journal.stalled_observations =
1513 journal.stalled_observations.saturating_add(1);
1514 write_journal(&paths, &journal)?;
1515 if journal.stalled_observations
1516 >= observation_policy.maximum_stalled_observations
1517 {
1518 return Err(EnsureWorkflowError::Stalled {
1519 action: action.name().to_string(),
1520 observations: journal.stalled_observations,
1521 progress_identity,
1522 });
1523 }
1524 return Err(EnsureWorkflowError::Platform(source));
1525 }
1526 };
1527 if !create_outcome_is_exact(action, record, &state, &outcome) {
1528 return Err(EnsureWorkflowError::JournalIntegrity);
1529 }
1530 if let Some(created) = &outcome.created_principal {
1531 state
1532 .pending_principals
1533 .insert(action.name().to_string(), created.clone());
1534 write_state(&paths, &state)?;
1535 }
1536 let record = journal
1537 .effects
1538 .get_mut(index)
1539 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1540 record.created_principal = outcome.created_principal;
1541 record.receipt = outcome.receipt;
1542 record.post_cycles = outcome.post_cycles;
1543 record.progress_identity = Some(observed.progress_identity);
1544 record.state = EffectState::Issued;
1545 if !maintenance_continuation {
1547 journal.stalled_observations = 0;
1548 }
1549 write_journal(&paths, &journal)?;
1550 continue;
1551 }
1552
1553 if observed.retry == EffectRetry::ReplayExactIssuedCommand
1554 && !replayed_issued_commands.contains(&index)
1555 {
1556 if !matches!(
1557 action,
1558 EnsureAction::FleetProtocol { action, .. }
1559 if matches!(
1560 action.as_ref(),
1561 crate::fleet_ensure::model::CurrentFleetProtocolAction::ProvisionComponents { .. }
1562 )
1563 ) {
1564 return Err(EnsureWorkflowError::JournalIntegrity);
1565 }
1566 let retained_receipt = record.receipt.clone();
1567 let outcome = platform
1568 .apply(&journal.operation_id, action, record, &state)
1569 .map_err(EnsureWorkflowError::Platform)?;
1570 if outcome.created_principal.is_some()
1571 || outcome.receipt != retained_receipt
1572 || outcome.post_cycles.is_some()
1573 {
1574 return Err(EnsureWorkflowError::JournalIntegrity);
1575 }
1576 let record = journal
1577 .effects
1578 .get_mut(index)
1579 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1580 record.progress_identity = Some(observed.progress_identity);
1581 journal.stalled_observations = 0;
1582 write_journal(&paths, &journal)?;
1583 replayed_issued_commands.insert(index);
1584 continue;
1585 }
1586
1587 if record.progress_identity.as_deref() == Some(&observed.progress_identity) {
1588 journal.stalled_observations =
1589 journal.stalled_observations.saturating_add(1);
1590 } else {
1591 record.progress_identity = Some(observed.progress_identity);
1592 journal.stalled_observations = 0;
1593 }
1594 let progress_identity = record
1595 .progress_identity
1596 .clone()
1597 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1598 write_journal(&paths, &journal)?;
1599 if journal.stalled_observations
1600 >= observation_policy.maximum_stalled_observations
1601 {
1602 if let Some((failure, origin)) = observed
1603 .provisioning_failure
1604 .and_then(|failure| failure.origin.map(|origin| (failure, origin)))
1605 {
1606 return Err(EnsureWorkflowError::ProvisioningRetryPending {
1607 root: failure.fleet_subnet_root,
1608 stage: origin.stage,
1609 target: origin.target,
1610 operation_id: origin.operation_id,
1611 diagnostic_code: origin.diagnostic_code,
1612 observations: journal.stalled_observations,
1613 failed_at_ns: origin.failed_at_ns,
1614 });
1615 }
1616 return Err(EnsureWorkflowError::Stalled {
1617 action: action.name().to_string(),
1618 observations: journal.stalled_observations,
1619 progress_identity,
1620 });
1621 }
1622 if observation_policy.paced {
1623 report_progress_state(
1624 platform,
1625 &retained_plan,
1626 &journal,
1627 action_progress_phase(action),
1628 FleetEnsureProgressState::AwaitingProgress {
1629 elapsed_seconds: observation_started.elapsed().as_secs(),
1630 provisioning: observed.provisioning_progress,
1631 },
1632 );
1633 platform.pace_effect_observation(action, journal.stalled_observations);
1634 }
1635 }
1636 }
1637 if journal.effects.len() == actions.len()
1638 && journal
1639 .effects
1640 .iter()
1641 .all(|record| record.state == EffectState::Applied)
1642 {
1643 break;
1644 }
1645 if !deferred_controller_observation {
1646 return Err(EnsureWorkflowError::JournalIntegrity);
1647 }
1648 }
1649
1650 report_progress(
1651 platform,
1652 &retained_plan,
1653 &journal,
1654 FleetEnsurePhase::TerminalVerification,
1655 );
1656 if retained_plan.scope == FleetEnsurePlanScope::ReinstallPreparation {
1657 let actual = reinstall::complete(&retained_plan, &journal, &state, platform)?;
1658 journal.completion = FleetEnsureCompletion::Prepared;
1659 journal.stalled_observations = 0;
1660 write_journal(&paths, &journal)?;
1661 report_progress_state(
1662 platform,
1663 &retained_plan,
1664 &journal,
1665 FleetEnsurePhase::Infrastructure,
1666 FleetEnsureProgressState::PrerequisiteComplete,
1667 );
1668 return Ok(FleetEnsureReport {
1669 funding_review: None,
1670 actual_conservation: Some(actual),
1671 effects_applied: applied_count(&journal),
1672 plan: retained_plan,
1673 terminal: true,
1674 });
1675 }
1676 if retained_plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite {
1677 let actual = root_reinstall::complete(&retained_plan, &journal, &state, platform)?;
1678 publish_terminal_state(operation_desired, &retained_plan, &journal, &mut state)
1679 .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
1680 state.active_registry = None;
1681 write_state(&paths, &state)?;
1682 journal.completion = FleetEnsureCompletion::Converged;
1683 journal.stalled_observations = 0;
1684 write_journal(&paths, &journal)?;
1685 report_progress_state(
1686 platform,
1687 &retained_plan,
1688 &journal,
1689 FleetEnsurePhase::Infrastructure,
1690 FleetEnsureProgressState::PrerequisiteComplete,
1691 );
1692 return Ok(FleetEnsureReport {
1693 funding_review: None,
1694 actual_conservation: Some(actual),
1695 effects_applied: applied_count(&journal),
1696 plan: retained_plan,
1697 terminal: true,
1698 });
1699 }
1700 if retained_plan.scope == FleetEnsurePlanScope::RootStartPrerequisite {
1701 let actual_conservation = complete_root_start_prerequisite(
1702 operation_desired,
1703 &retained_plan,
1704 platform,
1705 &state,
1706 &journal,
1707 )?;
1708 journal.completion = FleetEnsureCompletion::Converged;
1709 journal.stalled_observations = 0;
1710 write_journal(&paths, &journal)?;
1711 report_progress_state(
1712 platform,
1713 &retained_plan,
1714 &journal,
1715 FleetEnsurePhase::Infrastructure,
1716 FleetEnsureProgressState::PrerequisiteComplete,
1717 );
1718 return Ok(FleetEnsureReport {
1719 funding_review: None,
1720 actual_conservation: Some(actual_conservation),
1721 effects_applied: applied_count(&journal),
1722 plan: retained_plan,
1723 terminal: true,
1724 });
1725 }
1726
1727 terminal_state = state.clone();
1728 publish_terminal_state(
1729 operation_desired,
1730 &retained_plan,
1731 &journal,
1732 &mut terminal_state,
1733 )
1734 .map_err(|TerminalStatePublicationError::JournalIntegrity| {
1735 EnsureWorkflowError::JournalIntegrity
1736 })?;
1737 if completed_infrastructure_reinstall(operation_desired, &retained_plan, &journal) {
1738 terminal_state.active_registry = None;
1739 }
1740 project_current_fleet_inventory(&terminal_state)?;
1741 let mut terminal_observation = platform
1742 .observe(&retained_plan.operation_id, &terminal_state)
1743 .map_err(EnsureWorkflowError::Platform)?;
1744 {
1745 let artifacts = resolve_desired_artifacts(root, operation_desired)?;
1746 let observation_started = Instant::now();
1747 let converged = loop {
1748 let protocol_actions = platform
1749 .protocol_actions(&retained_plan.operation_id, &terminal_state)
1750 .map_err(EnsureWorkflowError::Platform)?;
1751 match compile_plan(
1752 operation_desired,
1753 &artifacts,
1754 &protocol_actions,
1755 operation_desired_sha256,
1756 requested_fleet,
1757 &terminal_observation,
1758 retained_plan.planned_at_time,
1759 &retained_plan.operation_id,
1760 None,
1761 ) {
1762 Ok(plan) => {
1763 if journal.stalled_observations != 0 {
1764 journal.stalled_observations = 0;
1765 write_journal(&paths, &journal)?;
1766 }
1767 break plan;
1768 }
1769 Err(EnsurePolicyError::PendingRootOwnedBalance { name }) => {
1770 journal.stalled_observations =
1771 journal.stalled_observations.saturating_add(1);
1772 retain_observed_cycles(&mut state, &terminal_observation);
1773 write_state(&paths, &state)?;
1774 write_journal(&paths, &journal)?;
1775 if journal.stalled_observations
1776 >= operation_desired.maximum_stalled_observations
1777 {
1778 return Err(EnsureWorkflowError::RootOwnedObservationStalled {
1779 last_lifecycle: root_owned_lifecycle_label(
1780 &terminal_observation,
1781 &name,
1782 )
1783 .to_string(),
1784 observations: journal.stalled_observations,
1785 target: name,
1786 });
1787 }
1788 report_progress_state(
1789 platform,
1790 &retained_plan,
1791 &journal,
1792 FleetEnsurePhase::TerminalVerification,
1793 FleetEnsureProgressState::AwaitingProgress {
1794 elapsed_seconds: observation_started.elapsed().as_secs(),
1795 provisioning: None,
1796 },
1797 );
1798 platform.pace_root_owned_observation(&name, journal.stalled_observations);
1799 terminal_observation = platform
1800 .observe(&retained_plan.operation_id, &terminal_state)
1801 .map_err(EnsureWorkflowError::Platform)?;
1802 }
1803 Err(error @ EnsurePolicyError::EstatePoolCapacity { .. }) => {
1804 retain_observed_cycles(&mut terminal_state, &terminal_observation);
1805 retain_completed_reinstalls(&mut terminal_state, &retained_plan, &journal);
1806 write_state(&paths, &terminal_state)?;
1807 journal.completion = FleetEnsureCompletion::ReplanRequired;
1808 journal.stalled_observations = 0;
1809 write_journal(&paths, &journal)?;
1810 return Err(error.into());
1811 }
1812 Err(error) => return Err(error.into()),
1813 }
1814 };
1815 if converged
1816 .canisters
1817 .iter()
1818 .any(|canister| !canister.actions.is_empty())
1819 || !converged.protocol_actions.is_empty()
1820 {
1821 if retained_plan.continuation.is_some() {
1822 let appended = continuation::append(
1823 &paths,
1824 operation_desired,
1825 &retained_plan,
1826 &mut journal,
1827 &terminal_state,
1828 &terminal_observation,
1829 converged,
1830 platform,
1831 );
1832 match appended {
1833 Ok(()) => {
1834 write_state(&paths, &terminal_state)?;
1835 state = terminal_state.clone();
1836 report_progress_state(
1837 platform,
1838 &retained_plan,
1839 &journal,
1840 FleetEnsurePhase::TerminalVerification,
1841 FleetEnsureProgressState::PrerequisiteComplete,
1842 );
1843 continue;
1844 }
1845 Err(EnsureWorkflowError::SuccessorReviewRequired { reason, review }) => {
1846 retain_observed_cycles(&mut terminal_state, &terminal_observation);
1847 retain_completed_reinstalls(
1848 &mut terminal_state,
1849 &retained_plan,
1850 &journal,
1851 );
1852 write_state(&paths, &terminal_state)?;
1853 journal.completion = FleetEnsureCompletion::ReplanRequired;
1854 write_journal(&paths, &journal)?;
1855 report_progress_state(
1856 platform,
1857 &retained_plan,
1858 &journal,
1859 FleetEnsurePhase::TerminalVerification,
1860 FleetEnsureProgressState::ReviewRequired {
1861 reason,
1862 review: review.clone(),
1863 },
1864 );
1865 return Err(EnsureWorkflowError::SuccessorReviewRequired {
1866 reason,
1867 review,
1868 });
1869 }
1870 Err(error) => return Err(error),
1871 }
1872 }
1873 retain_observed_cycles(&mut terminal_state, &terminal_observation);
1874 retain_completed_reinstalls(&mut terminal_state, &retained_plan, &journal);
1875 write_state(&paths, &terminal_state)?;
1876 journal.completion = FleetEnsureCompletion::ReplanRequired;
1877 journal.stalled_observations = 0;
1878 write_journal(&paths, &journal)?;
1879 let reason =
1880 crate::fleet_ensure::model::FleetEnsureSuccessorReviewReason::AdditionalEffect;
1881 let review = Some(Box::new(
1882 crate::fleet_ensure::ops::recovery::review_details(&converged),
1883 ));
1884 report_progress_state(
1885 platform,
1886 &retained_plan,
1887 &journal,
1888 FleetEnsurePhase::TerminalVerification,
1889 FleetEnsureProgressState::ReviewRequired {
1890 reason,
1891 review: review.clone(),
1892 },
1893 );
1894 return Err(EnsureWorkflowError::SuccessorReviewRequired { reason, review });
1895 }
1896 }
1897 break;
1898 }
1899 let terminal_inventory_operation_id =
1900 completed_inventory_operation(&retained_plan, &journal, &terminal_state)?;
1901 let terminal_inventory = platform
1902 .terminal_inventory(terminal_inventory_operation_id, &terminal_state)
1903 .map_err(EnsureWorkflowError::Platform)?;
1904 let terminal_cycles = terminal_inventory.controlled_cycles_by_principal.clone();
1905 merge_terminal_inventory(&mut terminal_state, terminal_inventory)?;
1906 project_current_fleet_inventory(&terminal_state)?;
1907 let mut final_observation = platform
1908 .observe(&retained_plan.operation_id, &terminal_state)
1909 .map_err(EnsureWorkflowError::Platform)?;
1910 attach_terminal_cycles(&mut final_observation, terminal_cycles)?;
1911 reinstall::verify_terminal_estate(&retained_plan, &final_observation)?;
1912 reinstall::verify_terminal_authority(&retained_plan, &terminal_state, platform)?;
1913 let actual_conservation = verify_terminal_conservation(
1914 &retained_plan,
1915 &journal,
1916 &terminal_state,
1917 &final_observation,
1918 )?;
1919 terminal_state.completed_reinstall_action_sha256.clear();
1920 terminal_state.completed_reinstall_operation_id = None;
1921 terminal_state.completed_reinstalls.clear();
1922 write_state(&paths, &terminal_state)?;
1924 journal.completion = FleetEnsureCompletion::Converged;
1925 journal.stalled_observations = 0;
1926 write_journal(&paths, &journal)?;
1927 report_progress(
1928 platform,
1929 &retained_plan,
1930 &journal,
1931 FleetEnsurePhase::Complete,
1932 );
1933 Ok(FleetEnsureReport {
1934 funding_review: None,
1935 actual_conservation: Some(actual_conservation),
1936 effects_applied: applied_count(&journal),
1937 plan: retained_plan,
1938 terminal: true,
1939 })
1940}
1941
1942fn action_progress_phase(action: &EnsureAction) -> FleetEnsurePhase {
1943 match action {
1944 EnsureAction::FleetProtocol { action, .. } => match action.as_ref() {
1945 CurrentFleetProtocolAction::ReconcilePoolAsset { .. } => {
1946 FleetEnsurePhase::ImportReconciliation
1947 }
1948 CurrentFleetProtocolAction::ProvisionComponents { .. } => {
1949 FleetEnsurePhase::WorkloadProvisioning
1950 }
1951 CurrentFleetProtocolAction::MaintainPoolReadiness { .. }
1952 | CurrentFleetProtocolAction::ObservePoolReadiness { .. } => {
1953 FleetEnsurePhase::PoolReadiness
1954 }
1955 _ => FleetEnsurePhase::ControlPlane,
1956 },
1957 _ => FleetEnsurePhase::Infrastructure,
1958 }
1959}
1960
1961fn report_progress<P: EnsurePlatform>(
1962 platform: &mut P,
1963 plan: &FleetEnsurePlan,
1964 journal: &FleetEnsureJournalRecord,
1965 phase: FleetEnsurePhase,
1966) {
1967 let state = if phase == FleetEnsurePhase::Complete {
1968 FleetEnsureProgressState::Complete
1969 } else {
1970 FleetEnsureProgressState::Advancing
1971 };
1972 report_progress_state(platform, plan, journal, phase, state);
1973}
1974
1975fn report_progress_state<P: EnsurePlatform>(
1976 platform: &mut P,
1977 plan: &FleetEnsurePlan,
1978 journal: &FleetEnsureJournalRecord,
1979 phase: FleetEnsurePhase,
1980 state: FleetEnsureProgressState,
1981) {
1982 platform.report_progress(FleetEnsureProgress {
1983 state,
1984 operation_id: plan.operation_id.clone(),
1985 plan_sha256: plan.plan_sha256.clone(),
1986 phase,
1987 applied_effects: applied_count(journal),
1988 reviewed_effects: continuation::actions(plan, journal).len()
1989 + journal
1990 .funding_reviews
1991 .iter()
1992 .filter(|review| review.effect.is_some())
1993 .count(),
1994 });
1995}
1996
1997const fn merge_observed_cycles(
1998 record: &mut EffectRecord,
1999 source_cycles: Option<u128>,
2000 destination_cycles: Option<u128>,
2001) {
2002 if let Some(cycles) = source_cycles {
2003 record.post_cycles = Some(cycles);
2004 }
2005 if let Some(cycles) = destination_cycles {
2006 record.destination_post_cycles = Some(cycles);
2007 }
2008}
2009
2010pub(super) fn retain_applied_create_authority<E>(
2011 desired: &crate::fleet_ensure::model::DesiredFleet,
2012 action: &EnsureAction,
2013 record: &EffectRecord,
2014 state: &mut FleetEnsureStateRecord,
2015) -> Result<bool, EnsureWorkflowError<E>>
2016where
2017 E: std::error::Error + 'static,
2018{
2019 let EnsureAction::Create {
2020 name,
2021 requested_initial_cycles,
2022 ..
2023 } = action
2024 else {
2025 return Ok(false);
2026 };
2027 if record.state != EffectState::Applied {
2028 return Err(EnsureWorkflowError::JournalIntegrity);
2029 }
2030 let configured = desired
2031 .canisters
2032 .iter()
2033 .find(|canister| {
2034 canister.name == *name
2035 && canister.presence == crate::fleet_ensure::model::DesiredPresence::Present
2036 })
2037 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2038 let principal = record
2039 .created_principal
2040 .as_deref()
2041 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2042 let cycles = record
2043 .post_cycles
2044 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2045 let maximum_observation_burn_cycles =
2046 crate::fleet_ensure::ops::maximum_creation_observation_burn(desired)
2047 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2048 if !create_identity_is_exact(name, record, state)
2049 || !create_balance_is_terminal(
2050 Some(cycles),
2051 *requested_initial_cycles,
2052 maximum_observation_burn_cycles,
2053 )
2054 || state
2055 .retained_cycles_by_principal
2056 .get(principal)
2057 .is_some_and(|retained| *retained != cycles)
2058 {
2059 return Err(EnsureWorkflowError::JournalIntegrity);
2060 }
2061 validate_fresh_pool_creation_balance(
2062 desired,
2063 configured,
2064 name,
2065 cycles,
2066 maximum_observation_burn_cycles,
2067 *requested_initial_cycles,
2068 )?;
2069
2070 let prior_topology = state.topology.get(name);
2071 let topology_identity_matches = prior_topology.is_some_and(|topology| {
2072 topology.kind == configured.kind
2073 && topology.parent == configured.parent
2074 && topology.protocol_binding == configured.protocol_binding
2075 && topology.role
2076 == configured
2077 .protocol_binding
2078 .as_ref()
2079 .map(|binding| binding.role.to_string())
2080 });
2081 let topology = crate::fleet_ensure::model::FleetEnsureTopologyRecord {
2082 kind: configured.kind,
2083 module_hash: if topology_identity_matches {
2084 prior_topology.and_then(|topology| topology.module_hash.clone())
2085 } else {
2086 None
2087 },
2088 parent: configured.parent.clone(),
2089 protocol_binding: configured.protocol_binding.clone(),
2090 role: configured
2091 .protocol_binding
2092 .as_ref()
2093 .map(|binding| binding.role.to_string()),
2094 };
2095 let changed = state.retained_cycles_by_principal.get(principal) != Some(&cycles)
2096 || state.topology.get(name) != Some(&topology);
2097 state
2098 .retained_cycles_by_principal
2099 .insert(principal.to_string(), cycles);
2100 state.topology.insert(name.clone(), topology);
2101 Ok(changed)
2102}
2103
2104fn validate_fresh_pool_creation_balance<E>(
2105 desired: &crate::fleet_ensure::model::DesiredFleet,
2106 configured: &crate::fleet_ensure::model::DesiredCanister,
2107 name: &str,
2108 cycles: u128,
2109 maximum_observation_burn_cycles: u128,
2110 requested_initial_cycles: u128,
2111) -> Result<(), EnsureWorkflowError<E>>
2112where
2113 E: std::error::Error + 'static,
2114{
2115 if configured.kind == crate::fleet_ensure::model::DesiredCanisterKind::Pool
2116 && desired
2117 .bootstrap
2118 .as_ref()
2119 .is_some_and(|bootstrap| bootstrap.fresh_estate)
2120 && configured.principal.is_none()
2121 && configured.controllers.is_empty()
2122 && configured.parent.as_ref().is_some_and(|parent| {
2123 configured.controller_canisters.as_slice() == std::slice::from_ref(parent)
2124 })
2125 {
2126 let readiness_floor_cycles = configured
2127 .minimum_cycles
2128 .parse::<Cycles>()
2129 .map(|cycles| cycles.to_u128())
2130 .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
2131 let remaining_controller_burn_cycles = desired
2132 .maximum_update_burn_cycles
2133 .parse::<Cycles>()
2134 .map(|cycles| cycles.to_u128())
2135 .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
2136 let required_pre_finalization_balance_cycles = readiness_floor_cycles
2137 .checked_add(remaining_controller_burn_cycles)
2138 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2139 if cycles < required_pre_finalization_balance_cycles {
2140 return Err(EnsureWorkflowError::FreshPoolCreationUnderfunded(Box::new(
2141 FreshPoolCreationUnderfundedError {
2142 canister: name.to_string(),
2143 live_balance_cycles: cycles,
2144 maximum_observation_burn_cycles,
2145 pre_finalization_shortfall_cycles: required_pre_finalization_balance_cycles
2146 - cycles,
2147 readiness_floor_cycles,
2148 readiness_shortfall_cycles: readiness_floor_cycles.saturating_sub(cycles),
2149 remaining_controller_burn_cycles,
2150 requested_creation_funding_cycles: requested_initial_cycles,
2151 required_pre_finalization_balance_cycles,
2152 },
2153 )));
2154 }
2155 }
2156 Ok(())
2157}
2158
2159fn retain_applied_funding_cycles(
2160 state: &mut FleetEnsureStateRecord,
2161 action: &EnsureAction,
2162 record: &EffectRecord,
2163) -> bool {
2164 let EnsureAction::Fund { principal, .. } = action else {
2165 return false;
2166 };
2167 if record.state != EffectState::Applied || record.receipt.is_none() {
2168 return false;
2169 }
2170 let Some(cycles) = record.post_cycles else {
2171 return false;
2172 };
2173 let principal = principal.strip_prefix("created:").map_or_else(
2174 || Some(principal.as_str()),
2175 |name| state.pending_principals.get(name).map(String::as_str),
2176 );
2177 let Some(principal) = principal else {
2178 return false;
2179 };
2180 if state.retained_cycles_by_principal.get(principal) == Some(&cycles) {
2181 return false;
2182 }
2183 state
2184 .retained_cycles_by_principal
2185 .insert(principal.to_string(), cycles);
2186 true
2187}
2188
2189fn retain_created_canister_for_replan<E>(
2190 desired: &crate::fleet_ensure::model::DesiredFleet,
2191 name: &str,
2192 record: &EffectRecord,
2193 state: &mut FleetEnsureStateRecord,
2194) -> Result<(), EnsureWorkflowError<E>>
2195where
2196 E: std::error::Error + 'static,
2197{
2198 let configured = desired
2199 .canisters
2200 .iter()
2201 .find(|canister| {
2202 canister.name == name
2203 && canister.presence == crate::fleet_ensure::model::DesiredPresence::Present
2204 })
2205 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2206 let principal = record
2207 .created_principal
2208 .as_deref()
2209 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2210 let cycles = record
2211 .post_cycles
2212 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2213 if state.pending_principals.get(name).map(String::as_str) != Some(principal) {
2214 return Err(EnsureWorkflowError::JournalIntegrity);
2215 }
2216 state.pending_principals.remove(name);
2217 state
2218 .principals
2219 .insert(name.to_string(), principal.to_string());
2220 state
2221 .retained_cycles_by_principal
2222 .insert(principal.to_string(), cycles);
2223 state.topology.insert(
2224 name.to_string(),
2225 crate::fleet_ensure::model::FleetEnsureTopologyRecord {
2226 kind: configured.kind,
2227 module_hash: None,
2228 parent: configured.parent.clone(),
2229 protocol_binding: configured.protocol_binding.clone(),
2230 role: configured
2231 .protocol_binding
2232 .as_ref()
2233 .map(|binding| binding.role.to_string()),
2234 },
2235 );
2236 Ok(())
2237}
2238
2239fn create_record_is_exact(
2240 action: &EnsureAction,
2241 record: &EffectRecord,
2242 state: &FleetEnsureStateRecord,
2243) -> bool {
2244 let EnsureAction::Create {
2245 name,
2246 requested_initial_cycles,
2247 ..
2248 } = action
2249 else {
2250 return true;
2251 };
2252 match record.state {
2253 EffectState::Intent => {
2254 record.created_principal.is_none()
2255 && record.receipt.is_none()
2256 && record.post_cycles.is_none()
2257 }
2258 EffectState::Issued => {
2259 create_identity_is_exact(name, record, state)
2260 && record.post_cycles == Some(*requested_initial_cycles)
2261 }
2262 EffectState::Applied => create_identity_is_exact(name, record, state),
2263 }
2264}
2265
2266fn create_identity_is_exact(
2267 name: &str,
2268 record: &EffectRecord,
2269 state: &FleetEnsureStateRecord,
2270) -> bool {
2271 let Some(created_principal) = record
2272 .created_principal
2273 .as_deref()
2274 .filter(|value| !value.is_empty())
2275 else {
2276 return false;
2277 };
2278 if record.receipt.as_deref().is_none_or(str::is_empty) {
2279 return false;
2280 }
2281 state_create_principal_matches(name, created_principal, state, true)
2282}
2283
2284fn state_create_principal_matches(
2285 name: &str,
2286 created_principal: &str,
2287 state: &FleetEnsureStateRecord,
2288 require_retained: bool,
2289) -> bool {
2290 let pending = state.pending_principals.get(name).map(String::as_str);
2291 let terminal = state.principals.get(name).map(String::as_str);
2292 let retained = pending.is_some() || terminal.is_some();
2293 let pending_matches = pending.is_none_or(|principal| principal == created_principal);
2294 let terminal_matches = terminal.is_none_or(|principal| principal == created_principal);
2295 (!require_retained || retained) && pending_matches && terminal_matches
2296}
2297
2298fn create_outcome_is_exact(
2299 action: &EnsureAction,
2300 record: &EffectRecord,
2301 state: &FleetEnsureStateRecord,
2302 outcome: &crate::fleet_ensure::ops::EffectOutcome,
2303) -> bool {
2304 if matches!(action, EnsureAction::FundEstate { .. }) {
2305 return outcome.created_principal.is_none()
2306 && outcome.post_cycles.is_none()
2307 && outcome
2308 .receipt
2309 .as_deref()
2310 .is_some_and(|receipt| !receipt.is_empty());
2311 }
2312 let EnsureAction::Create {
2313 name,
2314 requested_initial_cycles,
2315 ..
2316 } = action
2317 else {
2318 return true;
2319 };
2320 let Some(created_principal) = outcome
2321 .created_principal
2322 .as_deref()
2323 .filter(|value| !value.is_empty())
2324 else {
2325 return false;
2326 };
2327 let Some(receipt) = outcome.receipt.as_deref().filter(|value| !value.is_empty()) else {
2328 return false;
2329 };
2330 if outcome.post_cycles != Some(*requested_initial_cycles) {
2331 return false;
2332 }
2333 let principal_matches = record
2334 .created_principal
2335 .as_deref()
2336 .is_none_or(|retained| retained == created_principal);
2337 let receipt_matches = record
2338 .receipt
2339 .as_deref()
2340 .is_none_or(|retained| retained == receipt);
2341 let state_matches = state_create_principal_matches(name, created_principal, state, false);
2342 principal_matches && receipt_matches && state_matches
2343}
2344
2345fn verify_fresh_plan<P>(
2346 root: &Path,
2347 desired: &crate::fleet_ensure::model::DesiredFleet,
2348 desired_sha256: &str,
2349 requested_fleet: &str,
2350 retained_plan: &FleetEnsurePlan,
2351 platform: &mut P,
2352 state: &FleetEnsureStateRecord,
2353) -> Result<(FleetObservation, u128), EnsureWorkflowError<P::Error>>
2354where
2355 P: EnsurePlatform,
2356{
2357 if retained_plan.reinstall.is_some() {
2358 return reinstall::verify_before_apply(root, desired, retained_plan, platform, state);
2359 }
2360 if retained_plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite {
2361 return root_reinstall::verify_before_apply(root, desired, retained_plan, platform, state);
2362 }
2363 if retained_plan.scope == FleetEnsurePlanScope::RootStartPrerequisite {
2364 let targets = reviewed_root_start_targets(retained_plan)?;
2365 let management = platform
2366 .observe_root_management(state, &targets)
2367 .map_err(EnsureWorkflowError::Platform)?
2368 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2369 let current = recompile_root_start_prerequisite_plan(
2370 RootStartPlanInput {
2371 state,
2372 authority: retained_plan.root_start_authority.as_deref(),
2373 created_at_time: retained_plan.planned_at_time,
2374 desired,
2375 desired_sha256,
2376 observation: &management,
2377 requested_fleet,
2378 },
2379 &targets,
2380 )?;
2381 if !compatible_root_start_prerequisite(retained_plan, ¤t, desired) {
2382 return Err(EnsureWorkflowError::DriftedBeforeApply);
2383 }
2384 let observation = root_management_fleet_observation(&management, &targets)?;
2385 return Ok((observation, current.conservation.observed_controlled_cycles));
2386 }
2387 let mut observation = platform
2388 .observe(&retained_plan.operation_id, state)
2389 .map_err(EnsureWorkflowError::Platform)?;
2390 let terminal_inventory_operation_id =
2391 reviewed_terminal_inventory_operation(retained_plan, state)?;
2392 if let Some(operation_id) = terminal_inventory_operation_id {
2393 attach_terminal_inventory_cycles(operation_id, state, platform, &mut observation)?;
2394 }
2395 let artifacts = resolve_desired_artifacts(root, desired)?;
2396 let mut retained_observations = 0_u32;
2397 let mut current = loop {
2398 let protocol_actions = platform
2399 .protocol_actions(&retained_plan.operation_id, state)
2400 .map_err(EnsureWorkflowError::Platform)?;
2401 match compile_plan(
2402 desired,
2403 &artifacts,
2404 &protocol_actions,
2405 desired_sha256,
2406 requested_fleet,
2407 &observation,
2408 retained_plan.planned_at_time,
2409 &retained_plan.operation_id,
2410 None,
2411 ) {
2412 Ok(plan) => break plan,
2413 Err(EnsurePolicyError::PendingRootOwnedBalance { name }) => {
2414 retained_observations = retained_observations.saturating_add(1);
2415 if retained_observations >= desired.maximum_stalled_observations {
2416 return Err(EnsureWorkflowError::RootOwnedObservationStalled {
2417 last_lifecycle: root_owned_lifecycle_label(&observation, &name).to_string(),
2418 observations: retained_observations,
2419 target: name,
2420 });
2421 }
2422 platform.pace_root_owned_observation(&name, retained_observations);
2423 observation = platform
2424 .observe(&retained_plan.operation_id, state)
2425 .map_err(EnsureWorkflowError::Platform)?;
2426 if let Some(operation_id) = terminal_inventory_operation_id {
2427 attach_terminal_inventory_cycles(
2428 operation_id,
2429 state,
2430 platform,
2431 &mut observation,
2432 )?;
2433 }
2434 }
2435 Err(error) => return Err(error.into()),
2436 }
2437 };
2438 if observation.operator_cycles < retained_plan.conservation.maximum_operator_debit_cycles {
2439 return Err(EnsureWorkflowError::InsufficientOperatorCycles {
2440 actual: observation.operator_cycles,
2441 required: retained_plan.conservation.maximum_operator_debit_cycles,
2442 });
2443 }
2444 bind_terminal_inventory_operation(
2445 &mut current,
2446 retained_plan.terminal_inventory_operation_id.clone(),
2447 );
2448 if current.plan_sha256 != retained_plan.plan_sha256
2449 && !compatible_after_bounded_observation(retained_plan, ¤t, desired, &observation)
2450 {
2451 return Err(EnsureWorkflowError::DriftedBeforeApply);
2452 }
2453 Ok((observation, current.conservation.observed_controlled_cycles))
2454}
2455
2456fn estate_funding_requirement<E>(
2457 plan: &FleetEnsurePlan,
2458 state: &FleetEnsureStateRecord,
2459 observation: &FleetObservation,
2460) -> Result<Option<EstateFundingRequiredRecord>, EnsureWorkflowError<E>>
2461where
2462 E: std::error::Error + 'static,
2463{
2464 for reviewed in &plan.conservation.estate_funding_domains {
2465 let observed = observation
2466 .estate_funding_domains
2467 .get(&reviewed.root)
2468 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2469 if observed.cycles_ledger != reviewed.cycles_ledger {
2470 return Err(EnsureWorkflowError::PlanIntegrity);
2471 }
2472 let root_principal = observed
2473 .root_principal
2474 .as_deref()
2475 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2476 if reviewed_estate_root_principal(reviewed, state) != Some(root_principal) {
2477 return Err(EnsureWorkflowError::PlanIntegrity);
2478 }
2479 let available_cycles = observed
2480 .balance_cycles
2481 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2482 let reviewed_available = reviewed.available_cycles.unwrap_or_default();
2483 let maximum_reviewed_balance = reviewed_available
2484 .checked_add(reviewed.maximum_funding_cycles)
2485 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2486 if available_cycles < reviewed_available || available_cycles > maximum_reviewed_balance {
2487 return Err(EnsureWorkflowError::DriftedBeforeApply);
2488 }
2489 if reviewed.required_creation_count == 0 {
2490 if available_cycles != reviewed_available {
2491 return Err(EnsureWorkflowError::DriftedBeforeApply);
2492 }
2493 continue;
2494 }
2495 if available_cycles < reviewed.maximum_creation_debit_cycles {
2496 return Ok(Some(EstateFundingRequiredRecord {
2497 attempt_count: None,
2498 available_cycles,
2499 creation_amount_cycles: reviewed.creation_amount_cycles,
2500 creation_execution_margin_cycles: reviewed.creation_execution_margin_cycles,
2501 readiness_floor_cycles: reviewed.readiness_floor_cycles,
2502 cycles_ledger: reviewed.cycles_ledger.clone(),
2503 ledger_fee_cycles: reviewed.ledger_fee_cycles,
2504 management_creation_fee_cycles: reviewed.management_creation_fee_cycles,
2505 maximum_creation_debit_cycles: reviewed.maximum_creation_debit_cycles,
2506 maximum_creation_fee_cycles: reviewed.maximum_creation_fee_cycles,
2507 operation_id: plan.operation_id.clone(),
2508 pending_creation_operation_id: None,
2509 plan_sha256: plan.plan_sha256.clone(),
2510 required_creation_count: reviewed.required_creation_count,
2511 root: reviewed.root.clone(),
2512 root_principal: root_principal.to_string(),
2513 last_attempt_at_ns: None,
2514 retry_at_ns: None,
2515 shortfall_cycles: reviewed
2516 .maximum_creation_debit_cycles
2517 .saturating_sub(available_cycles),
2518 }));
2519 }
2520 }
2521 Ok(None)
2522}
2523
2524fn protocol_estate_funding_requirement<E>(
2525 plan: &FleetEnsurePlan,
2526 state: &FleetEnsureStateRecord,
2527 funding: &canic_core::dto::component_provisioning::RootEstateFundingRequired,
2528) -> Result<EstateFundingRequiredRecord, EnsureWorkflowError<E>>
2529where
2530 E: std::error::Error + 'static,
2531{
2532 let root_principal = funding.root.to_text();
2533 let reviewed = plan
2534 .conservation
2535 .estate_funding_domains
2536 .iter()
2537 .find(|domain| reviewed_estate_root_principal(domain, state) == Some(&root_principal))
2538 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2539 let creation_amount_cycles = funding.creation_amount.to_u128();
2540 let ledger_fee_cycles = funding.ledger_fee.to_u128();
2541 let management_creation_fee_cycles = funding.management_creation_fee.to_u128();
2542 let required_cycles = creation_amount_cycles
2543 .checked_add(ledger_fee_cycles)
2544 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2545 let maximum_creation_fee_cycles = management_creation_fee_cycles
2546 .checked_add(ledger_fee_cycles)
2547 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2548 let exact_reviewed_authority = reviewed.required_creation_count > 0
2549 && reviewed.cycles_ledger == funding.cycles_ledger.to_text()
2550 && reviewed.creation_amount_cycles == creation_amount_cycles
2551 && reviewed.creation_execution_margin_cycles == funding.execution_margin.to_u128()
2552 && reviewed.readiness_floor_cycles == funding.readiness_floor.to_u128()
2553 && reviewed.ledger_fee_cycles == ledger_fee_cycles
2554 && reviewed.management_creation_fee_cycles == management_creation_fee_cycles;
2555 let exact_runtime_arithmetic = funding.required.to_u128() == required_cycles
2556 && funding.available < funding.required
2557 && funding.shortfall.to_u128()
2558 == required_cycles.saturating_sub(funding.available.to_u128());
2559 if !exact_reviewed_authority || !exact_runtime_arithmetic {
2560 return Err(EnsureWorkflowError::PlanIntegrity);
2561 }
2562 Ok(EstateFundingRequiredRecord {
2563 attempt_count: Some(funding.attempt_count),
2564 available_cycles: funding.available.to_u128(),
2565 creation_amount_cycles,
2566 cycles_ledger: funding.cycles_ledger.to_text(),
2567 creation_execution_margin_cycles: funding.execution_margin.to_u128(),
2568 readiness_floor_cycles: funding.readiness_floor.to_u128(),
2569 ledger_fee_cycles,
2570 management_creation_fee_cycles,
2571 maximum_creation_debit_cycles: required_cycles,
2572 maximum_creation_fee_cycles,
2573 operation_id: plan.operation_id.clone(),
2574 pending_creation_operation_id: Some(canic_core::cdk::utils::hash::hex_bytes(
2575 funding.operation_id,
2576 )),
2577 plan_sha256: plan.plan_sha256.clone(),
2578 required_creation_count: 1,
2579 root: reviewed.root.clone(),
2580 root_principal,
2581 last_attempt_at_ns: funding.last_attempt_at_ns,
2582 retry_at_ns: Some(funding.retry_at_ns),
2583 shortfall_cycles: funding.shortfall.to_u128(),
2584 })
2585}
2586
2587fn estate_funding_error(required: &EstateFundingRequiredRecord) -> EstateFundingRequiredError {
2588 EstateFundingRequiredError {
2589 attempt_count: required.attempt_count,
2590 available_cycles: required.available_cycles,
2591 creation_amount_cycles: required.creation_amount_cycles,
2592 creation_execution_margin_cycles: required.creation_execution_margin_cycles,
2593 readiness_floor_cycles: required.readiness_floor_cycles,
2594 cycles_ledger: required.cycles_ledger.clone(),
2595 ledger_fee_cycles: required.ledger_fee_cycles,
2596 management_creation_fee_cycles: required.management_creation_fee_cycles,
2597 maximum_creation_debit_cycles: required.maximum_creation_debit_cycles,
2598 maximum_creation_fee_cycles: required.maximum_creation_fee_cycles,
2599 pending_creation_operation_id: required.pending_creation_operation_id.clone(),
2600 required_creation_count: required.required_creation_count,
2601 root: required.root.clone(),
2602 root_principal: required.root_principal.clone(),
2603 last_attempt_at_ns: required.last_attempt_at_ns,
2604 retry_at_ns: required.retry_at_ns,
2605 shortfall_cycles: required.shortfall_cycles,
2606 }
2607}
2608
2609fn retain_estate_funding_pause<E>(
2610 paths: &EnsurePaths,
2611 journal: &mut FleetEnsureJournalRecord,
2612 required: Option<&EstateFundingRequiredRecord>,
2613) -> Result<bool, EnsureWorkflowError<E>>
2614where
2615 E: std::error::Error + 'static,
2616{
2617 if journal.estate_funding_required.as_ref() == required {
2618 return Ok(false);
2619 }
2620 journal.estate_funding_required = required.cloned();
2621 write_journal(paths, journal)?;
2622 Ok(true)
2623}
2624
2625fn reviewed_estate_root_principal<'a>(
2626 domain: &'a crate::fleet_ensure::model::EstateFundingDomainPlan,
2627 state: &'a FleetEnsureStateRecord,
2628) -> Option<&'a str> {
2629 domain
2630 .root_principal
2631 .as_deref()
2632 .or_else(|| {
2633 state
2634 .pending_principals
2635 .get(&domain.root)
2636 .map(String::as_str)
2637 })
2638 .or_else(|| state.principals.get(&domain.root).map(String::as_str))
2639}
2640
2641fn prior_fleet_protocol_effect_started(
2642 actions: &[&EnsureAction],
2643 journal: &FleetEnsureJournalRecord,
2644 current_index: usize,
2645) -> bool {
2646 actions
2647 .iter()
2648 .take(current_index)
2649 .zip(&journal.effects)
2650 .any(|(action, effect)| {
2651 matches!(action, EnsureAction::FleetProtocol { .. })
2652 && matches!(effect.state, EffectState::Issued | EffectState::Applied)
2653 })
2654}
2655
2656fn completed_inventory_operation<'a, E>(
2657 plan: &'a FleetEnsurePlan,
2658 journal: &FleetEnsureJournalRecord,
2659 state: &FleetEnsureStateRecord,
2660) -> Result<&'a str, EnsureWorkflowError<E>>
2661where
2662 E: std::error::Error + 'static,
2663{
2664 if plan.protocol_actions.is_empty()
2665 && journal.successor_phases.is_empty()
2666 && state.active_registry.is_some()
2667 {
2668 reviewed_terminal_inventory_operation(plan, state)?
2669 .ok_or(EnsureWorkflowError::PlanIntegrity)
2670 } else {
2671 Ok(&plan.operation_id)
2672 }
2673}
2674
2675fn reviewed_terminal_inventory_operation<'a, E>(
2676 plan: &'a FleetEnsurePlan,
2677 state: &FleetEnsureStateRecord,
2678) -> Result<Option<&'a str>, EnsureWorkflowError<E>>
2679where
2680 E: std::error::Error + 'static,
2681{
2682 match (
2683 state.active_registry.is_some(),
2684 plan.terminal_inventory_operation_id.as_deref(),
2685 ) {
2686 (false, None) => Ok(None),
2687 (true, Some(operation_id)) if !operation_id.is_empty() => Ok(Some(operation_id)),
2688 _ => Err(EnsureWorkflowError::PlanIntegrity),
2689 }
2690}
2691
2692fn complete_root_start_prerequisite<P>(
2693 desired: &crate::fleet_ensure::model::DesiredFleet,
2694 retained_plan: &FleetEnsurePlan,
2695 platform: &mut P,
2696 state: &FleetEnsureStateRecord,
2697 journal: &FleetEnsureJournalRecord,
2698) -> Result<ActualCycleConservation, EnsureWorkflowError<P::Error>>
2699where
2700 P: EnsurePlatform,
2701{
2702 let targets = reviewed_root_start_targets(retained_plan)?;
2703 let management = platform
2704 .observe_root_management(state, &targets)
2705 .map_err(EnsureWorkflowError::Platform)?
2706 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2707 let current = recompile_root_start_prerequisite_plan(
2708 RootStartPlanInput {
2709 state,
2710 authority: retained_plan.root_start_authority.as_deref(),
2711 created_at_time: retained_plan.planned_at_time,
2712 desired,
2713 desired_sha256: &retained_plan.desired_sha256,
2714 observation: &management,
2715 requested_fleet: &retained_plan.fleet,
2716 },
2717 &targets,
2718 )?;
2719 if !compatible_root_start_prerequisite(retained_plan, ¤t, desired) {
2720 return Err(EnsureWorkflowError::ConvergenceDrift);
2721 }
2722 if management.roots.iter().any(|(name, observed)| {
2723 targets.contains(name)
2724 && observed.live.status != crate::fleet_ensure::model::CanisterRuntimeStatus::Running
2725 }) {
2726 return Err(EnsureWorkflowError::ConvergenceDrift);
2727 }
2728 let terminal = root_management_fleet_observation(&management, &targets)?;
2729 verify_terminal_conservation(retained_plan, journal, state, &terminal)
2730}
2731
2732fn verify_terminal_root_start_replay<P>(
2733 desired: &crate::fleet_ensure::model::DesiredFleet,
2734 plan: &FleetEnsurePlan,
2735 journal: &FleetEnsureJournalRecord,
2736 requested_fleet: &str,
2737 platform: &mut P,
2738 state: &FleetEnsureStateRecord,
2739) -> Result<(), EnsureWorkflowError<P::Error>>
2740where
2741 P: EnsurePlatform,
2742{
2743 verify_journal(journal, plan, requested_fleet, state)?;
2744 let actions = ordered_actions(plan);
2745 if actions.len() != journal.effects.len()
2746 || journal
2747 .effects
2748 .iter()
2749 .any(|effect| effect.state != EffectState::Applied)
2750 {
2751 return Err(EnsureWorkflowError::JournalIntegrity);
2752 }
2753 for (action, record) in actions.into_iter().zip(&journal.effects) {
2754 if action_sha256(action) != record.action_sha256
2755 || !platform
2756 .observe_effect(&journal.operation_id, action, record, state)
2757 .map_err(EnsureWorkflowError::Platform)?
2758 .applied
2759 {
2760 return Err(EnsureWorkflowError::ConvergenceDrift);
2761 }
2762 }
2763 let targets = reviewed_root_start_targets(plan)?;
2764 let management = platform
2765 .observe_root_management(state, &targets)
2766 .map_err(EnsureWorkflowError::Platform)?
2767 .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2768 let current = recompile_root_start_prerequisite_plan(
2769 RootStartPlanInput {
2770 state,
2771 authority: plan.root_start_authority.as_deref(),
2772 created_at_time: plan.planned_at_time,
2773 desired,
2774 desired_sha256: &plan.desired_sha256,
2775 observation: &management,
2776 requested_fleet: &plan.fleet,
2777 },
2778 &targets,
2779 )?;
2780 if normalized_plan(plan) != normalized_plan(¤t)
2781 || management.roots.iter().any(|(name, observed)| {
2782 targets.contains(name)
2783 && observed.live.status
2784 != crate::fleet_ensure::model::CanisterRuntimeStatus::Running
2785 })
2786 {
2787 return Err(EnsureWorkflowError::ConvergenceDrift);
2788 }
2789 Ok(())
2790}
2791
2792fn reviewed_root_start_targets<E>(
2793 plan: &FleetEnsurePlan,
2794) -> Result<BTreeSet<String>, EnsureWorkflowError<E>>
2795where
2796 E: std::error::Error + 'static,
2797{
2798 if plan.scope != FleetEnsurePlanScope::RootStartPrerequisite
2799 || !plan.protocol_actions.is_empty()
2800 || plan.canisters.is_empty()
2801 || plan.conservation.maximum_new_funding_cycles != 0
2802 || plan.conservation.maximum_operator_debit_cycles != 0
2803 || plan.conservation.maximum_unavoidable_fee_cycles != 0
2804 || !plan.conservation.estate_funding_domains.is_empty()
2805 || plan.conservation.scheduled_transfer_cycles != 0
2806 || plan.conservation.observed_controlled_cycles
2807 != plan.conservation.retained_in_reused_canisters_cycles
2808 || plan
2809 .conservation
2810 .observed_controlled_cycles
2811 .checked_sub(plan.conservation.maximum_execution_burn_cycles)
2812 != Some(plan.conservation.expected_post_operation_cycles)
2813 {
2814 return Err(EnsureWorkflowError::PlanIntegrity);
2815 }
2816 let mut targets = BTreeSet::new();
2817 for canister in &plan.canisters {
2818 if canister.disposition != CanisterDisposition::Reuse
2819 || canister.principal.is_none()
2820 || !matches!(
2821 canister.actions.as_slice(),
2822 [EnsureAction::Start { name, principal }]
2823 if name == &canister.name
2824 && Some(principal) == canister.principal.as_ref()
2825 )
2826 || !targets.insert(canister.name.clone())
2827 {
2828 return Err(EnsureWorkflowError::PlanIntegrity);
2829 }
2830 }
2831 Ok(targets)
2832}
2833
2834fn root_management_fleet_observation<E>(
2835 management: &RootManagementObservation,
2836 targets: &BTreeSet<String>,
2837) -> Result<FleetObservation, EnsureWorkflowError<E>>
2838where
2839 E: std::error::Error + 'static,
2840{
2841 let canisters = targets
2842 .iter()
2843 .map(|name| {
2844 management
2845 .roots
2846 .get(name)
2847 .map(|observed| (name.clone(), Some(observed.live.clone())))
2848 .ok_or(EnsureWorkflowError::PlanIntegrity)
2849 })
2850 .collect::<Result<BTreeMap<_, _>, _>>()?;
2851 Ok(FleetObservation {
2852 additional_controlled_cycles: BTreeMap::new(),
2853 canisters,
2854 estate_funding_domains: BTreeMap::new(),
2855 ledger_fee_cycles: 0,
2856 operator_cycles: management.operator_cycles,
2857 protocol_ready: BTreeMap::new(),
2858 })
2859}
2860
2861fn compatible_root_start_prerequisite(
2862 retained: &FleetEnsurePlan,
2863 current: &FleetEnsurePlan,
2864 desired: &crate::fleet_ensure::model::DesiredFleet,
2865) -> bool {
2866 let Ok(maximum_movement) = desired
2867 .maximum_update_burn_cycles
2868 .parse::<Cycles>()
2869 .map(|cycles| cycles.to_u128())
2870 else {
2871 return false;
2872 };
2873 retained.canisters.len() == current.canisters.len()
2874 && retained
2875 .canisters
2876 .iter()
2877 .zip(¤t.canisters)
2878 .all(|(retained, current)| {
2879 retained.observed_cycles.abs_diff(current.observed_cycles) <= maximum_movement
2880 })
2881 && normalized_plan(retained) == normalized_plan(current)
2882}
2883
2884fn attach_terminal_inventory_cycles<P>(
2885 operation_id: &str,
2886 state: &FleetEnsureStateRecord,
2887 platform: &mut P,
2888 observation: &mut FleetObservation,
2889) -> Result<(), EnsureWorkflowError<P::Error>>
2890where
2891 P: EnsurePlatform,
2892{
2893 let inventory = platform
2894 .terminal_inventory(operation_id, state)
2895 .map_err(EnsureWorkflowError::Platform)?;
2896 attach_terminal_cycles(observation, inventory.controlled_cycles_by_principal)
2897}
2898
2899pub(super) fn completed_infrastructure_reinstall(
2900 desired: &crate::fleet_ensure::model::DesiredFleet,
2901 plan: &FleetEnsurePlan,
2902 journal: &FleetEnsureJournalRecord,
2903) -> bool {
2904 if desired.protocol.is_none() {
2905 return false;
2906 }
2907 let infrastructure = desired.canisters.iter().filter(|canister| {
2908 canister.presence == crate::fleet_ensure::model::DesiredPresence::Present
2909 && matches!(
2910 canister.kind,
2911 crate::fleet_ensure::model::DesiredCanisterKind::Coordinator
2912 | crate::fleet_ensure::model::DesiredCanisterKind::Root
2913 | crate::fleet_ensure::model::DesiredCanisterKind::Store
2914 )
2915 });
2916 let mut count = 0_usize;
2917 for configured in infrastructure {
2918 count += 1;
2919 let Some(action) = plan
2920 .canisters
2921 .iter()
2922 .find(|canister| {
2923 canister.name == configured.name
2924 && canister.disposition == CanisterDisposition::Reinstall
2925 })
2926 .and_then(|canister| {
2927 canister.actions.iter().find(|action| {
2928 matches!(
2929 action,
2930 EnsureAction::Install {
2931 mode: crate::fleet_ensure::model::InstallMode::Reinstall,
2932 name,
2933 ..
2934 } if name == &configured.name
2935 )
2936 })
2937 })
2938 else {
2939 return false;
2940 };
2941 let action_sha256 = action_sha256(action);
2942 if !journal.effects.iter().any(|effect| {
2943 effect.action_sha256 == action_sha256 && effect.state == EffectState::Applied
2944 }) {
2945 return false;
2946 }
2947 }
2948 count > 0
2949}
2950
2951fn compatible_after_bounded_observation(
2952 retained: &FleetEnsurePlan,
2953 current: &FleetEnsurePlan,
2954 desired: &crate::fleet_ensure::model::DesiredFleet,
2955 observation: &FleetObservation,
2956) -> bool {
2957 let Ok(maximum_observation_movement) = desired
2958 .maximum_observation_burn_cycles
2959 .parse::<Cycles>()
2960 .map(|cycles| cycles.to_u128())
2961 else {
2962 return false;
2963 };
2964 if retained.canisters.len() != current.canisters.len() {
2965 return false;
2966 }
2967 for (retained_canister, current_canister) in retained.canisters.iter().zip(¤t.canisters) {
2968 let movement = retained_canister
2969 .observed_cycles
2970 .abs_diff(current_canister.observed_cycles);
2971 if movement > maximum_observation_movement {
2972 return false;
2973 }
2974 }
2975 if !retained_funding_remains_sufficient(retained, desired, observation) {
2976 return false;
2977 }
2978 normalized_plan(retained) == normalized_plan(current)
2979}
2980
2981fn retained_funding_remains_sufficient(
2982 retained: &FleetEnsurePlan,
2983 desired: &crate::fleet_ensure::model::DesiredFleet,
2984 observation: &FleetObservation,
2985) -> bool {
2986 retained
2987 .canisters
2988 .iter()
2989 .flat_map(|canister| &canister.actions)
2990 .filter_map(|action| match action {
2991 EnsureAction::Fund {
2992 amount,
2993 name,
2994 pool_funding,
2995 principal,
2996 ..
2997 } => Some((amount, name, pool_funding, principal)),
2998 _ => None,
2999 })
3000 .all(|(amount, name, pool_funding, principal)| {
3001 if let Some(root) = pool_funding {
3002 return observation
3003 .estate_funding_domains
3004 .values()
3005 .find(|domain| domain.root_principal.as_ref() == Some(&root.root))
3006 .and_then(|domain| domain.pool.as_ref())
3007 .is_some_and(|pool| {
3008 pool.assets.iter().any(|asset| {
3009 asset.principal == *principal
3010 && asset.lifecycle == root.lifecycle
3011 && asset
3012 .cycles
3013 .checked_add(*amount)
3014 .is_some_and(|funded| funded >= pool.readiness_floor_cycles)
3015 })
3016 });
3017 }
3018 let Some(live) = observation.canisters.get(name).and_then(Option::as_ref) else {
3019 return false;
3020 };
3021 let Some(configured) = desired
3022 .canisters
3023 .iter()
3024 .find(|configured| configured.name == *name)
3025 else {
3026 return false;
3027 };
3028 let Ok(minimum) = configured
3029 .minimum_cycles
3030 .parse::<Cycles>()
3031 .map(|cycles| cycles.to_u128())
3032 else {
3033 return false;
3034 };
3035 live.cycles
3036 .checked_add(*amount)
3037 .is_some_and(|funded| funded >= minimum)
3038 })
3039}
3040
3041fn normalized_plan(plan: &FleetEnsurePlan) -> FleetEnsurePlan {
3042 let mut normalized = plan.clone();
3043 normalized.plan_sha256.clear();
3044 normalized.reviewed_desired = None;
3045 if let Some(review) = &mut normalized.recovery_review {
3046 review.continuation_reserve_cycles = 0;
3048 for funding in &mut review.known_pool_funding {
3049 funding.amount_cycles = 0;
3050 funding.funding_deficit_cycles = 0;
3051 }
3052 }
3053 normalized.conservation = CycleConservation {
3054 estate_funding_domains: plan
3055 .conservation
3056 .estate_funding_domains
3057 .iter()
3058 .cloned()
3059 .map(|mut domain| {
3060 domain.available_cycles = None;
3061 domain.maximum_funding_cycles = 0;
3062 domain.root_principal = None;
3063 domain.shortfall_cycles = 0;
3064 domain
3065 })
3066 .collect(),
3067 expected_post_operation_cycles: 0,
3068 maximum_execution_burn_cycles: 0,
3069 maximum_new_funding_cycles: 0,
3070 maximum_operator_debit_cycles: 0,
3071 maximum_unavoidable_fee_cycles: 0,
3072 observed_controlled_cycles: 0,
3073 retained_in_reused_canisters_cycles: 0,
3074 scheduled_transfer_cycles: 0,
3075 };
3076 for canister in &mut normalized.canisters {
3077 canister.observed_cycles = 0;
3078 for action in &mut canister.actions {
3079 if let EnsureAction::Fund {
3080 amount,
3081 funding_deficit_cycles,
3082 ..
3083 } = action
3084 {
3085 *amount = 0;
3086 *funding_deficit_cycles = 0;
3087 }
3088 }
3089 }
3090 normalized
3091}
3092
3093fn attach_terminal_cycles<E>(
3094 observation: &mut FleetObservation,
3095 additional: BTreeMap<String, u128>,
3096) -> Result<(), EnsureWorkflowError<E>>
3097where
3098 E: std::error::Error + 'static,
3099{
3100 let mut retained = BTreeMap::new();
3101 for (principal, cycles) in additional {
3102 let Some(configured) = observation
3103 .canisters
3104 .values_mut()
3105 .filter_map(Option::as_mut)
3106 .find(|canister| canister.principal == principal)
3107 else {
3108 retained.insert(principal, cycles);
3109 continue;
3110 };
3111 match configured.root_owned_lifecycle {
3112 Some(crate::fleet_ensure::model::RootOwnedCanisterLifecycle::Workload) => {
3113 configured.cycles = configured.cycles.min(cycles);
3116 }
3117 Some(crate::fleet_ensure::model::RootOwnedCanisterLifecycle::Idle)
3118 if configured.cycles == cycles => {}
3119 Some(crate::fleet_ensure::model::RootOwnedCanisterLifecycle::Idle) => {
3120 return Err(EnsureWorkflowError::TerminalInventory(
3121 TerminalInventoryError::IdleCycleObservationConflict {
3122 canister: principal,
3123 expected: configured.cycles,
3124 observed: cycles,
3125 },
3126 ));
3127 }
3128 _ => {
3129 return Err(EnsureWorkflowError::TerminalInventory(
3130 TerminalInventoryError::CycleObservationLifecycleConflict {
3131 canister: principal,
3132 },
3133 ));
3134 }
3135 }
3136 }
3137 observation.additional_controlled_cycles = retained;
3138 Ok(())
3139}
3140
3141fn funding_plan<'a, E: std::error::Error + 'static>(
3142 plan: &'a FleetEnsurePlan,
3143 journal: &FleetEnsureJournalRecord,
3144) -> Result<std::borrow::Cow<'a, FleetEnsurePlan>, EnsureWorkflowError<E>> {
3145 if journal.funding_reviews.is_empty() {
3146 return Ok(std::borrow::Cow::Borrowed(plan));
3147 }
3148 let mut funded = plan.clone();
3149 let (amount, fee) = funding::totals::<E>(journal, None)?;
3150 let conservation = &mut funded.conservation;
3151 conservation.maximum_operator_debit_cycles = conservation
3152 .maximum_operator_debit_cycles
3153 .checked_add(amount)
3154 .and_then(|n| n.checked_add(fee))
3155 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3156 conservation.maximum_new_funding_cycles = conservation
3157 .maximum_new_funding_cycles
3158 .checked_add(amount)
3159 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3160 conservation.maximum_unavoidable_fee_cycles = conservation
3161 .maximum_unavoidable_fee_cycles
3162 .checked_add(fee)
3163 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3164 for domain in &mut conservation.estate_funding_domains {
3165 domain.maximum_funding_cycles = domain
3166 .maximum_funding_cycles
3167 .checked_add(funding::totals::<E>(journal, Some(&domain.root))?.0)
3168 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3169 }
3170 Ok(std::borrow::Cow::Owned(funded))
3171}
3172
3173fn verify_terminal_conservation<E>(
3174 plan: &FleetEnsurePlan,
3175 journal: &FleetEnsureJournalRecord,
3176 state: &FleetEnsureStateRecord,
3177 terminal: &FleetObservation,
3178) -> Result<ActualCycleConservation, EnsureWorkflowError<E>>
3179where
3180 E: std::error::Error + 'static,
3181{
3182 let funded = funding_plan(plan, journal)?;
3183 let conservation = &funded.conservation;
3184 let final_controlled_cycles = controlled_cycles(terminal)?;
3185 let operator_debit_cycles = journal
3186 .initial_operator_cycles
3187 .checked_sub(terminal.operator_cycles)
3188 .ok_or_else(|| {
3189 EnsureWorkflowError::Conservation(
3190 "operator balance increased during apply; review a new plan".to_string(),
3191 )
3192 })?;
3193 if operator_debit_cycles > conservation.maximum_operator_debit_cycles {
3194 return Err(EnsureWorkflowError::Conservation(format!(
3195 "operator debit {operator_debit_cycles} exceeded reviewed maximum {}",
3196 conservation.maximum_operator_debit_cycles
3197 )));
3198 }
3199 let received_new_funding_cycles = operator_debit_cycles
3200 .checked_sub(conservation.maximum_unavoidable_fee_cycles)
3201 .ok_or_else(|| {
3202 EnsureWorkflowError::Conservation(
3203 "operator debit was below the exact reviewed fee total".to_string(),
3204 )
3205 })?;
3206 if received_new_funding_cycles > conservation.maximum_new_funding_cycles {
3207 return Err(EnsureWorkflowError::Conservation(format!(
3208 "received funding {received_new_funding_cycles} exceeded reviewed maximum {}",
3209 conservation.maximum_new_funding_cycles
3210 )));
3211 }
3212 let (estate_funding_cycles, exact_estate_creation_fee_cycles) =
3213 reconcile_estate_funding(plan, journal, state, terminal)?;
3214 let observed_settlement_credit_cycles = settlement::observed_credit(plan, journal)?;
3215 let available = journal
3216 .initial_controlled_cycles
3217 .checked_add(received_new_funding_cycles)
3218 .and_then(|cycles| cycles.checked_add(observed_settlement_credit_cycles))
3219 .ok_or_else(|| {
3220 EnsureWorkflowError::Conservation(
3221 "terminal controlled-cycle arithmetic overflowed".to_string(),
3222 )
3223 })?;
3224 let after_estate_fees = available
3225 .checked_sub(exact_estate_creation_fee_cycles)
3226 .ok_or_else(|| {
3227 EnsureWorkflowError::Conservation(
3228 "terminal controlled cycles cannot cover reviewed estate creation fees".to_string(),
3229 )
3230 })?;
3231 let measured_execution_burn_cycles = after_estate_fees
3232 .checked_sub(final_controlled_cycles)
3233 .ok_or_else(|| {
3234 EnsureWorkflowError::Conservation(
3235 "terminal estate exceeds starting cycles, received funding and bounded observed credits"
3236 .to_string(),
3237 )
3238 })?;
3239 if measured_execution_burn_cycles > conservation.maximum_execution_burn_cycles {
3240 return Err(EnsureWorkflowError::Conservation(format!(
3241 "measured execution burn {measured_execution_burn_cycles} exceeded reviewed maximum {}",
3242 conservation.maximum_execution_burn_cycles
3243 )));
3244 }
3245 Ok(ActualCycleConservation {
3246 estate_funding_cycles,
3247 exact_estate_creation_fee_cycles,
3248 exact_unavoidable_fee_cycles: conservation.maximum_unavoidable_fee_cycles,
3249 final_controlled_cycles,
3250 measured_execution_burn_cycles,
3251 observed_starting_cycles: journal.initial_controlled_cycles,
3252 observed_settlement_credit_cycles,
3253 operator_debit_cycles,
3254 received_new_funding_cycles,
3255 })
3256}
3257
3258fn resolved_initial_pool_assets<E>(
3259 domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3260 state: &FleetEnsureStateRecord,
3261) -> Result<BTreeSet<String>, EnsureWorkflowError<E>>
3262where
3263 E: std::error::Error + 'static,
3264{
3265 domain
3266 .initial_pool_assets
3267 .iter()
3268 .map(|identity| {
3269 identity.strip_prefix("created:").map_or_else(
3270 || Ok(identity.clone()),
3271 |name| {
3272 state
3273 .pending_principals
3274 .get(name)
3275 .or_else(|| state.principals.get(name))
3276 .cloned()
3277 .ok_or(EnsureWorkflowError::JournalIntegrity)
3278 },
3279 )
3280 })
3281 .collect()
3282}
3283
3284fn exact_estate_creation_costs<E>(
3285 state: &FleetEnsureStateRecord,
3286 terminal: &FleetObservation,
3287 domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3288) -> Result<(u128, u128), EnsureWorkflowError<E>>
3289where
3290 E: std::error::Error + 'static,
3291{
3292 let observed = terminal
3293 .estate_funding_domains
3294 .get(&domain.root)
3295 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3296 let Some(pool) = observed.pool.as_ref() else {
3297 return if domain.required_creation_count == 0 {
3298 Ok((0, 0))
3299 } else {
3300 Err(EnsureWorkflowError::JournalIntegrity)
3301 };
3302 };
3303 if pool.pending_creation.is_some() {
3304 return Err(EnsureWorkflowError::JournalIntegrity);
3305 }
3306
3307 estate_creation_costs(state, terminal, domain)
3308}
3309
3310fn estate_creation_costs<E: std::error::Error + 'static>(
3311 state: &FleetEnsureStateRecord,
3312 observation: &FleetObservation,
3313 domain: &EstateFundingDomainPlan,
3314) -> Result<(u128, u128), EnsureWorkflowError<E>> {
3315 let pool = observation
3316 .estate_funding_domains
3317 .get(&domain.root)
3318 .and_then(|domain| domain.pool.as_ref())
3319 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3320 let initial = resolved_initial_pool_assets(domain, state)?;
3321 let terminal_principals = pool
3322 .assets
3323 .iter()
3324 .map(|asset| asset.principal.clone())
3325 .collect::<BTreeSet<_>>();
3326 if terminal_principals.len() != pool.assets.len() || !initial.is_subset(&terminal_principals) {
3327 return Err(EnsureWorkflowError::JournalIntegrity);
3328 }
3329 let created = pool
3330 .assets
3331 .iter()
3332 .filter(|asset| !initial.contains(&asset.principal))
3333 .collect::<Vec<_>>();
3334 let creation_count_is_bounded = created.len()
3335 <= usize::try_from(domain.required_creation_count)
3336 .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
3337 let pool_policy_is_exact = created.is_empty() || terminal_pool_policy_is_exact(pool, domain);
3340 if !creation_count_is_bounded || !pool_policy_is_exact {
3341 return Err(EnsureWorkflowError::JournalIntegrity);
3342 }
3343
3344 let mut operation_ids = BTreeSet::new();
3345 let mut block_indexes = BTreeSet::new();
3346 let mut creation_debit = 0_u128;
3347 let mut creation_fee = 0_u128;
3348 for asset in created {
3349 let (asset_debit, asset_fee) = exact_estate_creation_receipt_costs(
3350 asset,
3351 domain,
3352 &mut operation_ids,
3353 &mut block_indexes,
3354 )?;
3355 creation_debit = creation_debit.checked_add(asset_debit).ok_or_else(|| {
3356 EnsureWorkflowError::Conservation(
3357 "estate creation-debit arithmetic overflowed".to_string(),
3358 )
3359 })?;
3360 creation_fee = creation_fee.checked_add(asset_fee).ok_or_else(|| {
3361 EnsureWorkflowError::Conservation(
3362 "estate creation-fee arithmetic overflowed".to_string(),
3363 )
3364 })?;
3365 }
3366 if creation_debit > domain.maximum_creation_debit_cycles
3367 || creation_fee > domain.maximum_creation_fee_cycles
3368 {
3369 return Err(EnsureWorkflowError::JournalIntegrity);
3370 }
3371 Ok((creation_debit, creation_fee))
3372}
3373
3374const fn terminal_pool_policy_is_exact(
3375 pool: &crate::fleet_ensure::model::EstatePoolInventoryObservation,
3376 domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3377) -> bool {
3378 let expected_maximum_size = domain.pool_maximum_size;
3379 let expected_readiness_floor = domain.readiness_floor_cycles;
3380 let expected_execution_margin = domain.creation_execution_margin_cycles;
3381 pool.maximum_size == expected_maximum_size
3382 && pool.readiness_floor_cycles == expected_readiness_floor
3383 && pool.creation_execution_margin_cycles == expected_execution_margin
3384}
3385
3386#[derive(Eq, PartialEq)]
3387struct EstateCreationReceiptAuthority<'a> {
3388 creation_execution_margin_cycles: u128,
3389 cycles_ledger: &'a str,
3390 ledger_amount_cycles: u128,
3391 ledger_fee_cycles: u128,
3392 management_creation_fee_cycles: u128,
3393 readiness_floor_cycles: u128,
3394}
3395
3396fn observed_estate_creation_authority(
3397 receipt: &crate::fleet_ensure::model::EstatePoolCreationReceiptObservation,
3398) -> EstateCreationReceiptAuthority<'_> {
3399 EstateCreationReceiptAuthority {
3400 creation_execution_margin_cycles: receipt.creation_execution_margin_cycles,
3401 cycles_ledger: &receipt.cycles_ledger,
3402 ledger_amount_cycles: receipt.ledger_amount_cycles,
3403 ledger_fee_cycles: receipt.ledger_fee_cycles,
3404 management_creation_fee_cycles: receipt.management_creation_fee_cycles,
3405 readiness_floor_cycles: receipt.readiness_floor_cycles,
3406 }
3407}
3408
3409fn reviewed_estate_creation_authority(
3410 domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3411) -> EstateCreationReceiptAuthority<'_> {
3412 EstateCreationReceiptAuthority {
3413 creation_execution_margin_cycles: domain.creation_execution_margin_cycles,
3414 cycles_ledger: &domain.cycles_ledger,
3415 ledger_amount_cycles: domain.creation_amount_cycles,
3416 ledger_fee_cycles: domain.ledger_fee_cycles,
3417 management_creation_fee_cycles: domain.management_creation_fee_cycles,
3418 readiness_floor_cycles: domain.readiness_floor_cycles,
3419 }
3420}
3421
3422fn first_estate_creation_observation_is_exact(
3423 receipt: &crate::fleet_ensure::model::EstatePoolCreationReceiptObservation,
3424) -> bool {
3425 receipt
3426 .ledger_amount_cycles
3427 .checked_sub(receipt.management_creation_fee_cycles)
3428 .zip(receipt.first_observed_cycles)
3429 .is_some_and(|(funded_native_cycles, observed_cycles)| {
3430 observed_cycles >= receipt.readiness_floor_cycles
3431 && observed_cycles <= funded_native_cycles
3432 && funded_native_cycles - observed_cycles
3433 <= receipt.creation_execution_margin_cycles
3434 })
3435}
3436
3437fn exact_estate_creation_receipt_costs<E>(
3438 asset: &crate::fleet_ensure::model::EstatePoolAssetObservation,
3439 domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3440 operation_ids: &mut BTreeSet<String>,
3441 block_indexes: &mut BTreeSet<u64>,
3442) -> Result<(u128, u128), EnsureWorkflowError<E>>
3443where
3444 E: std::error::Error + 'static,
3445{
3446 let receipt = asset
3447 .creation_receipt
3448 .as_ref()
3449 .filter(|_| asset.origin == crate::fleet_ensure::model::EstatePoolAssetOrigin::Created)
3450 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3451 let authority_is_exact =
3452 observed_estate_creation_authority(receipt) == reviewed_estate_creation_authority(domain);
3453 if !is_sha256_hex(&receipt.operation_id)
3454 || !authority_is_exact
3455 || !first_estate_creation_observation_is_exact(receipt)
3456 || !operation_ids.insert(receipt.operation_id.clone())
3457 || !block_indexes.insert(receipt.block_index)
3458 {
3459 return Err(EnsureWorkflowError::JournalIntegrity);
3460 }
3461 let debit = receipt
3462 .ledger_amount_cycles
3463 .checked_add(receipt.ledger_fee_cycles)
3464 .ok_or_else(|| {
3465 EnsureWorkflowError::Conservation(
3466 "estate creation-debit arithmetic overflowed".to_string(),
3467 )
3468 })?;
3469 let fee = receipt
3470 .management_creation_fee_cycles
3471 .checked_add(receipt.ledger_fee_cycles)
3472 .ok_or_else(|| {
3473 EnsureWorkflowError::Conservation(
3474 "estate creation-fee arithmetic overflowed".to_string(),
3475 )
3476 })?;
3477 Ok((debit, fee))
3478}
3479
3480fn reconcile_estate_funding<E>(
3481 plan: &FleetEnsurePlan,
3482 journal: &FleetEnsureJournalRecord,
3483 state: &FleetEnsureStateRecord,
3484 terminal: &FleetObservation,
3485) -> Result<(u128, u128), EnsureWorkflowError<E>>
3486where
3487 E: std::error::Error + 'static,
3488{
3489 plan.conservation
3490 .estate_funding_domains
3491 .iter()
3492 .try_fold((0_u128, 0_u128), |(funding_total, fee_total), domain| {
3493 let initial = journal
3494 .initial_estate_funding_cycles_by_root
3495 .get(&domain.root)
3496 .copied()
3497 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3498 let observed = terminal
3499 .estate_funding_domains
3500 .get(&domain.root)
3501 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3502 if observed.cycles_ledger != domain.cycles_ledger
3503 || reviewed_estate_root_principal(domain, state)
3504 != observed.root_principal.as_deref()
3505 {
3506 return Err(EnsureWorkflowError::JournalIntegrity);
3507 }
3508 let terminal_balance = observed
3509 .balance_cycles
3510 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3511 let (creation_debit, creation_fee) = exact_estate_creation_costs(
3512 state,
3513 terminal,
3514 domain,
3515 )?;
3516 let funding = applied_estate_funding_for_domain(plan, journal, state, domain)?
3517 .checked_add(funding::totals::<E>(journal, Some(&domain.root))?.0)
3518 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3519 let expected_terminal_balance = initial
3520 .checked_add(funding)
3521 .and_then(|available| available.checked_sub(creation_debit))
3522 .ok_or_else(|| {
3523 EnsureWorkflowError::Conservation(format!(
3524 "Root {} funding-account balance cannot reconcile its reviewed funding and creation debit",
3525 domain.root
3526 ))
3527 })?;
3528 if terminal_balance != expected_terminal_balance {
3529 return Err(EnsureWorkflowError::Conservation(format!(
3530 "Root {} funding-account balance {terminal_balance} differs from reviewed terminal balance {expected_terminal_balance}",
3531 domain.root
3532 )));
3533 }
3534 Ok((
3535 funding_total.checked_add(funding).ok_or_else(|| {
3536 EnsureWorkflowError::Conservation(
3537 "estate funding arithmetic overflowed".to_string(),
3538 )
3539 })?,
3540 fee_total
3541 .checked_add(creation_fee)
3542 .ok_or_else(|| {
3543 EnsureWorkflowError::Conservation(
3544 "estate creation-fee arithmetic overflowed".to_string(),
3545 )
3546 })?,
3547 ))
3548 })
3549}
3550
3551fn applied_estate_funding_for_domain<E>(
3552 plan: &FleetEnsurePlan,
3553 journal: &FleetEnsureJournalRecord,
3554 state: &FleetEnsureStateRecord,
3555 domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3556) -> Result<u128, EnsureWorkflowError<E>>
3557where
3558 E: std::error::Error + 'static,
3559{
3560 let action = plan
3561 .canisters
3562 .iter()
3563 .find(|canister| canister.name == domain.root)
3564 .into_iter()
3565 .flat_map(|canister| &canister.actions)
3566 .find(|action| matches!(action, EnsureAction::FundEstate { .. }));
3567 let Some(action) = action else {
3568 return if domain.maximum_funding_cycles == 0 {
3569 Ok(0)
3570 } else {
3571 Err(EnsureWorkflowError::JournalIntegrity)
3572 };
3573 };
3574 let EnsureAction::FundEstate {
3575 amount,
3576 expected_post_cycles,
3577 ledger,
3578 ledger_fee_cycles,
3579 principal,
3580 ..
3581 } = action
3582 else {
3583 unreachable!("filtered estate-funding action")
3584 };
3585 let root_principal = principal.strip_prefix("created:").map_or_else(
3586 || Some(principal.as_str()),
3587 |name| {
3588 state
3589 .pending_principals
3590 .get(name)
3591 .or_else(|| state.principals.get(name))
3592 .map(String::as_str)
3593 },
3594 );
3595 let expected_destination = domain
3596 .available_cycles
3597 .unwrap_or_default()
3598 .checked_add(domain.shortfall_cycles)
3599 .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3600 let action_authority_is_exact = *amount == domain.shortfall_cycles
3601 && *expected_post_cycles == expected_destination
3602 && ledger == &domain.cycles_ledger
3603 && *ledger_fee_cycles == domain.ledger_fee_cycles
3604 && root_principal == reviewed_estate_root_principal(domain, state);
3605 let effect = journal
3606 .effects
3607 .iter()
3608 .find(|effect| effect.action_sha256 == action_sha256(action));
3609 let Some(effect) = effect.filter(|effect| {
3610 effect.state == EffectState::Applied
3611 && effect
3612 .receipt
3613 .as_deref()
3614 .is_some_and(|receipt| !receipt.is_empty())
3615 }) else {
3616 return Err(EnsureWorkflowError::JournalIntegrity);
3617 };
3618 let balance_evidence_is_exact = crate::fleet_ensure::ops::estate_funding_applied(
3619 crate::fleet_ensure::ops::EstateFundingObservation {
3620 amount: *amount,
3621 destination_after: effect.destination_post_cycles.unwrap_or_default(),
3622 destination_before: effect.destination_pre_cycles,
3623 expected_destination_after: *expected_post_cycles,
3624 ledger_fee_cycles: *ledger_fee_cycles,
3625 source_after: effect.post_cycles.unwrap_or_default(),
3626 source_before: effect.pre_cycles,
3627 },
3628 );
3629 if !action_authority_is_exact || !balance_evidence_is_exact {
3630 return Err(EnsureWorkflowError::JournalIntegrity);
3631 }
3632 Ok(*amount)
3633}
3634
3635fn controlled_cycles<E>(observation: &FleetObservation) -> Result<u128, EnsureWorkflowError<E>>
3636where
3637 E: std::error::Error + 'static,
3638{
3639 observation
3640 .canisters
3641 .values()
3642 .filter_map(Option::as_ref)
3643 .try_fold(0_u128, |total, live| total.checked_add(live.cycles))
3644 .and_then(|total| {
3645 observation
3646 .additional_controlled_cycles
3647 .values()
3648 .try_fold(total, |subtotal, cycles| subtotal.checked_add(*cycles))
3649 })
3650 .and_then(|total| {
3651 observation
3652 .estate_funding_domains
3653 .values()
3654 .filter_map(|domain| domain.balance_cycles)
3655 .try_fold(total, u128::checked_add)
3656 })
3657 .ok_or_else(|| {
3658 EnsureWorkflowError::Conservation("controlled-cycle total overflowed".to_string())
3659 })
3660}
3661
3662fn verified_plan<E>(plan: FleetEnsurePlan) -> Result<FleetEnsurePlan, EnsureWorkflowError<E>>
3663where
3664 E: std::error::Error + 'static,
3665{
3666 if expected_plan_sha256(&plan) != plan.plan_sha256 {
3667 return Err(EnsureWorkflowError::PlanIntegrity);
3668 }
3669 Ok(plan)
3670}
3671
3672fn verify_journal<E>(
3673 journal: &FleetEnsureJournalRecord,
3674 plan: &FleetEnsurePlan,
3675 requested_fleet: &str,
3676 state: &FleetEnsureStateRecord,
3677) -> Result<(), EnsureWorkflowError<E>>
3678where
3679 E: std::error::Error + 'static,
3680{
3681 if journal.fleet != requested_fleet
3682 || journal.operation_id != plan.operation_id
3683 || journal.plan_sha256 != plan.plan_sha256
3684 || !initial_estate_funding_is_exact(journal, plan)
3685 || !estate_funding_pause_is_exact(journal, plan, state)
3686 {
3687 return Err(EnsureWorkflowError::JournalIntegrity);
3688 }
3689 funding::verify(plan, journal, state)?;
3690 continuation::verify_records(plan, journal)?;
3691 let actions = continuation::actions(plan, journal);
3692 let effect_count_matches = journal.effects.len() <= actions.len();
3693 let action_hashes_match = journal.effects.iter().zip(actions).all(|(effect, action)| {
3694 effect.action_sha256 == action_sha256(action)
3695 && maintenance_attempts_are_exact(effect, action)
3696 && publication_attempts_are_exact(effect, action)
3697 });
3698 if !(effect_count_matches && action_hashes_match) {
3699 return Err(EnsureWorkflowError::JournalIntegrity);
3700 }
3701 Ok(())
3702}
3703
3704fn publication_attempts_are_exact(effect: &EffectRecord, action: &EnsureAction) -> bool {
3705 match action.fixture_publication_attempt_limit() {
3706 Some(maximum) => maximum > 0 && effect.publication_attempts <= maximum,
3707 None => effect.publication_attempts == 0,
3708 }
3709}
3710
3711fn maintenance_attempts_are_exact(effect: &EffectRecord, action: &EnsureAction) -> bool {
3712 if let EnsureAction::FleetProtocol { action, .. } = action
3713 && let CurrentFleetProtocolAction::MaintainPoolReadiness {
3714 maximum_updates, ..
3715 } = action.as_ref()
3716 {
3717 effect.maintenance_attempts <= *maximum_updates
3718 } else {
3719 effect.maintenance_attempts == 0
3720 }
3721}
3722
3723fn initial_estate_funding_is_exact(
3724 journal: &FleetEnsureJournalRecord,
3725 plan: &FleetEnsurePlan,
3726) -> bool {
3727 journal.initial_estate_funding_cycles_by_root.len()
3728 == plan.conservation.estate_funding_domains.len()
3729 && plan
3730 .conservation
3731 .estate_funding_domains
3732 .iter()
3733 .all(|domain| {
3734 journal
3735 .initial_estate_funding_cycles_by_root
3736 .get(&domain.root)
3737 .copied()
3738 == Some(domain.available_cycles.unwrap_or_default())
3739 })
3740}
3741
3742fn estate_funding_pause_is_exact(
3743 journal: &FleetEnsureJournalRecord,
3744 plan: &FleetEnsurePlan,
3745 state: &FleetEnsureStateRecord,
3746) -> bool {
3747 let Some(required) = journal.estate_funding_required.as_ref() else {
3748 return true;
3749 };
3750 journal.completion == FleetEnsureCompletion::InProgress
3751 && estate_funding_record_is_exact(required, plan, state)
3752}
3753
3754fn estate_funding_record_is_exact(
3755 required: &EstateFundingRequiredRecord,
3756 plan: &FleetEnsurePlan,
3757 state: &FleetEnsureStateRecord,
3758) -> bool {
3759 let Some(reviewed) = plan
3760 .conservation
3761 .estate_funding_domains
3762 .iter()
3763 .find(|domain| domain.root == required.root)
3764 else {
3765 return false;
3766 };
3767 let operation_authority_is_exact =
3768 required.operation_id == plan.operation_id && required.plan_sha256 == plan.plan_sha256;
3769 let funding_authority_is_exact =
3770 EstateFundingPauseAuthority::from(required) == EstateFundingPauseAuthority::from(reviewed);
3771 let root_authority_is_exact =
3772 reviewed_estate_root_principal(reviewed, state) == Some(required.root_principal.as_str());
3773 let shortfall_is_exact = required.available_cycles < required.maximum_creation_debit_cycles
3774 && required.shortfall_cycles
3775 == required
3776 .maximum_creation_debit_cycles
3777 .saturating_sub(required.available_cycles);
3778 if !(operation_authority_is_exact
3779 && funding_authority_is_exact
3780 && root_authority_is_exact
3781 && shortfall_is_exact)
3782 {
3783 return false;
3784 }
3785 if required.pending_creation_operation_id.is_some() {
3786 current_creation_funding_pause_is_exact(required, reviewed)
3787 } else {
3788 planned_estate_funding_pause_is_exact(required, reviewed)
3789 }
3790}
3791
3792#[derive(Eq, PartialEq)]
3793struct EstateFundingPauseAuthority<'a> {
3794 creation_amount_cycles: u128,
3795 creation_execution_margin_cycles: u128,
3796 cycles_ledger: &'a str,
3797 ledger_fee_cycles: u128,
3798 management_creation_fee_cycles: u128,
3799 readiness_floor_cycles: u128,
3800 root: &'a str,
3801}
3802
3803impl<'a> From<&'a EstateFundingRequiredRecord> for EstateFundingPauseAuthority<'a> {
3804 fn from(required: &'a EstateFundingRequiredRecord) -> Self {
3805 Self {
3806 creation_amount_cycles: required.creation_amount_cycles,
3807 creation_execution_margin_cycles: required.creation_execution_margin_cycles,
3808 cycles_ledger: &required.cycles_ledger,
3809 ledger_fee_cycles: required.ledger_fee_cycles,
3810 management_creation_fee_cycles: required.management_creation_fee_cycles,
3811 readiness_floor_cycles: required.readiness_floor_cycles,
3812 root: &required.root,
3813 }
3814 }
3815}
3816
3817impl<'a> From<&'a EstateFundingDomainPlan> for EstateFundingPauseAuthority<'a> {
3818 fn from(reviewed: &'a EstateFundingDomainPlan) -> Self {
3819 Self {
3820 creation_amount_cycles: reviewed.creation_amount_cycles,
3821 creation_execution_margin_cycles: reviewed.creation_execution_margin_cycles,
3822 cycles_ledger: &reviewed.cycles_ledger,
3823 ledger_fee_cycles: reviewed.ledger_fee_cycles,
3824 management_creation_fee_cycles: reviewed.management_creation_fee_cycles,
3825 readiness_floor_cycles: reviewed.readiness_floor_cycles,
3826 root: &reviewed.root,
3827 }
3828 }
3829}
3830
3831fn current_creation_funding_pause_is_exact(
3832 required: &EstateFundingRequiredRecord,
3833 reviewed: &EstateFundingDomainPlan,
3834) -> bool {
3835 let operation_id_is_exact = required
3836 .pending_creation_operation_id
3837 .as_deref()
3838 .is_some_and(is_sha256_hex);
3839 let attempt_evidence_is_exact = required.attempt_count.is_some()
3840 && required
3841 .retry_at_ns
3842 .is_some_and(|retry_at_ns| retry_at_ns > 0)
3843 && required
3844 .last_attempt_at_ns
3845 .is_none_or(|attempted_at_ns| attempted_at_ns > 0);
3846 let single_creation_debit = required
3847 .creation_amount_cycles
3848 .checked_add(required.ledger_fee_cycles);
3849 let single_creation_fees = required
3850 .management_creation_fee_cycles
3851 .checked_add(required.ledger_fee_cycles);
3852 let debit_authority_is_exact = Some(required.maximum_creation_debit_cycles)
3853 == single_creation_debit
3854 && Some(required.maximum_creation_fee_cycles) == single_creation_fees;
3855 reviewed.required_creation_count > 0
3856 && required.required_creation_count == 1
3857 && operation_id_is_exact
3858 && attempt_evidence_is_exact
3859 && debit_authority_is_exact
3860}
3861
3862const fn planned_estate_funding_pause_is_exact(
3863 required: &EstateFundingRequiredRecord,
3864 reviewed: &EstateFundingDomainPlan,
3865) -> bool {
3866 let runtime_attempt_is_absent = required.attempt_count.is_none()
3867 && required.last_attempt_at_ns.is_none()
3868 && required.retry_at_ns.is_none();
3869 let reviewed_debit_is_exact = required.maximum_creation_debit_cycles
3870 == reviewed.maximum_creation_debit_cycles
3871 && required.maximum_creation_fee_cycles == reviewed.maximum_creation_fee_cycles
3872 && required.required_creation_count == reviewed.required_creation_count;
3873 runtime_attempt_is_absent && reviewed_debit_is_exact
3874}
3875
3876fn is_sha256_hex(value: &str) -> bool {
3877 value.len() == 64
3878 && value
3879 .bytes()
3880 .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
3881}
3882
3883#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3884enum TerminalStatePublicationError {
3885 JournalIntegrity,
3886}
3887
3888fn publish_terminal_state(
3889 desired: &crate::fleet_ensure::model::DesiredFleet,
3890 plan: &FleetEnsurePlan,
3891 journal: &FleetEnsureJournalRecord,
3892 state: &mut FleetEnsureStateRecord,
3893) -> Result<(), TerminalStatePublicationError> {
3894 let maximum_observation_burn_cycles =
3895 crate::fleet_ensure::ops::maximum_creation_observation_burn(desired)
3896 .ok_or(TerminalStatePublicationError::JournalIntegrity)?;
3897 let actions = continuation::actions(plan, journal);
3898 if actions.len() != journal.effects.len() {
3899 return Err(TerminalStatePublicationError::JournalIntegrity);
3900 }
3901 for (action, record) in actions.into_iter().zip(&journal.effects) {
3902 if action_sha256(action) != record.action_sha256 || record.state != EffectState::Applied {
3903 return Err(TerminalStatePublicationError::JournalIntegrity);
3904 }
3905 let EnsureAction::Create {
3906 name,
3907 requested_initial_cycles,
3908 ..
3909 } = action
3910 else {
3911 continue;
3912 };
3913 let Some(created_principal) = record.created_principal.as_deref() else {
3914 return Err(TerminalStatePublicationError::JournalIntegrity);
3915 };
3916 let Some(post_cycles) = record.post_cycles else {
3917 return Err(TerminalStatePublicationError::JournalIntegrity);
3918 };
3919 if !create_balance_is_terminal(
3920 Some(post_cycles),
3921 *requested_initial_cycles,
3922 maximum_observation_burn_cycles,
3923 ) || !create_identity_is_exact(name, record, state)
3924 || state
3925 .retained_cycles_by_principal
3926 .get(created_principal)
3927 .is_some_and(|retained| *retained != post_cycles)
3928 {
3929 return Err(TerminalStatePublicationError::JournalIntegrity);
3930 }
3931 state
3932 .retained_cycles_by_principal
3933 .insert(created_principal.to_string(), post_cycles);
3934 }
3935
3936 let prior_topology = state.topology.clone();
3937 for canister in &plan.canisters {
3938 if !desired
3939 .canisters
3940 .iter()
3941 .any(|configured| configured.name == canister.name)
3942 {
3943 if !is_reviewed_dynamic_pool_funding(plan, canister) {
3946 return Err(TerminalStatePublicationError::JournalIntegrity);
3947 }
3948 continue;
3949 }
3950
3951 match canister.disposition {
3952 CanisterDisposition::Create | CanisterDisposition::Replace => {
3953 if let Some(principal) = state.pending_principals.remove(&canister.name) {
3954 state.principals.insert(canister.name.clone(), principal);
3955 }
3956 }
3957 CanisterDisposition::Reinstall | CanisterDisposition::Reuse => {
3958 if let Some(principal) = state
3959 .pending_principals
3960 .remove(&canister.name)
3961 .or_else(|| canister.principal.clone())
3962 {
3963 state.principals.insert(canister.name.clone(), principal);
3964 }
3965 }
3966 CanisterDisposition::Delete => {
3967 state.principals.remove(&canister.name);
3968 state.pending_principals.remove(&canister.name);
3969 }
3970 }
3971 }
3972 state.topology = desired
3973 .canisters
3974 .iter()
3975 .filter(|canister| {
3976 canister.presence == crate::fleet_ensure::model::DesiredPresence::Present
3977 })
3978 .map(|canister| {
3979 (
3980 canister.name.clone(),
3981 crate::fleet_ensure::model::FleetEnsureTopologyRecord {
3982 kind: canister.kind,
3983 module_hash: projected_module_hash(plan, &prior_topology, canister),
3984 parent: canister.parent.clone(),
3985 protocol_binding: canister.protocol_binding.clone(),
3986 role: canister
3987 .protocol_binding
3988 .as_ref()
3989 .map(|binding| binding.role.to_string()),
3990 },
3991 )
3992 })
3993 .collect();
3994 retain_configured_principal_bindings(state);
3995 Ok(())
3996}
3997
3998fn is_reviewed_dynamic_pool_funding(
3999 plan: &FleetEnsurePlan,
4000 canister: &crate::fleet_ensure::model::CanisterPlan,
4001) -> bool {
4002 canister.disposition == CanisterDisposition::Reuse
4003 && !canister.actions.is_empty()
4004 && canister
4005 .actions
4006 .iter()
4007 .all(|action| is_reviewed_pool_funding(action, plan, canister.principal.as_deref()))
4008}
4009
4010fn is_reviewed_pool_funding(
4011 action: &EnsureAction,
4012 plan: &FleetEnsurePlan,
4013 expected_principal: Option<&str>,
4014) -> bool {
4015 let EnsureAction::Fund {
4016 pool_funding: Some(root),
4017 principal,
4018 ..
4019 } = action
4020 else {
4021 return false;
4022 };
4023 if Some(principal.as_str()) != expected_principal {
4024 return false;
4025 }
4026 plan.protocol_actions.iter().any(|candidate| {
4027 let EnsureAction::FleetProtocol {
4028 principal: target,
4029 action,
4030 ..
4031 } = candidate
4032 else {
4033 return false;
4034 };
4035 let CurrentFleetProtocolAction::ReconcilePoolAsset { request, .. } = action.as_ref() else {
4036 return false;
4037 };
4038 target == &root.root && request.canister_id.to_text() == *principal
4039 })
4040}
4041
4042fn projected_module_hash(
4043 plan: &FleetEnsurePlan,
4044 prior_topology: &BTreeMap<String, crate::fleet_ensure::model::FleetEnsureTopologyRecord>,
4045 canister: &crate::fleet_ensure::model::DesiredCanister,
4046) -> Option<String> {
4047 let installed = plan
4048 .canisters
4049 .iter()
4050 .find(|planned| planned.name == canister.name)
4051 .and_then(|planned| {
4052 planned
4053 .actions
4054 .iter()
4055 .rev()
4056 .find_map(|action| match action {
4057 EnsureAction::Install { wasm_sha256, .. } => Some(wasm_sha256.clone()),
4058 _ => None,
4059 })
4060 });
4061 installed.or_else(|| {
4062 prior_topology
4063 .get(&canister.name)
4064 .filter(|retained| retained.kind == canister.kind && retained.parent == canister.parent)
4065 .and_then(|retained| retained.module_hash.clone())
4066 })
4067}
4068
4069fn merge_terminal_inventory<E>(
4070 state: &mut FleetEnsureStateRecord,
4071 inventory: crate::fleet_ensure::ops::TerminalFleetInventory,
4072) -> Result<(), EnsureWorkflowError<E>>
4073where
4074 E: std::error::Error + 'static,
4075{
4076 let entries = inventory.entries;
4077 let mut names_by_principal = state
4078 .principals
4079 .iter()
4080 .map(|(name, principal)| (principal.clone(), name.clone()))
4081 .collect::<BTreeMap<_, _>>();
4082 if names_by_principal.len() != state.principals.len() {
4083 return Err(EnsureWorkflowError::TerminalInventory(
4084 TerminalInventoryError::DuplicateRetainedPrincipal,
4085 ));
4086 }
4087 for entry in &entries {
4088 names_by_principal
4089 .entry(entry.pid.clone())
4090 .or_insert_with(|| format!("observed:{}", entry.pid));
4091 }
4092 for entry in entries {
4093 let name = names_by_principal.get(&entry.pid).cloned().ok_or_else(|| {
4094 EnsureWorkflowError::TerminalInventory(TerminalInventoryError::LostPrincipalBinding {
4095 canister: entry.pid.clone(),
4096 })
4097 })?;
4098 let parent = entry
4099 .parent_pid
4100 .as_ref()
4101 .map(|principal| {
4102 names_by_principal.get(principal).cloned().ok_or_else(|| {
4103 EnsureWorkflowError::TerminalInventory(TerminalInventoryError::UnknownParent {
4104 canister: entry.pid.clone(),
4105 parent: principal.clone(),
4106 })
4107 })
4108 })
4109 .transpose()?;
4110 let existing = state.topology.get(&name);
4111 let kind = terminal_inventory_kind(existing, &entry);
4112 let parent_conflict = existing
4113 .and_then(|topology| topology.parent.as_ref())
4114 .zip(parent.as_ref())
4115 .filter(|(expected, actual)| expected != actual);
4116 if let Some((retained_parent, terminal_parent)) = parent_conflict
4117 && !verified_pool_to_component_parent_transition(existing, kind, &entry)
4118 {
4119 return Err(EnsureWorkflowError::TerminalInventory(
4120 TerminalInventoryError::RetainedParentConflict {
4121 canister: entry.pid,
4122 retained_parent: retained_parent.clone(),
4123 terminal_parent: terminal_parent.clone(),
4124 },
4125 ));
4126 }
4127 state.principals.insert(name.clone(), entry.pid);
4128 state.topology.insert(
4129 name,
4130 crate::fleet_ensure::model::FleetEnsureTopologyRecord {
4131 kind,
4132 module_hash: entry.module_hash,
4133 parent,
4134 protocol_binding: entry.protocol_binding,
4135 role: entry.role,
4136 },
4137 );
4138 }
4139 state.active_registry = inventory.active_registry;
4140 Ok(())
4141}
4142
4143fn terminal_inventory_kind(
4144 existing: Option<&crate::fleet_ensure::model::FleetEnsureTopologyRecord>,
4145 entry: &crate::registry::RegistryEntry,
4146) -> crate::fleet_ensure::model::DesiredCanisterKind {
4147 existing.map_or_else(
4148 || {
4149 if entry.module_hash.is_some() {
4150 crate::fleet_ensure::model::DesiredCanisterKind::Component
4151 } else {
4152 crate::fleet_ensure::model::DesiredCanisterKind::Auxiliary
4153 }
4154 },
4155 |topology| {
4156 if topology.kind == crate::fleet_ensure::model::DesiredCanisterKind::Pool
4157 && entry.module_hash.is_some()
4158 && entry.protocol_binding.is_some()
4159 {
4160 crate::fleet_ensure::model::DesiredCanisterKind::Component
4161 } else {
4162 topology.kind
4163 }
4164 },
4165 )
4166}
4167
4168fn verified_pool_to_component_parent_transition(
4169 existing: Option<&crate::fleet_ensure::model::FleetEnsureTopologyRecord>,
4170 terminal_kind: crate::fleet_ensure::model::DesiredCanisterKind,
4171 entry: &crate::registry::RegistryEntry,
4172) -> bool {
4173 existing.is_some_and(|topology| {
4174 topology.kind == crate::fleet_ensure::model::DesiredCanisterKind::Pool
4175 && terminal_kind == crate::fleet_ensure::model::DesiredCanisterKind::Component
4176 && entry.module_hash.is_some()
4177 && entry.protocol_binding.is_some()
4178 })
4179}
4180
4181fn applied_count(journal: &FleetEnsureJournalRecord) -> u32 {
4182 u32::try_from(
4183 journal
4184 .effects
4185 .iter()
4186 .chain(
4187 journal
4188 .funding_reviews
4189 .iter()
4190 .filter_map(|review| review.effect.as_ref()),
4191 )
4192 .filter(|effect| matches!(effect.state, EffectState::Applied))
4193 .count(),
4194 )
4195 .unwrap_or(u32::MAX)
4196}
4197
4198fn verify_store_activation_binding<P: EnsurePlatform>(
4199 plan: &FleetEnsurePlan,
4200 action: &EnsureAction,
4201 state: &FleetEnsureStateRecord,
4202 platform: &mut P,
4203) -> Result<(), EnsureWorkflowError<P::Error>> {
4204 use crate::fleet_ensure::model::DesiredCanisterInit;
4205 let EnsureAction::Install {
4206 canic_init: Some(DesiredCanisterInit::Store { root }),
4207 ..
4208 } = action
4209 else {
4210 return Ok(());
4211 };
4212 let root_will_be_installed = plan.canisters.iter().flat_map(|canister| &canister.actions)
4213 .any(|action| matches!(action,
4214 EnsureAction::Install { name, canic_init: Some(DesiredCanisterInit::Root { root: installed_root }), .. }
4215 if name == root && installed_root == root));
4216 if !root_will_be_installed {
4217 platform
4218 .require_retained_root_activation(&plan.operation_id, root, state)
4219 .map_err(EnsureWorkflowError::Platform)?;
4220 }
4221 Ok(())
4222}
4223
4224pub(super) fn ordered_actions(plan: &FleetEnsurePlan) -> Vec<&EnsureAction> {
4225 let mut actions = plan
4226 .canisters
4227 .iter()
4228 .flat_map(|canister| canister.actions.iter())
4229 .chain(plan.protocol_actions.iter())
4230 .collect::<Vec<_>>();
4231 if plan.scope == FleetEnsurePlanScope::ReinstallPreparation
4232 && plan
4233 .reinstall
4234 .as_ref()
4235 .is_some_and(|intent| intent.activation_reset.is_some())
4236 {
4237 actions.sort_by_key(|action| matches!(action, EnsureAction::Start { .. }));
4239 return actions;
4240 }
4241 if plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite {
4242 return actions;
4243 }
4244 let temporary_pool_observation_finalizations = plan
4245 .reviewed_desired
4246 .as_deref()
4247 .map(|reviewed| {
4248 let desired = reviewed.desired();
4249 plan.canisters
4250 .iter()
4251 .filter_map(|canister_plan| {
4252 let configured = desired.canisters.iter().find(|configured| {
4253 configured.name == canister_plan.name
4254 && configured.kind
4255 == crate::fleet_ensure::model::DesiredCanisterKind::Pool
4256 })?;
4257 let temporary_create = desired
4258 .bootstrap
4259 .as_ref()
4260 .is_some_and(|bootstrap| bootstrap.fresh_estate)
4261 && configured.principal.is_none()
4262 && configured.controllers.is_empty()
4263 && configured.controller_canisters.len() == 1
4264 && canister_plan.actions.iter().any(|action| {
4265 matches!(
4266 action,
4267 EnsureAction::Create {
4268 controller_canisters,
4269 controllers,
4270 ..
4271 } if controller_canisters == &configured.controller_canisters
4272 && controllers == std::slice::from_ref(&desired.operator)
4273 )
4274 });
4275 temporary_create.then_some(canister_plan.name.as_str())
4276 })
4277 .collect::<BTreeSet<_>>()
4278 })
4279 .unwrap_or_default();
4280 actions.sort_by_key(|action| {
4281 if matches!(action, EnsureAction::SetControllers { name, .. } if temporary_pool_observation_finalizations.contains(name.as_str()))
4282 {
4283 6
4284 } else {
4285 action_order(action)
4286 }
4287 });
4288 actions
4289}
4290
4291fn deferred_create_observation_is_exact(
4292 desired: &crate::fleet_ensure::model::DesiredFleet,
4293 actions: &[&EnsureAction],
4294 index: usize,
4295 action: &EnsureAction,
4296 record: &EffectRecord,
4297 state: &FleetEnsureStateRecord,
4298) -> bool {
4299 let EnsureAction::Create {
4300 controller_canisters,
4301 controllers,
4302 name,
4303 ..
4304 } = action
4305 else {
4306 return false;
4307 };
4308 let Some(configured) = desired.canisters.iter().find(|canister| {
4309 canister.name == *name
4310 && canister.kind == crate::fleet_ensure::model::DesiredCanisterKind::Pool
4311 }) else {
4312 return false;
4313 };
4314 let Some(parent) = configured.parent.as_deref() else {
4315 return false;
4316 };
4317 let fresh_root_only_create = desired
4318 .bootstrap
4319 .as_ref()
4320 .is_some_and(|bootstrap| bootstrap.fresh_estate)
4321 && configured.principal.is_none()
4322 && configured.controllers.is_empty()
4323 && configured.controller_canisters.as_slice() == [parent]
4324 && controller_canisters == &configured.controller_canisters
4325 && controllers.is_empty();
4326 let exact_issued_create = record.state == EffectState::Issued
4327 && create_identity_is_exact(name, record, state)
4328 && record.post_cycles.is_some();
4329 let later_root_install = actions[index.saturating_add(1)..].iter().any(|candidate| {
4330 matches!(
4331 candidate,
4332 EnsureAction::Install {
4333 canic_init: Some(crate::fleet_ensure::model::DesiredCanisterInit::Root { root }),
4334 name: install_name,
4335 ..
4336 } if root == parent && install_name == parent
4337 )
4338 });
4339 let infrastructure_only = actions.iter().all(|candidate| {
4340 matches!(
4341 candidate,
4342 EnsureAction::Create { .. }
4343 | EnsureAction::FundEstate { .. }
4344 | EnsureAction::Install { .. }
4345 | EnsureAction::SetControllers { .. }
4346 )
4347 });
4348 fresh_root_only_create && exact_issued_create && later_root_install && infrastructure_only
4349}
4350
4351pub(super) const fn action_order(action: &EnsureAction) -> u8 {
4352 match action {
4353 EnsureAction::SealAuthority {
4354 authority_kind: crate::fleet_ensure::model::DesiredCanisterKind::Root,
4355 ..
4356 }
4357 | EnsureAction::Create { .. } => 0,
4358 EnsureAction::SealAuthority { .. }
4359 | EnsureAction::Fund { .. }
4360 | EnsureAction::FundEstate { .. } => 1,
4361 EnsureAction::Install {
4362 canic_init: Some(crate::fleet_ensure::model::DesiredCanisterInit::Coordinator),
4363 ..
4364 } => 2,
4365 EnsureAction::SetControllers { .. } => 3,
4366 EnsureAction::Install {
4367 canic_init: Some(crate::fleet_ensure::model::DesiredCanisterInit::Store { .. }),
4368 ..
4369 } => 4,
4370 EnsureAction::Install { .. } => 5,
4371 EnsureAction::Start { .. } => 6,
4372 EnsureAction::FleetProtocol { .. } | EnsureAction::Protocol { .. } => 7,
4373 EnsureAction::Transfer { .. } => 8,
4374 EnsureAction::Stop { .. } => 9,
4375 EnsureAction::Delete { .. } => 10,
4376 }
4377}
4378
4379#[cfg(test)]
4380mod tests {
4381 use super::*;
4382
4383 #[test]
4384 fn store_binding_requires_exact_retained_root_or_reviewed_root_install() {
4385 use crate::fleet_ensure::{
4386 model::{CanisterDisposition, CanisterPlan, DesiredCanisterInit, InstallMode},
4387 tests::MockPlatform,
4388 };
4389 let (state, _) = retained_evidence();
4390 let mut plan = estate_funding_plan();
4391 let desired = serde_json::from_value(serde_json::json!({
4392 "canisters": [], "cycles_ledger": "um5iw-rqaaa-aaaaq-qaaba-cai",
4393 "environment": "local", "fleet": "fleet", "ledger_fee_cycles": "0",
4394 "management_creation_fee_cycles": "0", "material_cycle_threshold": "1",
4395 "maximum_observation_burn_cycles": "1", "maximum_stalled_observations": 2,
4396 "maximum_update_burn_cycles": "2", "operator": "rrkah-fqaaa-aaaaa-aaaaq-cai",
4397 "schema_version": FLEET_ENSURE_SCHEMA_VERSION, "treasury": "root"
4398 }))
4399 .unwrap();
4400 let mut platform = MockPlatform::new(desired, []);
4401 platform.rejected_activation_root = Some("root".into());
4402 let store = EnsureAction::Install {
4403 canic_init: Some(DesiredCanisterInit::Store {
4404 root: "root".into(),
4405 }),
4406 reinstall_witness: None,
4407 init_arg: None,
4408 init_arg_sha256: None,
4409 init_candid: None,
4410 init_candid_sha256: None,
4411 mode: InstallMode::Install,
4412 name: "store".into(),
4413 principal: "store-principal".into(),
4414 wasm: "store.wasm".into(),
4415 wasm_sha256: "store-hash".into(),
4416 };
4417 assert!(matches!(
4418 verify_store_activation_binding(&plan, &store, &state, &mut platform),
4419 Err(EnsureWorkflowError::Platform(_))
4420 ));
4421 assert_eq!(
4422 platform.retained_activation_checks,
4423 [(plan.operation_id.clone(), "root".into())]
4424 );
4425 let mut root_install = store.clone();
4426 if let EnsureAction::Install {
4427 name, canic_init, ..
4428 } = &mut root_install
4429 {
4430 *name = "other-root".into();
4431 *canic_init = Some(DesiredCanisterInit::Root {
4432 root: "other-root".into(),
4433 });
4434 }
4435 plan.canisters.push(CanisterPlan {
4436 actions: vec![root_install],
4437 disposition: CanisterDisposition::Create,
4438 name: "root".into(),
4439 observed_cycles: 0,
4440 principal: None,
4441 });
4442 assert!(verify_store_activation_binding(&plan, &store, &state, &mut platform).is_err());
4443 if let EnsureAction::Install {
4444 name, canic_init, ..
4445 } = &mut plan.canisters.last_mut().unwrap().actions[0]
4446 {
4447 *name = "root".into();
4448 *canic_init = Some(DesiredCanisterInit::Root {
4449 root: "root".into(),
4450 });
4451 }
4452 let reads = platform.retained_activation_checks.len();
4453 verify_store_activation_binding(&plan, &store, &state, &mut platform).unwrap();
4454 assert_eq!(platform.retained_activation_checks.len(), reads);
4455 plan.canisters.pop();
4456 platform.rejected_activation_root = None;
4457 verify_store_activation_binding(&plan, &store, &state, &mut platform).unwrap();
4458 assert_eq!(platform.retained_activation_checks.len(), reads + 1);
4459 }
4460
4461 #[test]
4462 fn root_prerequisite_retains_activation_authority_across_desired_refresh() {
4463 let (mut state, mut journal) = retained_evidence();
4464 let mut plan = estate_funding_plan();
4465 plan.scope = FleetEnsurePlanScope::RootReinstallPrerequisite;
4466 plan.conservation.estate_funding_domains.clear();
4467 journal.effects.clear();
4468 journal.completion = FleetEnsureCompletion::Converged;
4469 let before = state.clone();
4470 assert!(matches!(
4471 reconcile_retained_planning_state::<std::io::Error>(
4472 &mut state,
4473 Some(&plan),
4474 Some(&journal),
4475 "fleet",
4476 "local",
4477 "refreshed-imports"
4478 ),
4479 Err(EnsureWorkflowError::RootActivationDesiredConflict { .. })
4480 ));
4481 assert_eq!(state, before);
4482 assert!(
4483 reconcile_retained_planning_state::<std::io::Error>(
4484 &mut state,
4485 Some(&plan),
4486 Some(&journal),
4487 "fleet",
4488 "local",
4489 "desired"
4490 )
4491 .unwrap()
4492 .is_some()
4493 );
4494 }
4495
4496 fn terminal_component_entry(canister: &str, parent: &str) -> crate::registry::RegistryEntry {
4497 crate::registry::RegistryEntry {
4498 pid: canister.to_string(),
4499 role: Some("managed_component".to_string()),
4500 parent_pid: Some(parent.to_string()),
4501 module_hash: Some("11".repeat(32)),
4502 protocol_binding: Some(crate::protocol_binding::RegistryProtocolBinding {
4503 release_identity: "0.110.test".to_string(),
4504 role: canic_core::ids::CanisterRole::from("managed_component"),
4505 capabilities: BTreeSet::new(),
4506 candid_sha256: [1; 32],
4507 protocol_profile_digest:
4508 canic_core::role_contract::ProtocolProfileDigest::from_bytes([2; 32]),
4509 }),
4510 }
4511 }
4512
4513 pub(super) fn retained_evidence() -> (FleetEnsureStateRecord, FleetEnsureJournalRecord) {
4514 let state = FleetEnsureStateRecord {
4515 active_registry: None,
4516 completed_reinstall_action_sha256: BTreeMap::from([(
4517 "root".to_string(),
4518 "action".to_string(),
4519 )]),
4520 completed_reinstall_operation_id: Some("operation".to_string()),
4521 completed_reinstalls: BTreeMap::from([("root".to_string(), 7)]),
4522 fleet: "fleet".to_string(),
4523 pending_principals: BTreeMap::new(),
4524 principals: BTreeMap::new(),
4525 retained_cycles_by_principal: BTreeMap::new(),
4526 schema_version: crate::fleet_ensure::model::FLEET_ENSURE_SCHEMA_VERSION,
4527 topology: BTreeMap::new(),
4528 };
4529 let journal = FleetEnsureJournalRecord {
4530 funding_reviews: Vec::new(),
4531 successor_phases: Vec::new(),
4532 completion: FleetEnsureCompletion::ReplanRequired,
4533 estate_funding_required: None,
4534 effects: vec![crate::fleet_ensure::model::EffectRecord {
4535 publication_attempts: 0,
4536 maintenance_attempts: 0,
4537 action_sha256: "action".to_string(),
4538 created_principal: None,
4539 destination_post_cycles: None,
4540 destination_pre_cycles: None,
4541 post_cycles: None,
4542 pre_cycles: None,
4543 pre_canister_version: Some(7),
4544 progress_identity: None,
4545 receipt: None,
4546 state: EffectState::Applied,
4547 }],
4548 fleet: "fleet".to_string(),
4549 initial_controlled_cycles: 0,
4550 initial_estate_funding_cycles_by_root: BTreeMap::new(),
4551 initial_operator_cycles: 0,
4552 operation_id: "operation".to_string(),
4553 plan_sha256: "plan".to_string(),
4554 schema_version: crate::fleet_ensure::model::FLEET_ENSURE_SCHEMA_VERSION,
4555 stalled_observations: 0,
4556 };
4557 (state, journal)
4558 }
4559
4560 pub(super) fn estate_funding_plan() -> FleetEnsurePlan {
4561 FleetEnsurePlan {
4562 continuation: None,
4563 canisters: Vec::new(),
4564 conservation: CycleConservation {
4565 estate_funding_domains: vec![crate::fleet_ensure::model::EstateFundingDomainPlan {
4566 allocated_workloads: 0,
4567 available_cycles: Some(40),
4568 available_pool_slots: 2,
4569 creation_amount_cycles: 45,
4570 creation_execution_margin_cycles: 5,
4571 readiness_floor_cycles: 35,
4572 cycles_ledger: "ledger".to_string(),
4573 eligible_ready_pool_assets: 0,
4574 initial_pool_assets: Vec::new(),
4575 ledger_fee_cycles: 5,
4576 management_creation_fee_cycles: 5,
4577 maximum_creation_debit_cycles: 100,
4578 maximum_creation_fee_cycles: 20,
4579 maximum_funding_cycles: 60,
4580 occupied_pool_assets: 0,
4581 pending_creation_count: 0,
4582 pending_creation: None,
4583 planned_initial_workloads: 2,
4584 pool_maximum_size: 2,
4585 required_creation_count: 2,
4586 root: "root".to_string(),
4587 root_principal: Some("rrkah-fqaaa-aaaaa-aaaaq-cai".to_string()),
4588 shortfall_cycles: 60,
4589 }],
4590 expected_post_operation_cycles: 80,
4591 maximum_execution_burn_cycles: 0,
4592 maximum_new_funding_cycles: 0,
4593 maximum_operator_debit_cycles: 0,
4594 maximum_unavoidable_fee_cycles: 0,
4595 observed_controlled_cycles: 40,
4596 retained_in_reused_canisters_cycles: 0,
4597 scheduled_transfer_cycles: 0,
4598 },
4599 desired_sha256: "desired".to_string(),
4600 environment: "local".to_string(),
4601 fleet: "fleet".to_string(),
4602 operation_id: "operation".to_string(),
4603 plan_sha256: "plan".to_string(),
4604 planned_at_time: 1,
4605 protocol_actions: Vec::new(),
4606 recovery_review: None,
4607 reinstall: None,
4608 root_reinstall_bindings: Vec::new(),
4609 root_start_authority: None,
4610 reviewed_desired: None,
4611 schema_version: FLEET_ENSURE_SCHEMA_VERSION,
4612 scope: FleetEnsurePlanScope::Full,
4613 terminal_inventory_operation_id: None,
4614 }
4615 }
4616
4617 pub(super) fn estate_funding_observation(balance_cycles: Option<u128>) -> FleetObservation {
4618 FleetObservation {
4619 additional_controlled_cycles: BTreeMap::new(),
4620 canisters: BTreeMap::new(),
4621 estate_funding_domains: BTreeMap::from([(
4622 "root".to_string(),
4623 crate::fleet_ensure::model::EstateFundingDomainObservation {
4624 balance_cycles,
4625 cycles_ledger: "ledger".to_string(),
4626 pool: None,
4627 root_principal: Some("rrkah-fqaaa-aaaaa-aaaaq-cai".to_string()),
4628 },
4629 )]),
4630 ledger_fee_cycles: 0,
4631 operator_cycles: 0,
4632 protocol_ready: BTreeMap::new(),
4633 }
4634 }
4635
4636 fn terminal_estate_creation_observation() -> FleetObservation {
4637 let mut observation = estate_funding_observation(Some(0));
4638 let receipt =
4639 |byte, block_index| crate::fleet_ensure::model::EstatePoolCreationReceiptObservation {
4640 block_index,
4641 operation_id: format!("{byte:02x}").repeat(32),
4642 cycles_ledger: "ledger".to_string(),
4643 ledger_amount_cycles: 45,
4644 ledger_fee_cycles: 5,
4645 readiness_floor_cycles: 35,
4646 creation_execution_margin_cycles: 5,
4647 management_creation_fee_cycles: 5,
4648 first_observed_cycles: Some(39),
4649 };
4650 observation
4651 .estate_funding_domains
4652 .get_mut("root")
4653 .expect("Root funding domain")
4654 .pool = Some(crate::fleet_ensure::model::EstatePoolInventoryObservation {
4655 assets: vec![
4656 crate::fleet_ensure::model::EstatePoolAssetObservation {
4657 creation_receipt: Some(receipt(1, 10)),
4658 cycles: 35,
4659 lifecycle: crate::fleet_ensure::model::EstatePoolAssetLifecycle::Ready,
4660 origin: crate::fleet_ensure::model::EstatePoolAssetOrigin::Created,
4661 principal: "created-one".to_string(),
4662 },
4663 crate::fleet_ensure::model::EstatePoolAssetObservation {
4664 creation_receipt: Some(receipt(2, 11)),
4665 cycles: 35,
4666 lifecycle: crate::fleet_ensure::model::EstatePoolAssetLifecycle::Workload,
4667 origin: crate::fleet_ensure::model::EstatePoolAssetOrigin::Created,
4668 principal: "created-two".to_string(),
4669 },
4670 ],
4671 maximum_size: 2,
4672 minimum_size: 0,
4673 pending_creation: None,
4674 readiness_floor_cycles: 35,
4675 creation_execution_margin_cycles: 5,
4676 });
4677 observation
4678 }
4679
4680 fn estate_creation_receipt(
4681 observation: &mut FleetObservation,
4682 asset_index: usize,
4683 ) -> &mut crate::fleet_ensure::model::EstatePoolCreationReceiptObservation {
4684 observation
4685 .estate_funding_domains
4686 .get_mut("root")
4687 .expect("Root domain")
4688 .pool
4689 .as_mut()
4690 .expect("pool")
4691 .assets[asset_index]
4692 .creation_receipt
4693 .as_mut()
4694 .expect("receipt")
4695 }
4696
4697 #[test]
4698 fn autonomous_reserve_creation_reconciles_without_host_provisioning() {
4699 let mut plan = estate_funding_plan();
4700 let domain = &mut plan.conservation.estate_funding_domains[0];
4701 domain.available_cycles = Some(100);
4702 domain.maximum_funding_cycles = 0;
4703 domain.shortfall_cycles = 0;
4704 let (state, mut journal) = retained_evidence();
4705 journal.effects.clear();
4706 journal
4707 .initial_estate_funding_cycles_by_root
4708 .insert("root".to_string(), 100);
4709 let terminal = terminal_estate_creation_observation();
4710 assert_eq!(
4711 reconcile_estate_funding::<std::io::Error>(&plan, &journal, &state, &terminal)
4712 .expect("reconcile exact Root-owned creation against the reviewed reserve"),
4713 (0, 20)
4714 );
4715 }
4716
4717 #[test]
4718 fn terminal_estate_creation_uses_exact_protected_receipts() {
4719 let plan = estate_funding_plan();
4720 let domain = &plan.conservation.estate_funding_domains[0];
4721 let (state, _) = retained_evidence();
4722 let exact = terminal_estate_creation_observation();
4723 assert_eq!(
4724 exact_estate_creation_costs::<std::io::Error>(&state, &exact, domain)
4725 .expect("reconcile exact creation receipts"),
4726 (100, 20)
4727 );
4728 for required_creation_count in [0, 1] {
4729 let mut insufficient = domain.clone();
4730 insufficient.required_creation_count = required_creation_count;
4731 assert!(matches!(
4732 exact_estate_creation_costs::<std::io::Error>(&state, &exact, &insufficient),
4733 Err(EnsureWorkflowError::JournalIntegrity)
4734 ));
4735 }
4736
4737 let mut drifted = Vec::new();
4738 let mut changed = exact.clone();
4739 estate_creation_receipt(&mut changed, 0).ledger_amount_cycles = 44;
4740 drifted.push(changed);
4741 let mut changed = exact.clone();
4742 estate_creation_receipt(&mut changed, 1).operation_id = "01".repeat(32);
4743 drifted.push(changed);
4744 let mut changed = exact.clone();
4745 estate_creation_receipt(&mut changed, 1).block_index = 10;
4746 drifted.push(changed);
4747 let mut changed = exact.clone();
4748 estate_creation_receipt(&mut changed, 0).first_observed_cycles = None;
4749 drifted.push(changed);
4750 let mut changed = exact.clone();
4751 estate_creation_receipt(&mut changed, 0).first_observed_cycles = Some(34);
4752 drifted.push(changed);
4753 let mut changed = exact.clone();
4754 estate_creation_receipt(&mut changed, 0).first_observed_cycles = Some(41);
4755 drifted.push(changed);
4756 let mut changed = exact;
4757 changed
4758 .estate_funding_domains
4759 .get_mut("root")
4760 .expect("Root domain")
4761 .pool
4762 .as_mut()
4763 .expect("pool")
4764 .assets[0]
4765 .origin = crate::fleet_ensure::model::EstatePoolAssetOrigin::Imported;
4766 drifted.push(changed);
4767 for observation in drifted {
4768 assert!(matches!(
4769 exact_estate_creation_costs::<std::io::Error>(&state, &observation, domain,),
4770 Err(EnsureWorkflowError::JournalIntegrity)
4771 ));
4772 }
4773 }
4774
4775 #[test]
4776 fn estate_funding_pause_is_exact_and_clears_only_after_sufficient_balance() {
4777 let plan = estate_funding_plan();
4778 let (state, mut journal) = retained_evidence();
4779 let required = estate_funding_requirement::<std::io::Error>(
4780 &plan,
4781 &state,
4782 &estate_funding_observation(Some(40)),
4783 )
4784 .expect("validate funding observation")
4785 .expect("underfunded estate pauses");
4786 assert_eq!(required.available_cycles, 40);
4787 assert_eq!(required.maximum_creation_debit_cycles, 100);
4788 assert_eq!(required.shortfall_cycles, 60);
4789 assert!(
4790 estate_funding_requirement::<std::io::Error>(
4791 &plan,
4792 &state,
4793 &estate_funding_observation(Some(100)),
4794 )
4795 .expect("validate sufficient funding observation")
4796 .is_none()
4797 );
4798
4799 journal.completion = FleetEnsureCompletion::InProgress;
4800 journal.effects.clear();
4801 journal.estate_funding_required = Some(required);
4802 journal.initial_estate_funding_cycles_by_root = BTreeMap::from([("root".to_string(), 40)]);
4803 assert!(initial_estate_funding_is_exact(&journal, &plan));
4804 assert!(estate_funding_pause_is_exact(&journal, &plan, &state));
4805
4806 journal
4807 .estate_funding_required
4808 .as_mut()
4809 .expect("funding pause")
4810 .shortfall_cycles = 59;
4811 assert!(!estate_funding_pause_is_exact(&journal, &plan, &state));
4812
4813 *journal
4814 .initial_estate_funding_cycles_by_root
4815 .get_mut("root")
4816 .expect("initial Root funding balance") = 41;
4817 assert!(!initial_estate_funding_is_exact(&journal, &plan));
4818 }
4819
4820 #[test]
4821 fn current_creation_funding_pause_is_bound_to_exact_runtime_evidence() {
4822 let mut plan = estate_funding_plan();
4823 plan.conservation.estate_funding_domains[0].cycles_ledger = "aaaaa-aa".to_string();
4824 let (state, mut journal) = retained_evidence();
4825 let funding = canic_core::dto::component_provisioning::RootEstateFundingRequired {
4826 available: Cycles::new(40),
4827 attempt_count: 2,
4828 creation_amount: Cycles::new(45),
4829 cycles_ledger: candid::Principal::from_text("aaaaa-aa").expect("Ledger Principal"),
4830 execution_margin: Cycles::new(5),
4831 last_attempt_at_ns: Some(10),
4832 ledger_fee: Cycles::new(5),
4833 management_creation_fee: Cycles::new(5),
4834 operation_id: [17; 32],
4835 readiness_floor: Cycles::new(35),
4836 required: Cycles::new(50),
4837 retry_at_ns: 20,
4838 root: candid::Principal::from_text("rrkah-fqaaa-aaaaa-aaaaq-cai")
4839 .expect("Root Principal"),
4840 shortfall: Cycles::new(10),
4841 };
4842 let required =
4843 protocol_estate_funding_requirement::<std::io::Error>(&plan, &state, &funding)
4844 .expect("bind runtime funding evidence");
4845 journal.completion = FleetEnsureCompletion::InProgress;
4846 journal.effects.clear();
4847 journal.estate_funding_required = Some(required);
4848 assert!(estate_funding_pause_is_exact(&journal, &plan, &state));
4849
4850 journal
4851 .estate_funding_required
4852 .as_mut()
4853 .expect("funding pause")
4854 .pending_creation_operation_id = Some("AA".repeat(32));
4855 assert!(!estate_funding_pause_is_exact(&journal, &plan, &state));
4856
4857 let mut drifted = funding;
4858 drifted.execution_margin = Cycles::new(6);
4859 assert!(matches!(
4860 protocol_estate_funding_requirement::<std::io::Error>(&plan, &state, &drifted),
4861 Err(EnsureWorkflowError::PlanIntegrity)
4862 ));
4863 }
4864
4865 #[test]
4866 fn estate_funding_observation_rejects_missing_or_mismatched_authority() {
4867 let plan = estate_funding_plan();
4868 let (state, _) = retained_evidence();
4869 let mut missing_balance = estate_funding_observation(None);
4870 assert!(matches!(
4871 estate_funding_requirement::<std::io::Error>(&plan, &state, &missing_balance),
4872 Err(EnsureWorkflowError::PlanIntegrity)
4873 ));
4874
4875 let observed = missing_balance
4876 .estate_funding_domains
4877 .get_mut("root")
4878 .expect("Root funding domain");
4879 observed.balance_cycles = Some(40);
4880 observed.root_principal = Some("aaaaa-aa".to_string());
4881 assert!(matches!(
4882 estate_funding_requirement::<std::io::Error>(&plan, &state, &missing_balance),
4883 Err(EnsureWorkflowError::PlanIntegrity)
4884 ));
4885
4886 for changed_balance in [39, 101] {
4887 assert!(matches!(
4888 estate_funding_requirement::<std::io::Error>(
4889 &plan,
4890 &state,
4891 &estate_funding_observation(Some(changed_balance)),
4892 ),
4893 Err(EnsureWorkflowError::DriftedBeforeApply)
4894 ));
4895 }
4896 }
4897
4898 #[test]
4899 fn non_creation_protocol_completion_does_not_consume_estate_creation_authority() {
4900 let mut plan = estate_funding_plan();
4901 plan.conservation.estate_funding_domains[0].maximum_creation_debit_cycles = 0;
4902 plan.conservation.estate_funding_domains[0].maximum_creation_fee_cycles = 0;
4903 plan.conservation.estate_funding_domains[0].maximum_funding_cycles = 0;
4904 plan.conservation.estate_funding_domains[0].required_creation_count = 0;
4905 plan.conservation.estate_funding_domains[0].shortfall_cycles = 0;
4906 let action = EnsureAction::FleetProtocol {
4907 action: Box::new(
4908 crate::fleet_ensure::model::CurrentFleetProtocolAction::PrepareStoreChunkSet {
4909 request: canic_control_plane::dto::template::TemplateChunkSetPrepareInput {
4910 template_id: canic_control_plane::ids::TemplateId::from("root"),
4911 version: canic_control_plane::ids::TemplateVersion::from("current"),
4912 payload_hash: vec![1; 32],
4913 payload_size_bytes: 1,
4914 chunk_hashes: vec![vec![2; 32]],
4915 },
4916 },
4917 ),
4918 candid: "store.did".to_string(),
4919 candid_sha256: "11".repeat(32),
4920 maximum_execution_burn_cycles: 0,
4921 name: "prepare-store-chunks".to_string(),
4922 principal: "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(),
4923 };
4924 let action_hash = action_sha256(&action);
4925 plan.protocol_actions.push(action);
4926 let (state, mut journal) = retained_evidence();
4927 journal.initial_estate_funding_cycles_by_root = BTreeMap::from([("root".to_string(), 40)]);
4928 journal.effects = vec![EffectRecord {
4929 publication_attempts: 0,
4930 maintenance_attempts: 0,
4931 action_sha256: action_hash,
4932 created_principal: None,
4933 destination_post_cycles: None,
4934 destination_pre_cycles: None,
4935 post_cycles: None,
4936 pre_cycles: None,
4937 pre_canister_version: None,
4938 progress_identity: Some("store chunks prepared".to_string()),
4939 receipt: Some("receipt".to_string()),
4940 state: EffectState::Applied,
4941 }];
4942
4943 assert_eq!(
4944 reconcile_estate_funding::<std::io::Error>(
4945 &plan,
4946 &journal,
4947 &state,
4948 &estate_funding_observation(Some(40)),
4949 )
4950 .expect("non-creation protocol preserves the Root Ledger balance"),
4951 (0, 0),
4952 );
4953 }
4954
4955 #[test]
4956 fn fresh_estate_funding_binds_the_created_root_from_durable_state() {
4957 let mut plan = estate_funding_plan();
4958 plan.conservation.estate_funding_domains[0].available_cycles = None;
4959 plan.conservation.estate_funding_domains[0].maximum_funding_cycles = 100;
4960 plan.conservation.estate_funding_domains[0].root_principal = None;
4961 let (mut state, _) = retained_evidence();
4962 state.pending_principals.insert(
4963 "root".to_string(),
4964 "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(),
4965 );
4966
4967 let required = estate_funding_requirement::<std::io::Error>(
4968 &plan,
4969 &state,
4970 &estate_funding_observation(Some(0)),
4971 )
4972 .expect("validate created Root funding authority")
4973 .expect("fresh underfunded estate pauses");
4974 assert_eq!(required.shortfall_cycles, 100);
4975 assert_eq!(required.root_principal, "rrkah-fqaaa-aaaaa-aaaaq-cai");
4976
4977 state
4978 .pending_principals
4979 .insert("root".to_string(), "aaaaa-aa".to_string());
4980 assert!(matches!(
4981 estate_funding_requirement::<std::io::Error>(
4982 &plan,
4983 &state,
4984 &estate_funding_observation(Some(0)),
4985 ),
4986 Err(EnsureWorkflowError::PlanIntegrity)
4987 ));
4988 }
4989
4990 #[test]
4991 fn unchanged_estate_funding_pause_does_not_rewrite_its_journal() {
4992 let root = crate::test_support::temp_dir("estate-funding-pause");
4993 let paths = EnsurePaths::under(&root, "local", "fleet");
4994 let plan = estate_funding_plan();
4995 let (state, mut journal) = retained_evidence();
4996 journal.completion = FleetEnsureCompletion::InProgress;
4997 journal.effects.clear();
4998 journal.initial_estate_funding_cycles_by_root = BTreeMap::from([("root".to_string(), 40)]);
4999 let required = estate_funding_requirement::<std::io::Error>(
5000 &plan,
5001 &state,
5002 &estate_funding_observation(Some(40)),
5003 )
5004 .expect("validate funding observation")
5005 .expect("underfunded estate pauses");
5006
5007 assert!(
5008 retain_estate_funding_pause::<std::io::Error>(&paths, &mut journal, Some(&required),)
5009 .expect("retain first funding pause")
5010 );
5011 let retained = std::fs::read(&paths.journal).expect("read first retained pause");
5012 assert!(
5013 !retain_estate_funding_pause::<std::io::Error>(&paths, &mut journal, Some(&required),)
5014 .expect("replay unchanged funding pause")
5015 );
5016 assert_eq!(
5017 std::fs::read(&paths.journal).expect("reread unchanged retained pause"),
5018 retained
5019 );
5020 assert!(
5021 retain_estate_funding_pause::<std::io::Error>(&paths, &mut journal, None)
5022 .expect("clear funded pause")
5023 );
5024 assert!(journal.estate_funding_required.is_none());
5025
5026 std::fs::remove_dir_all(root).expect("remove funding-pause fixture");
5027 }
5028
5029 #[test]
5030 fn estate_funding_is_rechecked_until_the_first_protocol_effect_starts() {
5031 let protocol_action = || EnsureAction::FleetProtocol {
5032 action: Box::new(
5033 crate::fleet_ensure::model::CurrentFleetProtocolAction::PrepareStoreChunkSet {
5034 request: canic_control_plane::dto::template::TemplateChunkSetPrepareInput {
5035 template_id: canic_control_plane::ids::TemplateId::from("root"),
5036 version: canic_control_plane::ids::TemplateVersion::from("current"),
5037 payload_hash: vec![1; 32],
5038 payload_size_bytes: 1,
5039 chunk_hashes: vec![vec![2; 32]],
5040 },
5041 },
5042 ),
5043 candid: "store.did".to_string(),
5044 candid_sha256: "11".repeat(32),
5045 maximum_execution_burn_cycles: 1,
5046 name: "prepare".to_string(),
5047 principal: "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(),
5048 };
5049 let first = protocol_action();
5050 let second = protocol_action();
5051 let actions = [&first, &second];
5052 let (_, mut journal) = retained_evidence();
5053 journal.effects = vec![EffectRecord {
5054 publication_attempts: 0,
5055 maintenance_attempts: 0,
5056 action_sha256: action_sha256(&first),
5057 created_principal: None,
5058 destination_post_cycles: None,
5059 destination_pre_cycles: None,
5060 post_cycles: None,
5061 pre_cycles: None,
5062 pre_canister_version: None,
5063 progress_identity: None,
5064 receipt: None,
5065 state: EffectState::Intent,
5066 }];
5067 assert!(!prior_fleet_protocol_effect_started(&actions, &journal, 1));
5068
5069 journal.effects[0].state = EffectState::Issued;
5070 assert!(prior_fleet_protocol_effect_started(&actions, &journal, 1));
5071 }
5072
5073 #[test]
5074 fn configured_projection_publishes_unique_bindings_without_losing_dynamic_cycles() {
5075 let asset = "rrkah-fqaaa-aaaaa-aaaaq-cai";
5076 let dynamic = "ryjl3-tyaaa-aaaaa-aaaba-cai";
5077 for kind in ["pool", "component"] {
5078 let desired = serde_json::from_value(serde_json::json!({
5079 "canisters": [{
5080 "controllers": [], "initial_cycles": "0", "kind": kind,
5081 "minimum_cycles": "0", "name": "configured", "presence": "present",
5082 "principal": asset, "replace": false, "subnet": "subnet"
5083 }],
5084 "cycles_ledger": "um5iw-rqaaa-aaaaq-qaaba-cai",
5085 "environment": "local", "fleet": "fleet", "ledger_fee_cycles": "0",
5086 "management_creation_fee_cycles": "0", "material_cycle_threshold": "1",
5087 "maximum_observation_burn_cycles": "1", "maximum_stalled_observations": 2,
5088 "maximum_update_burn_cycles": "2", "operator": asset,
5089 "schema_version": FLEET_ENSURE_SCHEMA_VERSION, "treasury": "configured"
5090 }))
5091 .expect("current desired projection");
5092 let (mut state, mut journal) = retained_evidence();
5093 journal.effects.clear();
5094 state.principals = BTreeMap::from([
5095 ("configured".to_string(), asset.to_string()),
5096 (format!("observed:{asset}"), asset.to_string()),
5097 (format!("observed:{dynamic}"), dynamic.to_string()),
5098 ]);
5099 state.retained_cycles_by_principal =
5100 BTreeMap::from([(asset.to_string(), 100), (dynamic.to_string(), 60)]);
5101 let cycles = state.retained_cycles_by_principal.clone();
5102 let journal_before = serde_json::to_vec(&journal).expect("journal before projection");
5103 let plan = estate_funding_plan();
5104
5105 publish_terminal_state(&desired, &plan, &journal, &mut state)
5106 .expect("publish current configured topology");
5107 assert_eq!(
5108 state.principals,
5109 BTreeMap::from([
5110 ("configured".to_string(), asset.to_string()),
5111 (format!("observed:{dynamic}"), dynamic.to_string()),
5112 ])
5113 );
5114 assert_eq!(state.retained_cycles_by_principal, cycles);
5115 let published = state.clone();
5116 publish_terminal_state(&desired, &plan, &journal, &mut state)
5117 .expect("repeat projection");
5118 assert_eq!(state, published);
5119 assert_eq!(
5120 serde_json::to_vec(&journal).expect("journal after projection"),
5121 journal_before
5122 );
5123 }
5124 }
5125
5126 #[test]
5127 fn configured_projection_preserves_duplicate_authority_rejection() {
5128 let asset = "rrkah-fqaaa-aaaaa-aaaaq-cai";
5129 for configured in [false, true] {
5130 let (mut state, _) = retained_evidence();
5131 state.principals = BTreeMap::from([
5132 ("first".to_string(), asset.to_string()),
5133 ("second".to_string(), asset.to_string()),
5134 ]);
5135 if configured {
5136 let topology = crate::fleet_ensure::model::FleetEnsureTopologyRecord {
5137 kind: crate::fleet_ensure::model::DesiredCanisterKind::Pool,
5138 module_hash: None,
5139 parent: None,
5140 protocol_binding: None,
5141 role: None,
5142 };
5143 state.topology = BTreeMap::from([
5144 ("first".to_string(), topology.clone()),
5145 ("second".to_string(), topology),
5146 ]);
5147 }
5148 let before = state.clone();
5149 retain_configured_principal_bindings(&mut state);
5150 assert_eq!(state, before);
5151 for entries in [Vec::new(), vec![terminal_component_entry(asset, "parent")]] {
5152 assert!(matches!(
5153 merge_terminal_inventory::<std::io::Error>(
5154 &mut state,
5155 crate::fleet_ensure::ops::TerminalFleetInventory {
5156 entries,
5157 ..Default::default()
5158 }
5159 ),
5160 Err(EnsureWorkflowError::TerminalInventory(
5161 TerminalInventoryError::DuplicateRetainedPrincipal
5162 ))
5163 ));
5164 assert_eq!(state, before);
5165 }
5166 }
5167 }
5168
5169 #[test]
5170 fn terminal_inventory_allows_only_verified_pool_to_component_parent_transition() {
5171 let asset = "rrkah-fqaaa-aaaaa-aaaaq-cai";
5172 let old_root = "r7inp-6aaaa-aaaaa-aaabq-cai";
5173 let hub = "ryjl3-tyaaa-aaaaa-aaaba-cai";
5174 let state = || {
5175 let (mut state, _) = retained_evidence();
5176 state.principals = BTreeMap::from([
5177 ("asset".to_string(), asset.to_string()),
5178 ("hub".to_string(), hub.to_string()),
5179 ("root".to_string(), old_root.to_string()),
5180 ]);
5181 state.topology.insert(
5182 "asset".to_string(),
5183 crate::fleet_ensure::model::FleetEnsureTopologyRecord {
5184 kind: crate::fleet_ensure::model::DesiredCanisterKind::Pool,
5185 module_hash: None,
5186 parent: Some("root".to_string()),
5187 protocol_binding: None,
5188 role: Some("canister_pool_asset".to_string()),
5189 },
5190 );
5191 state
5192 };
5193
5194 let mut promoted = state();
5195 merge_terminal_inventory::<std::io::Error>(
5196 &mut promoted,
5197 crate::fleet_ensure::ops::TerminalFleetInventory {
5198 active_registry: None,
5199 controlled_cycles_by_principal: BTreeMap::new(),
5200 entries: vec![terminal_component_entry(asset, hub)],
5201 },
5202 )
5203 .expect("verified pool workload assumes its terminal Component parent");
5204 let promoted = promoted.topology.get("asset").expect("promoted asset");
5205 assert_eq!(
5206 promoted.kind,
5207 crate::fleet_ensure::model::DesiredCanisterKind::Component
5208 );
5209 assert_eq!(promoted.parent.as_deref(), Some("hub"));
5210
5211 let mut already_component = state();
5212 already_component
5213 .topology
5214 .get_mut("asset")
5215 .expect("retained asset")
5216 .kind = crate::fleet_ensure::model::DesiredCanisterKind::Component;
5217 assert!(matches!(
5218 merge_terminal_inventory::<std::io::Error>(
5219 &mut already_component,
5220 crate::fleet_ensure::ops::TerminalFleetInventory {
5221 active_registry: None,
5222 controlled_cycles_by_principal: BTreeMap::new(),
5223 entries: vec![terminal_component_entry(asset, hub)],
5224 },
5225 ),
5226 Err(EnsureWorkflowError::TerminalInventory(
5227 TerminalInventoryError::RetainedParentConflict {
5228 canister,
5229 retained_parent,
5230 terminal_parent,
5231 }
5232 )) if canister == asset
5233 && retained_parent == "root"
5234 && terminal_parent == "hub"
5235 ));
5236
5237 for incomplete_entry in [
5238 crate::registry::RegistryEntry {
5239 module_hash: None,
5240 ..terminal_component_entry(asset, hub)
5241 },
5242 crate::registry::RegistryEntry {
5243 protocol_binding: None,
5244 ..terminal_component_entry(asset, hub)
5245 },
5246 ] {
5247 assert!(matches!(
5248 merge_terminal_inventory::<std::io::Error>(
5249 &mut state(),
5250 crate::fleet_ensure::ops::TerminalFleetInventory {
5251 active_registry: None,
5252 controlled_cycles_by_principal: BTreeMap::new(),
5253 entries: vec![incomplete_entry],
5254 },
5255 ),
5256 Err(EnsureWorkflowError::TerminalInventory(
5257 TerminalInventoryError::RetainedParentConflict {
5258 canister,
5259 retained_parent,
5260 terminal_parent,
5261 }
5262 )) if canister == asset
5263 && retained_parent == "root"
5264 && terminal_parent == "hub"
5265 ));
5266 }
5267 }
5268
5269 #[test]
5270 fn reinstall_recovery_review_requires_complete_exact_install_evidence() {
5271 use crate::fleet_ensure::model::{CanisterPlan, FleetReinstallRecord, InstallMode};
5272 let (state, mut journal) = retained_evidence();
5273 let mut plan = estate_funding_plan();
5274 plan.conservation.estate_funding_domains.clear();
5275 plan.reinstall = Some(Box::new(FleetReinstallRecord {
5276 target_artifacts_sha256: Some("11".repeat(32)),
5277 source: None,
5278 activation_reset: None,
5279 operation_id: plan.operation_id.clone(),
5280 source_operation_id: "source-operation".into(),
5281 authorities: Vec::new(),
5282 assets: Vec::new(),
5283 }));
5284 let action = EnsureAction::Install {
5285 canic_init: None,
5286 reinstall_witness: None,
5287 init_arg: None,
5288 init_arg_sha256: None,
5289 init_candid: None,
5290 init_candid_sha256: None,
5291 mode: InstallMode::Reinstall,
5292 name: "root".into(),
5293 principal: "rrkah-fqaaa-aaaaa-aaaaq-cai".into(),
5294 wasm: "root.wasm".into(),
5295 wasm_sha256: "22".repeat(32),
5296 };
5297 journal.effects[0].action_sha256 = action_sha256(&action);
5298 plan.canisters.push(CanisterPlan {
5299 actions: vec![action],
5300 disposition: CanisterDisposition::Reuse,
5301 name: "root".into(),
5302 observed_cycles: 0,
5303 principal: Some("rrkah-fqaaa-aaaaa-aaaaq-cai".into()),
5304 });
5305 let admit = |journal: &FleetEnsureJournalRecord| {
5306 reinstall_recovery_ready::<std::io::Error>(&plan, journal, &state)
5307 };
5308 assert!(admit(&journal).unwrap());
5309 let mut issued = journal.clone();
5310 issued.effects[0].state = EffectState::Issued;
5311 let mut missing = journal.clone();
5312 missing.effects.clear();
5313 let mut wrong_action = journal.clone();
5314 wrong_action.effects[0].action_sha256 = "other".into();
5315 let mut no_version = journal.clone();
5316 no_version.effects[0].pre_canister_version = None;
5317 let mut wrong_operation = journal.clone();
5318 wrong_operation.operation_id = "other".into();
5319 for invalid in [issued, missing, wrong_action, no_version, wrong_operation] {
5320 assert!(matches!(
5321 admit(&invalid),
5322 Err(EnsureWorkflowError::JournalIntegrity)
5323 ));
5324 }
5325 for completion in [
5326 FleetEnsureCompletion::InProgress,
5327 FleetEnsureCompletion::Converged,
5328 ] {
5329 let mut other = journal.clone();
5330 other.completion = completion;
5331 assert!(!admit(&other).unwrap());
5332 }
5333 }
5334
5335 #[test]
5336 fn completed_reinstall_evidence_requires_exact_operation_action_and_version() {
5337 let (state, journal) = retained_evidence();
5338 assert!(retained_reinstall_operation_matches(
5339 &state,
5340 "fleet",
5341 "operation",
5342 &journal,
5343 ));
5344
5345 let mut wrong_action = journal.clone();
5346 wrong_action.effects[0].action_sha256 = "other".to_string();
5347 assert!(!retained_reinstall_operation_matches(
5348 &state,
5349 "fleet",
5350 "operation",
5351 &wrong_action,
5352 ));
5353
5354 let mut wrong_version = journal.clone();
5355 wrong_version.effects[0].pre_canister_version = Some(8);
5356 assert!(!retained_reinstall_operation_matches(
5357 &state,
5358 "fleet",
5359 "operation",
5360 &wrong_version,
5361 ));
5362
5363 for (fleet, operation) in [("other", "operation"), ("fleet", "other")] {
5364 assert!(!retained_reinstall_operation_matches(
5365 &state, fleet, operation, &journal,
5366 ));
5367 }
5368
5369 let mut incomplete = journal;
5370 incomplete.effects[0].state = EffectState::Issued;
5371 assert!(!retained_reinstall_operation_matches(
5372 &state,
5373 "fleet",
5374 "operation",
5375 &incomplete,
5376 ));
5377 }
5378
5379 #[test]
5380 fn fresh_fleet_alternate_desired_cannot_clear_completed_reinstall_evidence() {
5381 let (state, journal) = retained_evidence();
5382 assert!(retained_reinstall_desired_conflict(
5383 &state,
5384 "fleet",
5385 "operation",
5386 "reviewed-desired",
5387 "alternate-desired",
5388 &journal,
5389 ));
5390 assert!(!retained_reinstall_desired_conflict(
5391 &state,
5392 "fleet",
5393 "operation",
5394 "reviewed-desired",
5395 "reviewed-desired",
5396 &journal,
5397 ));
5398
5399 let mut terminal = journal;
5400 terminal.completion = FleetEnsureCompletion::Converged;
5401 assert!(!retained_reinstall_desired_conflict(
5402 &state,
5403 "fleet",
5404 "operation",
5405 "reviewed-desired",
5406 "alternate-desired",
5407 &terminal,
5408 ));
5409 }
5410
5411 fn cycle_observation(
5412 lifecycle: crate::fleet_ensure::model::RootOwnedCanisterLifecycle,
5413 cycles: u128,
5414 ) -> FleetObservation {
5415 FleetObservation {
5416 additional_controlled_cycles: BTreeMap::new(),
5417 canisters: BTreeMap::from([(
5418 "asset".to_string(),
5419 Some(crate::fleet_ensure::model::LiveCanister {
5420 canister_version: Some(1),
5421 controllers: Vec::new(),
5422 cycles,
5423 module_sha256: None,
5424 principal: "controlled-principal".to_string(),
5425 reinstall_required: false,
5426 root_owned_lifecycle: Some(lifecycle),
5427 status: crate::fleet_ensure::model::CanisterRuntimeStatus::Stopped,
5428 }),
5429 )]),
5430 estate_funding_domains: BTreeMap::new(),
5431 ledger_fee_cycles: 0,
5432 operator_cycles: 0,
5433 protocol_ready: BTreeMap::new(),
5434 }
5435 }
5436
5437 #[test]
5438 fn fresh_fleet_idle_cycle_duplicate_requires_exact_balance() {
5439 use crate::fleet_ensure::model::RootOwnedCanisterLifecycle;
5440
5441 let additional = BTreeMap::from([("controlled-principal".to_string(), 100)]);
5442 let mut idle = cycle_observation(RootOwnedCanisterLifecycle::Idle, 100);
5443 attach_terminal_cycles::<std::io::Error>(&mut idle, additional.clone())
5444 .expect("merge exact Idle observation");
5445 assert!(idle.additional_controlled_cycles.is_empty());
5446
5447 let mut conflicting_idle = cycle_observation(RootOwnedCanisterLifecycle::Idle, 99);
5448 assert!(matches!(
5449 attach_terminal_cycles::<std::io::Error>(&mut conflicting_idle, additional.clone()),
5450 Err(EnsureWorkflowError::TerminalInventory(
5451 TerminalInventoryError::IdleCycleObservationConflict {
5452 canister,
5453 expected: 99,
5454 observed: 100,
5455 }
5456 )) if canister == "controlled-principal"
5457 ));
5458
5459 let mut workload = cycle_observation(RootOwnedCanisterLifecycle::Workload, 99);
5460 attach_terminal_cycles::<std::io::Error>(&mut workload, additional)
5461 .expect("merge conservative workload observation");
5462 assert_eq!(
5463 workload
5464 .canisters
5465 .get("asset")
5466 .and_then(Option::as_ref)
5467 .map(|live| live.cycles),
5468 Some(99),
5469 );
5470 }
5471}