Skip to main content

canic_host/fleet_ensure/workflow/
mod.rs

1//! Module: fleet_ensure::workflow
2//!
3//! Responsibility: plan and advance one durable idempotent Fleet convergence operation.
4//! Does not own: policy decisions, IC transport, or storage mechanics.
5//! Boundary: persists exact intent before each ops effect and reconciles it before any retry.
6
7pub mod attempt_recovery;
8#[cfg(test)]
9mod balance_observation;
10pub mod capacity_import;
11pub mod clean_reinstall;
12mod continuation;
13mod funding;
14pub mod funding_observation;
15#[cfg(test)]
16mod funding_tests;
17mod independent_effects;
18pub mod infrastructure_bootstrap;
19pub mod operator_mint;
20pub mod readiness;
21mod reinstall;
22pub mod release;
23mod retained_plan;
24mod root_reinstall;
25
26use crate::fleet_ensure::{
27    dto::{FleetEnsurePhase, FleetEnsureProgress, FleetEnsureProgressState},
28    inventory::project_current_fleet_inventory,
29    model::{
30        ActualCycleConservation, CanisterDisposition, CurrentFleetProtocolAction,
31        CycleConservation, EffectRecord, EffectState, EnsureAction, EstateFundingDomainPlan,
32        EstateFundingRequiredRecord, FLEET_ENSURE_SCHEMA_VERSION, FleetEnsureCompletion,
33        FleetEnsureJournalRecord, FleetEnsurePlan, FleetEnsurePlanScope, FleetEnsureReport,
34        FleetEnsureStateRecord, FleetObservation, RootManagementObservation,
35        create_balance_is_terminal,
36    },
37    ops::{
38        EffectRetry, EnsurePaths, EnsurePlatform, EnsureStateError, action_sha256,
39        continuation::verify_release_transition, effect_preparation::prepare_effect,
40        lock_operation, read_root_start_authority, read_state, reserve_fixture_publication_attempt,
41        resolve_desired_artifacts, retain_configured_principal_bindings, write_journal, write_plan,
42        write_state,
43    },
44    policy::{
45        EnsurePolicyError, RootStartPlanInput, compile_plan, compile_root_start_prerequisite_plan,
46        effect_observation_policy, expected_plan_sha256, operation_id,
47        recompile_root_start_prerequisite_plan, validate_creation_fee_scope,
48        validate_path_identity, validate_path_labels,
49    },
50};
51use canic_contracts::cycles::Cycles;
52use std::{
53    collections::{BTreeMap, BTreeSet},
54    path::Path,
55    time::Instant,
56};
57use thiserror::Error as ThisError;
58
59pub use reinstall::plan_reinstall;
60
61/// Exact no-effect evidence that a fresh pool cannot retain its readiness floor.
62
63#[derive(Debug, ThisError)]
64#[error(
65    "fresh pool canister {canister} requested {requested_creation_funding_cycles} creation cycles for readiness floor {readiness_floor_cycles}; its first live balance is {live_balance_cycles} cycles (readiness shortfall {readiness_shortfall_cycles}) and must retain {remaining_controller_burn_cycles} cycles for controller finalization (required pre-finalization balance {required_pre_finalization_balance_cycles}, pre-finalization shortfall {pre_finalization_shortfall_cycles}; maximum first-observation burn {maximum_observation_burn_cycles}); no controller or protocol action followed"
66)]
67pub struct FreshPoolCreationUnderfundedError {
68    pub canister: String,
69    pub live_balance_cycles: u128,
70    pub maximum_observation_burn_cycles: u128,
71    pub pre_finalization_shortfall_cycles: u128,
72    pub readiness_floor_cycles: u128,
73    pub readiness_shortfall_cycles: u128,
74    pub remaining_controller_burn_cycles: u128,
75    pub requested_creation_funding_cycles: u128,
76    pub required_pre_finalization_balance_cycles: u128,
77}
78
79/// Exact reviewed Root-account funding prerequisite that blocks protocol mutation.
80#[derive(Debug, ThisError)]
81#[error(
82    "Root {root} Cycles Ledger account {root_principal} on {cycles_ledger} has {available_cycles} cycles; {required_creation_count} autonomous creations require at most {maximum_creation_debit_cycles} cycles ({creation_amount_cycles} gross amount containing {readiness_floor_cycles} readiness floor, {management_creation_fee_cycles} management creation fee and {creation_execution_margin_cycles} execution margin, plus {ledger_fee_cycles} Ledger fee each; total reviewed fees {maximum_creation_fee_cycles}), shortfall {shortfall_cycles}; pending_creation={pending_creation_operation_id:?} attempts={attempt_count:?} last_attempt_at_ns={last_attempt_at_ns:?} retry_at_ns={retry_at_ns:?}; preserve this operation, repeat Fleet ensure without --apply and review funding_review, then apply its review_sha256 for the exact shortfall; do not fund the account out of band"
83)]
84pub struct EstateFundingRequiredError {
85    pub attempt_count: Option<u32>,
86    pub available_cycles: u128,
87    pub creation_amount_cycles: u128,
88    pub cycles_ledger: String,
89    pub creation_execution_margin_cycles: u128,
90    pub readiness_floor_cycles: u128,
91    pub ledger_fee_cycles: u128,
92    pub management_creation_fee_cycles: u128,
93    pub maximum_creation_debit_cycles: u128,
94    pub maximum_creation_fee_cycles: u128,
95    pub pending_creation_operation_id: Option<String>,
96    pub required_creation_count: u32,
97    pub root: String,
98    pub root_principal: String,
99    pub last_attempt_at_ns: Option<u64>,
100    pub retry_at_ns: Option<u64>,
101    pub shortfall_cycles: u128,
102}
103
104/// Exact terminal-inventory merge or cycle-observation invariant failure.
105#[derive(Debug, ThisError)]
106pub enum TerminalInventoryError {
107    #[error("one Principal is retained under more than one desired role")]
108    DuplicateRetainedPrincipal,
109
110    #[error("verified canister {canister} lost its Principal binding")]
111    LostPrincipalBinding { canister: String },
112
113    #[error("canister {canister} names unknown parent {parent}")]
114    UnknownParent { canister: String, parent: String },
115
116    #[error(
117        "canister {canister} retained parent {retained_parent} differs from terminal parent {terminal_parent}"
118    )]
119    RetainedParentConflict {
120        canister: String,
121        retained_parent: String,
122        terminal_parent: String,
123    },
124
125    #[error(
126        "terminal Root-owned Idle canister {canister} cycle observations differ: expected {expected}, observed {observed}"
127    )]
128    IdleCycleObservationConflict {
129        canister: String,
130        expected: u128,
131        observed: u128,
132    },
133
134    #[error(
135        "terminal cycle observation duplicates canister {canister} outside its exact Root-owned Idle or Workload lifecycle"
136    )]
137    CycleObservationLifecycleConflict { canister: String },
138}
139
140/// Fleet ensure planning or convergence failure.
141
142#[derive(Debug, ThisError)]
143pub enum EnsureWorkflowError<E>
144where
145    E: std::error::Error + 'static,
146{
147    #[error(transparent)]
148    InfrastructureBootstrap(
149        #[from] crate::fleet_ensure::ops::infrastructure_bootstrap::InfrastructureBootstrapError,
150    ),
151    #[error(transparent)]
152    RetainedContract(#[from] crate::fleet_ensure::ops::retained_contract::RetainedContractError),
153
154    #[error(transparent)]
155    FundingObservation(
156        #[from] crate::fleet_ensure::model::funding_observation::FundingObservationError,
157    ),
158
159    #[error(
160        "operator conversion must be resolved or its unapproved review cancelled before Fleet effects continue: {review_sha256}; inspect with fleet ensure --operator-mint"
161    )]
162    OperatorMintPending { review_sha256: String },
163
164    #[error(
165        "operator conversion required for retained funding review {review_sha256}; preserve the original withdrawal intent"
166    )]
167    OperatorFundingRequired { review_sha256: String },
168
169    #[error("operator conversion review, approval digest or current authority differs")]
170    OperatorMintReviewConflict,
171
172    #[error("operator conversion reply conflicts with a retained transaction outcome")]
173    OperatorMintReplyConflict,
174
175    #[error("operator conversion wire data is invalid: {0}")]
176    OperatorMintWire(#[from] crate::fleet_ensure::ops::operator_mint::OperatorMintWireError),
177
178    #[error(
179        "retained Fleet operation cannot be resumed: {source}; local source evidence identifies operation {operation_id}, journal plan reference {plan_sha256}, source document {source_document_sha256}; preserve plan, journal, state and artifacts; request an explicit --reinstall review without --apply using the selected corrected release; live authority, complete estate and cycle checks must pass before any reset; see docs/features/operations/fleet-ensure.md#unreadable-retained-plan"
180    )]
181    RetainedActivationReviewRequired {
182        operation_id: String,
183        plan_sha256: String,
184        source_document_sha256: String,
185        #[source]
186        source: Box<EnsureStateError>,
187    },
188
189    #[error(
190        "retained Fleet operation is unreadable: {source}; preserve plan, journal, state, artifacts and paid-effect receipts; local evidence does not establish a supported separate recovery review; resolve issued effects under their exact authority before a release transition; do not edit fields, replace digests or discard the operation; see docs/features/operations/fleet-ensure.md#unreadable-retained-plan"
191    )]
192    RetainedPlanUnreadable {
193        #[source]
194        source: Box<EnsureStateError>,
195    },
196
197    #[error(
198        "activation {operation_id} has an Issued provisioning action after its Applied host prefix (source document {source_document_sha256}); preserve the plan and journal: required live evidence for the bounded partial-activation recovery is unavailable"
199    )]
200    PartialActivationResetUnavailable {
201        operation_id: String,
202        source_document_sha256: String,
203    },
204
205    #[error(
206        "reinstall requires a converged Fleet with retained physical authority or an admitted single-Root partial-activation recovery source; the supplied source or target does not satisfy either contract; preserve the retained plan and journal"
207    )]
208    ReinstallConflict,
209
210    #[error(
211        "retained Fleet operation {operation_id} must recover before a new reinstall; preserve original plan {plan_sha256}, journal and receipts; run ensure without --reinstall to review retained funding, use --operator-mint if conversion is required, and resume the original reviewed operation before requesting a selected-release reinstall"
212    )]
213    RetainedOperationRecoveryRequired {
214        operation_id: String,
215        plan_sha256: String,
216    },
217
218    #[error("reviewed Fleet plan digest changed before its first effect")]
219    DriftedBeforeApply,
220
221    #[error(
222        "replan-required Fleet operation owns completed reinstall evidence for desired input {expected}; refusing alternate desired input {actual}"
223    )]
224    RetainedReinstallDesiredConflict { actual: String, expected: String },
225
226    #[error(
227        "Root prerequisite retains activation authority for desired input {expected}; refreshed input {actual} is refused before Store effects; preserve the retained plan and journal for recovery review"
228    )]
229    RootActivationDesiredConflict { actual: String, expected: String },
230
231    #[error("reviewed Fleet plan {expected} does not match retained plan {actual}")]
232    PlanDigestMismatch { actual: String, expected: String },
233
234    #[error("reviewed Fleet plan is missing; run `canic fleet ensure <fleet>` first")]
235    PlanMissing,
236
237    #[error("retained Fleet plan failed its content digest")]
238    PlanIntegrity,
239
240    #[error("retained Fleet journal does not match the reviewed Fleet operation")]
241    JournalIntegrity,
242
243    #[error(
244        "Fleet ensure action {action} made no progress for {observations} consecutive observations (last progress: {progress_identity}); operation remains resumable"
245    )]
246    Stalled {
247        action: String,
248        observations: u32,
249        progress_identity: String,
250    },
251
252    #[error("Root {root} provisioning has made no progress for {observations} observations: stage={stage:?}, target={target}, operation={operation}, diagnostic={diagnostic_code}, retry=Backoff, failed_at_ns={failed_at_ns}; the issued operation remains retained; resume the same reviewed apply after the dependency recovers", operation = canic_core::cdk::utils::hash::hex_bytes(.operation_id))]
253    ProvisioningRetryPending {
254        root: candid::Principal,
255        stage: canic_contracts::dto::component_provisioning::ProvisioningFailureStage,
256        target: candid::Principal,
257        operation_id: [u8; 32],
258        diagnostic_code: u16,
259        observations: u32,
260        failed_at_ns: u64,
261    },
262
263    #[error(
264        "Root-owned canister {target} remained {last_lifecycle} for {observations} consecutive passive observations; no command was reissued and the operation remains resumable"
265    )]
266    RootOwnedObservationStalled {
267        target: String,
268        observations: u32,
269        last_lifecycle: String,
270    },
271
272    #[error(
273        "selected Cycles Ledger account has {actual} cycles, below reviewed maximum debit {required}; shortfall {} cycles; no new execution journal or funding intent was retained; fund the selected operator account before retrying this reviewed plan; repeat fleet ensure with --operator-mint and without --apply for a read-only ICP quote",
274        required.saturating_sub(*actual)
275    )]
276    InsufficientOperatorCycles { actual: u128, required: u128 },
277
278    #[error("terminal Fleet cycle conservation failed: {0}")]
279    Conservation(String),
280
281    #[error(
282        "completed Fleet operation {operation_id} cannot replay accounting for plan {plan_sha256}: current operator balance {current_operator_cycles} is outside the reviewed range {minimum_operator_cycles}..={operator_source_cycles}; subsequent account activity cannot be attributed to this Fleet operation; preserve the completed plan, journal and receipts, run `canic fleet ensure {fleet}` without --apply to review a fresh plan, then apply only its newly reviewed digest and debit; do not repeat the completed deployment or edit its accounting"
283    )]
284    TerminalReplayBalanceChanged {
285        fleet: String,
286        operation_id: String,
287        plan_sha256: String,
288        current_operator_cycles: u128,
289        minimum_operator_cycles: u128,
290        operator_source_cycles: u128,
291    },
292
293    #[error("terminal Fleet inventory failed exact current-authority validation: {0}")]
294    TerminalInventory(#[source] TerminalInventoryError),
295
296    #[error(
297        "reviewed effects completed, but live Fleet state still differs from desired state; review a new plan"
298    )]
299    ConvergenceDrift,
300
301    #[error(
302        "fixture publication action {action} exhausted its {maximum_attempts} reviewed paid attempts; preserve the operation and reconcile its exact source status; no additional update was issued"
303    )]
304    FixturePublicationBound {
305        action: String,
306        maximum_attempts: u32,
307    },
308
309    #[error(
310        "Root pool maintenance {action} exhausted its reviewed limit of {maximum_updates} updates; retained receipts remain authoritative"
311    )]
312    PoolMaintenanceBound {
313        action: String,
314        maximum_updates: u32,
315    },
316
317    #[error(
318        "created canister {canister} retained one exact Ledger creation receipt, but its first live balance is {actual_cycles} cycles instead of requested target {requested_cycles} (deficit {deficit_cycles}; configured management creation fee {configured_fee_cycles}); no controller or protocol action followed, and the immutable operation is now replan-required"
319    )]
320    ReplanRequiredAfterCreateBalanceDrift {
321        actual_cycles: u128,
322        canister: String,
323        configured_fee_cycles: u128,
324        deficit_cycles: u128,
325        requested_cycles: u128,
326    },
327
328    #[error("Fleet successor requires a new review: {reason:?}; {}", .review.as_deref().map(ToString::to_string).unwrap_or_default())]
329    SuccessorReviewRequired {
330        reason: crate::fleet_ensure::model::FleetEnsureSuccessorReviewReason,
331        review: Option<Box<crate::fleet_ensure::model::FleetSuccessorReview>>,
332    },
333
334    #[error(transparent)]
335    FreshPoolCreationUnderfunded(Box<FreshPoolCreationUnderfundedError>),
336
337    #[error(transparent)]
338    EstateFundingRequired(Box<EstateFundingRequiredError>),
339
340    #[error("native Root funding needs explicit approval of review {review_sha256}")]
341    NativeFundingRequired { review_sha256: String },
342
343    #[error(transparent)]
344    Policy(#[from] EnsurePolicyError),
345
346    #[error(transparent)]
347    Inventory(#[from] crate::fleet_ensure::CurrentFleetInventoryError),
348
349    #[error("Fleet platform operation failed: {0}")]
350    Platform(#[source] E),
351
352    #[error(transparent)]
353    State(#[from] EnsureStateError),
354}
355
356/// Recover the selected target for an exact reinstall apply, including terminal replay.
357pub fn retained_reinstall_apply_plan<E: std::error::Error + 'static>(
358    root: &Path,
359    environment: &str,
360    requested_fleet: &str,
361    reviewed_plan_sha256: &str,
362) -> Result<Option<FleetEnsurePlan>, EnsureWorkflowError<E>> {
363    validate_path_labels(environment, requested_fleet)?;
364    let paths = EnsurePaths::under(root, environment, requested_fleet);
365    let _lock = lock_operation(&paths)?;
366    // The staged review owns apply; its source remains receipt-only evidence.
367    let plan = if let Some(review) = crate::fleet_ensure::ops::reinstall::adoption::review(&paths)?
368    {
369        Some(review)
370    } else {
371        if crate::fleet_ensure::ops::operation_selection::completed(
372            &paths,
373            environment,
374            requested_fleet,
375        )?
376        .is_some_and(|completed| completed.plan_sha256 != reviewed_plan_sha256)
377        {
378            return Ok(None);
379        }
380        retained_plan::read(&paths, environment, requested_fleet)?
381    };
382    let Some(plan) = plan else {
383        return Ok(None);
384    };
385    let plan = verified_plan(plan)?;
386    if plan.environment != environment || plan.fleet != requested_fleet {
387        return Err(EnsureWorkflowError::PlanIntegrity);
388    }
389    if plan.reinstall.is_some() && plan.plan_sha256 == reviewed_plan_sha256 {
390        Ok(Some(plan))
391    } else {
392        Ok(None)
393    }
394}
395
396/// Load and verify the sole immutable in-progress operation for one exact Fleet path.
397pub fn retained_in_progress_plan<E>(
398    root: &Path,
399    environment: &str,
400    requested_fleet: &str,
401) -> Result<Option<FleetEnsurePlan>, EnsureWorkflowError<E>>
402where
403    E: std::error::Error + 'static,
404{
405    validate_path_labels(environment, requested_fleet)?;
406    let paths = EnsurePaths::under(root, environment, requested_fleet);
407    let _lock = lock_operation(&paths)?;
408    if let Some(review) = crate::fleet_ensure::ops::reinstall::adoption::review(&paths)? {
409        if review.environment != environment || review.fleet != requested_fleet {
410            return Err(EnsureWorkflowError::PlanIntegrity);
411        }
412        return Ok(Some(review));
413    }
414    if crate::fleet_ensure::ops::operation_selection::completed(
415        &paths,
416        environment,
417        requested_fleet,
418    )?
419    .is_some()
420    {
421        return Ok(None);
422    }
423    let Some(journal) = retained_plan::journal(&paths, environment, requested_fleet)? else {
424        return Ok(None);
425    };
426    if !matches!(
427        journal.completion,
428        FleetEnsureCompletion::InProgress | FleetEnsureCompletion::Prepared
429    ) {
430        return Ok(None);
431    }
432    let retained = verified_plan(
433        retained_plan::read(&paths, environment, requested_fleet)?
434            .ok_or(EnsureWorkflowError::PlanMissing)?,
435    )?;
436    if retained.environment != environment || retained.fleet != requested_fleet {
437        return Err(EnsureWorkflowError::PlanIntegrity);
438    }
439    let state = read_state(&paths, requested_fleet)?;
440    let prepared_full = journal.completion == FleetEnsureCompletion::Prepared
441        && (retained.scope == FleetEnsurePlanScope::Full
442            || (retained.scope == FleetEnsurePlanScope::RootReinstallPrerequisite
443                && retained.reinstall.as_ref().is_some_and(|intent| {
444                    intent
445                        .activation_reset
446                        .as_ref()
447                        .is_some_and(|activation| activation.preparation.is_some())
448                })))
449        && retained.reinstall.as_ref().is_some_and(|intent| {
450            intent.operation_id == journal.operation_id
451                && intent.operation_id == retained.operation_id
452        });
453    if !prepared_full {
454        verify_journal_integrity(&journal, &retained, requested_fleet, &state)?;
455    }
456    Ok(Some(retained))
457}
458
459/// Build and retain one read-only plan from current desired state plus live observation.
460pub fn plan<P>(
461    root: &Path,
462    desired: &crate::fleet_ensure::model::DesiredFleet,
463    desired_sha256: &str,
464    requested_fleet: &str,
465    created_at_time: u64,
466    platform: &mut P,
467) -> Result<FleetEnsureReport, EnsureWorkflowError<P::Error>>
468where
469    P: EnsurePlatform,
470{
471    platform.with_planning_observations(|platform| {
472        plan_with_observations(
473            root,
474            desired,
475            desired_sha256,
476            requested_fleet,
477            created_at_time,
478            platform,
479        )
480    })
481}
482
483#[expect(
484    clippy::too_many_lines,
485    reason = "one locked read-only transaction reconciles retained authority, observation and the immutable plan"
486)]
487fn plan_with_observations<P>(
488    root: &Path,
489    desired: &crate::fleet_ensure::model::DesiredFleet,
490    desired_sha256: &str,
491    requested_fleet: &str,
492    created_at_time: u64,
493    platform: &mut P,
494) -> Result<FleetEnsureReport, EnsureWorkflowError<P::Error>>
495where
496    P: EnsurePlatform,
497{
498    validate_path_identity(desired, requested_fleet)?;
499    let paths = EnsurePaths::under(root, &desired.environment, requested_fleet);
500    let _lock = lock_operation(&paths)?;
501    if let Some(review) = crate::fleet_ensure::ops::reinstall::adoption::review(&paths)? {
502        if review.desired_sha256 != desired_sha256 {
503            return Err(EnsureWorkflowError::DriftedBeforeApply);
504        }
505        verify_release_transition(root, desired, review.scope)?;
506        return Ok(FleetEnsureReport {
507            funding_review: None,
508            actual_conservation: None,
509            effects_applied: 0,
510            plan: review,
511            terminal: false,
512        });
513    }
514    if let Some(mut journal) =
515        retained_plan::journal(&paths, &desired.environment, requested_fleet)?
516        && journal.completion == FleetEnsureCompletion::InProgress
517    {
518        let retained = verified_plan(
519            retained_plan::read(&paths, &desired.environment, requested_fleet)?
520                .ok_or(EnsureWorkflowError::PlanMissing)?,
521        )?;
522        let state = read_state(&paths, requested_fleet)?;
523        verify_journal(&journal, &retained, requested_fleet, &state)?;
524        let operation_desired = retained
525            .reviewed_desired
526            .as_deref()
527            .ok_or(EnsureWorkflowError::PlanIntegrity)?
528            .desired();
529        verify_release_transition(root, operation_desired, retained.scope)?;
530        if journal.estate_funding_required.is_some()
531            || funding::native_review_applicable(&retained, &journal)
532        {
533            platform
534                .bind_reviewed_desired(operation_desired)
535                .map_err(EnsureWorkflowError::Platform)?;
536        }
537        let funding_review = funding::prepare(
538            &paths,
539            &retained,
540            &mut journal,
541            &state,
542            created_at_time,
543            platform,
544        )?;
545        return Ok(FleetEnsureReport {
546            funding_review,
547            actual_conservation: None,
548            effects_applied: applied_count(&journal),
549            plan: retained,
550            terminal: false,
551        });
552    }
553    verify_release_transition(root, desired, FleetEnsurePlanScope::RootStartPrerequisite)?;
554    let mut state = read_state(&paths, requested_fleet)?;
555    let prior_plan = retained_plan::read(&paths, &desired.environment, requested_fleet)?
556        .map(verified_plan)
557        .transpose()?;
558    let prior_journal = retained_plan::journal(&paths, &desired.environment, requested_fleet)?;
559    if let Some(prior) = prior_plan
560        .as_ref()
561        .filter(|prior| prior.scope == FleetEnsurePlanScope::InfrastructureBootstrap)
562    {
563        crate::fleet_ensure::ops::infrastructure_bootstrap::convergence::prepare(
564            &paths,
565            prior,
566            prior_journal
567                .as_ref()
568                .ok_or(EnsureWorkflowError::JournalIntegrity)?,
569            &state,
570            desired,
571        )?;
572    }
573    if let Some(prior) = &prior_plan {
574        let recovery_ready = prior_journal
575            .as_ref()
576            .map(|journal| reinstall_recovery_ready(prior, journal, &state))
577            .transpose()?
578            .unwrap_or(false);
579        if recovery_ready && let Some(journal) = &prior_journal {
580            if prior.desired_sha256 != desired_sha256 {
581                return Err(EnsureWorkflowError::ReinstallConflict);
582            }
583            retain_completed_reinstalls(&mut state, prior, journal);
584        }
585        let activation_reset = prior
586            .reinstall
587            .as_ref()
588            .is_some_and(|intent| intent.activation_reset.is_some());
589        if activation_reset
590            && prior.scope == FleetEnsurePlanScope::RootReinstallPrerequisite
591            && let Some(journal) = prior_journal
592                .as_ref()
593                .filter(|journal| journal.completion == FleetEnsureCompletion::Converged)
594        {
595            verify_journal(journal, prior, requested_fleet, &state)?;
596            retain_completed_reinstalls(&mut state, prior, journal);
597            write_state(&paths, &state)?;
598        }
599        let unfinished_reset = prior.reinstall.is_some()
600            && !recovery_ready
601            && !prior_journal.as_ref().is_some_and(|journal| {
602                journal.plan_sha256 == prior.plan_sha256
603                    && journal.completion == FleetEnsureCompletion::Converged
604            });
605        if prior.scope == FleetEnsurePlanScope::ReinstallPreparation
606            || unfinished_reset
607            || (activation_reset && prior.scope == FleetEnsurePlanScope::RootReinstallPrerequisite)
608        {
609            if prior.desired_sha256 != desired_sha256 {
610                return Err(EnsureWorkflowError::ReinstallConflict);
611            }
612            return reinstall::continue_plan(
613                root,
614                &paths,
615                desired,
616                prior,
617                prior_journal.as_ref(),
618                &state,
619                platform,
620            );
621        }
622    }
623    let matching_prior = reconcile_retained_planning_state(
624        &mut state,
625        prior_plan.as_ref(),
626        prior_journal.as_ref(),
627        requested_fleet,
628        &desired.environment,
629        desired_sha256,
630    )?;
631    if let Some(prior) = matching_prior {
632        retain_plan_cycles(&mut state, prior);
633        if let Some(journal) = &prior_journal {
634            retain_completed_reinstalls(&mut state, prior, journal);
635        }
636    }
637    // Store and Component Registry preparation bind the setup operation identity.
638    // Its verified import boundary continues that identity into workload provisioning.
639    let operation_id = prior_plan
640        .as_ref()
641        .filter(|prior| prior.scope == FleetEnsurePlanScope::InfrastructureBootstrap)
642        .or_else(|| matching_prior.filter(|prior| prior.scope == FleetEnsurePlanScope::Full))
643        .map_or_else(
644            || operation_id(desired_sha256, &desired.environment, requested_fleet),
645            |prior| prior.operation_id.clone(),
646        );
647    if let Some(management) = platform
648        .observe_root_management(&state, &BTreeSet::new())
649        .map_err(EnsureWorkflowError::Platform)?
650    {
651        let root_start_authority = read_root_start_authority(&paths)?;
652        if let Some(plan) = compile_root_start_prerequisite_plan(RootStartPlanInput {
653            state: &state,
654            authority: root_start_authority.as_ref(),
655            created_at_time,
656            desired,
657            desired_sha256,
658            observation: &management,
659            requested_fleet,
660        })? {
661            write_plan(&paths, &plan)?;
662            return Ok(FleetEnsureReport {
663                funding_review: None,
664                actual_conservation: None,
665                effects_applied: 0,
666                plan,
667                terminal: false,
668            });
669        }
670        verify_release_transition(root, desired, FleetEnsurePlanScope::Full)?;
671        if let Some(plan) = crate::fleet_ensure::policy::root_reinstall::compile(
672            RootStartPlanInput {
673                state: &state,
674                authority: None,
675                created_at_time,
676                desired,
677                desired_sha256,
678                observation: &management,
679                requested_fleet,
680            },
681            &resolve_desired_artifacts(root, desired)?,
682        )? {
683            write_plan(&paths, &plan)?;
684            return Ok(FleetEnsureReport {
685                funding_review: None,
686                actual_conservation: None,
687                effects_applied: 0,
688                plan,
689                terminal: false,
690            });
691        }
692    }
693    verify_release_transition(root, desired, FleetEnsurePlanScope::Full)?;
694    let artifacts = resolve_desired_artifacts(root, desired)?;
695    let mut observation = platform
696        .observe(&operation_id, &state)
697        .map_err(EnsureWorkflowError::Platform)?;
698    retain_observed_cycles(&mut state, &observation);
699    // The prior Converged journal can remain visible until apply. Planning may refresh only
700    // backup-inert observation and prior-operation evidence, never successor topology authority.
701    write_state(&paths, &state)?;
702    let terminal_inventory_operation_id = current_terminal_inventory_operation(
703        &state,
704        prior_plan.as_ref(),
705        prior_journal.as_ref(),
706        requested_fleet,
707    )?;
708    if let Some(operation_id) = terminal_inventory_operation_id.as_deref() {
709        attach_terminal_inventory_cycles(operation_id, &state, platform, &mut observation)?;
710    }
711    let mut protocol_actions = platform
712        .protocol_actions(
713            terminal_inventory_operation_id
714                .as_deref()
715                .unwrap_or(&operation_id),
716            &state,
717        )
718        .map_err(EnsureWorkflowError::Platform)?;
719    let mut retained_observations = 0_u32;
720    let mut plan = loop {
721        match compile_plan(
722            desired,
723            &artifacts,
724            &protocol_actions,
725            desired_sha256,
726            requested_fleet,
727            &observation,
728            created_at_time,
729            &operation_id,
730            None,
731        ) {
732            Ok(plan) => break plan,
733            Err(EnsurePolicyError::PendingRootOwnedBalance { name }) => {
734                retained_observations = retained_observations.saturating_add(1);
735                if retained_observations >= desired.maximum_stalled_observations {
736                    return Err(EnsureWorkflowError::RootOwnedObservationStalled {
737                        last_lifecycle: root_owned_lifecycle_label(&observation, &name).to_string(),
738                        observations: retained_observations,
739                        target: name,
740                    });
741                }
742                platform.pace_root_owned_observation(&name, retained_observations);
743                observation = platform
744                    .observe(&operation_id, &state)
745                    .map_err(EnsureWorkflowError::Platform)?;
746                retain_observed_cycles(&mut state, &observation);
747                write_state(&paths, &state)?;
748                if let Some(operation_id) = terminal_inventory_operation_id.as_deref() {
749                    attach_terminal_inventory_cycles(
750                        operation_id,
751                        &state,
752                        platform,
753                        &mut observation,
754                    )?;
755                }
756                protocol_actions = platform
757                    .protocol_actions(
758                        terminal_inventory_operation_id
759                            .as_deref()
760                            .unwrap_or(&operation_id),
761                        &state,
762                    )
763                    .map_err(EnsureWorkflowError::Platform)?;
764            }
765            Err(error) => return Err(error.into()),
766        }
767    };
768    for action in ordered_actions(&plan) {
769        verify_store_activation_binding(&plan, action, &state, platform)?;
770    }
771    bind_terminal_inventory_operation(&mut plan, terminal_inventory_operation_id);
772    write_plan(&paths, &plan)?;
773    Ok(FleetEnsureReport {
774        funding_review: None,
775        actual_conservation: None,
776        effects_applied: 0,
777        plan,
778        terminal: false,
779    })
780}
781
782fn current_terminal_inventory_operation<E>(
783    state: &FleetEnsureStateRecord,
784    prior_plan: Option<&FleetEnsurePlan>,
785    prior_journal: Option<&FleetEnsureJournalRecord>,
786    requested_fleet: &str,
787) -> Result<Option<String>, EnsureWorkflowError<E>>
788where
789    E: std::error::Error + 'static,
790{
791    if state.active_registry.is_none() {
792        return Ok(None);
793    }
794    let prior_plan = prior_plan.ok_or(EnsureWorkflowError::PlanMissing)?;
795    let prior_journal = prior_journal.ok_or(EnsureWorkflowError::JournalIntegrity)?;
796    verify_journal(prior_journal, prior_plan, requested_fleet, state)?;
797    // The separately receipted setup/import boundary has no workload inventory yet.
798    // Planning admitted and archived its exact completion before refreshing observations.
799    if prior_plan.scope == FleetEnsurePlanScope::InfrastructureBootstrap
800        && prior_journal.completion == FleetEnsureCompletion::Converged
801    {
802        return Ok(None);
803    }
804    if prior_journal.completion != FleetEnsureCompletion::Converged
805        || prior_plan.scope != FleetEnsurePlanScope::Full
806    {
807        return Err(EnsureWorkflowError::JournalIntegrity);
808    }
809    let operation_id = prior_plan
810        .terminal_inventory_operation_id
811        .as_ref()
812        .unwrap_or(&prior_plan.operation_id);
813    if operation_id.is_empty() {
814        return Err(EnsureWorkflowError::JournalIntegrity);
815    }
816    Ok(Some(operation_id.clone()))
817}
818
819fn bind_terminal_inventory_operation(plan: &mut FleetEnsurePlan, operation_id: Option<String>) {
820    plan.terminal_inventory_operation_id = operation_id;
821    plan.plan_sha256 = expected_plan_sha256(plan);
822}
823
824fn reconcile_retained_planning_state<'a, E>(
825    state: &mut FleetEnsureStateRecord,
826    prior_plan: Option<&'a FleetEnsurePlan>,
827    prior_journal: Option<&FleetEnsureJournalRecord>,
828    requested_fleet: &str,
829    desired_environment: &str,
830    desired_sha256: &str,
831) -> Result<Option<&'a FleetEnsurePlan>, EnsureWorkflowError<E>>
832where
833    E: std::error::Error + 'static,
834{
835    if let Some((prior, journal)) = prior_plan.zip(prior_journal)
836        && prior.fleet == requested_fleet
837        && prior.environment == desired_environment
838        && retained_reinstall_desired_conflict(
839            state,
840            &prior.fleet,
841            &prior.operation_id,
842            &prior.desired_sha256,
843            desired_sha256,
844            journal,
845        )
846    {
847        return Err(EnsureWorkflowError::RetainedReinstallDesiredConflict {
848            actual: desired_sha256.to_string(),
849            expected: prior.desired_sha256.clone(),
850        });
851    }
852    if let Some((prior, journal)) = prior_plan.zip(prior_journal)
853        && prior.scope == FleetEnsurePlanScope::RootReinstallPrerequisite
854        && prior.desired_sha256 != desired_sha256
855    {
856        verify_journal(journal, prior, requested_fleet, state)?;
857        return Err(EnsureWorkflowError::RootActivationDesiredConflict {
858            actual: desired_sha256.to_string(),
859            expected: prior.desired_sha256.clone(),
860        });
861    }
862    let matching_prior = prior_plan.filter(|prior| {
863        prior.fleet == requested_fleet
864            && prior.environment == desired_environment
865            && prior.desired_sha256 == desired_sha256
866    });
867    let retained_operation_matches =
868        matching_prior
869            .zip(prior_journal)
870            .is_some_and(|(prior, journal)| {
871                retained_reinstall_operation_matches(
872                    state,
873                    &prior.fleet,
874                    &prior.operation_id,
875                    journal,
876                )
877            });
878    if !retained_operation_matches {
879        state.completed_reinstall_action_sha256.clear();
880        state.completed_reinstall_operation_id = None;
881        state.completed_reinstalls.clear();
882    }
883    Ok(matching_prior)
884}
885
886fn retain_completed_reinstalls(
887    state: &mut FleetEnsureStateRecord,
888    plan: &FleetEnsurePlan,
889    journal: &FleetEnsureJournalRecord,
890) {
891    if journal.fleet != plan.fleet || journal.operation_id != plan.operation_id {
892        return;
893    }
894    for action in ordered_actions(plan) {
895        let EnsureAction::Install {
896            mode: crate::fleet_ensure::model::InstallMode::Reinstall,
897            name,
898            principal,
899            wasm_sha256,
900            ..
901        } = action
902        else {
903            continue;
904        };
905        let action_sha256 = action_sha256(action);
906        let Some(effect) = journal
907            .effects
908            .iter()
909            .find(|effect| effect.action_sha256 == action_sha256)
910        else {
911            continue;
912        };
913        if effect.state == EffectState::Applied
914            && let Some(pre_canister_version) = effect.pre_canister_version
915            && state
916                .principals
917                .get(name)
918                .is_some_and(|retained| retained == principal)
919        {
920            state.completed_reinstall_operation_id = Some(journal.operation_id.clone());
921            state
922                .completed_reinstall_action_sha256
923                .insert(name.clone(), action_sha256);
924            state
925                .completed_reinstalls
926                .insert(name.clone(), pre_canister_version);
927            if let Some(topology) = state.topology.get_mut(name) {
928                topology.module_hash = Some(wasm_sha256.clone());
929            }
930        }
931    }
932}
933
934fn completed_reinstall_evidence_matches(
935    state: &FleetEnsureStateRecord,
936    journal: &FleetEnsureJournalRecord,
937) -> bool {
938    !state.completed_reinstalls.is_empty()
939        && state
940            .completed_reinstalls
941            .iter()
942            .all(|(name, pre_version)| {
943                state
944                    .completed_reinstall_action_sha256
945                    .get(name)
946                    .and_then(|action_sha256| {
947                        journal
948                            .effects
949                            .iter()
950                            .find(|effect| effect.action_sha256 == *action_sha256)
951                    })
952                    .is_some_and(|effect| {
953                        effect.state == EffectState::Applied
954                            && effect.pre_canister_version == Some(*pre_version)
955                    })
956            })
957        && state.completed_reinstall_action_sha256.len() == state.completed_reinstalls.len()
958}
959
960/// Admit another review only after every effect of the selected reset is accounted for.
961fn reinstall_recovery_ready<E: std::error::Error + 'static>(
962    plan: &FleetEnsurePlan,
963    journal: &FleetEnsureJournalRecord,
964    state: &FleetEnsureStateRecord,
965) -> Result<bool, EnsureWorkflowError<E>> {
966    if plan.scope != FleetEnsurePlanScope::Full
967        || plan.reinstall.is_none()
968        || journal.completion != FleetEnsureCompletion::ReplanRequired
969        || journal.plan_sha256 != plan.plan_sha256
970    {
971        return Ok(false);
972    }
973    verify_journal(journal, plan, &plan.fleet, state)?;
974    let actions = continuation::actions(plan, journal);
975    let complete = actions.len() == journal.effects.len()
976        && journal
977            .effects
978            .iter()
979            .all(|effect| effect.state == EffectState::Applied);
980    let installs_recorded = actions
981        .iter()
982        .zip(&journal.effects)
983        .all(|(action, effect)| {
984            !matches!(action, EnsureAction::Install { .. }) || effect.pre_canister_version.is_some()
985        });
986    if !complete || !installs_recorded {
987        return Err(EnsureWorkflowError::JournalIntegrity);
988    }
989    Ok(true)
990}
991
992fn retained_reinstall_operation_matches(
993    state: &FleetEnsureStateRecord,
994    prior_fleet: &str,
995    prior_operation_id: &str,
996    journal: &FleetEnsureJournalRecord,
997) -> bool {
998    journal.completion == FleetEnsureCompletion::ReplanRequired
999        && journal.fleet == prior_fleet
1000        && journal.operation_id == prior_operation_id
1001        && state.completed_reinstall_operation_id.as_deref() == Some(journal.operation_id.as_str())
1002        && completed_reinstall_evidence_matches(state, journal)
1003}
1004
1005fn retained_reinstall_desired_conflict(
1006    state: &FleetEnsureStateRecord,
1007    prior_fleet: &str,
1008    prior_operation_id: &str,
1009    prior_desired_sha256: &str,
1010    supplied_desired_sha256: &str,
1011    journal: &FleetEnsureJournalRecord,
1012) -> bool {
1013    prior_desired_sha256 != supplied_desired_sha256
1014        && retained_reinstall_operation_matches(state, prior_fleet, prior_operation_id, journal)
1015}
1016
1017fn retain_plan_cycles(state: &mut FleetEnsureStateRecord, plan: &FleetEnsurePlan) {
1018    for canister in &plan.canisters {
1019        if let Some(principal) = &canister.principal {
1020            state
1021                .retained_cycles_by_principal
1022                .entry(principal.clone())
1023                .or_insert(canister.observed_cycles);
1024        }
1025    }
1026}
1027
1028fn retain_observed_cycles(state: &mut FleetEnsureStateRecord, observation: &FleetObservation) {
1029    for live in observation.canisters.values().filter_map(Option::as_ref) {
1030        state
1031            .retained_cycles_by_principal
1032            .insert(live.principal.clone(), live.cycles);
1033    }
1034    state
1035        .retained_cycles_by_principal
1036        .extend(observation.additional_controlled_cycles.clone());
1037}
1038
1039fn root_owned_lifecycle_label(observation: &FleetObservation, target: &str) -> &'static str {
1040    observation
1041        .canisters
1042        .get(target)
1043        .and_then(Option::as_ref)
1044        .and_then(|live| live.root_owned_lifecycle)
1045        .map_or("unavailable", |lifecycle| lifecycle.label())
1046}
1047
1048/// Apply or resume exactly one reviewed plan until it converges or returns a typed blocker.
1049#[expect(
1050    clippy::too_many_lines,
1051    reason = "the sole effect driver keeps intent, reconciliation and terminal publication visibly ordered"
1052)]
1053pub fn apply<P>(
1054    root: &Path,
1055    desired: &crate::fleet_ensure::model::DesiredFleet,
1056    desired_sha256: &str,
1057    requested_fleet: &str,
1058    reviewed_plan_sha256: &str,
1059    platform: &mut P,
1060) -> Result<FleetEnsureReport, EnsureWorkflowError<P::Error>>
1061where
1062    P: EnsurePlatform,
1063{
1064    validate_path_identity(desired, requested_fleet)?;
1065    let paths = EnsurePaths::under(root, &desired.environment, requested_fleet);
1066    let _lock = lock_operation(&paths)?;
1067    let pending = crate::fleet_ensure::ops::reinstall::adoption::review(&paths)?;
1068    let adopting = pending.is_some();
1069    let retained_plan = verified_plan(match pending {
1070        Some(plan) => plan,
1071        None => retained_plan::read(&paths, &desired.environment, requested_fleet)?
1072            .ok_or(EnsureWorkflowError::PlanMissing)?,
1073    })?;
1074    let retained_journal = if adopting {
1075        None
1076    } else {
1077        retained_plan::journal(&paths, &desired.environment, requested_fleet)?
1078    };
1079    if retained_plan.plan_sha256 != reviewed_plan_sha256
1080        && !retained_journal
1081            .as_ref()
1082            .is_some_and(|journal| funding::accepts(&retained_plan, journal, reviewed_plan_sha256))
1083    {
1084        return Err(EnsureWorkflowError::PlanDigestMismatch {
1085            actual: retained_plan.plan_sha256,
1086            expected: reviewed_plan_sha256.to_string(),
1087        });
1088    }
1089    let mut state = read_state(&paths, requested_fleet)?;
1090    let in_progress = retained_journal
1091        .as_ref()
1092        .is_some_and(|journal| journal.completion == FleetEnsureCompletion::InProgress);
1093    if let Some(journal) = retained_journal.as_ref().filter(|_| in_progress) {
1094        verify_journal(journal, &retained_plan, requested_fleet, &state)?;
1095    }
1096    if !in_progress
1097        && retained_plan.reinstall.is_none()
1098        && retained_plan.desired_sha256 != desired_sha256
1099    {
1100        return Err(EnsureWorkflowError::DriftedBeforeApply);
1101    }
1102    let operation_desired = if in_progress || retained_plan.reinstall.is_some() {
1103        let reviewed = retained_plan
1104            .reviewed_desired
1105            .as_deref()
1106            .ok_or(EnsureWorkflowError::PlanIntegrity)?
1107            .desired();
1108        validate_path_identity(reviewed, requested_fleet)?;
1109        if reviewed.environment != retained_plan.environment {
1110            return Err(EnsureWorkflowError::PlanIntegrity);
1111        }
1112        reviewed
1113    } else {
1114        if retained_plan.desired_sha256 != desired_sha256 {
1115            return Err(EnsureWorkflowError::DriftedBeforeApply);
1116        }
1117        desired
1118    };
1119    if let Some(journal) = &retained_journal
1120        && let Some(actual) = crate::fleet_ensure::ops::terminal::read(
1121            &paths,
1122            &retained_plan,
1123            journal,
1124            &state,
1125            &funding_plan::<P::Error>(&retained_plan, journal)?.conservation,
1126        )?
1127    {
1128        verify_journal(journal, &retained_plan, requested_fleet, &state)?;
1129        if journal.completion != FleetEnsureCompletion::Converged {
1130            let mut completed = journal.clone();
1131            completed.completion = FleetEnsureCompletion::Converged;
1132            completed.stalled_observations = 0;
1133            write_journal(&paths, &completed)?;
1134        }
1135        return Ok(FleetEnsureReport {
1136            funding_review: None,
1137            actual_conservation: Some(actual),
1138            effects_applied: 0,
1139            plan: retained_plan,
1140            terminal: true,
1141        });
1142    }
1143    if let Some(journal) = &retained_journal
1144        && let Some(actual) = crate::fleet_ensure::ops::infrastructure_bootstrap::terminal::read(
1145            &paths,
1146            &retained_plan,
1147            journal,
1148            &state,
1149        )?
1150    {
1151        verify_journal(journal, &retained_plan, requested_fleet, &state)?;
1152        if journal.completion != FleetEnsureCompletion::Converged {
1153            let mut completed = journal.clone();
1154            completed.completion = FleetEnsureCompletion::Converged;
1155            completed.stalled_observations = 0;
1156            write_journal(&paths, &completed)?;
1157        }
1158        return Ok(FleetEnsureReport {
1159            funding_review: None,
1160            actual_conservation: Some(actual),
1161            effects_applied: 0,
1162            plan: retained_plan,
1163            terminal: true,
1164        });
1165    }
1166    if retained_plan.scope == FleetEnsurePlanScope::InfrastructureBootstrap {
1167        crate::fleet_ensure::ops::infrastructure_bootstrap::verify_plan(root, &retained_plan)?;
1168    }
1169    verify_release_transition(root, operation_desired, retained_plan.scope)?;
1170    if !retained_journal.as_ref().is_some_and(|journal| {
1171        journal.completion == FleetEnsureCompletion::Converged
1172            && journal.plan_sha256 == retained_plan.plan_sha256
1173    }) {
1174        validate_creation_fee_scope(
1175            operation_desired,
1176            &retained_plan.canisters,
1177            &retained_plan.conservation.estate_funding_domains,
1178        )?;
1179    }
1180    platform
1181        .bind_reviewed_desired(operation_desired)
1182        .map_err(EnsureWorkflowError::Platform)?;
1183    if let Some(source) = &retained_plan.infrastructure_bootstrap {
1184        platform
1185            .bind_infrastructure_bootstrap(source)
1186            .map_err(EnsureWorkflowError::Platform)?;
1187    }
1188    let operation_desired_sha256 = retained_plan.desired_sha256.as_str();
1189    if let Some(review) = retained_journal.as_ref().and_then(
1190        crate::fleet_ensure::ops::infrastructure_bootstrap::registration_recovery::approved,
1191    ) && !platform
1192        .bind_bootstrap_registration_recovery(review)
1193        .map_err(EnsureWorkflowError::Platform)?
1194    {
1195        return Err(EnsureWorkflowError::PlanIntegrity);
1196    }
1197    continuation::verify_inputs(root, operation_desired, &retained_plan)?;
1198    if let Some(journal) = &retained_journal {
1199        continuation::verify_canonical(&retained_plan, journal, &state, platform)?;
1200        if reinstall_recovery_ready(&retained_plan, journal, &state)? {
1201            return Err(EnsureWorkflowError::SuccessorReviewRequired {
1202                reason:
1203                    crate::fleet_ensure::model::FleetEnsureSuccessorReviewReason::AdditionalEffect,
1204                review: None,
1205            });
1206        }
1207    }
1208    let completed_journal = retained_journal.as_ref().filter(|journal| {
1209        (journal.completion == FleetEnsureCompletion::Converged
1210            || (journal.completion == FleetEnsureCompletion::Prepared
1211                && retained_plan.scope == FleetEnsurePlanScope::ReinstallPreparation))
1212            && journal.plan_sha256 == retained_plan.plan_sha256
1213    });
1214    if retained_plan.scope == FleetEnsurePlanScope::ReinstallPreparation
1215        && let Some(journal) = completed_journal
1216    {
1217        let actual = reinstall::complete(&retained_plan, journal, &state, platform)?;
1218        return Ok(FleetEnsureReport {
1219            funding_review: None,
1220            actual_conservation: Some(actual),
1221            effects_applied: 0,
1222            plan: retained_plan,
1223            terminal: true,
1224        });
1225    }
1226    if retained_plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite
1227        && let Some(journal) = completed_journal
1228    {
1229        verify_journal(journal, &retained_plan, requested_fleet, &state)?;
1230        let actual = root_reinstall::complete(&retained_plan, journal, &state, platform)?;
1231        report_progress_state(
1232            platform,
1233            &retained_plan,
1234            journal,
1235            FleetEnsurePhase::Infrastructure,
1236            FleetEnsureProgressState::PrerequisiteComplete,
1237        );
1238        return Ok(FleetEnsureReport {
1239            funding_review: None,
1240            actual_conservation: Some(actual),
1241            effects_applied: 0,
1242            plan: retained_plan,
1243            terminal: true,
1244        });
1245    }
1246    if retained_plan.scope == FleetEnsurePlanScope::RootStartPrerequisite
1247        && let Some(journal) = completed_journal
1248    {
1249        verify_terminal_root_start_replay(
1250            operation_desired,
1251            &retained_plan,
1252            journal,
1253            requested_fleet,
1254            platform,
1255            &state,
1256        )?;
1257        report_progress_state(
1258            platform,
1259            &retained_plan,
1260            journal,
1261            FleetEnsurePhase::Infrastructure,
1262            FleetEnsureProgressState::PrerequisiteComplete,
1263        );
1264        return Ok(FleetEnsureReport {
1265            funding_review: None,
1266            actual_conservation: None,
1267            effects_applied: applied_count(journal),
1268            plan: retained_plan,
1269            terminal: true,
1270        });
1271    }
1272    let mut journal = match retained_journal {
1273        Some(journal) if journal.completion == FleetEnsureCompletion::InProgress => {
1274            verify_journal(&journal, &retained_plan, requested_fleet, &state)?;
1275            journal
1276        }
1277        _ => {
1278            let (observation, initial_controlled_cycles) = verify_fresh_plan(
1279                root,
1280                operation_desired,
1281                operation_desired_sha256,
1282                requested_fleet,
1283                &retained_plan,
1284                platform,
1285                &state,
1286            )?;
1287            // Every fresh scope crosses this boundary, including reinstall verification.
1288            // Retained intents must instead reconcile their exact, possibly paid request.
1289            let required = retained_plan.conservation.maximum_operator_debit_cycles;
1290            if observation.operator_cycles < required {
1291                return Err(EnsureWorkflowError::InsufficientOperatorCycles {
1292                    actual: observation.operator_cycles,
1293                    required,
1294                });
1295            }
1296            root_reinstall::verify_initial_funding(
1297                &retained_plan,
1298                observation.operator_cycles,
1299                platform,
1300            )?;
1301            let journal = FleetEnsureJournalRecord {
1302                release: None,
1303                bootstrap_registration_recovery: None,
1304                funding_observations: BTreeMap::new(),
1305                funding_reviews: Vec::new(),
1306                successor_phases: Vec::new(),
1307                completion: FleetEnsureCompletion::InProgress,
1308                estate_funding_required: None,
1309                effects: Vec::new(),
1310                fleet: requested_fleet.to_string(),
1311                initial_controlled_cycles,
1312                initial_estate_funding_cycles_by_root: retained_plan
1313                    .conservation
1314                    .estate_funding_domains
1315                    .iter()
1316                    .map(|domain| {
1317                        (
1318                            domain.root.clone(),
1319                            domain.available_cycles.unwrap_or_default(),
1320                        )
1321                    })
1322                    .collect(),
1323                initial_operator_cycles: observation.operator_cycles,
1324                operation_id: retained_plan.operation_id.clone(),
1325                plan_sha256: retained_plan.plan_sha256.clone(),
1326                schema_version: FLEET_ENSURE_SCHEMA_VERSION,
1327                stalled_observations: 0,
1328            };
1329            // Cross the nonterminal boundary before any effect-owned state can be retained.
1330            if adopting {
1331                crate::fleet_ensure::ops::reinstall::adoption::adopt(
1332                    &paths,
1333                    &retained_plan,
1334                    &journal,
1335                )?;
1336            } else {
1337                write_journal(&paths, &journal)?;
1338            }
1339            journal
1340        }
1341    };
1342
1343    funding::resume(
1344        &paths,
1345        &retained_plan,
1346        &mut journal,
1347        &state,
1348        reviewed_plan_sha256,
1349        platform,
1350    )?;
1351    if journal
1352        .estate_funding_required
1353        .as_ref()
1354        .is_some_and(|pause| funding::covered(&journal, pause))
1355    {
1356        retain_estate_funding_pause(&paths, &mut journal, None)?;
1357    }
1358    let mut terminal_state;
1359    let mut bootstrap_coordinator_verified = false;
1360    // Only the immediately following protocol phase may consume this evidence.
1361    // It is invocation-local and never survives interruption or a preceding effect.
1362    let mut continuation_observation: Option<ContinuationObservation> = None;
1363    loop {
1364        let execution_actions = continuation::actions(&retained_plan, &journal)
1365            .into_iter()
1366            .cloned()
1367            .collect::<Vec<_>>();
1368        let actions = execution_actions.iter().collect::<Vec<_>>();
1369        let mut replayed_issued_commands = BTreeSet::new();
1370        let mut reported_phase = None;
1371        loop {
1372            let mut deferred_controller_observation = false;
1373            for (index, action) in actions.iter().enumerate() {
1374                let observation_policy = effect_observation_policy(operation_desired, action)?;
1375                let action_hash = action_sha256(action);
1376                let retained_effect = journal.effects.get(index);
1377                let continued_observation = retained_effect
1378                    .is_none_or(|effect| effect.state != EffectState::Applied)
1379                    .then(|| continuation_observation.take())
1380                    .flatten()
1381                    .filter(|evidence| evidence.first_action_sha256 == action_hash);
1382                if retained_effect.is_none_or(|effect| effect.state != EffectState::Applied) {
1383                    if retained_plan.infrastructure_bootstrap.is_some()
1384                        && index < ordered_actions(&retained_plan).len()
1385                        && !bootstrap_coordinator_verified
1386                        && operation_desired
1387                            .bootstrap
1388                            .as_ref()
1389                            .is_some_and(|bootstrap| action.name() != bootstrap.coordinator)
1390                    {
1391                        if !platform
1392                            .verify_bootstrap_coordinator(&state)
1393                            .map_err(EnsureWorkflowError::Platform)?
1394                        {
1395                            return Err(EnsureWorkflowError::ConvergenceDrift);
1396                        }
1397                        bootstrap_coordinator_verified = true;
1398                    }
1399                    let phase = action_progress_phase(action);
1400                    if reported_phase != Some(phase) {
1401                        report_progress(platform, &retained_plan, &journal, phase);
1402                        reported_phase = Some(phase);
1403                    }
1404                }
1405                if retained_plan.infrastructure_bootstrap.is_none()
1406                    && matches!(action, EnsureAction::FleetProtocol { .. })
1407                    && retained_effect.is_none_or(|effect| effect.state == EffectState::Intent)
1408                    && !prior_fleet_protocol_effect_started(&actions, &journal, index)
1409                {
1410                    let funding_observation = match continued_observation {
1411                        Some(evidence) => evidence.observation,
1412                        None => platform
1413                            .observe(&journal.operation_id, &state)
1414                            .map_err(EnsureWorkflowError::Platform)?,
1415                    };
1416                    let funded_plan = funding_plan(&retained_plan, &journal)?;
1417                    let required =
1418                        estate_funding_requirement(&funded_plan, &state, &funding_observation)?;
1419                    retain_estate_funding_pause(&paths, &mut journal, required.as_ref())?;
1420                    if let Some(required) = required {
1421                        report_progress_state(
1422                            platform,
1423                            &retained_plan,
1424                            &journal,
1425                            action_progress_phase(action),
1426                            FleetEnsureProgressState::FundingRequired,
1427                        );
1428                        return Err(EnsureWorkflowError::EstateFundingRequired(Box::new(
1429                            estate_funding_error(&required),
1430                        )));
1431                    }
1432                }
1433                independent_effects::apply_batch(
1434                    &paths,
1435                    &retained_plan,
1436                    &actions,
1437                    index,
1438                    &mut journal,
1439                    &state,
1440                    platform,
1441                    observation_policy.maximum_stalled_observations,
1442                )?;
1443                let mut initial_observation = if journal.effects.len() <= index {
1444                    crate::fleet_ensure::ops::infrastructure_bootstrap::inspection::reserve_effect(
1445                        &paths,
1446                        &retained_plan,
1447                        action,
1448                    )?;
1449                    let prepared = prepare_effect(platform, &journal.operation_id, action, &state)
1450                        .map_err(EnsureWorkflowError::Platform)?;
1451                    journal.effects.push(prepared.record);
1452                    write_journal(&paths, &journal)?;
1453                    prepared.observation
1454                } else {
1455                    None
1456                };
1457                let observation_started = Instant::now();
1458                loop {
1459                    let record = journal
1460                        .effects
1461                        .get(index)
1462                        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1463                    if record.action_sha256 != action_hash {
1464                        return Err(EnsureWorkflowError::DriftedBeforeApply);
1465                    }
1466                    if !create_record_is_exact(action, record, &state) {
1467                        return Err(EnsureWorkflowError::JournalIntegrity);
1468                    }
1469                    if matches!(record.state, EffectState::Applied) {
1470                        let retained_create = journal
1471                            .effects
1472                            .get(index)
1473                            .ok_or(EnsureWorkflowError::JournalIntegrity)
1474                            .and_then(|record| {
1475                                retain_applied_create_authority(
1476                                    operation_desired,
1477                                    action,
1478                                    record,
1479                                    &mut state,
1480                                )
1481                            })?;
1482                        let retained_funding = journal.effects.get(index).is_some_and(|record| {
1483                            retain_applied_funding_cycles(&mut state, action, record)
1484                        });
1485                        if retained_create || retained_funding {
1486                            write_state(&paths, &state)?;
1487                        }
1488                        break;
1489                    }
1490
1491                    let observed = if let Some(observed) = initial_observation.take() {
1492                        observed
1493                    } else {
1494                        crate::fleet_ensure::ops::infrastructure_bootstrap::inspection::reserve_effect(
1495                            &paths, &retained_plan, action,
1496                        )?;
1497                        platform
1498                            .observe_effect(&journal.operation_id, action, record, &state)
1499                            .map_err(EnsureWorkflowError::Platform)?
1500                    };
1501                    let source_cycles = if observed.post_cycles.is_some() {
1502                        observed.post_cycles
1503                    } else {
1504                        platform
1505                            .action_cycles(action, &state)
1506                            .map_err(EnsureWorkflowError::Platform)?
1507                    };
1508                    let destination_cycles = platform
1509                        .action_destination_cycles(action, &state)
1510                        .map_err(EnsureWorkflowError::Platform)?;
1511                    let replan_after_create_balance =
1512                        observed.retry == EffectRetry::ReplanRequiredAfterCreateBalanceDrift;
1513                    let retained_create_balance_is_exact = !replan_after_create_balance
1514                        || (journal.effects.len() == index + 1
1515                            && journal.effects[..index]
1516                                .iter()
1517                                .all(|effect| effect.state == EffectState::Applied)
1518                            && record.state == EffectState::Issued
1519                            && create_identity_is_exact(action.name(), record, &state)
1520                            && observed.post_cycles.is_some()
1521                            && observed.post_cycles == source_cycles);
1522                    let protocol_funding_required = if record.state == EffectState::Issued {
1523                        observed
1524                            .estate_funding_required
1525                            .as_ref()
1526                            .map(|funding| {
1527                                protocol_estate_funding_requirement(&retained_plan, &state, funding)
1528                            })
1529                            .transpose()?
1530                    } else {
1531                        None
1532                    };
1533                    if let Some(required) = protocol_funding_required.as_ref()
1534                        && !funding::covered(&journal, required)
1535                    {
1536                        retain_estate_funding_pause(&paths, &mut journal, Some(required))?;
1537                        report_progress_state(
1538                            platform,
1539                            &retained_plan,
1540                            &journal,
1541                            action_progress_phase(action),
1542                            FleetEnsureProgressState::FundingRequired,
1543                        );
1544                        return Err(EnsureWorkflowError::EstateFundingRequired(Box::new(
1545                            estate_funding_error(required),
1546                        )));
1547                    }
1548                    if journal
1549                        .estate_funding_required
1550                        .as_ref()
1551                        .is_some_and(|required| required.pending_creation_operation_id.is_some())
1552                    {
1553                        retain_estate_funding_pause(&paths, &mut journal, None)?;
1554                    }
1555                    let record = journal
1556                        .effects
1557                        .get_mut(index)
1558                        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1559                    merge_observed_cycles(record, source_cycles, destination_cycles);
1560                    if replan_after_create_balance {
1561                        if !retained_create_balance_is_exact {
1562                            return Err(EnsureWorkflowError::JournalIntegrity);
1563                        }
1564                        let EnsureAction::Create {
1565                            name,
1566                            requested_initial_cycles,
1567                            ..
1568                        } = action
1569                        else {
1570                            return Err(EnsureWorkflowError::JournalIntegrity);
1571                        };
1572                        let actual_cycles = record
1573                            .post_cycles
1574                            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1575                        record.progress_identity = Some(observed.progress_identity);
1576                        record.state = EffectState::Applied;
1577                        retain_created_canister_for_replan(
1578                            operation_desired,
1579                            name,
1580                            record,
1581                            &mut state,
1582                        )?;
1583                        journal.completion = FleetEnsureCompletion::ReplanRequired;
1584                        journal.stalled_observations = 0;
1585                        write_journal(&paths, &journal)?;
1586                        write_state(&paths, &state)?;
1587                        let configured_fee_cycles = operation_desired
1588                            .management_creation_fee_cycles
1589                            .parse::<Cycles>()
1590                            .map(|cycles| cycles.to_u128())
1591                            .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
1592                        return Err(EnsureWorkflowError::ReplanRequiredAfterCreateBalanceDrift {
1593                            actual_cycles,
1594                            canister: name.clone(),
1595                            configured_fee_cycles,
1596                            deficit_cycles: requested_initial_cycles.saturating_sub(actual_cycles),
1597                            requested_cycles: *requested_initial_cycles,
1598                        });
1599                    }
1600                    if observed.applied {
1601                        record.progress_identity = Some(observed.progress_identity);
1602                        record.state = EffectState::Applied;
1603                        journal.stalled_observations = 0;
1604                        write_journal(&paths, &journal)?;
1605                        let record = journal
1606                            .effects
1607                            .get(index)
1608                            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1609                        let retained_create = retain_applied_create_authority(
1610                            operation_desired,
1611                            action,
1612                            record,
1613                            &mut state,
1614                        )?;
1615                        let retained_funding =
1616                            retain_applied_funding_cycles(&mut state, action, record);
1617                        if retained_create || retained_funding {
1618                            write_state(&paths, &state)?;
1619                        }
1620                        // Emit once per durable transition, including a reconciled lost reply.
1621                        report_progress(
1622                            platform,
1623                            &retained_plan,
1624                            &journal,
1625                            action_progress_phase(action),
1626                        );
1627                        break;
1628                    }
1629
1630                    if observed.retry == EffectRetry::DeferUntilControllerObservation {
1631                        if !deferred_create_observation_is_exact(
1632                            operation_desired,
1633                            &actions,
1634                            index,
1635                            action,
1636                            record,
1637                            &state,
1638                        ) {
1639                            return Err(EnsureWorkflowError::JournalIntegrity);
1640                        }
1641                        let progress_identity = observed.progress_identity.clone();
1642                        if record.progress_identity.as_deref() == Some(&observed.progress_identity)
1643                        {
1644                            journal.stalled_observations =
1645                                journal.stalled_observations.saturating_add(1);
1646                        } else {
1647                            record.progress_identity = Some(observed.progress_identity);
1648                            journal.stalled_observations = 0;
1649                        }
1650                        write_journal(&paths, &journal)?;
1651                        if journal.stalled_observations
1652                            >= observation_policy.maximum_stalled_observations
1653                        {
1654                            return Err(EnsureWorkflowError::Stalled {
1655                                action: action.name().to_string(),
1656                                observations: journal.stalled_observations,
1657                                progress_identity,
1658                            });
1659                        }
1660                        deferred_controller_observation = true;
1661                        break;
1662                    }
1663
1664                    let maintenance_continuation =
1665                        observed.retry == EffectRetry::ContinuePoolMaintenance;
1666                    if matches!(record.state, EffectState::Intent) || maintenance_continuation {
1667                        crate::fleet_ensure::policy::infrastructure_bootstrap::validate_effect_headroom(
1668                            &retained_plan,
1669                            action,
1670                            source_cycles,
1671                        )?;
1672                        if let Some(maximum_attempts) = action.fixture_publication_attempt_limit() {
1673                            if !reserve_fixture_publication_attempt(record, maximum_attempts) {
1674                                return Err(EnsureWorkflowError::FixturePublicationBound {
1675                                    action: action.name().to_string(),
1676                                    maximum_attempts,
1677                                });
1678                            }
1679                            // Reserve before the external call, even if its response never returns.
1680                            write_journal(&paths, &journal)?;
1681                        }
1682                        let record = journal
1683                            .effects
1684                            .get_mut(index)
1685                            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1686                        if let EnsureAction::FleetProtocol {
1687                            action: current, ..
1688                        } = action
1689                            && let CurrentFleetProtocolAction::MaintainPoolReadiness {
1690                                maximum_updates,
1691                                ..
1692                            } = current.as_ref()
1693                        {
1694                            if record.maintenance_attempts >= *maximum_updates {
1695                                return Err(EnsureWorkflowError::PoolMaintenanceBound {
1696                                    action: action.name().to_string(),
1697                                    maximum_updates: *maximum_updates,
1698                                });
1699                            }
1700                            if record.progress_identity.as_deref()
1701                                == Some(&observed.progress_identity)
1702                            {
1703                                journal.stalled_observations =
1704                                    journal.stalled_observations.saturating_add(1);
1705                            } else {
1706                                journal.stalled_observations = 0;
1707                            }
1708                            record.maintenance_attempts += 1;
1709                            let attempts = record.maintenance_attempts;
1710                            // A failed or lost call consumes its reviewed attempt across process restart.
1711                            write_journal(&paths, &journal)?;
1712                            if attempts > 1 {
1713                                platform
1714                                    .pace_effect_observation(action, journal.stalled_observations);
1715                            }
1716                        } else if maintenance_continuation {
1717                            return Err(EnsureWorkflowError::JournalIntegrity);
1718                        }
1719                        let record = journal
1720                            .effects
1721                            .get(index)
1722                            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1723                        if retained_plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite {
1724                            root_reinstall::verify_effect_authority(
1725                                &retained_plan,
1726                                action,
1727                                record.pre_cycles,
1728                                &state,
1729                                platform,
1730                            )?;
1731                        }
1732                        verify_store_activation_binding(&retained_plan, action, &state, platform)?;
1733                        reinstall::verify_effect_authority(
1734                            &retained_plan,
1735                            action,
1736                            &state,
1737                            platform,
1738                        )?;
1739                        let outcome =
1740                            match platform.apply(&journal.operation_id, action, record, &state) {
1741                                Ok(outcome) => outcome,
1742                                Err(source) => {
1743                                    let progress_identity = record
1744                                        .progress_identity
1745                                        .clone()
1746                                        .unwrap_or_else(|| "effect-call-failed".to_string());
1747                                    journal.stalled_observations =
1748                                        journal.stalled_observations.saturating_add(1);
1749                                    write_journal(&paths, &journal)?;
1750                                    if journal.stalled_observations
1751                                        >= observation_policy.maximum_stalled_observations
1752                                    {
1753                                        return Err(EnsureWorkflowError::Stalled {
1754                                            action: action.name().to_string(),
1755                                            observations: journal.stalled_observations,
1756                                            progress_identity,
1757                                        });
1758                                    }
1759                                    return Err(EnsureWorkflowError::Platform(source));
1760                                }
1761                            };
1762                        if !create_outcome_is_exact(action, record, &state, &outcome) {
1763                            return Err(EnsureWorkflowError::JournalIntegrity);
1764                        }
1765                        if let Some(created) = &outcome.created_principal {
1766                            state
1767                                .pending_principals
1768                                .insert(action.name().to_string(), created.clone());
1769                            write_state(&paths, &state)?;
1770                        }
1771                        let record = journal
1772                            .effects
1773                            .get_mut(index)
1774                            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1775                        record.created_principal = outcome.created_principal;
1776                        record.receipt = outcome.receipt;
1777                        record.post_cycles = outcome.post_cycles;
1778                        record.progress_identity = Some(observed.progress_identity);
1779                        record.state = EffectState::Issued;
1780                        // A successful maintenance call can report no progress while Root work is in flight.
1781                        if !maintenance_continuation {
1782                            journal.stalled_observations = 0;
1783                        }
1784                        write_journal(&paths, &journal)?;
1785                        continue;
1786                    }
1787
1788                    if observed.retry == EffectRetry::ReplayExactIssuedCommand
1789                        && !replayed_issued_commands.contains(&index)
1790                    {
1791                        if !matches!(
1792                            action,
1793                            EnsureAction::FleetProtocol { action, .. }
1794                                if matches!(
1795                                    action.as_ref(),
1796                                    crate::fleet_ensure::model::CurrentFleetProtocolAction::ProvisionComponents { .. }
1797                                )
1798                        ) {
1799                            return Err(EnsureWorkflowError::JournalIntegrity);
1800                        }
1801                        let retained_receipt = record.receipt.clone();
1802                        let outcome = platform
1803                            .apply(&journal.operation_id, action, record, &state)
1804                            .map_err(EnsureWorkflowError::Platform)?;
1805                        if outcome.created_principal.is_some()
1806                            || outcome.receipt != retained_receipt
1807                            || outcome.post_cycles.is_some()
1808                        {
1809                            return Err(EnsureWorkflowError::JournalIntegrity);
1810                        }
1811                        let record = journal
1812                            .effects
1813                            .get_mut(index)
1814                            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1815                        record.progress_identity = Some(observed.progress_identity);
1816                        journal.stalled_observations = 0;
1817                        write_journal(&paths, &journal)?;
1818                        replayed_issued_commands.insert(index);
1819                        continue;
1820                    }
1821
1822                    if record.progress_identity.as_deref() == Some(&observed.progress_identity) {
1823                        journal.stalled_observations =
1824                            journal.stalled_observations.saturating_add(1);
1825                    } else {
1826                        record.progress_identity = Some(observed.progress_identity);
1827                        journal.stalled_observations = 0;
1828                    }
1829                    let progress_identity = record
1830                        .progress_identity
1831                        .clone()
1832                        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1833                    write_journal(&paths, &journal)?;
1834                    if journal.stalled_observations
1835                        >= observation_policy.maximum_stalled_observations
1836                    {
1837                        if let Some((failure, origin)) = observed
1838                            .provisioning_failure
1839                            .and_then(|failure| failure.origin.map(|origin| (failure, origin)))
1840                        {
1841                            return Err(EnsureWorkflowError::ProvisioningRetryPending {
1842                                root: failure.fleet_subnet_root,
1843                                stage: origin.stage,
1844                                target: origin.target,
1845                                operation_id: origin.operation_id,
1846                                diagnostic_code: origin.diagnostic_code,
1847                                observations: journal.stalled_observations,
1848                                failed_at_ns: origin.failed_at_ns,
1849                            });
1850                        }
1851                        return Err(EnsureWorkflowError::Stalled {
1852                            action: action.name().to_string(),
1853                            observations: journal.stalled_observations,
1854                            progress_identity,
1855                        });
1856                    }
1857                    if observation_policy.paced {
1858                        report_progress_state(
1859                            platform,
1860                            &retained_plan,
1861                            &journal,
1862                            action_progress_phase(action),
1863                            FleetEnsureProgressState::AwaitingProgress {
1864                                elapsed_seconds: observation_started.elapsed().as_secs(),
1865                                provisioning: observed.provisioning_progress,
1866                            },
1867                        );
1868                        platform.pace_effect_observation(action, journal.stalled_observations);
1869                    }
1870                }
1871            }
1872            if journal.effects.len() == actions.len()
1873                && journal
1874                    .effects
1875                    .iter()
1876                    .all(|record| record.state == EffectState::Applied)
1877            {
1878                break;
1879            }
1880            if !deferred_controller_observation {
1881                return Err(EnsureWorkflowError::JournalIntegrity);
1882            }
1883        }
1884
1885        report_progress(
1886            platform,
1887            &retained_plan,
1888            &journal,
1889            FleetEnsurePhase::TerminalVerification,
1890        );
1891        if retained_plan.scope == FleetEnsurePlanScope::InfrastructureBootstrap {
1892            if infrastructure_bootstrap::advance(
1893                root,
1894                &retained_plan,
1895                &mut journal,
1896                &mut state,
1897                platform,
1898            )? {
1899                continue;
1900            }
1901            let phase = journal
1902                .successor_phases
1903                .first()
1904                .and_then(|phase| phase.plan.as_ref())
1905                .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1906            let expected =
1907                crate::fleet_ensure::ops::infrastructure_bootstrap::registration::registry(phase)?;
1908            if !platform
1909                .verify_bootstrap_registry(&expected)
1910                .map_err(EnsureWorkflowError::Platform)?
1911            {
1912                return Err(EnsureWorkflowError::ConvergenceDrift);
1913            }
1914            let (actual, observed) = infrastructure_bootstrap::complete(
1915                root,
1916                &retained_plan,
1917                &journal,
1918                &state,
1919                platform,
1920            )?;
1921            if !platform
1922                .verify_bootstrap_registry(&expected)
1923                .map_err(EnsureWorkflowError::Platform)?
1924            {
1925                return Err(EnsureWorkflowError::ConvergenceDrift);
1926            }
1927            publish_terminal_state(operation_desired, &retained_plan, &journal, &mut state)
1928                .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
1929            crate::fleet_ensure::ops::infrastructure_bootstrap::registration::bind_state(
1930                root, &mut state, phase,
1931            )?;
1932            write_state(&paths, &state)?;
1933            crate::fleet_ensure::ops::infrastructure_bootstrap::terminal::retain(
1934                &paths,
1935                &retained_plan,
1936                &journal,
1937                &state,
1938                actual.clone(),
1939                &observed,
1940            )?;
1941            journal.completion = FleetEnsureCompletion::Converged;
1942            journal.stalled_observations = 0;
1943            write_journal(&paths, &journal)?;
1944            report_progress_state(
1945                platform,
1946                &retained_plan,
1947                &journal,
1948                FleetEnsurePhase::Infrastructure,
1949                FleetEnsureProgressState::PrerequisiteComplete,
1950            );
1951            return Ok(FleetEnsureReport {
1952                funding_review: None,
1953                actual_conservation: Some(actual),
1954                effects_applied: applied_count(&journal),
1955                plan: retained_plan,
1956                terminal: true,
1957            });
1958        }
1959        if retained_plan.scope == FleetEnsurePlanScope::ReinstallPreparation {
1960            let actual = reinstall::complete(&retained_plan, &journal, &state, platform)?;
1961            journal.completion = FleetEnsureCompletion::Prepared;
1962            journal.stalled_observations = 0;
1963            write_journal(&paths, &journal)?;
1964            report_progress_state(
1965                platform,
1966                &retained_plan,
1967                &journal,
1968                FleetEnsurePhase::Infrastructure,
1969                FleetEnsureProgressState::PrerequisiteComplete,
1970            );
1971            return Ok(FleetEnsureReport {
1972                funding_review: None,
1973                actual_conservation: Some(actual),
1974                effects_applied: applied_count(&journal),
1975                plan: retained_plan,
1976                terminal: true,
1977            });
1978        }
1979        if retained_plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite {
1980            let actual = root_reinstall::complete(&retained_plan, &journal, &state, platform)?;
1981            publish_terminal_state(operation_desired, &retained_plan, &journal, &mut state)
1982                .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
1983            state.active_registry = None;
1984            write_state(&paths, &state)?;
1985            journal.completion = FleetEnsureCompletion::Converged;
1986            journal.stalled_observations = 0;
1987            write_journal(&paths, &journal)?;
1988            report_progress_state(
1989                platform,
1990                &retained_plan,
1991                &journal,
1992                FleetEnsurePhase::Infrastructure,
1993                FleetEnsureProgressState::PrerequisiteComplete,
1994            );
1995            return Ok(FleetEnsureReport {
1996                funding_review: None,
1997                actual_conservation: Some(actual),
1998                effects_applied: applied_count(&journal),
1999                plan: retained_plan,
2000                terminal: true,
2001            });
2002        }
2003        if retained_plan.scope == FleetEnsurePlanScope::RootStartPrerequisite {
2004            let actual_conservation = complete_root_start_prerequisite(
2005                operation_desired,
2006                &retained_plan,
2007                platform,
2008                &state,
2009                &journal,
2010            )?;
2011            journal.completion = FleetEnsureCompletion::Converged;
2012            journal.stalled_observations = 0;
2013            write_journal(&paths, &journal)?;
2014            report_progress_state(
2015                platform,
2016                &retained_plan,
2017                &journal,
2018                FleetEnsurePhase::Infrastructure,
2019                FleetEnsureProgressState::PrerequisiteComplete,
2020            );
2021            return Ok(FleetEnsureReport {
2022                funding_review: None,
2023                actual_conservation: Some(actual_conservation),
2024                effects_applied: applied_count(&journal),
2025                plan: retained_plan,
2026                terminal: true,
2027            });
2028        }
2029
2030        terminal_state = state.clone();
2031        publish_terminal_state(
2032            operation_desired,
2033            &retained_plan,
2034            &journal,
2035            &mut terminal_state,
2036        )
2037        .map_err(|TerminalStatePublicationError::JournalIntegrity| {
2038            EnsureWorkflowError::JournalIntegrity
2039        })?;
2040        if completed_infrastructure_reinstall(operation_desired, &retained_plan, &journal) {
2041            terminal_state.active_registry = None;
2042        }
2043        project_current_fleet_inventory(&terminal_state)?;
2044        {
2045            // Replanning after a completed phase is one read-only decision.
2046            // Pacing expires its evidence; appending or issuing work happens only
2047            // after this scope has closed.
2048            let (terminal_observation, converged) =
2049                platform.with_planning_observations(|platform| {
2050                    let mut terminal_observation = platform
2051                        .observe(&retained_plan.operation_id, &terminal_state)
2052                        .map_err(EnsureWorkflowError::Platform)?;
2053                    let artifacts = resolve_desired_artifacts(root, operation_desired)?;
2054                    let observation_started = Instant::now();
2055                    let converged = loop {
2056                        let protocol_actions = platform
2057                            .protocol_actions(protocol_operation(&retained_plan), &terminal_state)
2058                            .map_err(EnsureWorkflowError::Platform)?;
2059                        match compile_plan(
2060                            operation_desired,
2061                            &artifacts,
2062                            &protocol_actions,
2063                            operation_desired_sha256,
2064                            requested_fleet,
2065                            &terminal_observation,
2066                            retained_plan.planned_at_time,
2067                            &retained_plan.operation_id,
2068                            None,
2069                        ) {
2070                            Ok(plan) => {
2071                                if journal.stalled_observations != 0 {
2072                                    journal.stalled_observations = 0;
2073                                    write_journal(&paths, &journal)?;
2074                                }
2075                                break plan;
2076                            }
2077                            Err(EnsurePolicyError::PendingRootOwnedBalance { name }) => {
2078                                journal.stalled_observations =
2079                                    journal.stalled_observations.saturating_add(1);
2080                                retain_observed_cycles(&mut state, &terminal_observation);
2081                                write_state(&paths, &state)?;
2082                                write_journal(&paths, &journal)?;
2083                                if journal.stalled_observations
2084                                    >= operation_desired.maximum_stalled_observations
2085                                {
2086                                    return Err(EnsureWorkflowError::RootOwnedObservationStalled {
2087                                        last_lifecycle: root_owned_lifecycle_label(
2088                                            &terminal_observation,
2089                                            &name,
2090                                        )
2091                                        .to_string(),
2092                                        observations: journal.stalled_observations,
2093                                        target: name,
2094                                    });
2095                                }
2096                                report_progress_state(
2097                                    platform,
2098                                    &retained_plan,
2099                                    &journal,
2100                                    FleetEnsurePhase::TerminalVerification,
2101                                    FleetEnsureProgressState::AwaitingProgress {
2102                                        elapsed_seconds: observation_started.elapsed().as_secs(),
2103                                        provisioning: None,
2104                                    },
2105                                );
2106                                platform.pace_root_owned_observation(
2107                                    &name,
2108                                    journal.stalled_observations,
2109                                );
2110                                terminal_observation = platform
2111                                    .observe(&retained_plan.operation_id, &terminal_state)
2112                                    .map_err(EnsureWorkflowError::Platform)?;
2113                            }
2114                            Err(error @ EnsurePolicyError::EstatePoolCapacity { .. }) => {
2115                                retain_observed_cycles(&mut terminal_state, &terminal_observation);
2116                                retain_completed_reinstalls(
2117                                    &mut terminal_state,
2118                                    &retained_plan,
2119                                    &journal,
2120                                );
2121                                write_state(&paths, &terminal_state)?;
2122                                journal.completion = FleetEnsureCompletion::ReplanRequired;
2123                                journal.stalled_observations = 0;
2124                                write_journal(&paths, &journal)?;
2125                                return Err(error.into());
2126                            }
2127                            Err(error) => return Err(error.into()),
2128                        }
2129                    };
2130                    Ok::<_, EnsureWorkflowError<P::Error>>((terminal_observation, converged))
2131                })?;
2132            if converged
2133                .canisters
2134                .iter()
2135                .any(|canister| !canister.actions.is_empty())
2136                || !converged.protocol_actions.is_empty()
2137            {
2138                if retained_plan.continuation.is_some() {
2139                    let appended = continuation::append(
2140                        &paths,
2141                        operation_desired,
2142                        &retained_plan,
2143                        &mut journal,
2144                        &terminal_state,
2145                        &terminal_observation,
2146                        converged,
2147                        platform,
2148                    );
2149                    match appended {
2150                        Ok(()) => {
2151                            write_state(&paths, &terminal_state)?;
2152                            state = terminal_state.clone();
2153                            continuation_observation = journal
2154                                .successor_phases
2155                                .last()
2156                                .and_then(|phase| phase.plan.as_ref())
2157                                .and_then(|phase| phase.protocol_actions.first())
2158                                .map(|first| ContinuationObservation {
2159                                    first_action_sha256: action_sha256(first),
2160                                    observation: terminal_observation,
2161                                });
2162                            report_progress_state(
2163                                platform,
2164                                &retained_plan,
2165                                &journal,
2166                                FleetEnsurePhase::TerminalVerification,
2167                                FleetEnsureProgressState::PrerequisiteComplete,
2168                            );
2169                            continue;
2170                        }
2171                        Err(EnsureWorkflowError::SuccessorReviewRequired { reason, review }) => {
2172                            retain_observed_cycles(&mut terminal_state, &terminal_observation);
2173                            retain_completed_reinstalls(
2174                                &mut terminal_state,
2175                                &retained_plan,
2176                                &journal,
2177                            );
2178                            write_state(&paths, &terminal_state)?;
2179                            journal.completion = FleetEnsureCompletion::ReplanRequired;
2180                            write_journal(&paths, &journal)?;
2181                            report_progress_state(
2182                                platform,
2183                                &retained_plan,
2184                                &journal,
2185                                FleetEnsurePhase::TerminalVerification,
2186                                FleetEnsureProgressState::ReviewRequired {
2187                                    reason,
2188                                    review: review.clone(),
2189                                },
2190                            );
2191                            return Err(EnsureWorkflowError::SuccessorReviewRequired {
2192                                reason,
2193                                review,
2194                            });
2195                        }
2196                        Err(error) => return Err(error),
2197                    }
2198                }
2199                retain_observed_cycles(&mut terminal_state, &terminal_observation);
2200                retain_completed_reinstalls(&mut terminal_state, &retained_plan, &journal);
2201                write_state(&paths, &terminal_state)?;
2202                journal.completion = FleetEnsureCompletion::ReplanRequired;
2203                journal.stalled_observations = 0;
2204                write_journal(&paths, &journal)?;
2205                let reason =
2206                    crate::fleet_ensure::model::FleetEnsureSuccessorReviewReason::AdditionalEffect;
2207                let review = Some(Box::new(
2208                    crate::fleet_ensure::ops::recovery::review_details(&converged),
2209                ));
2210                report_progress_state(
2211                    platform,
2212                    &retained_plan,
2213                    &journal,
2214                    FleetEnsurePhase::TerminalVerification,
2215                    FleetEnsureProgressState::ReviewRequired {
2216                        reason,
2217                        review: review.clone(),
2218                    },
2219                );
2220                return Err(EnsureWorkflowError::SuccessorReviewRequired { reason, review });
2221            }
2222        }
2223        break;
2224    }
2225    let terminal_inventory_operation_id =
2226        completed_inventory_operation(&retained_plan, &terminal_state)?;
2227    let terminal_inventory = platform
2228        .terminal_inventory(terminal_inventory_operation_id, &terminal_state)
2229        .map_err(EnsureWorkflowError::Platform)?;
2230    let terminal_cycles = terminal_inventory.controlled_cycles_by_principal.clone();
2231    merge_terminal_inventory(&mut terminal_state, terminal_inventory)?;
2232    project_current_fleet_inventory(&terminal_state)?;
2233    let mut final_observation = platform
2234        .observe(&retained_plan.operation_id, &terminal_state)
2235        .map_err(EnsureWorkflowError::Platform)?;
2236    attach_terminal_cycles(&mut final_observation, terminal_cycles)?;
2237    reinstall::verify_terminal_estate(&retained_plan, &final_observation)?;
2238    reinstall::verify_terminal_authority(&retained_plan, &terminal_state, platform)?;
2239    let actual_conservation = verify_terminal_conservation(
2240        &retained_plan,
2241        &journal,
2242        &terminal_state,
2243        &final_observation,
2244    )?;
2245    terminal_state.completed_reinstall_action_sha256.clear();
2246    terminal_state.completed_reinstall_operation_id = None;
2247    terminal_state.completed_reinstalls.clear();
2248    // Publish the fully validated topology before making its terminal journal visible to backup.
2249    write_state(&paths, &terminal_state)?;
2250    journal.completion = FleetEnsureCompletion::Converged;
2251    journal.stalled_observations = 0;
2252    crate::fleet_ensure::ops::terminal::retain(
2253        &paths,
2254        &retained_plan,
2255        &journal,
2256        &terminal_state,
2257        &actual_conservation,
2258        &funding_plan::<P::Error>(&retained_plan, &journal)?.conservation,
2259    )?;
2260    write_journal(&paths, &journal)?;
2261    report_progress(
2262        platform,
2263        &retained_plan,
2264        &journal,
2265        FleetEnsurePhase::Complete,
2266    );
2267    Ok(FleetEnsureReport {
2268        funding_review: None,
2269        actual_conservation: Some(actual_conservation),
2270        effects_applied: applied_count(&journal),
2271        plan: retained_plan,
2272        terminal: true,
2273    })
2274}
2275
2276fn action_progress_phase(action: &EnsureAction) -> FleetEnsurePhase {
2277    match action {
2278        EnsureAction::FleetProtocol { action, .. } => match action.as_ref() {
2279            CurrentFleetProtocolAction::ReconcilePoolAsset { .. } => {
2280                FleetEnsurePhase::ImportReconciliation
2281            }
2282            CurrentFleetProtocolAction::ProvisionComponents { .. } => {
2283                FleetEnsurePhase::WorkloadProvisioning
2284            }
2285            CurrentFleetProtocolAction::MaintainPoolReadiness { .. }
2286            | CurrentFleetProtocolAction::ObservePoolReadiness { .. } => {
2287                FleetEnsurePhase::PoolReadiness
2288            }
2289            _ => FleetEnsurePhase::ControlPlane,
2290        },
2291        _ => FleetEnsurePhase::Infrastructure,
2292    }
2293}
2294
2295fn report_progress<P: EnsurePlatform>(
2296    platform: &mut P,
2297    plan: &FleetEnsurePlan,
2298    journal: &FleetEnsureJournalRecord,
2299    phase: FleetEnsurePhase,
2300) {
2301    let state = if phase == FleetEnsurePhase::Complete {
2302        FleetEnsureProgressState::Complete
2303    } else {
2304        FleetEnsureProgressState::Advancing
2305    };
2306    report_progress_state(platform, plan, journal, phase, state);
2307}
2308
2309fn report_progress_state<P: EnsurePlatform>(
2310    platform: &mut P,
2311    plan: &FleetEnsurePlan,
2312    journal: &FleetEnsureJournalRecord,
2313    phase: FleetEnsurePhase,
2314    state: FleetEnsureProgressState,
2315) {
2316    platform.report_progress(FleetEnsureProgress {
2317        next_action: if matches!(
2318            state,
2319            FleetEnsureProgressState::Advancing
2320                | FleetEnsureProgressState::AwaitingProgress { .. }
2321                | FleetEnsureProgressState::FundingRequired
2322        ) && phase != FleetEnsurePhase::TerminalVerification
2323        {
2324            crate::fleet_ensure::ops::progress::next_action(
2325                &continuation::actions(plan, journal),
2326                journal,
2327            )
2328        } else {
2329            None
2330        },
2331        state,
2332        operation_id: plan.operation_id.clone(),
2333        plan_sha256: plan.plan_sha256.clone(),
2334        phase,
2335        applied_effects: applied_count(journal),
2336        reviewed_effects: continuation::actions(plan, journal).len()
2337            + journal
2338                .funding_reviews
2339                .iter()
2340                .filter(|review| review.effect.is_some())
2341                .count(),
2342    });
2343}
2344
2345const fn merge_observed_cycles(
2346    record: &mut EffectRecord,
2347    source_cycles: Option<u128>,
2348    destination_cycles: Option<u128>,
2349) {
2350    if let Some(cycles) = source_cycles {
2351        record.post_cycles = Some(cycles);
2352    }
2353    if let Some(cycles) = destination_cycles {
2354        record.destination_post_cycles = Some(cycles);
2355    }
2356}
2357
2358pub(super) fn retain_applied_create_authority<E>(
2359    desired: &crate::fleet_ensure::model::DesiredFleet,
2360    action: &EnsureAction,
2361    record: &EffectRecord,
2362    state: &mut FleetEnsureStateRecord,
2363) -> Result<bool, EnsureWorkflowError<E>>
2364where
2365    E: std::error::Error + 'static,
2366{
2367    let EnsureAction::Create {
2368        name,
2369        requested_initial_cycles,
2370        ..
2371    } = action
2372    else {
2373        return Ok(false);
2374    };
2375    if record.state != EffectState::Applied {
2376        return Err(EnsureWorkflowError::JournalIntegrity);
2377    }
2378    let configured = desired
2379        .canisters
2380        .iter()
2381        .find(|canister| {
2382            canister.name == *name
2383                && canister.presence == crate::fleet_ensure::model::DesiredPresence::Present
2384        })
2385        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2386    let principal = record
2387        .created_principal
2388        .as_deref()
2389        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2390    let cycles = record
2391        .post_cycles
2392        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2393    let maximum_observation_burn_cycles =
2394        crate::fleet_ensure::ops::maximum_creation_observation_burn(desired)
2395            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2396    if !create_identity_is_exact(name, record, state)
2397        || !create_balance_is_terminal(
2398            Some(cycles),
2399            *requested_initial_cycles,
2400            maximum_observation_burn_cycles,
2401        )
2402        || state
2403            .retained_cycles_by_principal
2404            .get(principal)
2405            .is_some_and(|retained| *retained != cycles)
2406    {
2407        return Err(EnsureWorkflowError::JournalIntegrity);
2408    }
2409    validate_fresh_pool_creation_balance(
2410        desired,
2411        configured,
2412        name,
2413        cycles,
2414        maximum_observation_burn_cycles,
2415        *requested_initial_cycles,
2416    )?;
2417
2418    let prior_topology = state.topology.get(name);
2419    let topology_identity_matches = prior_topology.is_some_and(|topology| {
2420        topology.kind == configured.kind
2421            && topology.parent == configured.parent
2422            && topology.protocol_binding == configured.protocol_binding
2423            && topology.role
2424                == configured
2425                    .protocol_binding
2426                    .as_ref()
2427                    .map(|binding| binding.role.to_string())
2428    });
2429    let topology = crate::fleet_ensure::model::FleetEnsureTopologyRecord {
2430        kind: configured.kind,
2431        module_hash: if topology_identity_matches {
2432            prior_topology.and_then(|topology| topology.module_hash.clone())
2433        } else {
2434            None
2435        },
2436        parent: configured.parent.clone(),
2437        protocol_binding: configured.protocol_binding.clone(),
2438        role: configured
2439            .protocol_binding
2440            .as_ref()
2441            .map(|binding| binding.role.to_string()),
2442    };
2443    let changed = state.retained_cycles_by_principal.get(principal) != Some(&cycles)
2444        || state.topology.get(name) != Some(&topology);
2445    state
2446        .retained_cycles_by_principal
2447        .insert(principal.to_string(), cycles);
2448    state.topology.insert(name.clone(), topology);
2449    Ok(changed)
2450}
2451
2452fn validate_fresh_pool_creation_balance<E>(
2453    desired: &crate::fleet_ensure::model::DesiredFleet,
2454    configured: &crate::fleet_ensure::model::DesiredCanister,
2455    name: &str,
2456    cycles: u128,
2457    maximum_observation_burn_cycles: u128,
2458    requested_initial_cycles: u128,
2459) -> Result<(), EnsureWorkflowError<E>>
2460where
2461    E: std::error::Error + 'static,
2462{
2463    if configured.kind == crate::fleet_ensure::model::DesiredCanisterKind::Pool
2464        && desired
2465            .bootstrap
2466            .as_ref()
2467            .is_some_and(|bootstrap| bootstrap.fresh_estate)
2468        && configured.principal.is_none()
2469        && configured.controllers.is_empty()
2470        && configured.parent.as_ref().is_some_and(|parent| {
2471            configured.controller_canisters.as_slice() == std::slice::from_ref(parent)
2472        })
2473    {
2474        let readiness_floor_cycles = configured
2475            .minimum_cycles
2476            .parse::<Cycles>()
2477            .map(|cycles| cycles.to_u128())
2478            .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
2479        let remaining_controller_burn_cycles = desired
2480            .maximum_update_burn_cycles
2481            .parse::<Cycles>()
2482            .map(|cycles| cycles.to_u128())
2483            .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
2484        let required_pre_finalization_balance_cycles = readiness_floor_cycles
2485            .checked_add(remaining_controller_burn_cycles)
2486            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2487        if cycles < required_pre_finalization_balance_cycles {
2488            return Err(EnsureWorkflowError::FreshPoolCreationUnderfunded(Box::new(
2489                FreshPoolCreationUnderfundedError {
2490                    canister: name.to_string(),
2491                    live_balance_cycles: cycles,
2492                    maximum_observation_burn_cycles,
2493                    pre_finalization_shortfall_cycles: required_pre_finalization_balance_cycles
2494                        - cycles,
2495                    readiness_floor_cycles,
2496                    readiness_shortfall_cycles: readiness_floor_cycles.saturating_sub(cycles),
2497                    remaining_controller_burn_cycles,
2498                    requested_creation_funding_cycles: requested_initial_cycles,
2499                    required_pre_finalization_balance_cycles,
2500                },
2501            )));
2502        }
2503    }
2504    Ok(())
2505}
2506
2507fn retain_applied_funding_cycles(
2508    state: &mut FleetEnsureStateRecord,
2509    action: &EnsureAction,
2510    record: &EffectRecord,
2511) -> bool {
2512    let EnsureAction::Fund { principal, .. } = action else {
2513        return false;
2514    };
2515    if record.state != EffectState::Applied || record.receipt.is_none() {
2516        return false;
2517    }
2518    let Some(cycles) = record.post_cycles else {
2519        return false;
2520    };
2521    let principal = principal.strip_prefix("created:").map_or_else(
2522        || Some(principal.as_str()),
2523        |name| state.pending_principals.get(name).map(String::as_str),
2524    );
2525    let Some(principal) = principal else {
2526        return false;
2527    };
2528    if state.retained_cycles_by_principal.get(principal) == Some(&cycles) {
2529        return false;
2530    }
2531    state
2532        .retained_cycles_by_principal
2533        .insert(principal.to_string(), cycles);
2534    true
2535}
2536
2537fn retain_created_canister_for_replan<E>(
2538    desired: &crate::fleet_ensure::model::DesiredFleet,
2539    name: &str,
2540    record: &EffectRecord,
2541    state: &mut FleetEnsureStateRecord,
2542) -> Result<(), EnsureWorkflowError<E>>
2543where
2544    E: std::error::Error + 'static,
2545{
2546    let configured = desired
2547        .canisters
2548        .iter()
2549        .find(|canister| {
2550            canister.name == name
2551                && canister.presence == crate::fleet_ensure::model::DesiredPresence::Present
2552        })
2553        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2554    let principal = record
2555        .created_principal
2556        .as_deref()
2557        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2558    let cycles = record
2559        .post_cycles
2560        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2561    if state.pending_principals.get(name).map(String::as_str) != Some(principal) {
2562        return Err(EnsureWorkflowError::JournalIntegrity);
2563    }
2564    state.pending_principals.remove(name);
2565    state
2566        .principals
2567        .insert(name.to_string(), principal.to_string());
2568    state
2569        .retained_cycles_by_principal
2570        .insert(principal.to_string(), cycles);
2571    state.topology.insert(
2572        name.to_string(),
2573        crate::fleet_ensure::model::FleetEnsureTopologyRecord {
2574            kind: configured.kind,
2575            module_hash: None,
2576            parent: configured.parent.clone(),
2577            protocol_binding: configured.protocol_binding.clone(),
2578            role: configured
2579                .protocol_binding
2580                .as_ref()
2581                .map(|binding| binding.role.to_string()),
2582        },
2583    );
2584    Ok(())
2585}
2586
2587fn create_record_is_exact(
2588    action: &EnsureAction,
2589    record: &EffectRecord,
2590    state: &FleetEnsureStateRecord,
2591) -> bool {
2592    let EnsureAction::Create {
2593        name,
2594        requested_initial_cycles,
2595        ..
2596    } = action
2597    else {
2598        return true;
2599    };
2600    match record.state {
2601        EffectState::Intent => {
2602            record.created_principal.is_none()
2603                && record.receipt.is_none()
2604                && record.post_cycles.is_none()
2605        }
2606        EffectState::Issued => {
2607            create_identity_is_exact(name, record, state)
2608                && record.post_cycles == Some(*requested_initial_cycles)
2609        }
2610        EffectState::Applied => create_identity_is_exact(name, record, state),
2611    }
2612}
2613
2614fn create_identity_is_exact(
2615    name: &str,
2616    record: &EffectRecord,
2617    state: &FleetEnsureStateRecord,
2618) -> bool {
2619    let Some(created_principal) = record
2620        .created_principal
2621        .as_deref()
2622        .filter(|value| !value.is_empty())
2623    else {
2624        return false;
2625    };
2626    if record.receipt.as_deref().is_none_or(str::is_empty) {
2627        return false;
2628    }
2629    state_create_principal_matches(name, created_principal, state, true)
2630}
2631
2632fn state_create_principal_matches(
2633    name: &str,
2634    created_principal: &str,
2635    state: &FleetEnsureStateRecord,
2636    require_retained: bool,
2637) -> bool {
2638    let pending = state.pending_principals.get(name).map(String::as_str);
2639    let terminal = state.principals.get(name).map(String::as_str);
2640    let retained = pending.is_some() || terminal.is_some();
2641    let pending_matches = pending.is_none_or(|principal| principal == created_principal);
2642    let terminal_matches = terminal.is_none_or(|principal| principal == created_principal);
2643    (!require_retained || retained) && pending_matches && terminal_matches
2644}
2645
2646fn create_outcome_is_exact(
2647    action: &EnsureAction,
2648    record: &EffectRecord,
2649    state: &FleetEnsureStateRecord,
2650    outcome: &crate::fleet_ensure::ops::EffectOutcome,
2651) -> bool {
2652    if matches!(action, EnsureAction::FundEstate { .. }) {
2653        return outcome.created_principal.is_none()
2654            && outcome.post_cycles.is_none()
2655            && outcome
2656                .receipt
2657                .as_deref()
2658                .is_some_and(|receipt| !receipt.is_empty());
2659    }
2660    let EnsureAction::Create {
2661        name,
2662        requested_initial_cycles,
2663        ..
2664    } = action
2665    else {
2666        return true;
2667    };
2668    let Some(created_principal) = outcome
2669        .created_principal
2670        .as_deref()
2671        .filter(|value| !value.is_empty())
2672    else {
2673        return false;
2674    };
2675    let Some(receipt) = outcome.receipt.as_deref().filter(|value| !value.is_empty()) else {
2676        return false;
2677    };
2678    if outcome.post_cycles != Some(*requested_initial_cycles) {
2679        return false;
2680    }
2681    let principal_matches = record
2682        .created_principal
2683        .as_deref()
2684        .is_none_or(|retained| retained == created_principal);
2685    let receipt_matches = record
2686        .receipt
2687        .as_deref()
2688        .is_none_or(|retained| retained == receipt);
2689    let state_matches = state_create_principal_matches(name, created_principal, state, false);
2690    principal_matches && receipt_matches && state_matches
2691}
2692
2693fn verify_fresh_plan<P>(
2694    root: &Path,
2695    desired: &crate::fleet_ensure::model::DesiredFleet,
2696    desired_sha256: &str,
2697    requested_fleet: &str,
2698    retained_plan: &FleetEnsurePlan,
2699    platform: &mut P,
2700    state: &FleetEnsureStateRecord,
2701) -> Result<(FleetObservation, u128), EnsureWorkflowError<P::Error>>
2702where
2703    P: EnsurePlatform,
2704{
2705    if retained_plan.scope == FleetEnsurePlanScope::InfrastructureBootstrap {
2706        return infrastructure_bootstrap::verify_before_apply(root, retained_plan, state, platform);
2707    }
2708    if retained_plan.reinstall.is_some() {
2709        return reinstall::verify_before_apply(root, desired, retained_plan, platform, state);
2710    }
2711    if retained_plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite {
2712        return root_reinstall::verify_before_apply(root, desired, retained_plan, platform, state);
2713    }
2714    if retained_plan.scope == FleetEnsurePlanScope::RootStartPrerequisite {
2715        let targets = reviewed_root_start_targets(retained_plan)?;
2716        let management = platform
2717            .observe_root_management(state, &targets)
2718            .map_err(EnsureWorkflowError::Platform)?
2719            .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2720        let current = recompile_root_start_prerequisite_plan(
2721            RootStartPlanInput {
2722                state,
2723                authority: retained_plan.root_start_authority.as_deref(),
2724                created_at_time: retained_plan.planned_at_time,
2725                desired,
2726                desired_sha256,
2727                observation: &management,
2728                requested_fleet,
2729            },
2730            &targets,
2731        )?;
2732        if !compatible_root_start_prerequisite(retained_plan, &current, desired) {
2733            return Err(EnsureWorkflowError::DriftedBeforeApply);
2734        }
2735        let observation = root_management_fleet_observation(&management, &targets)?;
2736        return Ok((observation, current.conservation.observed_controlled_cycles));
2737    }
2738    let mut observation = platform
2739        .observe(&retained_plan.operation_id, state)
2740        .map_err(EnsureWorkflowError::Platform)?;
2741    let terminal_inventory_operation_id =
2742        fresh_plan_inventory_operation(root, retained_plan, state)?;
2743    if let Some(operation_id) = terminal_inventory_operation_id {
2744        attach_terminal_inventory_cycles(operation_id, state, platform, &mut observation)?;
2745    }
2746    let artifacts = resolve_desired_artifacts(root, desired)?;
2747    let mut retained_observations = 0_u32;
2748    let mut current = loop {
2749        let protocol_actions = platform
2750            .protocol_actions(protocol_operation(retained_plan), state)
2751            .map_err(EnsureWorkflowError::Platform)?;
2752        match compile_plan(
2753            desired,
2754            &artifacts,
2755            &protocol_actions,
2756            desired_sha256,
2757            requested_fleet,
2758            &observation,
2759            retained_plan.planned_at_time,
2760            &retained_plan.operation_id,
2761            None,
2762        ) {
2763            Ok(plan) => break plan,
2764            Err(EnsurePolicyError::PendingRootOwnedBalance { name }) => {
2765                retained_observations = retained_observations.saturating_add(1);
2766                if retained_observations >= desired.maximum_stalled_observations {
2767                    return Err(EnsureWorkflowError::RootOwnedObservationStalled {
2768                        last_lifecycle: root_owned_lifecycle_label(&observation, &name).to_string(),
2769                        observations: retained_observations,
2770                        target: name,
2771                    });
2772                }
2773                platform.pace_root_owned_observation(&name, retained_observations);
2774                observation = platform
2775                    .observe(&retained_plan.operation_id, state)
2776                    .map_err(EnsureWorkflowError::Platform)?;
2777                if let Some(operation_id) = terminal_inventory_operation_id {
2778                    attach_terminal_inventory_cycles(
2779                        operation_id,
2780                        state,
2781                        platform,
2782                        &mut observation,
2783                    )?;
2784                }
2785            }
2786            Err(error) => return Err(error.into()),
2787        }
2788    };
2789    bind_terminal_inventory_operation(
2790        &mut current,
2791        retained_plan.terminal_inventory_operation_id.clone(),
2792    );
2793    if current.plan_sha256 != retained_plan.plan_sha256
2794        && !compatible_after_bounded_observation(retained_plan, &current, desired, &observation)
2795    {
2796        return Err(EnsureWorkflowError::DriftedBeforeApply);
2797    }
2798    Ok((observation, current.conservation.observed_controlled_cycles))
2799}
2800
2801fn estate_funding_requirement<E>(
2802    plan: &FleetEnsurePlan,
2803    state: &FleetEnsureStateRecord,
2804    observation: &FleetObservation,
2805) -> Result<Option<EstateFundingRequiredRecord>, EnsureWorkflowError<E>>
2806where
2807    E: std::error::Error + 'static,
2808{
2809    for reviewed in &plan.conservation.estate_funding_domains {
2810        let observed = observation
2811            .estate_funding_domains
2812            .get(&reviewed.root)
2813            .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2814        if observed.cycles_ledger != reviewed.cycles_ledger {
2815            return Err(EnsureWorkflowError::PlanIntegrity);
2816        }
2817        let root_principal = observed
2818            .root_principal
2819            .as_deref()
2820            .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2821        if reviewed_estate_root_principal(reviewed, state) != Some(root_principal) {
2822            return Err(EnsureWorkflowError::PlanIntegrity);
2823        }
2824        let available_cycles = observed
2825            .balance_cycles
2826            .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2827        let reviewed_available = reviewed.available_cycles.unwrap_or_default();
2828        // A surplus stays controlled and grants no additional spending authority.
2829        // An unexplained Ledger debit still requires exact reconciliation.
2830        if available_cycles < reviewed_available {
2831            return Err(EnsureWorkflowError::DriftedBeforeApply);
2832        }
2833        if reviewed.required_creation_count == 0 {
2834            continue;
2835        }
2836        if available_cycles < reviewed.maximum_creation_debit_cycles {
2837            return Ok(Some(EstateFundingRequiredRecord {
2838                attempt_count: None,
2839                available_cycles,
2840                creation_amount_cycles: reviewed.creation_amount_cycles,
2841                creation_execution_margin_cycles: reviewed.creation_execution_margin_cycles,
2842                readiness_floor_cycles: reviewed.readiness_floor_cycles,
2843                cycles_ledger: reviewed.cycles_ledger.clone(),
2844                ledger_fee_cycles: reviewed.ledger_fee_cycles,
2845                management_creation_fee_cycles: reviewed.management_creation_fee_cycles,
2846                maximum_creation_debit_cycles: reviewed.maximum_creation_debit_cycles,
2847                maximum_creation_fee_cycles: reviewed.maximum_creation_fee_cycles,
2848                operation_id: plan.operation_id.clone(),
2849                pending_creation_operation_id: None,
2850                plan_sha256: plan.plan_sha256.clone(),
2851                required_creation_count: reviewed.required_creation_count,
2852                root: reviewed.root.clone(),
2853                root_principal: root_principal.to_string(),
2854                last_attempt_at_ns: None,
2855                retry_at_ns: None,
2856                shortfall_cycles: reviewed
2857                    .maximum_creation_debit_cycles
2858                    .saturating_sub(available_cycles),
2859            }));
2860        }
2861    }
2862    Ok(None)
2863}
2864
2865fn protocol_estate_funding_requirement<E>(
2866    plan: &FleetEnsurePlan,
2867    state: &FleetEnsureStateRecord,
2868    funding: &canic_contracts::dto::component_provisioning::RootEstateFundingRequired,
2869) -> Result<EstateFundingRequiredRecord, EnsureWorkflowError<E>>
2870where
2871    E: std::error::Error + 'static,
2872{
2873    let root_principal = funding.root.to_text();
2874    let reviewed = plan
2875        .conservation
2876        .estate_funding_domains
2877        .iter()
2878        .find(|domain| reviewed_estate_root_principal(domain, state) == Some(&root_principal))
2879        .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2880    let creation_amount_cycles = funding.creation_amount.to_u128();
2881    let ledger_fee_cycles = funding.ledger_fee.to_u128();
2882    let management_creation_fee_cycles = funding.management_creation_fee.to_u128();
2883    let required_cycles = creation_amount_cycles
2884        .checked_add(ledger_fee_cycles)
2885        .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2886    let maximum_creation_fee_cycles = management_creation_fee_cycles
2887        .checked_add(ledger_fee_cycles)
2888        .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2889    let exact_reviewed_authority = reviewed.required_creation_count > 0
2890        && reviewed.cycles_ledger == funding.cycles_ledger.to_text()
2891        && reviewed.creation_amount_cycles == creation_amount_cycles
2892        && reviewed.creation_execution_margin_cycles == funding.execution_margin.to_u128()
2893        && reviewed.readiness_floor_cycles == funding.readiness_floor.to_u128()
2894        && reviewed.ledger_fee_cycles == ledger_fee_cycles
2895        && reviewed.management_creation_fee_cycles == management_creation_fee_cycles;
2896    let exact_runtime_arithmetic = funding.required.to_u128() == required_cycles
2897        && funding.available < funding.required
2898        && funding.shortfall.to_u128()
2899            == required_cycles.saturating_sub(funding.available.to_u128());
2900    if !exact_reviewed_authority || !exact_runtime_arithmetic {
2901        return Err(EnsureWorkflowError::PlanIntegrity);
2902    }
2903    Ok(EstateFundingRequiredRecord {
2904        attempt_count: Some(funding.attempt_count),
2905        available_cycles: funding.available.to_u128(),
2906        creation_amount_cycles,
2907        cycles_ledger: funding.cycles_ledger.to_text(),
2908        creation_execution_margin_cycles: funding.execution_margin.to_u128(),
2909        readiness_floor_cycles: funding.readiness_floor.to_u128(),
2910        ledger_fee_cycles,
2911        management_creation_fee_cycles,
2912        maximum_creation_debit_cycles: required_cycles,
2913        maximum_creation_fee_cycles,
2914        operation_id: plan.operation_id.clone(),
2915        pending_creation_operation_id: Some(canic_core::cdk::utils::hash::hex_bytes(
2916            funding.operation_id,
2917        )),
2918        plan_sha256: plan.plan_sha256.clone(),
2919        required_creation_count: 1,
2920        root: reviewed.root.clone(),
2921        root_principal,
2922        last_attempt_at_ns: funding.last_attempt_at_ns,
2923        retry_at_ns: Some(funding.retry_at_ns),
2924        shortfall_cycles: funding.shortfall.to_u128(),
2925    })
2926}
2927
2928fn estate_funding_error(required: &EstateFundingRequiredRecord) -> EstateFundingRequiredError {
2929    EstateFundingRequiredError {
2930        attempt_count: required.attempt_count,
2931        available_cycles: required.available_cycles,
2932        creation_amount_cycles: required.creation_amount_cycles,
2933        creation_execution_margin_cycles: required.creation_execution_margin_cycles,
2934        readiness_floor_cycles: required.readiness_floor_cycles,
2935        cycles_ledger: required.cycles_ledger.clone(),
2936        ledger_fee_cycles: required.ledger_fee_cycles,
2937        management_creation_fee_cycles: required.management_creation_fee_cycles,
2938        maximum_creation_debit_cycles: required.maximum_creation_debit_cycles,
2939        maximum_creation_fee_cycles: required.maximum_creation_fee_cycles,
2940        pending_creation_operation_id: required.pending_creation_operation_id.clone(),
2941        required_creation_count: required.required_creation_count,
2942        root: required.root.clone(),
2943        root_principal: required.root_principal.clone(),
2944        last_attempt_at_ns: required.last_attempt_at_ns,
2945        retry_at_ns: required.retry_at_ns,
2946        shortfall_cycles: required.shortfall_cycles,
2947    }
2948}
2949
2950fn retain_estate_funding_pause<E>(
2951    paths: &EnsurePaths,
2952    journal: &mut FleetEnsureJournalRecord,
2953    required: Option<&EstateFundingRequiredRecord>,
2954) -> Result<bool, EnsureWorkflowError<E>>
2955where
2956    E: std::error::Error + 'static,
2957{
2958    if journal.estate_funding_required.as_ref() == required {
2959        return Ok(false);
2960    }
2961    journal.estate_funding_required = required.cloned();
2962    write_journal(paths, journal)?;
2963    Ok(true)
2964}
2965
2966fn reviewed_estate_root_principal<'a>(
2967    domain: &'a crate::fleet_ensure::model::EstateFundingDomainPlan,
2968    state: &'a FleetEnsureStateRecord,
2969) -> Option<&'a str> {
2970    domain
2971        .root_principal
2972        .as_deref()
2973        .or_else(|| {
2974            state
2975                .pending_principals
2976                .get(&domain.root)
2977                .map(String::as_str)
2978        })
2979        .or_else(|| state.principals.get(&domain.root).map(String::as_str))
2980}
2981
2982/// Evidence captured after all preceding effects, bound to one successor's first action.
2983struct ContinuationObservation {
2984    first_action_sha256: String,
2985    observation: FleetObservation,
2986}
2987
2988fn prior_fleet_protocol_effect_started(
2989    actions: &[&EnsureAction],
2990    journal: &FleetEnsureJournalRecord,
2991    current_index: usize,
2992) -> bool {
2993    actions
2994        .iter()
2995        .take(current_index)
2996        .zip(&journal.effects)
2997        .any(|(action, effect)| {
2998            matches!(action, EnsureAction::FleetProtocol { .. })
2999                && matches!(effect.state, EffectState::Issued | EffectState::Applied)
3000        })
3001}
3002
3003// A later Host operation observes the completed Component owner's exact receipts.
3004// Changing Host retry settings must not allocate a new infrastructure protocol identity.
3005fn protocol_operation(plan: &FleetEnsurePlan) -> &str {
3006    plan.terminal_inventory_operation_id
3007        .as_deref()
3008        .unwrap_or(&plan.operation_id)
3009}
3010
3011fn completed_inventory_operation<'a, E>(
3012    plan: &'a FleetEnsurePlan,
3013    state: &FleetEnsureStateRecord,
3014) -> Result<&'a str, EnsureWorkflowError<E>>
3015where
3016    E: std::error::Error + 'static,
3017{
3018    if plan.terminal_inventory_operation_id.is_some() {
3019        reviewed_terminal_inventory_operation(plan, state)?
3020            .ok_or(EnsureWorkflowError::PlanIntegrity)
3021    } else {
3022        Ok(&plan.operation_id)
3023    }
3024}
3025
3026fn fresh_plan_inventory_operation<'a, E>(
3027    root: &Path,
3028    plan: &'a FleetEnsurePlan,
3029    state: &FleetEnsureStateRecord,
3030) -> Result<Option<&'a str>, EnsureWorkflowError<E>>
3031where
3032    E: std::error::Error + 'static,
3033{
3034    if state.active_registry.is_some() && plan.terminal_inventory_operation_id.is_none() {
3035        crate::fleet_ensure::ops::infrastructure_bootstrap::convergence::verify_origin(
3036            &EnsurePaths::under(root, &plan.environment, &plan.fleet),
3037            plan,
3038            state,
3039        )?;
3040        Ok(None)
3041    } else {
3042        reviewed_terminal_inventory_operation(plan, state)
3043    }
3044}
3045
3046fn reviewed_terminal_inventory_operation<'a, E>(
3047    plan: &'a FleetEnsurePlan,
3048    state: &FleetEnsureStateRecord,
3049) -> Result<Option<&'a str>, EnsureWorkflowError<E>>
3050where
3051    E: std::error::Error + 'static,
3052{
3053    match (
3054        state.active_registry.is_some(),
3055        plan.terminal_inventory_operation_id.as_deref(),
3056    ) {
3057        (false, None) => Ok(None),
3058        (true, Some(operation_id)) if !operation_id.is_empty() => Ok(Some(operation_id)),
3059        _ => Err(EnsureWorkflowError::PlanIntegrity),
3060    }
3061}
3062
3063fn complete_root_start_prerequisite<P>(
3064    desired: &crate::fleet_ensure::model::DesiredFleet,
3065    retained_plan: &FleetEnsurePlan,
3066    platform: &mut P,
3067    state: &FleetEnsureStateRecord,
3068    journal: &FleetEnsureJournalRecord,
3069) -> Result<ActualCycleConservation, EnsureWorkflowError<P::Error>>
3070where
3071    P: EnsurePlatform,
3072{
3073    let targets = reviewed_root_start_targets(retained_plan)?;
3074    let management = platform
3075        .observe_root_management(state, &targets)
3076        .map_err(EnsureWorkflowError::Platform)?
3077        .ok_or(EnsureWorkflowError::PlanIntegrity)?;
3078    let current = recompile_root_start_prerequisite_plan(
3079        RootStartPlanInput {
3080            state,
3081            authority: retained_plan.root_start_authority.as_deref(),
3082            created_at_time: retained_plan.planned_at_time,
3083            desired,
3084            desired_sha256: &retained_plan.desired_sha256,
3085            observation: &management,
3086            requested_fleet: &retained_plan.fleet,
3087        },
3088        &targets,
3089    )?;
3090    if !compatible_root_start_prerequisite(retained_plan, &current, desired) {
3091        return Err(EnsureWorkflowError::ConvergenceDrift);
3092    }
3093    if management.roots.iter().any(|(name, observed)| {
3094        targets.contains(name)
3095            && observed.live.status != crate::fleet_ensure::model::CanisterRuntimeStatus::Running
3096    }) {
3097        return Err(EnsureWorkflowError::ConvergenceDrift);
3098    }
3099    let terminal = root_management_fleet_observation(&management, &targets)?;
3100    verify_terminal_conservation(retained_plan, journal, state, &terminal)
3101}
3102
3103fn verify_terminal_root_start_replay<P>(
3104    desired: &crate::fleet_ensure::model::DesiredFleet,
3105    plan: &FleetEnsurePlan,
3106    journal: &FleetEnsureJournalRecord,
3107    requested_fleet: &str,
3108    platform: &mut P,
3109    state: &FleetEnsureStateRecord,
3110) -> Result<(), EnsureWorkflowError<P::Error>>
3111where
3112    P: EnsurePlatform,
3113{
3114    verify_journal(journal, plan, requested_fleet, state)?;
3115    let actions = ordered_actions(plan);
3116    if actions.len() != journal.effects.len()
3117        || journal
3118            .effects
3119            .iter()
3120            .any(|effect| effect.state != EffectState::Applied)
3121    {
3122        return Err(EnsureWorkflowError::JournalIntegrity);
3123    }
3124    for (action, record) in actions.into_iter().zip(&journal.effects) {
3125        if action_sha256(action) != record.action_sha256
3126            || !platform
3127                .observe_effect(&journal.operation_id, action, record, state)
3128                .map_err(EnsureWorkflowError::Platform)?
3129                .applied
3130        {
3131            return Err(EnsureWorkflowError::ConvergenceDrift);
3132        }
3133    }
3134    let targets = reviewed_root_start_targets(plan)?;
3135    let management = platform
3136        .observe_root_management(state, &targets)
3137        .map_err(EnsureWorkflowError::Platform)?
3138        .ok_or(EnsureWorkflowError::PlanIntegrity)?;
3139    let current = recompile_root_start_prerequisite_plan(
3140        RootStartPlanInput {
3141            state,
3142            authority: plan.root_start_authority.as_deref(),
3143            created_at_time: plan.planned_at_time,
3144            desired,
3145            desired_sha256: &plan.desired_sha256,
3146            observation: &management,
3147            requested_fleet: &plan.fleet,
3148        },
3149        &targets,
3150    )?;
3151    if normalized_plan(plan) != normalized_plan(&current)
3152        || management.roots.iter().any(|(name, observed)| {
3153            targets.contains(name)
3154                && observed.live.status
3155                    != crate::fleet_ensure::model::CanisterRuntimeStatus::Running
3156        })
3157    {
3158        return Err(EnsureWorkflowError::ConvergenceDrift);
3159    }
3160    Ok(())
3161}
3162
3163fn reviewed_root_start_targets<E>(
3164    plan: &FleetEnsurePlan,
3165) -> Result<BTreeSet<String>, EnsureWorkflowError<E>>
3166where
3167    E: std::error::Error + 'static,
3168{
3169    if plan.scope != FleetEnsurePlanScope::RootStartPrerequisite
3170        || !plan.protocol_actions.is_empty()
3171        || plan.canisters.is_empty()
3172        || plan.conservation.maximum_new_funding_cycles != 0
3173        || plan.conservation.maximum_operator_debit_cycles != 0
3174        || plan.conservation.maximum_unavoidable_fee_cycles != 0
3175        || !plan.conservation.estate_funding_domains.is_empty()
3176        || plan.conservation.scheduled_transfer_cycles != 0
3177        || plan.conservation.observed_controlled_cycles
3178            != plan.conservation.retained_in_reused_canisters_cycles
3179        || plan
3180            .conservation
3181            .observed_controlled_cycles
3182            .checked_sub(plan.conservation.maximum_execution_burn_cycles)
3183            != Some(plan.conservation.expected_post_operation_cycles)
3184    {
3185        return Err(EnsureWorkflowError::PlanIntegrity);
3186    }
3187    let mut targets = BTreeSet::new();
3188    for canister in &plan.canisters {
3189        if canister.disposition != CanisterDisposition::Reuse
3190            || canister.principal.is_none()
3191            || !matches!(
3192                canister.actions.as_slice(),
3193                [EnsureAction::Start { name, principal }]
3194                    if name == &canister.name
3195                        && Some(principal) == canister.principal.as_ref()
3196            )
3197            || !targets.insert(canister.name.clone())
3198        {
3199            return Err(EnsureWorkflowError::PlanIntegrity);
3200        }
3201    }
3202    Ok(targets)
3203}
3204
3205fn root_management_fleet_observation<E>(
3206    management: &RootManagementObservation,
3207    targets: &BTreeSet<String>,
3208) -> Result<FleetObservation, EnsureWorkflowError<E>>
3209where
3210    E: std::error::Error + 'static,
3211{
3212    let canisters = targets
3213        .iter()
3214        .map(|name| {
3215            management
3216                .roots
3217                .get(name)
3218                .map(|observed| (name.clone(), Some(observed.live.clone())))
3219                .ok_or(EnsureWorkflowError::PlanIntegrity)
3220        })
3221        .collect::<Result<BTreeMap<_, _>, _>>()?;
3222    Ok(FleetObservation {
3223        additional_controlled_cycles: BTreeMap::new(),
3224        canisters,
3225        estate_funding_domains: BTreeMap::new(),
3226        ledger_fee_cycles: 0,
3227        operator_cycles: management.operator_cycles,
3228        protocol_ready: BTreeMap::new(),
3229    })
3230}
3231
3232fn compatible_root_start_prerequisite(
3233    retained: &FleetEnsurePlan,
3234    current: &FleetEnsurePlan,
3235    desired: &crate::fleet_ensure::model::DesiredFleet,
3236) -> bool {
3237    let Ok(maximum_movement) = desired
3238        .maximum_update_burn_cycles
3239        .parse::<Cycles>()
3240        .map(|cycles| cycles.to_u128())
3241    else {
3242        return false;
3243    };
3244    retained.canisters.len() == current.canisters.len()
3245        && retained
3246            .canisters
3247            .iter()
3248            .zip(&current.canisters)
3249            .all(|(retained, current)| {
3250                retained
3251                    .observed_cycles
3252                    .saturating_sub(current.observed_cycles)
3253                    <= maximum_movement
3254            })
3255        && normalized_plan(retained) == normalized_plan(current)
3256}
3257
3258fn attach_terminal_inventory_cycles<P>(
3259    operation_id: &str,
3260    state: &FleetEnsureStateRecord,
3261    platform: &mut P,
3262    observation: &mut FleetObservation,
3263) -> Result<(), EnsureWorkflowError<P::Error>>
3264where
3265    P: EnsurePlatform,
3266{
3267    let inventory = platform
3268        .terminal_inventory(operation_id, state)
3269        .map_err(EnsureWorkflowError::Platform)?;
3270    attach_terminal_cycles(observation, inventory.controlled_cycles_by_principal)
3271}
3272
3273pub(super) fn completed_infrastructure_reinstall(
3274    desired: &crate::fleet_ensure::model::DesiredFleet,
3275    plan: &FleetEnsurePlan,
3276    journal: &FleetEnsureJournalRecord,
3277) -> bool {
3278    if desired.protocol.is_none() {
3279        return false;
3280    }
3281    let infrastructure = desired.canisters.iter().filter(|canister| {
3282        canister.presence == crate::fleet_ensure::model::DesiredPresence::Present
3283            && matches!(
3284                canister.kind,
3285                crate::fleet_ensure::model::DesiredCanisterKind::Coordinator
3286                    | crate::fleet_ensure::model::DesiredCanisterKind::Root
3287                    | crate::fleet_ensure::model::DesiredCanisterKind::Store
3288            )
3289    });
3290    let mut count = 0_usize;
3291    for configured in infrastructure {
3292        count += 1;
3293        let Some(action) = plan
3294            .canisters
3295            .iter()
3296            .find(|canister| {
3297                canister.name == configured.name
3298                    && canister.disposition == CanisterDisposition::Reinstall
3299            })
3300            .and_then(|canister| {
3301                canister.actions.iter().find(|action| {
3302                    matches!(
3303                        action,
3304                        EnsureAction::Install {
3305                            mode: crate::fleet_ensure::model::InstallMode::Reinstall,
3306                            name,
3307                            ..
3308                        } if name == &configured.name
3309                    )
3310                })
3311            })
3312        else {
3313            return false;
3314        };
3315        let action_sha256 = action_sha256(action);
3316        if !journal.effects.iter().any(|effect| {
3317            effect.action_sha256 == action_sha256 && effect.state == EffectState::Applied
3318        }) {
3319            return false;
3320        }
3321    }
3322    count > 0
3323}
3324
3325fn compatible_after_bounded_observation(
3326    retained: &FleetEnsurePlan,
3327    current: &FleetEnsurePlan,
3328    desired: &crate::fleet_ensure::model::DesiredFleet,
3329    observation: &FleetObservation,
3330) -> bool {
3331    let Ok(maximum_observation_movement) = desired
3332        .maximum_observation_burn_cycles
3333        .parse::<Cycles>()
3334        .map(|cycles| cycles.to_u128())
3335    else {
3336        return false;
3337    };
3338    if retained.canisters.len() != current.canisters.len() {
3339        return false;
3340    }
3341    for (retained_canister, current_canister) in retained.canisters.iter().zip(&current.canisters) {
3342        let movement = retained_canister
3343            .observed_cycles
3344            .saturating_sub(current_canister.observed_cycles);
3345        if movement > maximum_observation_movement {
3346            return false;
3347        }
3348    }
3349    if !retained_funding_remains_sufficient(retained, desired, observation) {
3350        return false;
3351    }
3352    normalized_plan(retained) == normalized_plan(current)
3353}
3354
3355fn retained_funding_remains_sufficient(
3356    retained: &FleetEnsurePlan,
3357    desired: &crate::fleet_ensure::model::DesiredFleet,
3358    observation: &FleetObservation,
3359) -> bool {
3360    retained
3361        .canisters
3362        .iter()
3363        .flat_map(|canister| &canister.actions)
3364        .filter_map(|action| match action {
3365            EnsureAction::Fund {
3366                amount,
3367                name,
3368                pool_funding,
3369                principal,
3370                ..
3371            } => Some((amount, name, pool_funding, principal)),
3372            _ => None,
3373        })
3374        .all(|(amount, name, pool_funding, principal)| {
3375            if let Some(root) = pool_funding {
3376                return observation
3377                    .estate_funding_domains
3378                    .values()
3379                    .find(|domain| domain.root_principal.as_ref() == Some(&root.root))
3380                    .and_then(|domain| domain.pool.as_ref())
3381                    .is_some_and(|pool| {
3382                        pool.assets.iter().any(|asset| {
3383                            asset.principal == *principal
3384                                && asset.lifecycle == root.lifecycle
3385                                && asset
3386                                    .cycles
3387                                    .checked_add(*amount)
3388                                    .is_some_and(|funded| funded >= pool.readiness_floor_cycles)
3389                        })
3390                    });
3391            }
3392            let Some(live) = observation.canisters.get(name).and_then(Option::as_ref) else {
3393                return false;
3394            };
3395            let Some(configured) = desired
3396                .canisters
3397                .iter()
3398                .find(|configured| configured.name == *name)
3399            else {
3400                return false;
3401            };
3402            let Ok(minimum) = configured
3403                .minimum_cycles
3404                .parse::<Cycles>()
3405                .map(|cycles| cycles.to_u128())
3406            else {
3407                return false;
3408            };
3409            live.cycles
3410                .checked_add(*amount)
3411                .is_some_and(|funded| funded >= minimum)
3412        })
3413}
3414
3415fn normalized_plan(plan: &FleetEnsurePlan) -> FleetEnsurePlan {
3416    let mut normalized = plan.clone();
3417    normalized.plan_sha256.clear();
3418    normalized.reviewed_desired = None;
3419    if let Some(review) = &mut normalized.recovery_review {
3420        // These estimates move with bounded observations; they grant no funding authority.
3421        review.continuation_reserve_cycles = 0;
3422        for funding in &mut review.known_pool_funding {
3423            funding.amount_cycles = 0;
3424            funding.funding_deficit_cycles = 0;
3425        }
3426    }
3427    normalized.conservation = CycleConservation {
3428        estate_funding_domains: plan
3429            .conservation
3430            .estate_funding_domains
3431            .iter()
3432            .cloned()
3433            .map(|mut domain| {
3434                domain.available_cycles = None;
3435                domain.maximum_funding_cycles = 0;
3436                domain.root_principal = None;
3437                domain.shortfall_cycles = 0;
3438                domain
3439            })
3440            .collect(),
3441        expected_post_operation_cycles: 0,
3442        maximum_execution_burn_cycles: 0,
3443        maximum_new_funding_cycles: 0,
3444        maximum_operator_debit_cycles: 0,
3445        maximum_unavoidable_fee_cycles: 0,
3446        observed_controlled_cycles: 0,
3447        retained_in_reused_canisters_cycles: 0,
3448        scheduled_transfer_cycles: 0,
3449    };
3450    for canister in &mut normalized.canisters {
3451        canister.observed_cycles = 0;
3452        for action in &mut canister.actions {
3453            if let EnsureAction::Fund {
3454                amount,
3455                funding_deficit_cycles,
3456                ..
3457            } = action
3458            {
3459                *amount = 0;
3460                *funding_deficit_cycles = 0;
3461            }
3462        }
3463    }
3464    normalized
3465}
3466
3467fn attach_terminal_cycles<E>(
3468    observation: &mut FleetObservation,
3469    additional: BTreeMap<String, u128>,
3470) -> Result<(), EnsureWorkflowError<E>>
3471where
3472    E: std::error::Error + 'static,
3473{
3474    let mut retained = BTreeMap::new();
3475    for (principal, cycles) in additional {
3476        let Some(configured) = observation
3477            .canisters
3478            .values_mut()
3479            .filter_map(Option::as_mut)
3480            .find(|canister| canister.principal == principal)
3481        else {
3482            retained.insert(principal, cycles);
3483            continue;
3484        };
3485        match configured.root_owned_lifecycle {
3486            Some(crate::fleet_ensure::model::RootOwnedCanisterLifecycle::Workload) => {
3487                // Both protected observations identify the same running workload, but
3488                // execution may burn cycles between them. Retain the conservative lower value.
3489                configured.cycles = configured.cycles.min(cycles);
3490            }
3491            Some(crate::fleet_ensure::model::RootOwnedCanisterLifecycle::Idle)
3492                if cycles >= configured.cycles =>
3493            {
3494                // A stopped asset can receive donations between protected reads.
3495                // Count the latest balance once, without attributing its origin.
3496                configured.cycles = cycles;
3497            }
3498            Some(crate::fleet_ensure::model::RootOwnedCanisterLifecycle::Idle) => {
3499                return Err(EnsureWorkflowError::TerminalInventory(
3500                    TerminalInventoryError::IdleCycleObservationConflict {
3501                        canister: principal,
3502                        expected: configured.cycles,
3503                        observed: cycles,
3504                    },
3505                ));
3506            }
3507            _ => {
3508                return Err(EnsureWorkflowError::TerminalInventory(
3509                    TerminalInventoryError::CycleObservationLifecycleConflict {
3510                        canister: principal,
3511                    },
3512                ));
3513            }
3514        }
3515    }
3516    observation.additional_controlled_cycles = retained;
3517    Ok(())
3518}
3519
3520fn funding_plan<'a, E: std::error::Error + 'static>(
3521    plan: &'a FleetEnsurePlan,
3522    journal: &FleetEnsureJournalRecord,
3523) -> Result<std::borrow::Cow<'a, FleetEnsurePlan>, EnsureWorkflowError<E>> {
3524    if journal.funding_reviews.is_empty()
3525        && journal.funding_observations.is_empty()
3526        && journal.bootstrap_registration_recovery.is_none()
3527    {
3528        return Ok(std::borrow::Cow::Borrowed(plan));
3529    }
3530    funding_observation::verify(plan, journal)?;
3531    let mut funded = plan.clone();
3532    funded.conservation =
3533        crate::fleet_ensure::ops::infrastructure_bootstrap::registration_recovery::conservation(
3534            plan, journal,
3535        )?;
3536    let (amount, fee) = funding::totals::<E>(journal, None)?;
3537    let conservation = &mut funded.conservation;
3538    conservation.maximum_execution_burn_cycles = conservation
3539        .maximum_execution_burn_cycles
3540        .checked_add(funding_observation::total(journal)?)
3541        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3542    conservation.maximum_operator_debit_cycles = conservation
3543        .maximum_operator_debit_cycles
3544        .checked_add(amount)
3545        .and_then(|n| n.checked_add(fee))
3546        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3547    conservation.maximum_new_funding_cycles = conservation
3548        .maximum_new_funding_cycles
3549        .checked_add(amount)
3550        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3551    conservation.maximum_unavoidable_fee_cycles = conservation
3552        .maximum_unavoidable_fee_cycles
3553        .checked_add(fee)
3554        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3555    for domain in &mut conservation.estate_funding_domains {
3556        domain.maximum_funding_cycles = domain
3557            .maximum_funding_cycles
3558            .checked_add(funding::totals::<E>(journal, Some(&domain.root))?.0)
3559            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3560    }
3561    Ok(std::borrow::Cow::Owned(funded))
3562}
3563
3564fn verify_terminal_conservation<E>(
3565    plan: &FleetEnsurePlan,
3566    journal: &FleetEnsureJournalRecord,
3567    state: &FleetEnsureStateRecord,
3568    terminal: &FleetObservation,
3569) -> Result<ActualCycleConservation, EnsureWorkflowError<E>>
3570where
3571    E: std::error::Error + 'static,
3572{
3573    verify_terminal_conservation_with_total(
3574        plan,
3575        journal,
3576        state,
3577        terminal,
3578        controlled_cycles(terminal)?,
3579    )
3580}
3581
3582fn verify_terminal_conservation_with_total<E: std::error::Error + 'static>(
3583    plan: &FleetEnsurePlan,
3584    journal: &FleetEnsureJournalRecord,
3585    state: &FleetEnsureStateRecord,
3586    terminal: &FleetObservation,
3587    final_controlled_cycles: u128,
3588) -> Result<ActualCycleConservation, EnsureWorkflowError<E>> {
3589    let funded = funding_plan(plan, journal)?;
3590    let conservation = &funded.conservation;
3591    let operator_source_cycles =
3592        crate::fleet_ensure::policy::operator_mint::operator_source(journal)
3593            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3594    let minimum_operator_cycles =
3595        operator_source_cycles.saturating_sub(conservation.maximum_operator_debit_cycles);
3596    if journal.completion == FleetEnsureCompletion::Converged
3597        && !(minimum_operator_cycles..=operator_source_cycles).contains(&terminal.operator_cycles)
3598    {
3599        return Err(EnsureWorkflowError::TerminalReplayBalanceChanged {
3600            fleet: plan.fleet.clone(),
3601            operation_id: plan.operation_id.clone(),
3602            plan_sha256: plan.plan_sha256.clone(),
3603            current_operator_cycles: terminal.operator_cycles,
3604            minimum_operator_cycles,
3605            operator_source_cycles,
3606        });
3607    }
3608    let operator_debit_cycles = operator_source_cycles
3609        .checked_sub(terminal.operator_cycles)
3610        .ok_or_else(|| {
3611            EnsureWorkflowError::Conservation(
3612                "operator balance increased during apply; review a new plan".to_string(),
3613            )
3614        })?;
3615    if operator_debit_cycles > conservation.maximum_operator_debit_cycles {
3616        return Err(EnsureWorkflowError::Conservation(format!(
3617            "operator debit {operator_debit_cycles} exceeded reviewed maximum {}",
3618            conservation.maximum_operator_debit_cycles
3619        )));
3620    }
3621    let received_new_funding_cycles = operator_debit_cycles
3622        .checked_sub(conservation.maximum_unavoidable_fee_cycles)
3623        .ok_or_else(|| {
3624            EnsureWorkflowError::Conservation(
3625                "operator debit was below the exact reviewed fee total".to_string(),
3626            )
3627        })?;
3628    if received_new_funding_cycles > conservation.maximum_new_funding_cycles {
3629        return Err(EnsureWorkflowError::Conservation(format!(
3630            "received funding {received_new_funding_cycles} exceeded reviewed maximum {}",
3631            conservation.maximum_new_funding_cycles
3632        )));
3633    }
3634    let (estate_funding_cycles, exact_estate_creation_fee_cycles) =
3635        reconcile_estate_funding(plan, journal, state, terminal)?;
3636    let available = journal
3637        .initial_controlled_cycles
3638        .checked_add(received_new_funding_cycles)
3639        .ok_or_else(|| {
3640            EnsureWorkflowError::Conservation(
3641                "terminal controlled-cycle arithmetic overflowed".to_string(),
3642            )
3643        })?;
3644    let after_estate_fees = available
3645        .checked_sub(exact_estate_creation_fee_cycles)
3646        .ok_or_else(|| {
3647            EnsureWorkflowError::Conservation(
3648                "terminal controlled cycles cannot cover reviewed estate creation fees".to_string(),
3649            )
3650        })?;
3651    // Native balances are open to third-party deposits. These are net observations,
3652    // not authenticated funding or a measurement of gross execution consumption.
3653    let observed_net_cycle_debit_cycles = after_estate_fees.saturating_sub(final_controlled_cycles);
3654    let observed_net_cycle_credit_cycles =
3655        final_controlled_cycles.saturating_sub(after_estate_fees);
3656    if observed_net_cycle_debit_cycles > conservation.maximum_execution_burn_cycles {
3657        return Err(EnsureWorkflowError::Conservation(format!(
3658            "observed net cycle debit {observed_net_cycle_debit_cycles} exceeded reviewed maximum {}",
3659            conservation.maximum_execution_burn_cycles
3660        )));
3661    }
3662    Ok(ActualCycleConservation {
3663        estate_funding_cycles,
3664        exact_estate_creation_fee_cycles,
3665        exact_unavoidable_fee_cycles: conservation.maximum_unavoidable_fee_cycles,
3666        final_controlled_cycles,
3667        observed_net_cycle_debit_cycles,
3668        observed_starting_cycles: journal.initial_controlled_cycles,
3669        observed_net_cycle_credit_cycles,
3670        operator_debit_cycles,
3671        received_new_funding_cycles,
3672    })
3673}
3674
3675fn resolved_initial_pool_assets<E>(
3676    domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3677    state: &FleetEnsureStateRecord,
3678) -> Result<BTreeSet<String>, EnsureWorkflowError<E>>
3679where
3680    E: std::error::Error + 'static,
3681{
3682    domain
3683        .initial_pool_assets
3684        .iter()
3685        .map(|identity| {
3686            identity.strip_prefix("created:").map_or_else(
3687                || Ok(identity.clone()),
3688                |name| {
3689                    state
3690                        .pending_principals
3691                        .get(name)
3692                        .or_else(|| state.principals.get(name))
3693                        .cloned()
3694                        .ok_or(EnsureWorkflowError::JournalIntegrity)
3695                },
3696            )
3697        })
3698        .collect()
3699}
3700
3701fn exact_estate_creation_costs<E>(
3702    state: &FleetEnsureStateRecord,
3703    terminal: &FleetObservation,
3704    domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3705) -> Result<(u128, u128), EnsureWorkflowError<E>>
3706where
3707    E: std::error::Error + 'static,
3708{
3709    let observed = terminal
3710        .estate_funding_domains
3711        .get(&domain.root)
3712        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3713    let Some(pool) = observed.pool.as_ref() else {
3714        return if domain.required_creation_count == 0 {
3715            Ok((0, 0))
3716        } else {
3717            Err(EnsureWorkflowError::JournalIntegrity)
3718        };
3719    };
3720    if pool.pending_creation.is_some() {
3721        return Err(EnsureWorkflowError::JournalIntegrity);
3722    }
3723
3724    estate_creation_costs(state, terminal, domain)
3725}
3726
3727fn estate_creation_costs<E: std::error::Error + 'static>(
3728    state: &FleetEnsureStateRecord,
3729    observation: &FleetObservation,
3730    domain: &EstateFundingDomainPlan,
3731) -> Result<(u128, u128), EnsureWorkflowError<E>> {
3732    let pool = observation
3733        .estate_funding_domains
3734        .get(&domain.root)
3735        .and_then(|domain| domain.pool.as_ref())
3736        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3737    let initial = resolved_initial_pool_assets(domain, state)?;
3738    let terminal_principals = pool
3739        .assets
3740        .iter()
3741        .map(|asset| asset.principal.clone())
3742        .collect::<BTreeSet<_>>();
3743    if terminal_principals.len() != pool.assets.len() || !initial.is_subset(&terminal_principals) {
3744        return Err(EnsureWorkflowError::JournalIntegrity);
3745    }
3746    let created = pool
3747        .assets
3748        .iter()
3749        .filter(|asset| !initial.contains(&asset.principal))
3750        .collect::<Vec<_>>();
3751    let creation_count_is_bounded = created.len()
3752        <= usize::try_from(domain.required_creation_count)
3753            .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
3754    // Root maintenance can spend its reviewed reserve without provisioning a new Workload.
3755    // The per-Root budget and exact receipts bind that autonomous creation authority.
3756    let pool_policy_is_exact = created.is_empty() || terminal_pool_policy_is_exact(pool, domain);
3757    if !creation_count_is_bounded || !pool_policy_is_exact {
3758        return Err(EnsureWorkflowError::JournalIntegrity);
3759    }
3760
3761    let mut operation_ids = BTreeSet::new();
3762    let mut block_indexes = BTreeSet::new();
3763    let mut creation_debit = 0_u128;
3764    let mut creation_fee = 0_u128;
3765    for asset in created {
3766        let (asset_debit, asset_fee) = exact_estate_creation_receipt_costs(
3767            asset,
3768            domain,
3769            &mut operation_ids,
3770            &mut block_indexes,
3771        )?;
3772        creation_debit = creation_debit.checked_add(asset_debit).ok_or_else(|| {
3773            EnsureWorkflowError::Conservation(
3774                "estate creation-debit arithmetic overflowed".to_string(),
3775            )
3776        })?;
3777        creation_fee = creation_fee.checked_add(asset_fee).ok_or_else(|| {
3778            EnsureWorkflowError::Conservation(
3779                "estate creation-fee arithmetic overflowed".to_string(),
3780            )
3781        })?;
3782    }
3783    if creation_debit > domain.maximum_creation_debit_cycles
3784        || creation_fee > domain.maximum_creation_fee_cycles
3785    {
3786        return Err(EnsureWorkflowError::JournalIntegrity);
3787    }
3788    Ok((creation_debit, creation_fee))
3789}
3790
3791const fn terminal_pool_policy_is_exact(
3792    pool: &crate::fleet_ensure::model::EstatePoolInventoryObservation,
3793    domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3794) -> bool {
3795    let expected_maximum_size = domain.pool_maximum_size;
3796    let expected_readiness_floor = domain.readiness_floor_cycles;
3797    let expected_execution_margin = domain.creation_execution_margin_cycles;
3798    pool.maximum_size == expected_maximum_size
3799        && pool.readiness_floor_cycles == expected_readiness_floor
3800        && pool.creation_execution_margin_cycles == expected_execution_margin
3801}
3802
3803#[derive(Eq, PartialEq)]
3804struct EstateCreationReceiptAuthority<'a> {
3805    creation_execution_margin_cycles: u128,
3806    cycles_ledger: &'a str,
3807    ledger_amount_cycles: u128,
3808    ledger_fee_cycles: u128,
3809    management_creation_fee_cycles: u128,
3810    readiness_floor_cycles: u128,
3811}
3812
3813fn observed_estate_creation_authority(
3814    receipt: &crate::fleet_ensure::model::EstatePoolCreationReceiptObservation,
3815) -> EstateCreationReceiptAuthority<'_> {
3816    EstateCreationReceiptAuthority {
3817        creation_execution_margin_cycles: receipt.creation_execution_margin_cycles,
3818        cycles_ledger: &receipt.cycles_ledger,
3819        ledger_amount_cycles: receipt.ledger_amount_cycles,
3820        ledger_fee_cycles: receipt.ledger_fee_cycles,
3821        management_creation_fee_cycles: receipt.management_creation_fee_cycles,
3822        readiness_floor_cycles: receipt.readiness_floor_cycles,
3823    }
3824}
3825
3826fn reviewed_estate_creation_authority(
3827    domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3828) -> EstateCreationReceiptAuthority<'_> {
3829    EstateCreationReceiptAuthority {
3830        creation_execution_margin_cycles: domain.creation_execution_margin_cycles,
3831        cycles_ledger: &domain.cycles_ledger,
3832        ledger_amount_cycles: domain.creation_amount_cycles,
3833        ledger_fee_cycles: domain.ledger_fee_cycles,
3834        management_creation_fee_cycles: domain.management_creation_fee_cycles,
3835        readiness_floor_cycles: domain.readiness_floor_cycles,
3836    }
3837}
3838
3839fn first_estate_creation_observation_is_exact(
3840    receipt: &crate::fleet_ensure::model::EstatePoolCreationReceiptObservation,
3841) -> bool {
3842    receipt
3843        .ledger_amount_cycles
3844        .checked_sub(receipt.management_creation_fee_cycles)
3845        .zip(receipt.first_observed_cycles)
3846        .is_some_and(|(funded_native_cycles, observed_cycles)| {
3847            observed_cycles >= receipt.readiness_floor_cycles
3848                && funded_native_cycles.saturating_sub(observed_cycles)
3849                    <= receipt.creation_execution_margin_cycles
3850        })
3851}
3852
3853fn exact_estate_creation_receipt_costs<E>(
3854    asset: &crate::fleet_ensure::model::EstatePoolAssetObservation,
3855    domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3856    operation_ids: &mut BTreeSet<String>,
3857    block_indexes: &mut BTreeSet<u64>,
3858) -> Result<(u128, u128), EnsureWorkflowError<E>>
3859where
3860    E: std::error::Error + 'static,
3861{
3862    let receipt = asset
3863        .creation_receipt
3864        .as_ref()
3865        .filter(|_| asset.origin == crate::fleet_ensure::model::EstatePoolAssetOrigin::Created)
3866        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3867    let authority_is_exact =
3868        observed_estate_creation_authority(receipt) == reviewed_estate_creation_authority(domain);
3869    if !is_sha256_hex(&receipt.operation_id)
3870        || !authority_is_exact
3871        || !first_estate_creation_observation_is_exact(receipt)
3872        || !operation_ids.insert(receipt.operation_id.clone())
3873        || !block_indexes.insert(receipt.block_index)
3874    {
3875        return Err(EnsureWorkflowError::JournalIntegrity);
3876    }
3877    let debit = receipt
3878        .ledger_amount_cycles
3879        .checked_add(receipt.ledger_fee_cycles)
3880        .ok_or_else(|| {
3881            EnsureWorkflowError::Conservation(
3882                "estate creation-debit arithmetic overflowed".to_string(),
3883            )
3884        })?;
3885    let fee = receipt
3886        .management_creation_fee_cycles
3887        .checked_add(receipt.ledger_fee_cycles)
3888        .ok_or_else(|| {
3889            EnsureWorkflowError::Conservation(
3890                "estate creation-fee arithmetic overflowed".to_string(),
3891            )
3892        })?;
3893    Ok((debit, fee))
3894}
3895
3896fn reconcile_estate_funding<E>(
3897    plan: &FleetEnsurePlan,
3898    journal: &FleetEnsureJournalRecord,
3899    state: &FleetEnsureStateRecord,
3900    terminal: &FleetObservation,
3901) -> Result<(u128, u128), EnsureWorkflowError<E>>
3902where
3903    E: std::error::Error + 'static,
3904{
3905    plan.conservation
3906        .estate_funding_domains
3907        .iter()
3908        .try_fold((0_u128, 0_u128), |(funding_total, fee_total), domain| {
3909            let initial = journal
3910                .initial_estate_funding_cycles_by_root
3911                .get(&domain.root)
3912                .copied()
3913                .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3914            let observed = terminal
3915                .estate_funding_domains
3916                .get(&domain.root)
3917                .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3918            if observed.cycles_ledger != domain.cycles_ledger
3919                || reviewed_estate_root_principal(domain, state)
3920                    != observed.root_principal.as_deref()
3921            {
3922                return Err(EnsureWorkflowError::JournalIntegrity);
3923            }
3924            let terminal_balance = observed
3925                .balance_cycles
3926                .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3927            let (creation_debit, creation_fee) = exact_estate_creation_costs(
3928                state,
3929                terminal,
3930                domain,
3931            )?;
3932            let funding = applied_estate_funding_for_domain(plan, journal, state, domain)?
3933                .checked_add(funding::totals::<E>(journal, Some(&domain.root))?.0)
3934                .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3935            let expected_terminal_balance = initial
3936                .checked_add(funding)
3937                .and_then(|available| available.checked_sub(creation_debit))
3938                .ok_or_else(|| {
3939                    EnsureWorkflowError::Conservation(format!(
3940                        "Root {} funding-account balance cannot reconcile its reviewed funding and creation debit",
3941                        domain.root
3942                    ))
3943                })?;
3944            // Surplus Ledger cycles join the terminal net-credit observation;
3945            // authenticated funding and exact creation costs remain unchanged.
3946            if terminal_balance < expected_terminal_balance {
3947                return Err(EnsureWorkflowError::Conservation(format!(
3948                    "Root {} funding-account balance {terminal_balance} differs from reviewed terminal balance {expected_terminal_balance}",
3949                    domain.root
3950                )));
3951            }
3952            Ok((
3953                funding_total.checked_add(funding).ok_or_else(|| {
3954                    EnsureWorkflowError::Conservation(
3955                        "estate funding arithmetic overflowed".to_string(),
3956                    )
3957                })?,
3958                fee_total
3959                    .checked_add(creation_fee)
3960                    .ok_or_else(|| {
3961                        EnsureWorkflowError::Conservation(
3962                            "estate creation-fee arithmetic overflowed".to_string(),
3963                        )
3964                    })?,
3965            ))
3966        })
3967}
3968
3969fn applied_estate_funding_for_domain<E>(
3970    plan: &FleetEnsurePlan,
3971    journal: &FleetEnsureJournalRecord,
3972    state: &FleetEnsureStateRecord,
3973    domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3974) -> Result<u128, EnsureWorkflowError<E>>
3975where
3976    E: std::error::Error + 'static,
3977{
3978    let action = plan
3979        .canisters
3980        .iter()
3981        .find(|canister| canister.name == domain.root)
3982        .into_iter()
3983        .flat_map(|canister| &canister.actions)
3984        .find(|action| matches!(action, EnsureAction::FundEstate { .. }));
3985    let Some(action) = action else {
3986        return if domain.maximum_funding_cycles == 0 {
3987            Ok(0)
3988        } else {
3989            Err(EnsureWorkflowError::JournalIntegrity)
3990        };
3991    };
3992    let EnsureAction::FundEstate {
3993        amount,
3994        expected_post_cycles,
3995        ledger,
3996        ledger_fee_cycles,
3997        principal,
3998        ..
3999    } = action
4000    else {
4001        unreachable!("filtered estate-funding action")
4002    };
4003    let root_principal = principal.strip_prefix("created:").map_or_else(
4004        || Some(principal.as_str()),
4005        |name| {
4006            state
4007                .pending_principals
4008                .get(name)
4009                .or_else(|| state.principals.get(name))
4010                .map(String::as_str)
4011        },
4012    );
4013    let expected_destination = domain
4014        .available_cycles
4015        .unwrap_or_default()
4016        .checked_add(domain.shortfall_cycles)
4017        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
4018    let action_authority_is_exact = *amount == domain.shortfall_cycles
4019        && *expected_post_cycles == expected_destination
4020        && ledger == &domain.cycles_ledger
4021        && *ledger_fee_cycles == domain.ledger_fee_cycles
4022        && root_principal == reviewed_estate_root_principal(domain, state);
4023    let effect = journal
4024        .effects
4025        .iter()
4026        .find(|effect| effect.action_sha256 == action_sha256(action));
4027    let Some(effect) = effect.filter(|effect| {
4028        effect.state == EffectState::Applied
4029            && effect
4030                .receipt
4031                .as_deref()
4032                .is_some_and(|receipt| !receipt.is_empty())
4033    }) else {
4034        return Err(EnsureWorkflowError::JournalIntegrity);
4035    };
4036    let balance_evidence_is_exact = crate::fleet_ensure::ops::estate_funding_applied(
4037        crate::fleet_ensure::ops::EstateFundingObservation {
4038            amount: *amount,
4039            destination_after: effect.destination_post_cycles.unwrap_or_default(),
4040            destination_before: effect.destination_pre_cycles,
4041            expected_destination_after: *expected_post_cycles,
4042            ledger_fee_cycles: *ledger_fee_cycles,
4043            source_after: effect.post_cycles.unwrap_or_default(),
4044            source_before: effect.pre_cycles,
4045        },
4046    );
4047    if !action_authority_is_exact || !balance_evidence_is_exact {
4048        return Err(EnsureWorkflowError::JournalIntegrity);
4049    }
4050    Ok(*amount)
4051}
4052
4053fn controlled_cycles<E>(observation: &FleetObservation) -> Result<u128, EnsureWorkflowError<E>>
4054where
4055    E: std::error::Error + 'static,
4056{
4057    observation
4058        .canisters
4059        .values()
4060        .filter_map(Option::as_ref)
4061        .try_fold(0_u128, |total, live| total.checked_add(live.cycles))
4062        .and_then(|total| {
4063            observation
4064                .additional_controlled_cycles
4065                .values()
4066                .try_fold(total, |subtotal, cycles| subtotal.checked_add(*cycles))
4067        })
4068        .and_then(|total| {
4069            observation
4070                .estate_funding_domains
4071                .values()
4072                .filter_map(|domain| domain.balance_cycles)
4073                .try_fold(total, u128::checked_add)
4074        })
4075        .ok_or_else(|| {
4076            EnsureWorkflowError::Conservation("controlled-cycle total overflowed".to_string())
4077        })
4078}
4079
4080fn verified_plan<E>(plan: FleetEnsurePlan) -> Result<FleetEnsurePlan, EnsureWorkflowError<E>>
4081where
4082    E: std::error::Error + 'static,
4083{
4084    if plan.reviewed_desired.is_none()
4085        || expected_plan_sha256(&plan) != plan.plan_sha256
4086        || (plan.scope == FleetEnsurePlanScope::InfrastructureBootstrap)
4087            != plan.infrastructure_bootstrap.is_some()
4088    {
4089        return Err(EnsureWorkflowError::PlanIntegrity);
4090    }
4091    Ok(plan)
4092}
4093
4094fn verify_journal<E>(
4095    journal: &FleetEnsureJournalRecord,
4096    plan: &FleetEnsurePlan,
4097    requested_fleet: &str,
4098    state: &FleetEnsureStateRecord,
4099) -> Result<(), EnsureWorkflowError<E>>
4100where
4101    E: std::error::Error + 'static,
4102{
4103    verify_journal_integrity(journal, plan, requested_fleet, state)?;
4104    if let Some(review) = journal
4105        .funding_reviews
4106        .iter()
4107        .filter_map(|r| r.operator_mint.as_ref())
4108        .find(|review| review.receipt.is_none())
4109    {
4110        return Err(EnsureWorkflowError::OperatorMintPending {
4111            review_sha256: review.review_sha256.clone(),
4112        });
4113    }
4114    Ok(())
4115}
4116
4117fn verify_journal_integrity<E>(
4118    journal: &FleetEnsureJournalRecord,
4119    plan: &FleetEnsurePlan,
4120    requested_fleet: &str,
4121    state: &FleetEnsureStateRecord,
4122) -> Result<(), EnsureWorkflowError<E>>
4123where
4124    E: std::error::Error + 'static,
4125{
4126    if journal.fleet != requested_fleet
4127        || journal.operation_id != plan.operation_id
4128        || journal.plan_sha256 != plan.plan_sha256
4129        || !initial_estate_funding_is_exact(journal, plan)
4130        || !estate_funding_pause_is_exact(journal, plan, state)
4131    {
4132        return Err(EnsureWorkflowError::JournalIntegrity);
4133    }
4134    crate::fleet_ensure::ops::infrastructure_bootstrap::registration_recovery::verify(
4135        plan, journal, state,
4136    )?;
4137    funding::verify(plan, journal, state)?;
4138    funding_observation::verify(plan, journal)?;
4139    operator_mint::verify(plan, journal)?;
4140    continuation::verify_records(plan, journal)?;
4141    let actions = continuation::actions(plan, journal);
4142    let effect_count_matches = journal.effects.len() <= actions.len();
4143    let action_hashes_match = journal.effects.iter().zip(actions).all(|(effect, action)| {
4144        effect.action_sha256 == action_sha256(action) && effect.attempts_match(action)
4145    });
4146    if !(effect_count_matches && action_hashes_match) {
4147        return Err(EnsureWorkflowError::JournalIntegrity);
4148    }
4149    Ok(())
4150}
4151
4152fn initial_estate_funding_is_exact(
4153    journal: &FleetEnsureJournalRecord,
4154    plan: &FleetEnsurePlan,
4155) -> bool {
4156    journal.initial_estate_funding_cycles_by_root.len()
4157        == plan.conservation.estate_funding_domains.len()
4158        && plan
4159            .conservation
4160            .estate_funding_domains
4161            .iter()
4162            .all(|domain| {
4163                journal
4164                    .initial_estate_funding_cycles_by_root
4165                    .get(&domain.root)
4166                    .copied()
4167                    == Some(domain.available_cycles.unwrap_or_default())
4168            })
4169}
4170
4171fn estate_funding_pause_is_exact(
4172    journal: &FleetEnsureJournalRecord,
4173    plan: &FleetEnsurePlan,
4174    state: &FleetEnsureStateRecord,
4175) -> bool {
4176    let Some(required) = journal.estate_funding_required.as_ref() else {
4177        return true;
4178    };
4179    journal.completion == FleetEnsureCompletion::InProgress
4180        && estate_funding_record_is_exact(required, plan, state)
4181}
4182
4183fn estate_funding_record_is_exact(
4184    required: &EstateFundingRequiredRecord,
4185    plan: &FleetEnsurePlan,
4186    state: &FleetEnsureStateRecord,
4187) -> bool {
4188    let Some(reviewed) = plan
4189        .conservation
4190        .estate_funding_domains
4191        .iter()
4192        .find(|domain| domain.root == required.root)
4193    else {
4194        return false;
4195    };
4196    let operation_authority_is_exact =
4197        required.operation_id == plan.operation_id && required.plan_sha256 == plan.plan_sha256;
4198    let funding_authority_is_exact =
4199        EstateFundingPauseAuthority::from(required) == EstateFundingPauseAuthority::from(reviewed);
4200    let root_authority_is_exact =
4201        reviewed_estate_root_principal(reviewed, state) == Some(required.root_principal.as_str());
4202    let shortfall_is_exact = required.available_cycles < required.maximum_creation_debit_cycles
4203        && required.shortfall_cycles
4204            == required
4205                .maximum_creation_debit_cycles
4206                .saturating_sub(required.available_cycles);
4207    if !(operation_authority_is_exact
4208        && funding_authority_is_exact
4209        && root_authority_is_exact
4210        && shortfall_is_exact)
4211    {
4212        return false;
4213    }
4214    if required.pending_creation_operation_id.is_some() {
4215        current_creation_funding_pause_is_exact(required, reviewed)
4216    } else {
4217        planned_estate_funding_pause_is_exact(required, reviewed)
4218    }
4219}
4220
4221#[derive(Eq, PartialEq)]
4222struct EstateFundingPauseAuthority<'a> {
4223    creation_amount_cycles: u128,
4224    creation_execution_margin_cycles: u128,
4225    cycles_ledger: &'a str,
4226    ledger_fee_cycles: u128,
4227    management_creation_fee_cycles: u128,
4228    readiness_floor_cycles: u128,
4229    root: &'a str,
4230}
4231
4232impl<'a> From<&'a EstateFundingRequiredRecord> for EstateFundingPauseAuthority<'a> {
4233    fn from(required: &'a EstateFundingRequiredRecord) -> Self {
4234        Self {
4235            creation_amount_cycles: required.creation_amount_cycles,
4236            creation_execution_margin_cycles: required.creation_execution_margin_cycles,
4237            cycles_ledger: &required.cycles_ledger,
4238            ledger_fee_cycles: required.ledger_fee_cycles,
4239            management_creation_fee_cycles: required.management_creation_fee_cycles,
4240            readiness_floor_cycles: required.readiness_floor_cycles,
4241            root: &required.root,
4242        }
4243    }
4244}
4245
4246impl<'a> From<&'a EstateFundingDomainPlan> for EstateFundingPauseAuthority<'a> {
4247    fn from(reviewed: &'a EstateFundingDomainPlan) -> Self {
4248        Self {
4249            creation_amount_cycles: reviewed.creation_amount_cycles,
4250            creation_execution_margin_cycles: reviewed.creation_execution_margin_cycles,
4251            cycles_ledger: &reviewed.cycles_ledger,
4252            ledger_fee_cycles: reviewed.ledger_fee_cycles,
4253            management_creation_fee_cycles: reviewed.management_creation_fee_cycles,
4254            readiness_floor_cycles: reviewed.readiness_floor_cycles,
4255            root: &reviewed.root,
4256        }
4257    }
4258}
4259
4260fn current_creation_funding_pause_is_exact(
4261    required: &EstateFundingRequiredRecord,
4262    reviewed: &EstateFundingDomainPlan,
4263) -> bool {
4264    let operation_id_is_exact = required
4265        .pending_creation_operation_id
4266        .as_deref()
4267        .is_some_and(is_sha256_hex);
4268    let attempt_evidence_is_exact = required.attempt_count.is_some()
4269        && required
4270            .retry_at_ns
4271            .is_some_and(|retry_at_ns| retry_at_ns > 0)
4272        && required
4273            .last_attempt_at_ns
4274            .is_none_or(|attempted_at_ns| attempted_at_ns > 0);
4275    let single_creation_debit = required
4276        .creation_amount_cycles
4277        .checked_add(required.ledger_fee_cycles);
4278    let single_creation_fees = required
4279        .management_creation_fee_cycles
4280        .checked_add(required.ledger_fee_cycles);
4281    let debit_authority_is_exact = Some(required.maximum_creation_debit_cycles)
4282        == single_creation_debit
4283        && Some(required.maximum_creation_fee_cycles) == single_creation_fees;
4284    reviewed.required_creation_count > 0
4285        && required.required_creation_count == 1
4286        && operation_id_is_exact
4287        && attempt_evidence_is_exact
4288        && debit_authority_is_exact
4289}
4290
4291const fn planned_estate_funding_pause_is_exact(
4292    required: &EstateFundingRequiredRecord,
4293    reviewed: &EstateFundingDomainPlan,
4294) -> bool {
4295    let runtime_attempt_is_absent = required.attempt_count.is_none()
4296        && required.last_attempt_at_ns.is_none()
4297        && required.retry_at_ns.is_none();
4298    let reviewed_debit_is_exact = required.maximum_creation_debit_cycles
4299        == reviewed.maximum_creation_debit_cycles
4300        && required.maximum_creation_fee_cycles == reviewed.maximum_creation_fee_cycles
4301        && required.required_creation_count == reviewed.required_creation_count;
4302    runtime_attempt_is_absent && reviewed_debit_is_exact
4303}
4304
4305fn is_sha256_hex(value: &str) -> bool {
4306    value.len() == 64
4307        && value
4308            .bytes()
4309            .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
4310}
4311
4312#[derive(Clone, Copy, Debug, Eq, PartialEq)]
4313enum TerminalStatePublicationError {
4314    JournalIntegrity,
4315}
4316
4317fn publish_terminal_state(
4318    desired: &crate::fleet_ensure::model::DesiredFleet,
4319    plan: &FleetEnsurePlan,
4320    journal: &FleetEnsureJournalRecord,
4321    state: &mut FleetEnsureStateRecord,
4322) -> Result<(), TerminalStatePublicationError> {
4323    let maximum_observation_burn_cycles =
4324        crate::fleet_ensure::ops::maximum_creation_observation_burn(desired)
4325            .ok_or(TerminalStatePublicationError::JournalIntegrity)?;
4326    let actions = continuation::actions(plan, journal);
4327    if actions.len() != journal.effects.len() {
4328        return Err(TerminalStatePublicationError::JournalIntegrity);
4329    }
4330    for (action, record) in actions.into_iter().zip(&journal.effects) {
4331        if action_sha256(action) != record.action_sha256 || record.state != EffectState::Applied {
4332            return Err(TerminalStatePublicationError::JournalIntegrity);
4333        }
4334        let EnsureAction::Create {
4335            name,
4336            requested_initial_cycles,
4337            ..
4338        } = action
4339        else {
4340            continue;
4341        };
4342        let Some(created_principal) = record.created_principal.as_deref() else {
4343            return Err(TerminalStatePublicationError::JournalIntegrity);
4344        };
4345        let Some(post_cycles) = record.post_cycles else {
4346            return Err(TerminalStatePublicationError::JournalIntegrity);
4347        };
4348        if !create_balance_is_terminal(
4349            Some(post_cycles),
4350            *requested_initial_cycles,
4351            maximum_observation_burn_cycles,
4352        ) || !create_identity_is_exact(name, record, state)
4353            || state
4354                .retained_cycles_by_principal
4355                .get(created_principal)
4356                .is_some_and(|retained| *retained != post_cycles)
4357        {
4358            return Err(TerminalStatePublicationError::JournalIntegrity);
4359        }
4360        state
4361            .retained_cycles_by_principal
4362            .insert(created_principal.to_string(), post_cycles);
4363    }
4364
4365    let prior_topology = state.topology.clone();
4366    for canister in &plan.canisters {
4367        if !desired
4368            .canisters
4369            .iter()
4370            .any(|configured| configured.name == canister.name)
4371        {
4372            // Dynamic pool assets remain in protected Root inventory. A funding-only
4373            // projection must not become a configured application identity.
4374            if !is_reviewed_dynamic_pool_funding(plan, canister) {
4375                return Err(TerminalStatePublicationError::JournalIntegrity);
4376            }
4377            continue;
4378        }
4379
4380        match canister.disposition {
4381            CanisterDisposition::Create | CanisterDisposition::Replace => {
4382                if let Some(principal) = state.pending_principals.remove(&canister.name) {
4383                    state.principals.insert(canister.name.clone(), principal);
4384                }
4385            }
4386            CanisterDisposition::Reinstall | CanisterDisposition::Reuse => {
4387                if let Some(principal) = state
4388                    .pending_principals
4389                    .remove(&canister.name)
4390                    .or_else(|| canister.principal.clone())
4391                {
4392                    state.principals.insert(canister.name.clone(), principal);
4393                }
4394            }
4395            CanisterDisposition::Delete => {
4396                state.principals.remove(&canister.name);
4397                state.pending_principals.remove(&canister.name);
4398            }
4399        }
4400    }
4401    publish_topology(desired, plan, &prior_topology, state);
4402    retain_configured_principal_bindings(state);
4403    Ok(())
4404}
4405
4406fn publish_topology(
4407    desired: &crate::fleet_ensure::model::DesiredFleet,
4408    plan: &FleetEnsurePlan,
4409    prior_topology: &BTreeMap<String, crate::fleet_ensure::model::FleetEnsureTopologyRecord>,
4410    state: &mut FleetEnsureStateRecord,
4411) {
4412    state.topology = desired
4413        .canisters
4414        .iter()
4415        .filter(|canister| {
4416            canister.presence == crate::fleet_ensure::model::DesiredPresence::Present
4417                && (plan.scope != FleetEnsurePlanScope::InfrastructureBootstrap
4418                    || plan
4419                        .canisters
4420                        .iter()
4421                        .any(|target| target.name == canister.name))
4422        })
4423        .map(|canister| {
4424            (
4425                canister.name.clone(),
4426                crate::fleet_ensure::model::FleetEnsureTopologyRecord {
4427                    kind: canister.kind,
4428                    module_hash: projected_module_hash(plan, prior_topology, canister),
4429                    parent: canister.parent.clone(),
4430                    protocol_binding: canister.protocol_binding.clone(),
4431                    role: canister
4432                        .protocol_binding
4433                        .as_ref()
4434                        .map(|binding| binding.role.to_string()),
4435                },
4436            )
4437        })
4438        .collect();
4439}
4440
4441fn is_reviewed_dynamic_pool_funding(
4442    plan: &FleetEnsurePlan,
4443    canister: &crate::fleet_ensure::model::CanisterPlan,
4444) -> bool {
4445    canister.disposition == CanisterDisposition::Reuse
4446        && !canister.actions.is_empty()
4447        && canister
4448            .actions
4449            .iter()
4450            .all(|action| is_reviewed_pool_funding(action, plan, canister.principal.as_deref()))
4451}
4452
4453fn is_reviewed_pool_funding(
4454    action: &EnsureAction,
4455    plan: &FleetEnsurePlan,
4456    expected_principal: Option<&str>,
4457) -> bool {
4458    let EnsureAction::Fund {
4459        pool_funding: Some(root),
4460        principal,
4461        ..
4462    } = action
4463    else {
4464        return false;
4465    };
4466    if Some(principal.as_str()) != expected_principal {
4467        return false;
4468    }
4469    plan.protocol_actions.iter().any(|candidate| {
4470        let EnsureAction::FleetProtocol {
4471            principal: target,
4472            action,
4473            ..
4474        } = candidate
4475        else {
4476            return false;
4477        };
4478        let CurrentFleetProtocolAction::ReconcilePoolAsset { request, .. } = action.as_ref() else {
4479            return false;
4480        };
4481        target == &root.root && request.canister_id.to_text() == *principal
4482    })
4483}
4484
4485fn projected_module_hash(
4486    plan: &FleetEnsurePlan,
4487    prior_topology: &BTreeMap<String, crate::fleet_ensure::model::FleetEnsureTopologyRecord>,
4488    canister: &crate::fleet_ensure::model::DesiredCanister,
4489) -> Option<String> {
4490    let installed = plan
4491        .canisters
4492        .iter()
4493        .find(|planned| planned.name == canister.name)
4494        .and_then(|planned| {
4495            planned
4496                .actions
4497                .iter()
4498                .rev()
4499                .find_map(|action| match action {
4500                    EnsureAction::Install { wasm_sha256, .. } => Some(wasm_sha256.clone()),
4501                    _ => None,
4502                })
4503        });
4504    installed.or_else(|| {
4505        prior_topology
4506            .get(&canister.name)
4507            .filter(|retained| retained.kind == canister.kind && retained.parent == canister.parent)
4508            .and_then(|retained| retained.module_hash.clone())
4509    })
4510}
4511
4512fn merge_terminal_inventory<E>(
4513    state: &mut FleetEnsureStateRecord,
4514    inventory: crate::fleet_ensure::ops::TerminalFleetInventory,
4515) -> Result<(), EnsureWorkflowError<E>>
4516where
4517    E: std::error::Error + 'static,
4518{
4519    let entries = inventory.entries;
4520    let mut names_by_principal = state
4521        .principals
4522        .iter()
4523        .map(|(name, principal)| (principal.clone(), name.clone()))
4524        .collect::<BTreeMap<_, _>>();
4525    if names_by_principal.len() != state.principals.len() {
4526        return Err(EnsureWorkflowError::TerminalInventory(
4527            TerminalInventoryError::DuplicateRetainedPrincipal,
4528        ));
4529    }
4530    for entry in &entries {
4531        names_by_principal
4532            .entry(entry.pid.clone())
4533            .or_insert_with(|| format!("observed:{}", entry.pid));
4534    }
4535    for entry in entries {
4536        let name = names_by_principal.get(&entry.pid).cloned().ok_or_else(|| {
4537            EnsureWorkflowError::TerminalInventory(TerminalInventoryError::LostPrincipalBinding {
4538                canister: entry.pid.clone(),
4539            })
4540        })?;
4541        let parent = entry
4542            .parent_pid
4543            .as_ref()
4544            .map(|principal| {
4545                names_by_principal.get(principal).cloned().ok_or_else(|| {
4546                    EnsureWorkflowError::TerminalInventory(TerminalInventoryError::UnknownParent {
4547                        canister: entry.pid.clone(),
4548                        parent: principal.clone(),
4549                    })
4550                })
4551            })
4552            .transpose()?;
4553        let existing = state.topology.get(&name);
4554        let kind = terminal_inventory_kind(existing, &entry);
4555        let parent_conflict = existing
4556            .and_then(|topology| topology.parent.as_ref())
4557            .zip(parent.as_ref())
4558            .filter(|(expected, actual)| expected != actual);
4559        if let Some((retained_parent, terminal_parent)) = parent_conflict
4560            && !verified_pool_to_component_parent_transition(existing, kind, &entry)
4561        {
4562            return Err(EnsureWorkflowError::TerminalInventory(
4563                TerminalInventoryError::RetainedParentConflict {
4564                    canister: entry.pid,
4565                    retained_parent: retained_parent.clone(),
4566                    terminal_parent: terminal_parent.clone(),
4567                },
4568            ));
4569        }
4570        state.principals.insert(name.clone(), entry.pid);
4571        state.topology.insert(
4572            name,
4573            crate::fleet_ensure::model::FleetEnsureTopologyRecord {
4574                kind,
4575                module_hash: entry.module_hash,
4576                parent,
4577                protocol_binding: entry.protocol_binding,
4578                role: entry.role,
4579            },
4580        );
4581    }
4582    state.active_registry = inventory.active_registry;
4583    Ok(())
4584}
4585
4586fn terminal_inventory_kind(
4587    existing: Option<&crate::fleet_ensure::model::FleetEnsureTopologyRecord>,
4588    entry: &crate::registry::RegistryEntry,
4589) -> crate::fleet_ensure::model::DesiredCanisterKind {
4590    existing.map_or_else(
4591        || {
4592            if entry.module_hash.is_some() {
4593                crate::fleet_ensure::model::DesiredCanisterKind::Component
4594            } else {
4595                crate::fleet_ensure::model::DesiredCanisterKind::Auxiliary
4596            }
4597        },
4598        |topology| {
4599            if topology.kind == crate::fleet_ensure::model::DesiredCanisterKind::Pool
4600                && entry.module_hash.is_some()
4601                && entry.protocol_binding.is_some()
4602            {
4603                crate::fleet_ensure::model::DesiredCanisterKind::Component
4604            } else {
4605                topology.kind
4606            }
4607        },
4608    )
4609}
4610
4611fn verified_pool_to_component_parent_transition(
4612    existing: Option<&crate::fleet_ensure::model::FleetEnsureTopologyRecord>,
4613    terminal_kind: crate::fleet_ensure::model::DesiredCanisterKind,
4614    entry: &crate::registry::RegistryEntry,
4615) -> bool {
4616    existing.is_some_and(|topology| {
4617        topology.kind == crate::fleet_ensure::model::DesiredCanisterKind::Pool
4618            && terminal_kind == crate::fleet_ensure::model::DesiredCanisterKind::Component
4619            && entry.module_hash.is_some()
4620            && entry.protocol_binding.is_some()
4621    })
4622}
4623
4624fn applied_count(journal: &FleetEnsureJournalRecord) -> u32 {
4625    u32::try_from(
4626        journal
4627            .effects
4628            .iter()
4629            .chain(
4630                journal
4631                    .funding_reviews
4632                    .iter()
4633                    .filter_map(|review| review.effect.as_ref()),
4634            )
4635            .filter(|effect| matches!(effect.state, EffectState::Applied))
4636            .count(),
4637    )
4638    .unwrap_or(u32::MAX)
4639}
4640
4641fn verify_store_activation_binding<P: EnsurePlatform>(
4642    plan: &FleetEnsurePlan,
4643    action: &EnsureAction,
4644    state: &FleetEnsureStateRecord,
4645    platform: &mut P,
4646) -> Result<(), EnsureWorkflowError<P::Error>> {
4647    use crate::fleet_ensure::model::DesiredCanisterInit;
4648    let EnsureAction::Install {
4649        canic_init: Some(DesiredCanisterInit::Store { root }),
4650        ..
4651    } = action
4652    else {
4653        return Ok(());
4654    };
4655    let root_will_be_installed = plan.canisters.iter().flat_map(|canister| &canister.actions)
4656        .any(|action| matches!(action,
4657            EnsureAction::Install { name, canic_init: Some(DesiredCanisterInit::Root { root: installed_root }), .. }
4658            if name == root && installed_root == root));
4659    if !root_will_be_installed {
4660        platform
4661            .require_retained_root_activation(&plan.operation_id, root, state)
4662            .map_err(EnsureWorkflowError::Platform)?;
4663    }
4664    Ok(())
4665}
4666
4667pub(super) fn ordered_actions(plan: &FleetEnsurePlan) -> Vec<&EnsureAction> {
4668    let mut actions = plan
4669        .canisters
4670        .iter()
4671        .flat_map(|canister| canister.actions.iter())
4672        .chain(plan.protocol_actions.iter())
4673        .collect::<Vec<_>>();
4674    if plan.scope == FleetEnsurePlanScope::ReinstallPreparation
4675        && plan
4676            .reinstall
4677            .as_ref()
4678            .is_some_and(|intent| intent.activation_reset.is_some())
4679    {
4680        // Preserve the compiler's Coordinator-first order, then settle every Root before restart.
4681        actions.sort_by_key(|action| matches!(action, EnsureAction::Start { .. }));
4682        return actions;
4683    }
4684    if matches!(
4685        plan.scope,
4686        FleetEnsurePlanScope::RootReinstallPrerequisite
4687            | FleetEnsurePlanScope::InfrastructureBootstrap
4688    ) {
4689        return actions;
4690    }
4691    let temporary_pool_observation_finalizations = plan
4692        .reviewed_desired
4693        .as_deref()
4694        .map(|reviewed| {
4695            let desired = reviewed.desired();
4696            plan.canisters
4697                .iter()
4698                .filter_map(|canister_plan| {
4699                    let configured = desired.canisters.iter().find(|configured| {
4700                        configured.name == canister_plan.name
4701                            && configured.kind
4702                                == crate::fleet_ensure::model::DesiredCanisterKind::Pool
4703                    })?;
4704                    let temporary_create = desired
4705                        .bootstrap
4706                        .as_ref()
4707                        .is_some_and(|bootstrap| bootstrap.fresh_estate)
4708                        && configured.principal.is_none()
4709                        && desired.bootstrap.as_ref().is_some_and(|bootstrap| {
4710                            configured.controllers
4711                                == bootstrap
4712                                    .recovery_controllers
4713                                    .iter()
4714                                    .map(candid::Principal::to_text)
4715                                    .collect::<Vec<_>>()
4716                        })
4717                        && configured.controller_canisters.len() == 1
4718                        && canister_plan.actions.iter().any(|action| {
4719                            matches!(
4720                                action,
4721                                EnsureAction::Create {
4722                                    controller_canisters,
4723                                    controllers,
4724                                    ..
4725                                } if controller_canisters == &configured.controller_canisters
4726                                    && controllers == &{
4727                                        let mut expected = configured.controllers.clone();
4728                                        expected.push(desired.operator.clone());
4729                                        expected.sort();
4730                                        expected
4731                                    }
4732                            )
4733                        });
4734                    temporary_create.then_some(canister_plan.name.as_str())
4735                })
4736                .collect::<BTreeSet<_>>()
4737        })
4738        .unwrap_or_default();
4739    actions.sort_by_key(|action| {
4740        if matches!(action, EnsureAction::SetControllers { name, .. } if temporary_pool_observation_finalizations.contains(name.as_str()))
4741        {
4742            6
4743        } else {
4744            action_order(action)
4745        }
4746    });
4747    actions
4748}
4749
4750fn deferred_create_observation_is_exact(
4751    desired: &crate::fleet_ensure::model::DesiredFleet,
4752    actions: &[&EnsureAction],
4753    index: usize,
4754    action: &EnsureAction,
4755    record: &EffectRecord,
4756    state: &FleetEnsureStateRecord,
4757) -> bool {
4758    let EnsureAction::Create {
4759        controller_canisters,
4760        controllers,
4761        name,
4762        ..
4763    } = action
4764    else {
4765        return false;
4766    };
4767    let Some(configured) = desired.canisters.iter().find(|canister| {
4768        canister.name == *name
4769            && canister.kind == crate::fleet_ensure::model::DesiredCanisterKind::Pool
4770    }) else {
4771        return false;
4772    };
4773    let Some(parent) = configured.parent.as_deref() else {
4774        return false;
4775    };
4776    let fresh_root_only_create = desired
4777        .bootstrap
4778        .as_ref()
4779        .is_some_and(|bootstrap| bootstrap.fresh_estate)
4780        && configured.principal.is_none()
4781        && configured.controllers.is_empty()
4782        && configured.controller_canisters.as_slice() == [parent]
4783        && controller_canisters == &configured.controller_canisters
4784        && controllers.is_empty();
4785    let exact_issued_create = record.state == EffectState::Issued
4786        && create_identity_is_exact(name, record, state)
4787        && record.post_cycles.is_some();
4788    let later_root_install = actions[index.saturating_add(1)..].iter().any(|candidate| {
4789        matches!(
4790            candidate,
4791            EnsureAction::Install {
4792                canic_init: Some(crate::fleet_ensure::model::DesiredCanisterInit::Root { root }),
4793                name: install_name,
4794                ..
4795            } if root == parent && install_name == parent
4796        )
4797    });
4798    let infrastructure_only = actions.iter().all(|candidate| {
4799        matches!(
4800            candidate,
4801            EnsureAction::Create { .. }
4802                | EnsureAction::FundEstate { .. }
4803                | EnsureAction::Install { .. }
4804                | EnsureAction::SetControllers { .. }
4805        )
4806    });
4807    fresh_root_only_create && exact_issued_create && later_root_install && infrastructure_only
4808}
4809
4810pub(super) const fn action_order(action: &EnsureAction) -> u8 {
4811    match action {
4812        EnsureAction::Create { .. } => 0,
4813        EnsureAction::Fund { .. } | EnsureAction::FundEstate { .. } => 1,
4814        EnsureAction::Install {
4815            canic_init: Some(crate::fleet_ensure::model::DesiredCanisterInit::Coordinator),
4816            ..
4817        } => 2,
4818        EnsureAction::SetControllers { .. } => 3,
4819        EnsureAction::Install {
4820            canic_init: Some(crate::fleet_ensure::model::DesiredCanisterInit::Store { .. }),
4821            ..
4822        }
4823        | EnsureAction::DeleteSnapshot { .. } => 4,
4824        EnsureAction::Uninstall { .. } | EnsureAction::Install { .. } => 5,
4825        EnsureAction::Start { .. } => 6,
4826        EnsureAction::FleetProtocol { .. } | EnsureAction::Protocol { .. } => 7,
4827        EnsureAction::Transfer { .. } => 8,
4828        EnsureAction::Stop { .. } => 9,
4829        EnsureAction::Delete { .. } => 10,
4830    }
4831}
4832
4833#[cfg(test)]
4834mod tests {
4835    use super::*;
4836
4837    #[test]
4838    fn store_binding_requires_exact_retained_root_or_reviewed_root_install() {
4839        use crate::fleet_ensure::{
4840            model::{CanisterDisposition, CanisterPlan, DesiredCanisterInit, InstallMode},
4841            tests::MockPlatform,
4842        };
4843        let (state, _) = retained_evidence();
4844        let mut plan = estate_funding_plan();
4845        let desired = serde_json::from_value(serde_json::json!({
4846            "canisters": [], "cycles_ledger": "um5iw-rqaaa-aaaaq-qaaba-cai",
4847            "environment": "local", "fleet": "fleet", "ledger_fee_cycles": "0",
4848            "management_creation_fee_cycles": "0", "material_cycle_threshold": "1",
4849            "maximum_observation_burn_cycles": "1", "maximum_stalled_observations": 2,
4850            "maximum_update_burn_cycles": "2", "operator": "rrkah-fqaaa-aaaaa-aaaaq-cai",
4851            "schema_version": FLEET_ENSURE_SCHEMA_VERSION, "treasury": "root"
4852        }))
4853        .unwrap();
4854        let mut platform = MockPlatform::new(desired, []);
4855        platform.rejected_activation_root = Some("root".into());
4856        let store = EnsureAction::Install {
4857            canic_init: Some(DesiredCanisterInit::Store {
4858                root: "root".into(),
4859            }),
4860            reinstall_witness: None,
4861            init_arg: None,
4862            init_arg_sha256: None,
4863            init_candid: None,
4864            init_candid_sha256: None,
4865            mode: InstallMode::Install,
4866            name: "store".into(),
4867            principal: "store-principal".into(),
4868            wasm: "store.wasm".into(),
4869            wasm_sha256: "store-hash".into(),
4870        };
4871        assert!(matches!(
4872            verify_store_activation_binding(&plan, &store, &state, &mut platform),
4873            Err(EnsureWorkflowError::Platform(_))
4874        ));
4875        assert_eq!(
4876            platform.retained_activation_checks,
4877            [(plan.operation_id.clone(), "root".into())]
4878        );
4879        let mut root_install = store.clone();
4880        if let EnsureAction::Install {
4881            name, canic_init, ..
4882        } = &mut root_install
4883        {
4884            *name = "other-root".into();
4885            *canic_init = Some(DesiredCanisterInit::Root {
4886                root: "other-root".into(),
4887            });
4888        }
4889        plan.canisters.push(CanisterPlan {
4890            actions: vec![root_install],
4891            disposition: CanisterDisposition::Create,
4892            name: "root".into(),
4893            observed_cycles: 0,
4894            principal: None,
4895        });
4896        assert!(verify_store_activation_binding(&plan, &store, &state, &mut platform).is_err());
4897        if let EnsureAction::Install {
4898            name, canic_init, ..
4899        } = &mut plan.canisters.last_mut().unwrap().actions[0]
4900        {
4901            *name = "root".into();
4902            *canic_init = Some(DesiredCanisterInit::Root {
4903                root: "root".into(),
4904            });
4905        }
4906        let reads = platform.retained_activation_checks.len();
4907        verify_store_activation_binding(&plan, &store, &state, &mut platform).unwrap();
4908        assert_eq!(platform.retained_activation_checks.len(), reads);
4909        plan.canisters.pop();
4910        platform.rejected_activation_root = None;
4911        verify_store_activation_binding(&plan, &store, &state, &mut platform).unwrap();
4912        assert_eq!(platform.retained_activation_checks.len(), reads + 1);
4913    }
4914
4915    #[test]
4916    fn root_prerequisite_retains_activation_authority_across_desired_refresh() {
4917        let (mut state, mut journal) = retained_evidence();
4918        let mut plan = estate_funding_plan();
4919        plan.scope = FleetEnsurePlanScope::RootReinstallPrerequisite;
4920        plan.conservation.estate_funding_domains.clear();
4921        journal.effects.clear();
4922        journal.completion = FleetEnsureCompletion::Converged;
4923        let before = state.clone();
4924        assert!(matches!(
4925            reconcile_retained_planning_state::<std::io::Error>(
4926                &mut state,
4927                Some(&plan),
4928                Some(&journal),
4929                "fleet",
4930                "local",
4931                "refreshed-imports"
4932            ),
4933            Err(EnsureWorkflowError::RootActivationDesiredConflict { .. })
4934        ));
4935        assert_eq!(state, before);
4936        assert!(
4937            reconcile_retained_planning_state::<std::io::Error>(
4938                &mut state,
4939                Some(&plan),
4940                Some(&journal),
4941                "fleet",
4942                "local",
4943                "desired"
4944            )
4945            .unwrap()
4946            .is_some()
4947        );
4948    }
4949
4950    fn terminal_component_entry(canister: &str, parent: &str) -> crate::registry::RegistryEntry {
4951        crate::registry::RegistryEntry {
4952            pid: canister.to_string(),
4953            role: Some("managed_component".to_string()),
4954            parent_pid: Some(parent.to_string()),
4955            module_hash: Some("11".repeat(32)),
4956            protocol_binding: Some(crate::protocol_binding::RegistryProtocolBinding {
4957                release_identity: "0.110.test".to_string(),
4958                role: canic_contracts::ids::CanisterRole::from("managed_component"),
4959                capabilities: BTreeSet::new(),
4960                candid_sha256: [1; 32],
4961                protocol_profile_digest:
4962                    canic_core::role_contract::ProtocolProfileDigest::from_bytes([2; 32]),
4963            }),
4964        }
4965    }
4966
4967    pub(super) fn retained_evidence() -> (FleetEnsureStateRecord, FleetEnsureJournalRecord) {
4968        let state = FleetEnsureStateRecord {
4969            active_registry: None,
4970            completed_reinstall_action_sha256: BTreeMap::from([(
4971                "root".to_string(),
4972                "action".to_string(),
4973            )]),
4974            completed_reinstall_operation_id: Some("operation".to_string()),
4975            completed_reinstalls: BTreeMap::from([("root".to_string(), 7)]),
4976            fleet: "fleet".to_string(),
4977            pending_principals: BTreeMap::new(),
4978            principals: BTreeMap::new(),
4979            retained_cycles_by_principal: BTreeMap::new(),
4980            schema_version: crate::fleet_ensure::model::FLEET_ENSURE_SCHEMA_VERSION,
4981            topology: BTreeMap::new(),
4982        };
4983        let journal = FleetEnsureJournalRecord {
4984            release: None,
4985            bootstrap_registration_recovery: None,
4986            funding_observations: BTreeMap::new(),
4987            funding_reviews: Vec::new(),
4988            successor_phases: Vec::new(),
4989            completion: FleetEnsureCompletion::ReplanRequired,
4990            estate_funding_required: None,
4991            effects: vec![crate::fleet_ensure::model::EffectRecord {
4992                publication_attempts: 0,
4993                maintenance_attempts: 0,
4994                action_sha256: "action".to_string(),
4995                created_principal: None,
4996                destination_post_cycles: None,
4997                destination_pre_cycles: None,
4998                post_cycles: None,
4999                pre_cycles: None,
5000                pre_canister_version: Some(7),
5001                progress_identity: None,
5002                receipt: None,
5003                state: EffectState::Applied,
5004            }],
5005            fleet: "fleet".to_string(),
5006            initial_controlled_cycles: 0,
5007            initial_estate_funding_cycles_by_root: BTreeMap::new(),
5008            initial_operator_cycles: 0,
5009            operation_id: "operation".to_string(),
5010            plan_sha256: "plan".to_string(),
5011            schema_version: crate::fleet_ensure::model::FLEET_ENSURE_SCHEMA_VERSION,
5012            stalled_observations: 0,
5013        };
5014        (state, journal)
5015    }
5016
5017    pub(super) fn estate_funding_plan() -> FleetEnsurePlan {
5018        FleetEnsurePlan {
5019            continuation: None,
5020            canisters: Vec::new(),
5021            conservation: CycleConservation {
5022                estate_funding_domains: vec![crate::fleet_ensure::model::EstateFundingDomainPlan {
5023                    allocated_workloads: 0,
5024                    available_cycles: Some(40),
5025                    available_pool_slots: 2,
5026                    creation_amount_cycles: 45,
5027                    creation_execution_margin_cycles: 5,
5028                    readiness_floor_cycles: 35,
5029                    cycles_ledger: "ledger".to_string(),
5030                    eligible_ready_pool_assets: 0,
5031                    initial_pool_assets: Vec::new(),
5032                    ledger_fee_cycles: 5,
5033                    management_creation_fee_cycles: 5,
5034                    maximum_creation_debit_cycles: 100,
5035                    maximum_creation_fee_cycles: 20,
5036                    maximum_funding_cycles: 60,
5037                    occupied_pool_assets: 0,
5038                    pending_creation_count: 0,
5039                    pending_creation: None,
5040                    planned_initial_workloads: 2,
5041                    pool_maximum_size: 2,
5042                    required_creation_count: 2,
5043                    root: "root".to_string(),
5044                    root_principal: Some("rrkah-fqaaa-aaaaa-aaaaq-cai".to_string()),
5045                    shortfall_cycles: 60,
5046                }],
5047                expected_post_operation_cycles: 80,
5048                maximum_execution_burn_cycles: 0,
5049                maximum_new_funding_cycles: 0,
5050                maximum_operator_debit_cycles: 0,
5051                maximum_unavoidable_fee_cycles: 0,
5052                observed_controlled_cycles: 40,
5053                retained_in_reused_canisters_cycles: 0,
5054                scheduled_transfer_cycles: 0,
5055            },
5056            desired_sha256: "desired".to_string(),
5057            environment: "local".to_string(),
5058            fleet: "fleet".to_string(),
5059            operation_id: "operation".to_string(),
5060            plan_sha256: "plan".to_string(),
5061            planned_at_time: 1,
5062            protocol_actions: Vec::new(),
5063            recovery_review: None,
5064            reinstall: None,
5065            infrastructure_bootstrap: None,
5066            root_reinstall_bindings: Vec::new(),
5067            root_start_authority: None,
5068            reviewed_desired: None,
5069            schema_version: FLEET_ENSURE_SCHEMA_VERSION,
5070            scope: FleetEnsurePlanScope::Full,
5071            terminal_inventory_operation_id: None,
5072        }
5073    }
5074
5075    pub(super) fn estate_funding_observation(balance_cycles: Option<u128>) -> FleetObservation {
5076        FleetObservation {
5077            additional_controlled_cycles: BTreeMap::new(),
5078            canisters: BTreeMap::new(),
5079            estate_funding_domains: BTreeMap::from([(
5080                "root".to_string(),
5081                crate::fleet_ensure::model::EstateFundingDomainObservation {
5082                    balance_cycles,
5083                    cycles_ledger: "ledger".to_string(),
5084                    pool: None,
5085                    root_principal: Some("rrkah-fqaaa-aaaaa-aaaaq-cai".to_string()),
5086                },
5087            )]),
5088            ledger_fee_cycles: 0,
5089            operator_cycles: 0,
5090            protocol_ready: BTreeMap::new(),
5091        }
5092    }
5093
5094    fn terminal_estate_creation_observation() -> FleetObservation {
5095        let mut observation = estate_funding_observation(Some(0));
5096        let receipt =
5097            |byte, block_index| crate::fleet_ensure::model::EstatePoolCreationReceiptObservation {
5098                block_index,
5099                operation_id: format!("{byte:02x}").repeat(32),
5100                cycles_ledger: "ledger".to_string(),
5101                ledger_amount_cycles: 45,
5102                ledger_fee_cycles: 5,
5103                readiness_floor_cycles: 35,
5104                creation_execution_margin_cycles: 5,
5105                management_creation_fee_cycles: 5,
5106                first_observed_cycles: Some(39),
5107            };
5108        observation
5109            .estate_funding_domains
5110            .get_mut("root")
5111            .expect("Root funding domain")
5112            .pool = Some(crate::fleet_ensure::model::EstatePoolInventoryObservation {
5113            assets: vec![
5114                crate::fleet_ensure::model::EstatePoolAssetObservation {
5115                    creation_receipt: Some(receipt(1, 10)),
5116                    cycles: 35,
5117                    lifecycle: crate::fleet_ensure::model::EstatePoolAssetLifecycle::Ready,
5118                    origin: crate::fleet_ensure::model::EstatePoolAssetOrigin::Created,
5119                    principal: "created-one".to_string(),
5120                },
5121                crate::fleet_ensure::model::EstatePoolAssetObservation {
5122                    creation_receipt: Some(receipt(2, 11)),
5123                    cycles: 35,
5124                    lifecycle: crate::fleet_ensure::model::EstatePoolAssetLifecycle::Workload,
5125                    origin: crate::fleet_ensure::model::EstatePoolAssetOrigin::Created,
5126                    principal: "created-two".to_string(),
5127                },
5128            ],
5129            maximum_size: 2,
5130            minimum_size: 0,
5131            pending_creation: None,
5132            readiness_floor_cycles: 35,
5133            creation_execution_margin_cycles: 5,
5134        });
5135        observation
5136    }
5137
5138    fn estate_creation_receipt(
5139        observation: &mut FleetObservation,
5140        asset_index: usize,
5141    ) -> &mut crate::fleet_ensure::model::EstatePoolCreationReceiptObservation {
5142        observation
5143            .estate_funding_domains
5144            .get_mut("root")
5145            .expect("Root domain")
5146            .pool
5147            .as_mut()
5148            .expect("pool")
5149            .assets[asset_index]
5150            .creation_receipt
5151            .as_mut()
5152            .expect("receipt")
5153    }
5154
5155    #[test]
5156    fn autonomous_reserve_creation_reconciles_without_host_provisioning() {
5157        let mut plan = estate_funding_plan();
5158        let domain = &mut plan.conservation.estate_funding_domains[0];
5159        domain.available_cycles = Some(100);
5160        domain.maximum_funding_cycles = 0;
5161        domain.shortfall_cycles = 0;
5162        let (state, mut journal) = retained_evidence();
5163        journal.effects.clear();
5164        journal
5165            .initial_estate_funding_cycles_by_root
5166            .insert("root".to_string(), 100);
5167        let terminal = terminal_estate_creation_observation();
5168        assert_eq!(
5169            reconcile_estate_funding::<std::io::Error>(&plan, &journal, &state, &terminal)
5170                .expect("reconcile exact Root-owned creation against the reviewed reserve"),
5171            (0, 20)
5172        );
5173    }
5174
5175    #[test]
5176    fn terminal_estate_creation_uses_exact_protected_receipts() {
5177        let plan = estate_funding_plan();
5178        let domain = &plan.conservation.estate_funding_domains[0];
5179        let (state, _) = retained_evidence();
5180        let exact = terminal_estate_creation_observation();
5181        assert_eq!(
5182            exact_estate_creation_costs::<std::io::Error>(&state, &exact, domain)
5183                .expect("reconcile exact creation receipts"),
5184            (100, 20)
5185        );
5186        for required_creation_count in [0, 1] {
5187            let mut insufficient = domain.clone();
5188            insufficient.required_creation_count = required_creation_count;
5189            assert!(matches!(
5190                exact_estate_creation_costs::<std::io::Error>(&state, &exact, &insufficient),
5191                Err(EnsureWorkflowError::JournalIntegrity)
5192            ));
5193        }
5194
5195        let mut drifted = Vec::new();
5196        let mut changed = exact.clone();
5197        estate_creation_receipt(&mut changed, 0).ledger_amount_cycles = 44;
5198        drifted.push(changed);
5199        let mut changed = exact.clone();
5200        estate_creation_receipt(&mut changed, 1).operation_id = "01".repeat(32);
5201        drifted.push(changed);
5202        let mut changed = exact.clone();
5203        estate_creation_receipt(&mut changed, 1).block_index = 10;
5204        drifted.push(changed);
5205        let mut changed = exact.clone();
5206        estate_creation_receipt(&mut changed, 0).first_observed_cycles = None;
5207        drifted.push(changed);
5208        let mut changed = exact.clone();
5209        estate_creation_receipt(&mut changed, 0).first_observed_cycles = Some(34);
5210        drifted.push(changed);
5211        let mut donated = exact.clone();
5212        estate_creation_receipt(&mut donated, 0).first_observed_cycles = Some(41);
5213        assert_eq!(
5214            exact_estate_creation_costs::<std::io::Error>(&state, &donated, domain)
5215                .expect("external donation leaves exact creation cost unchanged"),
5216            (100, 20)
5217        );
5218        let mut changed = exact;
5219        changed
5220            .estate_funding_domains
5221            .get_mut("root")
5222            .expect("Root domain")
5223            .pool
5224            .as_mut()
5225            .expect("pool")
5226            .assets[0]
5227            .origin = crate::fleet_ensure::model::EstatePoolAssetOrigin::Imported;
5228        drifted.push(changed);
5229        for observation in drifted {
5230            assert!(matches!(
5231                exact_estate_creation_costs::<std::io::Error>(&state, &observation, domain,),
5232                Err(EnsureWorkflowError::JournalIntegrity)
5233            ));
5234        }
5235    }
5236
5237    #[test]
5238    fn estate_funding_pause_is_exact_and_clears_only_after_sufficient_balance() {
5239        let plan = estate_funding_plan();
5240        let (state, mut journal) = retained_evidence();
5241        let required = estate_funding_requirement::<std::io::Error>(
5242            &plan,
5243            &state,
5244            &estate_funding_observation(Some(40)),
5245        )
5246        .expect("validate funding observation")
5247        .expect("underfunded estate pauses");
5248        assert_eq!(required.available_cycles, 40);
5249        assert_eq!(required.maximum_creation_debit_cycles, 100);
5250        assert_eq!(required.shortfall_cycles, 60);
5251        assert!(
5252            estate_funding_requirement::<std::io::Error>(
5253                &plan,
5254                &state,
5255                &estate_funding_observation(Some(100)),
5256            )
5257            .expect("validate sufficient funding observation")
5258            .is_none()
5259        );
5260
5261        journal.completion = FleetEnsureCompletion::InProgress;
5262        journal.effects.clear();
5263        journal.estate_funding_required = Some(required);
5264        journal.initial_estate_funding_cycles_by_root = BTreeMap::from([("root".to_string(), 40)]);
5265        assert!(initial_estate_funding_is_exact(&journal, &plan));
5266        assert!(estate_funding_pause_is_exact(&journal, &plan, &state));
5267
5268        journal
5269            .estate_funding_required
5270            .as_mut()
5271            .expect("funding pause")
5272            .shortfall_cycles = 59;
5273        assert!(!estate_funding_pause_is_exact(&journal, &plan, &state));
5274
5275        *journal
5276            .initial_estate_funding_cycles_by_root
5277            .get_mut("root")
5278            .expect("initial Root funding balance") = 41;
5279        assert!(!initial_estate_funding_is_exact(&journal, &plan));
5280    }
5281
5282    #[test]
5283    fn current_creation_funding_pause_is_bound_to_exact_runtime_evidence() {
5284        let mut plan = estate_funding_plan();
5285        plan.conservation.estate_funding_domains[0].cycles_ledger = "aaaaa-aa".to_string();
5286        let (state, mut journal) = retained_evidence();
5287        let funding = canic_contracts::dto::component_provisioning::RootEstateFundingRequired {
5288            available: Cycles::new(40),
5289            attempt_count: 2,
5290            creation_amount: Cycles::new(45),
5291            cycles_ledger: candid::Principal::from_text("aaaaa-aa").expect("Ledger Principal"),
5292            execution_margin: Cycles::new(5),
5293            last_attempt_at_ns: Some(10),
5294            ledger_fee: Cycles::new(5),
5295            management_creation_fee: Cycles::new(5),
5296            operation_id: [17; 32],
5297            readiness_floor: Cycles::new(35),
5298            required: Cycles::new(50),
5299            retry_at_ns: 20,
5300            root: candid::Principal::from_text("rrkah-fqaaa-aaaaa-aaaaq-cai")
5301                .expect("Root Principal"),
5302            shortfall: Cycles::new(10),
5303        };
5304        let required =
5305            protocol_estate_funding_requirement::<std::io::Error>(&plan, &state, &funding)
5306                .expect("bind runtime funding evidence");
5307        journal.completion = FleetEnsureCompletion::InProgress;
5308        journal.effects.clear();
5309        journal.estate_funding_required = Some(required);
5310        assert!(estate_funding_pause_is_exact(&journal, &plan, &state));
5311
5312        journal
5313            .estate_funding_required
5314            .as_mut()
5315            .expect("funding pause")
5316            .pending_creation_operation_id = Some("AA".repeat(32));
5317        assert!(!estate_funding_pause_is_exact(&journal, &plan, &state));
5318
5319        let mut drifted = funding;
5320        drifted.execution_margin = Cycles::new(6);
5321        assert!(matches!(
5322            protocol_estate_funding_requirement::<std::io::Error>(&plan, &state, &drifted),
5323            Err(EnsureWorkflowError::PlanIntegrity)
5324        ));
5325    }
5326
5327    #[test]
5328    fn estate_funding_observation_rejects_missing_or_mismatched_authority() {
5329        let plan = estate_funding_plan();
5330        let (state, _) = retained_evidence();
5331        let mut missing_balance = estate_funding_observation(None);
5332        assert!(matches!(
5333            estate_funding_requirement::<std::io::Error>(&plan, &state, &missing_balance),
5334            Err(EnsureWorkflowError::PlanIntegrity)
5335        ));
5336
5337        let observed = missing_balance
5338            .estate_funding_domains
5339            .get_mut("root")
5340            .expect("Root funding domain");
5341        observed.balance_cycles = Some(40);
5342        observed.root_principal = Some("aaaaa-aa".to_string());
5343        assert!(matches!(
5344            estate_funding_requirement::<std::io::Error>(&plan, &state, &missing_balance),
5345            Err(EnsureWorkflowError::PlanIntegrity)
5346        ));
5347
5348        for changed_balance in [0, 39] {
5349            assert!(matches!(
5350                estate_funding_requirement::<std::io::Error>(
5351                    &plan,
5352                    &state,
5353                    &estate_funding_observation(Some(changed_balance)),
5354                ),
5355                Err(EnsureWorkflowError::DriftedBeforeApply)
5356            ));
5357        }
5358    }
5359
5360    #[test]
5361    fn estate_funding_surplus_preserves_reviewed_creation_limits() {
5362        let plan = estate_funding_plan();
5363        let original = plan.clone();
5364        let (state, _) = retained_evidence();
5365        assert!(
5366            estate_funding_requirement::<std::io::Error>(
5367                &plan,
5368                &state,
5369                &estate_funding_observation(Some(150)),
5370            )
5371            .unwrap()
5372            .is_none()
5373        );
5374        assert_eq!(plan, original);
5375        let mut extra_creation = terminal_estate_creation_observation();
5376        let pool = extra_creation
5377            .estate_funding_domains
5378            .get_mut("root")
5379            .unwrap()
5380            .pool
5381            .as_mut()
5382            .unwrap();
5383        let mut extra = pool.assets[0].clone();
5384        extra.principal = "unreviewed-third-creation".into();
5385        pool.assets.push(extra);
5386        assert!(matches!(
5387            exact_estate_creation_costs::<std::io::Error>(
5388                &state,
5389                &extra_creation,
5390                &plan.conservation.estate_funding_domains[0]
5391            ),
5392            Err(EnsureWorkflowError::JournalIntegrity)
5393        ));
5394    }
5395
5396    #[test]
5397    fn non_creation_protocol_completion_does_not_consume_estate_creation_authority() {
5398        let mut plan = estate_funding_plan();
5399        plan.conservation.estate_funding_domains[0].maximum_creation_debit_cycles = 0;
5400        plan.conservation.estate_funding_domains[0].maximum_creation_fee_cycles = 0;
5401        plan.conservation.estate_funding_domains[0].maximum_funding_cycles = 0;
5402        plan.conservation.estate_funding_domains[0].required_creation_count = 0;
5403        plan.conservation.estate_funding_domains[0].shortfall_cycles = 0;
5404        let action = EnsureAction::FleetProtocol {
5405            action: Box::new(
5406                crate::fleet_ensure::model::CurrentFleetProtocolAction::PublishStoreChunk {
5407                    request: canic_contracts::dto::template::TemplateChunkInput {
5408                        preparation: None,
5409                        template_id: canic_contracts::ids::TemplateId::from("root"),
5410                        version: canic_contracts::ids::TemplateVersion::from("current"),
5411                        chunk_index: 0,
5412                        bytes: vec![2],
5413                    },
5414                },
5415            ),
5416            candid: "store.did".to_string(),
5417            candid_sha256: "11".repeat(32),
5418            maximum_execution_burn_cycles: 0,
5419            name: "publish-store-chunk".to_string(),
5420            principal: "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(),
5421        };
5422        let action_hash = action_sha256(&action);
5423        plan.protocol_actions.push(action);
5424        let (state, mut journal) = retained_evidence();
5425        journal.initial_estate_funding_cycles_by_root = BTreeMap::from([("root".to_string(), 40)]);
5426        journal.effects = vec![EffectRecord {
5427            publication_attempts: 0,
5428            maintenance_attempts: 0,
5429            action_sha256: action_hash,
5430            created_principal: None,
5431            destination_post_cycles: None,
5432            destination_pre_cycles: None,
5433            post_cycles: None,
5434            pre_cycles: None,
5435            pre_canister_version: None,
5436            progress_identity: Some("store chunks prepared".to_string()),
5437            receipt: Some("receipt".to_string()),
5438            state: EffectState::Applied,
5439        }];
5440
5441        assert_eq!(
5442            reconcile_estate_funding::<std::io::Error>(
5443                &plan,
5444                &journal,
5445                &state,
5446                &estate_funding_observation(Some(40)),
5447            )
5448            .expect("non-creation protocol preserves the Root Ledger balance"),
5449            (0, 0),
5450        );
5451        journal.initial_controlled_cycles = 40;
5452        journal.initial_operator_cycles = 0;
5453        let original = plan.clone();
5454        for balance in [40, 41, 150] {
5455            let observed = estate_funding_observation(Some(balance));
5456            assert!(
5457                estate_funding_requirement::<std::io::Error>(&plan, &state, &observed)
5458                    .unwrap()
5459                    .is_none()
5460            );
5461            let actual =
5462                verify_terminal_conservation::<std::io::Error>(&plan, &journal, &state, &observed)
5463                    .unwrap();
5464            assert_eq!(actual.final_controlled_cycles, balance);
5465            assert_eq!(actual.observed_net_cycle_credit_cycles, balance - 40);
5466            assert_eq!(actual.estate_funding_cycles, 0);
5467            assert_eq!(actual.operator_debit_cycles, 0);
5468            assert_eq!(actual.exact_estate_creation_fee_cycles, 0);
5469        }
5470        assert_eq!(plan, original);
5471        assert!(matches!(
5472            reconcile_estate_funding::<std::io::Error>(
5473                &plan,
5474                &journal,
5475                &state,
5476                &estate_funding_observation(Some(39))
5477            ),
5478            Err(EnsureWorkflowError::Conservation(_))
5479        ));
5480    }
5481
5482    #[test]
5483    fn fresh_estate_funding_binds_the_created_root_from_durable_state() {
5484        let mut plan = estate_funding_plan();
5485        plan.conservation.estate_funding_domains[0].available_cycles = None;
5486        plan.conservation.estate_funding_domains[0].maximum_funding_cycles = 100;
5487        plan.conservation.estate_funding_domains[0].root_principal = None;
5488        let (mut state, _) = retained_evidence();
5489        state.pending_principals.insert(
5490            "root".to_string(),
5491            "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(),
5492        );
5493
5494        let required = estate_funding_requirement::<std::io::Error>(
5495            &plan,
5496            &state,
5497            &estate_funding_observation(Some(0)),
5498        )
5499        .expect("validate created Root funding authority")
5500        .expect("fresh underfunded estate pauses");
5501        assert_eq!(required.shortfall_cycles, 100);
5502        assert_eq!(required.root_principal, "rrkah-fqaaa-aaaaa-aaaaq-cai");
5503
5504        state
5505            .pending_principals
5506            .insert("root".to_string(), "aaaaa-aa".to_string());
5507        assert!(matches!(
5508            estate_funding_requirement::<std::io::Error>(
5509                &plan,
5510                &state,
5511                &estate_funding_observation(Some(0)),
5512            ),
5513            Err(EnsureWorkflowError::PlanIntegrity)
5514        ));
5515    }
5516
5517    #[test]
5518    fn unchanged_estate_funding_pause_does_not_rewrite_its_journal() {
5519        let root = crate::test_support::temp_dir("estate-funding-pause");
5520        let paths = EnsurePaths::under(&root, "local", "fleet");
5521        let plan = estate_funding_plan();
5522        let (state, mut journal) = retained_evidence();
5523        journal.completion = FleetEnsureCompletion::InProgress;
5524        journal.effects.clear();
5525        journal.initial_estate_funding_cycles_by_root = BTreeMap::from([("root".to_string(), 40)]);
5526        let required = estate_funding_requirement::<std::io::Error>(
5527            &plan,
5528            &state,
5529            &estate_funding_observation(Some(40)),
5530        )
5531        .expect("validate funding observation")
5532        .expect("underfunded estate pauses");
5533
5534        assert!(
5535            retain_estate_funding_pause::<std::io::Error>(&paths, &mut journal, Some(&required),)
5536                .expect("retain first funding pause")
5537        );
5538        let retained = std::fs::read(&paths.journal).expect("read first retained pause");
5539        assert!(
5540            !retain_estate_funding_pause::<std::io::Error>(&paths, &mut journal, Some(&required),)
5541                .expect("replay unchanged funding pause")
5542        );
5543        assert_eq!(
5544            std::fs::read(&paths.journal).expect("reread unchanged retained pause"),
5545            retained
5546        );
5547        assert!(
5548            retain_estate_funding_pause::<std::io::Error>(&paths, &mut journal, None)
5549                .expect("clear funded pause")
5550        );
5551        assert!(journal.estate_funding_required.is_none());
5552
5553        std::fs::remove_dir_all(root).expect("remove funding-pause fixture");
5554    }
5555
5556    #[test]
5557    fn estate_funding_is_rechecked_until_the_first_protocol_effect_starts() {
5558        let protocol_action = || EnsureAction::FleetProtocol {
5559            action: Box::new(
5560                crate::fleet_ensure::model::CurrentFleetProtocolAction::PublishStoreChunk {
5561                    request: canic_contracts::dto::template::TemplateChunkInput {
5562                        preparation: None,
5563                        template_id: canic_contracts::ids::TemplateId::from("root"),
5564                        version: canic_contracts::ids::TemplateVersion::from("current"),
5565                        chunk_index: 0,
5566                        bytes: vec![2],
5567                    },
5568                },
5569            ),
5570            candid: "store.did".to_string(),
5571            candid_sha256: "11".repeat(32),
5572            maximum_execution_burn_cycles: 1,
5573            name: "prepare".to_string(),
5574            principal: "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(),
5575        };
5576        let first = protocol_action();
5577        let second = protocol_action();
5578        let actions = [&first, &second];
5579        let (_, mut journal) = retained_evidence();
5580        journal.effects = vec![EffectRecord {
5581            publication_attempts: 0,
5582            maintenance_attempts: 0,
5583            action_sha256: action_sha256(&first),
5584            created_principal: None,
5585            destination_post_cycles: None,
5586            destination_pre_cycles: None,
5587            post_cycles: None,
5588            pre_cycles: None,
5589            pre_canister_version: None,
5590            progress_identity: None,
5591            receipt: None,
5592            state: EffectState::Intent,
5593        }];
5594        assert!(!prior_fleet_protocol_effect_started(&actions, &journal, 1));
5595
5596        journal.effects[0].state = EffectState::Issued;
5597        assert!(prior_fleet_protocol_effect_started(&actions, &journal, 1));
5598    }
5599
5600    #[test]
5601    fn configured_projection_publishes_unique_bindings_without_losing_dynamic_cycles() {
5602        let asset = "rrkah-fqaaa-aaaaa-aaaaq-cai";
5603        let dynamic = "ryjl3-tyaaa-aaaaa-aaaba-cai";
5604        for kind in ["pool", "component"] {
5605            let desired = serde_json::from_value(serde_json::json!({
5606                "canisters": [{
5607                    "controllers": [], "initial_cycles": "0", "kind": kind,
5608                    "minimum_cycles": "0", "name": "configured", "presence": "present",
5609                    "principal": asset, "replace": false, "subnet": "subnet"
5610                }],
5611                "cycles_ledger": "um5iw-rqaaa-aaaaq-qaaba-cai",
5612                "environment": "local", "fleet": "fleet", "ledger_fee_cycles": "0",
5613                "management_creation_fee_cycles": "0", "material_cycle_threshold": "1",
5614                "maximum_observation_burn_cycles": "1", "maximum_stalled_observations": 2,
5615                "maximum_update_burn_cycles": "2", "operator": asset,
5616                "schema_version": FLEET_ENSURE_SCHEMA_VERSION, "treasury": "configured"
5617            }))
5618            .expect("current desired projection");
5619            let (mut state, mut journal) = retained_evidence();
5620            journal.effects.clear();
5621            state.principals = BTreeMap::from([
5622                ("configured".to_string(), asset.to_string()),
5623                (format!("observed:{asset}"), asset.to_string()),
5624                (format!("observed:{dynamic}"), dynamic.to_string()),
5625            ]);
5626            state.retained_cycles_by_principal =
5627                BTreeMap::from([(asset.to_string(), 100), (dynamic.to_string(), 60)]);
5628            let cycles = state.retained_cycles_by_principal.clone();
5629            let journal_before = serde_json::to_vec(&journal).expect("journal before projection");
5630            let plan = estate_funding_plan();
5631
5632            publish_terminal_state(&desired, &plan, &journal, &mut state)
5633                .expect("publish current configured topology");
5634            assert_eq!(
5635                state.principals,
5636                BTreeMap::from([
5637                    ("configured".to_string(), asset.to_string()),
5638                    (format!("observed:{dynamic}"), dynamic.to_string()),
5639                ])
5640            );
5641            assert_eq!(state.retained_cycles_by_principal, cycles);
5642            let published = state.clone();
5643            publish_terminal_state(&desired, &plan, &journal, &mut state)
5644                .expect("repeat projection");
5645            assert_eq!(state, published);
5646            assert_eq!(
5647                serde_json::to_vec(&journal).expect("journal after projection"),
5648                journal_before
5649            );
5650        }
5651    }
5652
5653    #[test]
5654    fn configured_projection_preserves_duplicate_authority_rejection() {
5655        let asset = "rrkah-fqaaa-aaaaa-aaaaq-cai";
5656        for configured in [false, true] {
5657            let (mut state, _) = retained_evidence();
5658            state.principals = BTreeMap::from([
5659                ("first".to_string(), asset.to_string()),
5660                ("second".to_string(), asset.to_string()),
5661            ]);
5662            if configured {
5663                let topology = crate::fleet_ensure::model::FleetEnsureTopologyRecord {
5664                    kind: crate::fleet_ensure::model::DesiredCanisterKind::Pool,
5665                    module_hash: None,
5666                    parent: None,
5667                    protocol_binding: None,
5668                    role: None,
5669                };
5670                state.topology = BTreeMap::from([
5671                    ("first".to_string(), topology.clone()),
5672                    ("second".to_string(), topology),
5673                ]);
5674            }
5675            let before = state.clone();
5676            retain_configured_principal_bindings(&mut state);
5677            assert_eq!(state, before);
5678            for entries in [Vec::new(), vec![terminal_component_entry(asset, "parent")]] {
5679                assert!(matches!(
5680                    merge_terminal_inventory::<std::io::Error>(
5681                        &mut state,
5682                        crate::fleet_ensure::ops::TerminalFleetInventory {
5683                            entries,
5684                            ..Default::default()
5685                        }
5686                    ),
5687                    Err(EnsureWorkflowError::TerminalInventory(
5688                        TerminalInventoryError::DuplicateRetainedPrincipal
5689                    ))
5690                ));
5691                assert_eq!(state, before);
5692            }
5693        }
5694    }
5695
5696    #[test]
5697    fn terminal_inventory_allows_only_verified_pool_to_component_parent_transition() {
5698        let asset = "rrkah-fqaaa-aaaaa-aaaaq-cai";
5699        let old_root = "r7inp-6aaaa-aaaaa-aaabq-cai";
5700        let hub = "ryjl3-tyaaa-aaaaa-aaaba-cai";
5701        let state = || {
5702            let (mut state, _) = retained_evidence();
5703            state.principals = BTreeMap::from([
5704                ("asset".to_string(), asset.to_string()),
5705                ("hub".to_string(), hub.to_string()),
5706                ("root".to_string(), old_root.to_string()),
5707            ]);
5708            state.topology.insert(
5709                "asset".to_string(),
5710                crate::fleet_ensure::model::FleetEnsureTopologyRecord {
5711                    kind: crate::fleet_ensure::model::DesiredCanisterKind::Pool,
5712                    module_hash: None,
5713                    parent: Some("root".to_string()),
5714                    protocol_binding: None,
5715                    role: Some("canister_pool_asset".to_string()),
5716                },
5717            );
5718            state
5719        };
5720
5721        let mut promoted = state();
5722        merge_terminal_inventory::<std::io::Error>(
5723            &mut promoted,
5724            crate::fleet_ensure::ops::TerminalFleetInventory {
5725                active_registry: None,
5726                controlled_cycles_by_principal: BTreeMap::new(),
5727                entries: vec![terminal_component_entry(asset, hub)],
5728            },
5729        )
5730        .expect("verified pool workload assumes its terminal Component parent");
5731        let promoted = promoted.topology.get("asset").expect("promoted asset");
5732        assert_eq!(
5733            promoted.kind,
5734            crate::fleet_ensure::model::DesiredCanisterKind::Component
5735        );
5736        assert_eq!(promoted.parent.as_deref(), Some("hub"));
5737
5738        let mut already_component = state();
5739        already_component
5740            .topology
5741            .get_mut("asset")
5742            .expect("retained asset")
5743            .kind = crate::fleet_ensure::model::DesiredCanisterKind::Component;
5744        assert!(matches!(
5745            merge_terminal_inventory::<std::io::Error>(
5746                &mut already_component,
5747                crate::fleet_ensure::ops::TerminalFleetInventory {
5748                    active_registry: None,
5749                    controlled_cycles_by_principal: BTreeMap::new(),
5750                    entries: vec![terminal_component_entry(asset, hub)],
5751                },
5752            ),
5753            Err(EnsureWorkflowError::TerminalInventory(
5754                TerminalInventoryError::RetainedParentConflict {
5755                    canister,
5756                    retained_parent,
5757                    terminal_parent,
5758                }
5759            )) if canister == asset
5760                && retained_parent == "root"
5761                && terminal_parent == "hub"
5762        ));
5763
5764        for incomplete_entry in [
5765            crate::registry::RegistryEntry {
5766                module_hash: None,
5767                ..terminal_component_entry(asset, hub)
5768            },
5769            crate::registry::RegistryEntry {
5770                protocol_binding: None,
5771                ..terminal_component_entry(asset, hub)
5772            },
5773        ] {
5774            assert!(matches!(
5775                merge_terminal_inventory::<std::io::Error>(
5776                    &mut state(),
5777                    crate::fleet_ensure::ops::TerminalFleetInventory {
5778                        active_registry: None,
5779                        controlled_cycles_by_principal: BTreeMap::new(),
5780                        entries: vec![incomplete_entry],
5781                    },
5782                ),
5783                Err(EnsureWorkflowError::TerminalInventory(
5784                    TerminalInventoryError::RetainedParentConflict {
5785                        canister,
5786                        retained_parent,
5787                        terminal_parent,
5788                    }
5789                )) if canister == asset
5790                    && retained_parent == "root"
5791                    && terminal_parent == "hub"
5792            ));
5793        }
5794    }
5795
5796    #[test]
5797    fn reinstall_recovery_review_requires_complete_exact_install_evidence() {
5798        use crate::fleet_ensure::model::{CanisterPlan, FleetReinstallRecord, InstallMode};
5799        let (state, mut journal) = retained_evidence();
5800        let mut plan = estate_funding_plan();
5801        plan.conservation.estate_funding_domains.clear();
5802        plan.reinstall = Some(Box::new(FleetReinstallRecord {
5803            activation_reset: None,
5804            operation_id: plan.operation_id.clone(),
5805            source_operation_id: "source-operation".into(),
5806            authorities: Vec::new(),
5807            assets: Vec::new(),
5808        }));
5809        let action = EnsureAction::Install {
5810            canic_init: None,
5811            reinstall_witness: None,
5812            init_arg: None,
5813            init_arg_sha256: None,
5814            init_candid: None,
5815            init_candid_sha256: None,
5816            mode: InstallMode::Reinstall,
5817            name: "root".into(),
5818            principal: "rrkah-fqaaa-aaaaa-aaaaq-cai".into(),
5819            wasm: "root.wasm".into(),
5820            wasm_sha256: "22".repeat(32),
5821        };
5822        journal.effects[0].action_sha256 = action_sha256(&action);
5823        plan.canisters.push(CanisterPlan {
5824            actions: vec![action],
5825            disposition: CanisterDisposition::Reuse,
5826            name: "root".into(),
5827            observed_cycles: 0,
5828            principal: Some("rrkah-fqaaa-aaaaa-aaaaq-cai".into()),
5829        });
5830        let admit = |journal: &FleetEnsureJournalRecord| {
5831            reinstall_recovery_ready::<std::io::Error>(&plan, journal, &state)
5832        };
5833        assert!(admit(&journal).unwrap());
5834        let mut issued = journal.clone();
5835        issued.effects[0].state = EffectState::Issued;
5836        let mut missing = journal.clone();
5837        missing.effects.clear();
5838        let mut wrong_action = journal.clone();
5839        wrong_action.effects[0].action_sha256 = "other".into();
5840        let mut no_version = journal.clone();
5841        no_version.effects[0].pre_canister_version = None;
5842        let mut wrong_operation = journal.clone();
5843        wrong_operation.operation_id = "other".into();
5844        for invalid in [issued, missing, wrong_action, no_version, wrong_operation] {
5845            assert!(matches!(
5846                admit(&invalid),
5847                Err(EnsureWorkflowError::JournalIntegrity)
5848            ));
5849        }
5850        for completion in [
5851            FleetEnsureCompletion::InProgress,
5852            FleetEnsureCompletion::Converged,
5853        ] {
5854            let mut other = journal.clone();
5855            other.completion = completion;
5856            assert!(!admit(&other).unwrap());
5857        }
5858    }
5859
5860    #[test]
5861    fn completed_reinstall_evidence_requires_exact_operation_action_and_version() {
5862        let (state, journal) = retained_evidence();
5863        assert!(retained_reinstall_operation_matches(
5864            &state,
5865            "fleet",
5866            "operation",
5867            &journal,
5868        ));
5869
5870        let mut wrong_action = journal.clone();
5871        wrong_action.effects[0].action_sha256 = "other".to_string();
5872        assert!(!retained_reinstall_operation_matches(
5873            &state,
5874            "fleet",
5875            "operation",
5876            &wrong_action,
5877        ));
5878
5879        let mut wrong_version = journal.clone();
5880        wrong_version.effects[0].pre_canister_version = Some(8);
5881        assert!(!retained_reinstall_operation_matches(
5882            &state,
5883            "fleet",
5884            "operation",
5885            &wrong_version,
5886        ));
5887
5888        for (fleet, operation) in [("other", "operation"), ("fleet", "other")] {
5889            assert!(!retained_reinstall_operation_matches(
5890                &state, fleet, operation, &journal,
5891            ));
5892        }
5893
5894        let mut incomplete = journal;
5895        incomplete.effects[0].state = EffectState::Issued;
5896        assert!(!retained_reinstall_operation_matches(
5897            &state,
5898            "fleet",
5899            "operation",
5900            &incomplete,
5901        ));
5902    }
5903
5904    #[test]
5905    fn fresh_fleet_alternate_desired_cannot_clear_completed_reinstall_evidence() {
5906        let (state, journal) = retained_evidence();
5907        assert!(retained_reinstall_desired_conflict(
5908            &state,
5909            "fleet",
5910            "operation",
5911            "reviewed-desired",
5912            "alternate-desired",
5913            &journal,
5914        ));
5915        assert!(!retained_reinstall_desired_conflict(
5916            &state,
5917            "fleet",
5918            "operation",
5919            "reviewed-desired",
5920            "reviewed-desired",
5921            &journal,
5922        ));
5923
5924        let mut terminal = journal;
5925        terminal.completion = FleetEnsureCompletion::Converged;
5926        assert!(!retained_reinstall_desired_conflict(
5927            &state,
5928            "fleet",
5929            "operation",
5930            "reviewed-desired",
5931            "alternate-desired",
5932            &terminal,
5933        ));
5934    }
5935
5936    fn cycle_observation(
5937        lifecycle: crate::fleet_ensure::model::RootOwnedCanisterLifecycle,
5938        cycles: u128,
5939    ) -> FleetObservation {
5940        FleetObservation {
5941            additional_controlled_cycles: BTreeMap::new(),
5942            canisters: BTreeMap::from([(
5943                "asset".to_string(),
5944                Some(crate::fleet_ensure::model::LiveCanister {
5945                    canister_version: Some(1),
5946                    controllers: Vec::new(),
5947                    cycles,
5948                    module_sha256: None,
5949                    principal: "controlled-principal".to_string(),
5950                    reinstall_required: false,
5951                    root_owned_lifecycle: Some(lifecycle),
5952                    status: crate::fleet_ensure::model::CanisterRuntimeStatus::Stopped,
5953                }),
5954            )]),
5955            estate_funding_domains: BTreeMap::new(),
5956            ledger_fee_cycles: 0,
5957            operator_cycles: 0,
5958            protocol_ready: BTreeMap::new(),
5959        }
5960    }
5961
5962    #[test]
5963    fn fresh_fleet_idle_observation_accepts_donations_without_double_counting() {
5964        use crate::fleet_ensure::model::RootOwnedCanisterLifecycle;
5965
5966        let additional = BTreeMap::from([("controlled-principal".to_string(), 100)]);
5967        let mut idle = cycle_observation(RootOwnedCanisterLifecycle::Idle, 100);
5968        attach_terminal_cycles::<std::io::Error>(&mut idle, additional.clone())
5969            .expect("merge exact Idle observation");
5970        assert!(idle.additional_controlled_cycles.is_empty());
5971
5972        let mut donated_idle = cycle_observation(RootOwnedCanisterLifecycle::Idle, 99);
5973        attach_terminal_cycles::<std::io::Error>(&mut donated_idle, additional.clone()).unwrap();
5974        assert_eq!(
5975            controlled_cycles::<std::io::Error>(&donated_idle).unwrap(),
5976            100
5977        );
5978        assert!(donated_idle.additional_controlled_cycles.is_empty());
5979
5980        let mut conflicting_idle = cycle_observation(RootOwnedCanisterLifecycle::Idle, 101);
5981        assert!(matches!(
5982            attach_terminal_cycles::<std::io::Error>(&mut conflicting_idle, additional.clone()),
5983            Err(EnsureWorkflowError::TerminalInventory(
5984                TerminalInventoryError::IdleCycleObservationConflict {
5985                    canister,
5986                    expected: 101,
5987                    observed: 100,
5988                }
5989            )) if canister == "controlled-principal"
5990        ));
5991
5992        let mut workload = cycle_observation(RootOwnedCanisterLifecycle::Workload, 99);
5993        attach_terminal_cycles::<std::io::Error>(&mut workload, additional)
5994            .expect("merge conservative workload observation");
5995        assert_eq!(
5996            workload
5997                .canisters
5998                .get("asset")
5999                .and_then(Option::as_ref)
6000                .map(|live| live.cycles),
6001            Some(99),
6002        );
6003    }
6004}