Skip to main content

canic_host/fleet_ensure/workflow/
mod.rs

1//! Module: fleet_ensure::workflow
2//!
3//! Responsibility: plan and advance one durable idempotent Fleet convergence operation.
4//! Does not own: policy decisions, IC transport, or storage mechanics.
5//! Boundary: persists exact intent before each ops effect and reconciles it before any retry.
6
7mod continuation;
8mod funding;
9#[cfg(test)]
10mod funding_tests;
11mod reinstall;
12mod retained_plan;
13mod root_reinstall;
14mod settlement;
15
16use crate::fleet_ensure::{
17    dto::{FleetEnsurePhase, FleetEnsureProgress, FleetEnsureProgressState},
18    inventory::project_current_fleet_inventory,
19    model::{
20        ActualCycleConservation, CanisterDisposition, CurrentFleetProtocolAction,
21        CycleConservation, EffectRecord, EffectState, EnsureAction, EstateFundingDomainPlan,
22        EstateFundingRequiredRecord, FLEET_ENSURE_SCHEMA_VERSION, FleetEnsureCompletion,
23        FleetEnsureJournalRecord, FleetEnsurePlan, FleetEnsurePlanScope, FleetEnsureReport,
24        FleetEnsureStateRecord, FleetObservation, RootManagementObservation,
25        create_balance_is_terminal,
26    },
27    ops::{
28        EffectRetry, EnsurePaths, EnsurePlatform, EnsureStateError, action_sha256,
29        compact_inline_plan, effect_preparation::prepare_effect, lock_operation, read_plan,
30        read_root_start_authority, read_state, reserve_fixture_publication_attempt,
31        resolve_desired_artifacts, retain_configured_principal_bindings, write_journal, write_plan,
32        write_state,
33    },
34    policy::{
35        EnsurePolicyError, RootStartPlanInput, compile_plan, compile_root_start_prerequisite_plan,
36        effect_observation_policy, expected_plan_sha256, operation_id,
37        recompile_root_start_prerequisite_plan, validate_creation_fee_scope,
38        validate_path_identity, validate_path_labels,
39    },
40};
41use canic_core::cdk::types::Cycles;
42use std::{
43    collections::{BTreeMap, BTreeSet},
44    path::Path,
45    time::Instant,
46};
47use thiserror::Error as ThisError;
48
49pub use reinstall::plan_reinstall;
50
51/// Exact no-effect evidence that a fresh pool cannot retain its readiness floor.
52
53#[derive(Debug, ThisError)]
54#[error(
55    "fresh pool canister {canister} requested {requested_creation_funding_cycles} creation cycles for readiness floor {readiness_floor_cycles}; its first live balance is {live_balance_cycles} cycles (readiness shortfall {readiness_shortfall_cycles}) and must retain {remaining_controller_burn_cycles} cycles for controller finalization (required pre-finalization balance {required_pre_finalization_balance_cycles}, pre-finalization shortfall {pre_finalization_shortfall_cycles}; maximum first-observation burn {maximum_observation_burn_cycles}); no controller or protocol action followed"
56)]
57pub struct FreshPoolCreationUnderfundedError {
58    pub canister: String,
59    pub live_balance_cycles: u128,
60    pub maximum_observation_burn_cycles: u128,
61    pub pre_finalization_shortfall_cycles: u128,
62    pub readiness_floor_cycles: u128,
63    pub readiness_shortfall_cycles: u128,
64    pub remaining_controller_burn_cycles: u128,
65    pub requested_creation_funding_cycles: u128,
66    pub required_pre_finalization_balance_cycles: u128,
67}
68
69/// Exact reviewed Root-account funding prerequisite that blocks protocol mutation.
70#[derive(Debug, ThisError)]
71#[error(
72    "Root {root} Cycles Ledger account {root_principal} on {cycles_ledger} has {available_cycles} cycles; {required_creation_count} autonomous creations require at most {maximum_creation_debit_cycles} cycles ({creation_amount_cycles} gross amount containing {readiness_floor_cycles} readiness floor, {management_creation_fee_cycles} management creation fee and {creation_execution_margin_cycles} execution margin, plus {ledger_fee_cycles} Ledger fee each; total reviewed fees {maximum_creation_fee_cycles}), shortfall {shortfall_cycles}; pending_creation={pending_creation_operation_id:?} attempts={attempt_count:?} last_attempt_at_ns={last_attempt_at_ns:?} retry_at_ns={retry_at_ns:?}; preserve this operation, repeat Fleet ensure without --apply and review funding_review, then apply its review_sha256 for the exact shortfall; do not fund the account out of band"
73)]
74pub struct EstateFundingRequiredError {
75    pub attempt_count: Option<u32>,
76    pub available_cycles: u128,
77    pub creation_amount_cycles: u128,
78    pub cycles_ledger: String,
79    pub creation_execution_margin_cycles: u128,
80    pub readiness_floor_cycles: u128,
81    pub ledger_fee_cycles: u128,
82    pub management_creation_fee_cycles: u128,
83    pub maximum_creation_debit_cycles: u128,
84    pub maximum_creation_fee_cycles: u128,
85    pub pending_creation_operation_id: Option<String>,
86    pub required_creation_count: u32,
87    pub root: String,
88    pub root_principal: String,
89    pub last_attempt_at_ns: Option<u64>,
90    pub retry_at_ns: Option<u64>,
91    pub shortfall_cycles: u128,
92}
93
94/// Exact terminal-inventory merge or cycle-observation invariant failure.
95#[derive(Debug, ThisError)]
96pub enum TerminalInventoryError {
97    #[error("one Principal is retained under more than one desired role")]
98    DuplicateRetainedPrincipal,
99
100    #[error("verified canister {canister} lost its Principal binding")]
101    LostPrincipalBinding { canister: String },
102
103    #[error("canister {canister} names unknown parent {parent}")]
104    UnknownParent { canister: String, parent: String },
105
106    #[error(
107        "canister {canister} retained parent {retained_parent} differs from terminal parent {terminal_parent}"
108    )]
109    RetainedParentConflict {
110        canister: String,
111        retained_parent: String,
112        terminal_parent: String,
113    },
114
115    #[error(
116        "terminal Root-owned Idle canister {canister} cycle observations differ: expected {expected}, observed {observed}"
117    )]
118    IdleCycleObservationConflict {
119        canister: String,
120        expected: u128,
121        observed: u128,
122    },
123
124    #[error(
125        "terminal cycle observation duplicates canister {canister} outside its exact Root-owned Idle or Workload lifecycle"
126    )]
127    CycleObservationLifecycleConflict { canister: String },
128}
129
130/// Fleet ensure planning or convergence failure.
131
132#[derive(Debug, ThisError)]
133pub enum EnsureWorkflowError<E>
134where
135    E: std::error::Error + 'static,
136{
137    #[error(
138        "retained Fleet operation cannot be resumed: {source}; local source evidence identifies operation {operation_id}, journal plan reference {plan_sha256}, source document {source_document_sha256}; preserve plan, journal, state and artifacts; request an explicit --reinstall review without --apply using the selected corrected release; live authority, complete estate and cycle checks must pass before any reset; see docs/features/operations/fleet-ensure.md#unreadable-retained-plan"
139    )]
140    RetainedActivationReviewRequired {
141        operation_id: String,
142        plan_sha256: String,
143        source_document_sha256: String,
144        #[source]
145        source: Box<EnsureStateError>,
146    },
147
148    #[error(
149        "retained Fleet operation is unreadable: {source}; preserve plan, journal, state, artifacts and paid-effect receipts; local evidence does not establish the supported partial-activation review path; resolve issued effects under their exact authority before a release transition; do not edit fields, replace digests or discard the operation; see docs/features/operations/fleet-ensure.md#unreadable-retained-plan"
150    )]
151    RetainedPlanUnreadable {
152        #[source]
153        source: Box<EnsureStateError>,
154    },
155
156    #[error(
157        "activation {operation_id} has an Issued provisioning action after its Applied host prefix (source document {source_document_sha256}); preserve the plan and journal: required live evidence for the bounded partial-activation recovery is unavailable"
158    )]
159    PartialActivationResetUnavailable {
160        operation_id: String,
161        source_document_sha256: String,
162    },
163
164    #[error(
165        "reinstall requires a converged Fleet with retained physical authority or an admitted single-Root partial-activation recovery source; the supplied source or target does not satisfy either contract; preserve the retained plan and journal"
166    )]
167    ReinstallConflict,
168
169    #[error("reviewed Fleet plan digest changed before its first effect")]
170    DriftedBeforeApply,
171
172    #[error(
173        "in-progress Fleet plan requires exact desired input {expected}; supplied input is {actual}"
174    )]
175    RetainedDesiredUnavailable { actual: String, expected: String },
176
177    #[error(
178        "replan-required Fleet operation owns completed reinstall evidence for desired input {expected}; refusing alternate desired input {actual}"
179    )]
180    RetainedReinstallDesiredConflict { actual: String, expected: String },
181
182    #[error(
183        "Root prerequisite retains activation authority for desired input {expected}; refreshed input {actual} is refused before Store effects; preserve the retained plan and journal for recovery review"
184    )]
185    RootActivationDesiredConflict { actual: String, expected: String },
186
187    #[error("reviewed Fleet plan {expected} does not match retained plan {actual}")]
188    PlanDigestMismatch { actual: String, expected: String },
189
190    #[error("reviewed Fleet plan is missing; run `canic fleet ensure <fleet>` first")]
191    PlanMissing,
192
193    #[error("retained Fleet plan failed its content digest")]
194    PlanIntegrity,
195
196    #[error("retained Fleet journal does not match the reviewed Fleet operation")]
197    JournalIntegrity,
198
199    #[error(
200        "Fleet ensure action {action} made no progress for {observations} consecutive observations (last progress: {progress_identity}); operation remains resumable"
201    )]
202    Stalled {
203        action: String,
204        observations: u32,
205        progress_identity: String,
206    },
207
208    #[error("Root {root} provisioning has made no progress for {observations} observations: stage={stage:?}, target={target}, operation={operation}, diagnostic={diagnostic_code}, retry=Backoff, failed_at_ns={failed_at_ns}; the issued operation remains retained; resume the same reviewed apply after the dependency recovers", operation = canic_core::cdk::utils::hash::hex_bytes(.operation_id))]
209    ProvisioningRetryPending {
210        root: candid::Principal,
211        stage: canic_core::dto::component_provisioning::ProvisioningFailureStage,
212        target: candid::Principal,
213        operation_id: [u8; 32],
214        diagnostic_code: u16,
215        observations: u32,
216        failed_at_ns: u64,
217    },
218
219    #[error(
220        "Root-owned canister {target} remained {last_lifecycle} for {observations} consecutive passive observations; no command was reissued and the operation remains resumable"
221    )]
222    RootOwnedObservationStalled {
223        target: String,
224        observations: u32,
225        last_lifecycle: String,
226    },
227
228    #[error(
229        "selected Cycles Ledger account has {actual} cycles, below reviewed maximum debit {required}"
230    )]
231    InsufficientOperatorCycles { actual: u128, required: u128 },
232
233    #[error("terminal Fleet cycle conservation failed: {0}")]
234    Conservation(String),
235
236    #[error("terminal Fleet inventory failed exact current-authority validation: {0}")]
237    TerminalInventory(#[source] TerminalInventoryError),
238
239    #[error(
240        "reviewed effects completed, but live Fleet state still differs from desired state; review a new plan"
241    )]
242    ConvergenceDrift,
243
244    #[error(
245        "fixture publication action {action} exhausted its {maximum_attempts} reviewed paid attempts; preserve the operation and reconcile its exact source status; no additional update was issued"
246    )]
247    FixturePublicationBound {
248        action: String,
249        maximum_attempts: u32,
250    },
251
252    #[error(
253        "Root pool maintenance {action} exhausted its reviewed limit of {maximum_updates} updates; retained receipts remain authoritative"
254    )]
255    PoolMaintenanceBound {
256        action: String,
257        maximum_updates: u32,
258    },
259
260    #[error(
261        "created canister {canister} retained one exact Ledger creation receipt, but its first live balance is {actual_cycles} cycles instead of requested target {requested_cycles} (deficit {deficit_cycles}; configured management creation fee {configured_fee_cycles}); no controller or protocol action followed, and the immutable operation is now replan-required"
262    )]
263    ReplanRequiredAfterCreateBalanceDrift {
264        actual_cycles: u128,
265        canister: String,
266        configured_fee_cycles: u128,
267        deficit_cycles: u128,
268        requested_cycles: u128,
269    },
270
271    #[error("Fleet successor requires a new review: {reason:?}; {}", .review.as_deref().map(ToString::to_string).unwrap_or_default())]
272    SuccessorReviewRequired {
273        reason: crate::fleet_ensure::model::FleetEnsureSuccessorReviewReason,
274        review: Option<Box<crate::fleet_ensure::model::FleetSuccessorReview>>,
275    },
276
277    #[error(transparent)]
278    FreshPoolCreationUnderfunded(Box<FreshPoolCreationUnderfundedError>),
279
280    #[error(transparent)]
281    EstateFundingRequired(Box<EstateFundingRequiredError>),
282
283    #[error("native Root funding needs explicit approval of review {review_sha256}")]
284    NativeFundingRequired { review_sha256: String },
285
286    #[error(transparent)]
287    Policy(#[from] EnsurePolicyError),
288
289    #[error(transparent)]
290    Inventory(#[from] crate::fleet_ensure::CurrentFleetInventoryError),
291
292    #[error("Fleet platform operation failed: {0}")]
293    Platform(#[source] E),
294
295    #[error(transparent)]
296    State(#[from] EnsureStateError),
297}
298
299/// Recover the selected target for an exact reinstall apply, including terminal replay.
300pub fn retained_reinstall_apply_plan<E: std::error::Error + 'static>(
301    root: &Path,
302    environment: &str,
303    requested_fleet: &str,
304    reviewed_plan_sha256: &str,
305) -> Result<Option<FleetEnsurePlan>, EnsureWorkflowError<E>> {
306    validate_path_labels(environment, requested_fleet)?;
307    let paths = EnsurePaths::under(root, environment, requested_fleet);
308    let _lock = lock_operation(&paths)?;
309    // The staged review owns apply; its source remains receipt-only evidence.
310    let plan = match crate::fleet_ensure::ops::reinstall::adoption::review(&paths)? {
311        Some(review) => Some(review),
312        None => retained_plan::read(&paths, environment, requested_fleet)?,
313    };
314    let Some(plan) = plan else {
315        return Ok(None);
316    };
317    let plan = verified_plan(plan)?;
318    if plan.environment != environment || plan.fleet != requested_fleet {
319        return Err(EnsureWorkflowError::PlanIntegrity);
320    }
321    if plan.reinstall.is_some() && plan.plan_sha256 == reviewed_plan_sha256 {
322        Ok(Some(plan))
323    } else {
324        Ok(None)
325    }
326}
327
328/// Load and verify the sole immutable in-progress operation for one exact Fleet path.
329pub fn retained_in_progress_plan<E>(
330    root: &Path,
331    environment: &str,
332    requested_fleet: &str,
333) -> Result<Option<FleetEnsurePlan>, EnsureWorkflowError<E>>
334where
335    E: std::error::Error + 'static,
336{
337    validate_path_labels(environment, requested_fleet)?;
338    let paths = EnsurePaths::under(root, environment, requested_fleet);
339    let _lock = lock_operation(&paths)?;
340    if let Some(review) = crate::fleet_ensure::ops::reinstall::adoption::review(&paths)? {
341        if review.environment != environment || review.fleet != requested_fleet {
342            return Err(EnsureWorkflowError::PlanIntegrity);
343        }
344        return Ok(Some(review));
345    }
346    let Some(journal) = retained_plan::journal(&paths, environment, requested_fleet)? else {
347        return Ok(None);
348    };
349    if !matches!(
350        journal.completion,
351        FleetEnsureCompletion::InProgress | FleetEnsureCompletion::Prepared
352    ) {
353        return Ok(None);
354    }
355    let retained = verified_plan(
356        retained_plan::read(&paths, environment, requested_fleet)?
357            .ok_or(EnsureWorkflowError::PlanMissing)?,
358    )?;
359    if retained.environment != environment || retained.fleet != requested_fleet {
360        return Err(EnsureWorkflowError::PlanIntegrity);
361    }
362    let state = read_state(&paths, requested_fleet)?;
363    let prepared_full = journal.completion == FleetEnsureCompletion::Prepared
364        && (retained.scope == FleetEnsurePlanScope::Full
365            || (retained.scope == FleetEnsurePlanScope::RootReinstallPrerequisite
366                && retained.reinstall.as_ref().is_some_and(|intent| {
367                    intent
368                        .activation_reset
369                        .as_ref()
370                        .is_some_and(|activation| activation.preparation.is_some())
371                })))
372        && retained.reinstall.as_ref().is_some_and(|intent| {
373            intent.operation_id == journal.operation_id
374                && intent.operation_id == retained.operation_id
375        });
376    if !prepared_full {
377        verify_journal(&journal, &retained, requested_fleet, &state)?;
378    }
379    Ok(Some(retained))
380}
381
382/// Build and retain one read-only plan from current desired state plus live observation.
383#[expect(
384    clippy::too_many_lines,
385    reason = "one locked read-only transaction reconciles retained authority, observation and the immutable plan"
386)]
387pub fn plan<P>(
388    root: &Path,
389    desired: &crate::fleet_ensure::model::DesiredFleet,
390    desired_sha256: &str,
391    requested_fleet: &str,
392    created_at_time: u64,
393    platform: &mut P,
394) -> Result<FleetEnsureReport, EnsureWorkflowError<P::Error>>
395where
396    P: EnsurePlatform,
397{
398    validate_path_identity(desired, requested_fleet)?;
399    let paths = EnsurePaths::under(root, &desired.environment, requested_fleet);
400    let _lock = lock_operation(&paths)?;
401    if let Some(review) = crate::fleet_ensure::ops::reinstall::adoption::review(&paths)? {
402        if review.desired_sha256 != desired_sha256 {
403            return Err(EnsureWorkflowError::DriftedBeforeApply);
404        }
405        return Ok(FleetEnsureReport {
406            funding_review: None,
407            actual_conservation: None,
408            effects_applied: 0,
409            plan: review,
410            terminal: false,
411        });
412    }
413    if let Some(mut journal) =
414        retained_plan::journal(&paths, &desired.environment, requested_fleet)?
415        && journal.completion == FleetEnsureCompletion::InProgress
416    {
417        let retained = verified_plan(
418            retained_plan::read(&paths, &desired.environment, requested_fleet)?
419                .ok_or(EnsureWorkflowError::PlanMissing)?,
420        )?;
421        let state = read_state(&paths, requested_fleet)?;
422        verify_journal(&journal, &retained, requested_fleet, &state)?;
423        if journal.estate_funding_required.is_some()
424            || funding::native_review_applicable(&retained, &journal)
425        {
426            if let Some(reviewed) = retained.reviewed_desired.as_deref() {
427                platform
428                    .bind_reviewed_desired(reviewed.desired())
429                    .map_err(EnsureWorkflowError::Platform)?;
430            } else if retained.desired_sha256 != desired_sha256 {
431                return Err(EnsureWorkflowError::RetainedDesiredUnavailable {
432                    actual: desired_sha256.to_string(),
433                    expected: retained.desired_sha256.clone(),
434                });
435            }
436        }
437        let funding_review = funding::prepare(
438            &paths,
439            &retained,
440            &mut journal,
441            &state,
442            created_at_time,
443            platform,
444        )?;
445        return Ok(FleetEnsureReport {
446            funding_review,
447            actual_conservation: None,
448            effects_applied: applied_count(&journal),
449            plan: retained,
450            terminal: false,
451        });
452    }
453    let mut state = read_state(&paths, requested_fleet)?;
454    let prior_plan = retained_plan::read(&paths, &desired.environment, requested_fleet)?
455        .map(verified_plan)
456        .transpose()?;
457    let prior_journal = retained_plan::journal(&paths, &desired.environment, requested_fleet)?;
458    if let Some(prior) = &prior_plan {
459        let recovery_ready = prior_journal
460            .as_ref()
461            .map(|journal| reinstall_recovery_ready(prior, journal, &state))
462            .transpose()?
463            .unwrap_or(false);
464        if recovery_ready && let Some(journal) = &prior_journal {
465            if prior.desired_sha256 != desired_sha256 {
466                return Err(EnsureWorkflowError::ReinstallConflict);
467            }
468            retain_completed_reinstalls(&mut state, prior, journal);
469        }
470        let activation_reset = prior
471            .reinstall
472            .as_ref()
473            .is_some_and(|intent| intent.activation_reset.is_some());
474        if activation_reset
475            && prior.scope == FleetEnsurePlanScope::RootReinstallPrerequisite
476            && let Some(journal) = prior_journal
477                .as_ref()
478                .filter(|journal| journal.completion == FleetEnsureCompletion::Converged)
479        {
480            verify_journal(journal, prior, requested_fleet, &state)?;
481            retain_completed_reinstalls(&mut state, prior, journal);
482            write_state(&paths, &state)?;
483        }
484        let unfinished_reset = prior.reinstall.is_some()
485            && !recovery_ready
486            && !prior_journal.as_ref().is_some_and(|journal| {
487                journal.plan_sha256 == prior.plan_sha256
488                    && journal.completion == FleetEnsureCompletion::Converged
489            });
490        if prior.scope == FleetEnsurePlanScope::ReinstallPreparation
491            || unfinished_reset
492            || (activation_reset && prior.scope == FleetEnsurePlanScope::RootReinstallPrerequisite)
493        {
494            if prior.desired_sha256 != desired_sha256 {
495                return Err(EnsureWorkflowError::ReinstallConflict);
496            }
497            return reinstall::continue_plan(
498                root,
499                &paths,
500                desired,
501                prior,
502                prior_journal.as_ref(),
503                &state,
504                platform,
505            );
506        }
507    }
508    let matching_prior = reconcile_retained_planning_state(
509        &mut state,
510        prior_plan.as_ref(),
511        prior_journal.as_ref(),
512        requested_fleet,
513        &desired.environment,
514        desired_sha256,
515    )?;
516    if let Some(prior) = matching_prior {
517        retain_plan_cycles(&mut state, prior);
518        if let Some(journal) = &prior_journal {
519            retain_completed_reinstalls(&mut state, prior, journal);
520        }
521    }
522    let operation_id = matching_prior
523        .filter(|prior| prior.scope == FleetEnsurePlanScope::Full)
524        .map_or_else(
525            || operation_id(desired_sha256, &desired.environment, requested_fleet),
526            |prior| prior.operation_id.clone(),
527        );
528    if let Some(management) = platform
529        .observe_root_management(&state, &BTreeSet::new())
530        .map_err(EnsureWorkflowError::Platform)?
531    {
532        let root_start_authority = read_root_start_authority(&paths)?;
533        if let Some(plan) = compile_root_start_prerequisite_plan(RootStartPlanInput {
534            state: &state,
535            authority: root_start_authority.as_ref(),
536            created_at_time,
537            desired,
538            desired_sha256,
539            observation: &management,
540            requested_fleet,
541        })? {
542            write_plan(&paths, &plan)?;
543            return Ok(FleetEnsureReport {
544                funding_review: None,
545                actual_conservation: None,
546                effects_applied: 0,
547                plan,
548                terminal: false,
549            });
550        }
551        if let Some(plan) = crate::fleet_ensure::policy::root_reinstall::compile(
552            RootStartPlanInput {
553                state: &state,
554                authority: None,
555                created_at_time,
556                desired,
557                desired_sha256,
558                observation: &management,
559                requested_fleet,
560            },
561            &resolve_desired_artifacts(root, desired)?,
562        )? {
563            write_plan(&paths, &plan)?;
564            return Ok(FleetEnsureReport {
565                funding_review: None,
566                actual_conservation: None,
567                effects_applied: 0,
568                plan,
569                terminal: false,
570            });
571        }
572    }
573    let artifacts = resolve_desired_artifacts(root, desired)?;
574    let mut observation = platform
575        .observe(&operation_id, &state)
576        .map_err(EnsureWorkflowError::Platform)?;
577    retain_observed_cycles(&mut state, &observation);
578    // The prior Converged journal can remain visible until apply. Planning may refresh only
579    // backup-inert observation and prior-operation evidence, never successor topology authority.
580    write_state(&paths, &state)?;
581    let terminal_inventory_operation_id = current_terminal_inventory_operation(
582        &state,
583        prior_plan.as_ref(),
584        prior_journal.as_ref(),
585        requested_fleet,
586    )?;
587    if let Some(operation_id) = terminal_inventory_operation_id.as_deref() {
588        attach_terminal_inventory_cycles(operation_id, &state, platform, &mut observation)?;
589    }
590    let mut protocol_actions = platform
591        .protocol_actions(&operation_id, &state)
592        .map_err(EnsureWorkflowError::Platform)?;
593    let mut retained_observations = 0_u32;
594    let mut plan = loop {
595        match compile_plan(
596            desired,
597            &artifacts,
598            &protocol_actions,
599            desired_sha256,
600            requested_fleet,
601            &observation,
602            created_at_time,
603            &operation_id,
604            None,
605        ) {
606            Ok(plan) => break plan,
607            Err(EnsurePolicyError::PendingRootOwnedBalance { name }) => {
608                retained_observations = retained_observations.saturating_add(1);
609                if retained_observations >= desired.maximum_stalled_observations {
610                    return Err(EnsureWorkflowError::RootOwnedObservationStalled {
611                        last_lifecycle: root_owned_lifecycle_label(&observation, &name).to_string(),
612                        observations: retained_observations,
613                        target: name,
614                    });
615                }
616                platform.pace_root_owned_observation(&name, retained_observations);
617                observation = platform
618                    .observe(&operation_id, &state)
619                    .map_err(EnsureWorkflowError::Platform)?;
620                retain_observed_cycles(&mut state, &observation);
621                write_state(&paths, &state)?;
622                if let Some(operation_id) = terminal_inventory_operation_id.as_deref() {
623                    attach_terminal_inventory_cycles(
624                        operation_id,
625                        &state,
626                        platform,
627                        &mut observation,
628                    )?;
629                }
630                protocol_actions = platform
631                    .protocol_actions(&operation_id, &state)
632                    .map_err(EnsureWorkflowError::Platform)?;
633            }
634            Err(error) => return Err(error.into()),
635        }
636    };
637    for action in ordered_actions(&plan) {
638        verify_store_activation_binding(&plan, action, &state, platform)?;
639    }
640    bind_terminal_inventory_operation(&mut plan, terminal_inventory_operation_id);
641    write_plan(&paths, &plan)?;
642    Ok(FleetEnsureReport {
643        funding_review: None,
644        actual_conservation: None,
645        effects_applied: 0,
646        plan,
647        terminal: false,
648    })
649}
650
651fn current_terminal_inventory_operation<E>(
652    state: &FleetEnsureStateRecord,
653    prior_plan: Option<&FleetEnsurePlan>,
654    prior_journal: Option<&FleetEnsureJournalRecord>,
655    requested_fleet: &str,
656) -> Result<Option<String>, EnsureWorkflowError<E>>
657where
658    E: std::error::Error + 'static,
659{
660    if state.active_registry.is_none() {
661        return Ok(None);
662    }
663    let prior_plan = prior_plan.ok_or(EnsureWorkflowError::PlanMissing)?;
664    let prior_journal = prior_journal.ok_or(EnsureWorkflowError::JournalIntegrity)?;
665    verify_journal(prior_journal, prior_plan, requested_fleet, state)?;
666    if prior_journal.completion != FleetEnsureCompletion::Converged
667        || prior_plan.scope != FleetEnsurePlanScope::Full
668    {
669        return Err(EnsureWorkflowError::JournalIntegrity);
670    }
671    let operation_id = prior_plan
672        .terminal_inventory_operation_id
673        .as_ref()
674        .unwrap_or(&prior_plan.operation_id);
675    if operation_id.is_empty() {
676        return Err(EnsureWorkflowError::JournalIntegrity);
677    }
678    Ok(Some(operation_id.clone()))
679}
680
681fn bind_terminal_inventory_operation(plan: &mut FleetEnsurePlan, operation_id: Option<String>) {
682    plan.terminal_inventory_operation_id = operation_id;
683    plan.plan_sha256 = expected_plan_sha256(plan);
684}
685
686fn reconcile_retained_planning_state<'a, E>(
687    state: &mut FleetEnsureStateRecord,
688    prior_plan: Option<&'a FleetEnsurePlan>,
689    prior_journal: Option<&FleetEnsureJournalRecord>,
690    requested_fleet: &str,
691    desired_environment: &str,
692    desired_sha256: &str,
693) -> Result<Option<&'a FleetEnsurePlan>, EnsureWorkflowError<E>>
694where
695    E: std::error::Error + 'static,
696{
697    if let Some((prior, journal)) = prior_plan.zip(prior_journal)
698        && prior.fleet == requested_fleet
699        && prior.environment == desired_environment
700        && retained_reinstall_desired_conflict(
701            state,
702            &prior.fleet,
703            &prior.operation_id,
704            &prior.desired_sha256,
705            desired_sha256,
706            journal,
707        )
708    {
709        return Err(EnsureWorkflowError::RetainedReinstallDesiredConflict {
710            actual: desired_sha256.to_string(),
711            expected: prior.desired_sha256.clone(),
712        });
713    }
714    if let Some((prior, journal)) = prior_plan.zip(prior_journal)
715        && prior.scope == FleetEnsurePlanScope::RootReinstallPrerequisite
716        && prior.desired_sha256 != desired_sha256
717    {
718        verify_journal(journal, prior, requested_fleet, state)?;
719        return Err(EnsureWorkflowError::RootActivationDesiredConflict {
720            actual: desired_sha256.to_string(),
721            expected: prior.desired_sha256.clone(),
722        });
723    }
724    let matching_prior = prior_plan.filter(|prior| {
725        prior.fleet == requested_fleet
726            && prior.environment == desired_environment
727            && prior.desired_sha256 == desired_sha256
728    });
729    let retained_operation_matches =
730        matching_prior
731            .zip(prior_journal)
732            .is_some_and(|(prior, journal)| {
733                retained_reinstall_operation_matches(
734                    state,
735                    &prior.fleet,
736                    &prior.operation_id,
737                    journal,
738                )
739            });
740    if !retained_operation_matches {
741        state.completed_reinstall_action_sha256.clear();
742        state.completed_reinstall_operation_id = None;
743        state.completed_reinstalls.clear();
744    }
745    Ok(matching_prior)
746}
747
748fn retain_completed_reinstalls(
749    state: &mut FleetEnsureStateRecord,
750    plan: &FleetEnsurePlan,
751    journal: &FleetEnsureJournalRecord,
752) {
753    if journal.fleet != plan.fleet || journal.operation_id != plan.operation_id {
754        return;
755    }
756    for action in ordered_actions(plan) {
757        let EnsureAction::Install {
758            mode: crate::fleet_ensure::model::InstallMode::Reinstall,
759            name,
760            principal,
761            wasm_sha256,
762            ..
763        } = action
764        else {
765            continue;
766        };
767        let action_sha256 = action_sha256(action);
768        let Some(effect) = journal
769            .effects
770            .iter()
771            .find(|effect| effect.action_sha256 == action_sha256)
772        else {
773            continue;
774        };
775        if effect.state == EffectState::Applied
776            && let Some(pre_canister_version) = effect.pre_canister_version
777            && state
778                .principals
779                .get(name)
780                .is_some_and(|retained| retained == principal)
781        {
782            state.completed_reinstall_operation_id = Some(journal.operation_id.clone());
783            state
784                .completed_reinstall_action_sha256
785                .insert(name.clone(), action_sha256);
786            state
787                .completed_reinstalls
788                .insert(name.clone(), pre_canister_version);
789            if let Some(topology) = state.topology.get_mut(name) {
790                topology.module_hash = Some(wasm_sha256.clone());
791            }
792        }
793    }
794}
795
796fn completed_reinstall_evidence_matches(
797    state: &FleetEnsureStateRecord,
798    journal: &FleetEnsureJournalRecord,
799) -> bool {
800    !state.completed_reinstalls.is_empty()
801        && state
802            .completed_reinstalls
803            .iter()
804            .all(|(name, pre_version)| {
805                state
806                    .completed_reinstall_action_sha256
807                    .get(name)
808                    .and_then(|action_sha256| {
809                        journal
810                            .effects
811                            .iter()
812                            .find(|effect| effect.action_sha256 == *action_sha256)
813                    })
814                    .is_some_and(|effect| {
815                        effect.state == EffectState::Applied
816                            && effect.pre_canister_version == Some(*pre_version)
817                    })
818            })
819        && state.completed_reinstall_action_sha256.len() == state.completed_reinstalls.len()
820}
821
822/// Admit another review only after every effect of the selected reset is accounted for.
823fn reinstall_recovery_ready<E: std::error::Error + 'static>(
824    plan: &FleetEnsurePlan,
825    journal: &FleetEnsureJournalRecord,
826    state: &FleetEnsureStateRecord,
827) -> Result<bool, EnsureWorkflowError<E>> {
828    if plan.scope != FleetEnsurePlanScope::Full
829        || plan.reinstall.is_none()
830        || journal.completion != FleetEnsureCompletion::ReplanRequired
831        || journal.plan_sha256 != plan.plan_sha256
832    {
833        return Ok(false);
834    }
835    verify_journal(journal, plan, &plan.fleet, state)?;
836    let actions = continuation::actions(plan, journal);
837    let complete = actions.len() == journal.effects.len()
838        && journal
839            .effects
840            .iter()
841            .all(|effect| effect.state == EffectState::Applied);
842    let installs_recorded = actions
843        .iter()
844        .zip(&journal.effects)
845        .all(|(action, effect)| {
846            !matches!(action, EnsureAction::Install { .. }) || effect.pre_canister_version.is_some()
847        });
848    if !complete || !installs_recorded {
849        return Err(EnsureWorkflowError::JournalIntegrity);
850    }
851    Ok(true)
852}
853
854fn retained_reinstall_operation_matches(
855    state: &FleetEnsureStateRecord,
856    prior_fleet: &str,
857    prior_operation_id: &str,
858    journal: &FleetEnsureJournalRecord,
859) -> bool {
860    journal.completion == FleetEnsureCompletion::ReplanRequired
861        && journal.fleet == prior_fleet
862        && journal.operation_id == prior_operation_id
863        && state.completed_reinstall_operation_id.as_deref() == Some(journal.operation_id.as_str())
864        && completed_reinstall_evidence_matches(state, journal)
865}
866
867fn retained_reinstall_desired_conflict(
868    state: &FleetEnsureStateRecord,
869    prior_fleet: &str,
870    prior_operation_id: &str,
871    prior_desired_sha256: &str,
872    supplied_desired_sha256: &str,
873    journal: &FleetEnsureJournalRecord,
874) -> bool {
875    prior_desired_sha256 != supplied_desired_sha256
876        && retained_reinstall_operation_matches(state, prior_fleet, prior_operation_id, journal)
877}
878
879fn retain_plan_cycles(state: &mut FleetEnsureStateRecord, plan: &FleetEnsurePlan) {
880    for canister in &plan.canisters {
881        if let Some(principal) = &canister.principal {
882            state
883                .retained_cycles_by_principal
884                .entry(principal.clone())
885                .or_insert(canister.observed_cycles);
886        }
887    }
888}
889
890fn retain_observed_cycles(state: &mut FleetEnsureStateRecord, observation: &FleetObservation) {
891    for live in observation.canisters.values().filter_map(Option::as_ref) {
892        state
893            .retained_cycles_by_principal
894            .insert(live.principal.clone(), live.cycles);
895    }
896    state
897        .retained_cycles_by_principal
898        .extend(observation.additional_controlled_cycles.clone());
899}
900
901fn root_owned_lifecycle_label(observation: &FleetObservation, target: &str) -> &'static str {
902    observation
903        .canisters
904        .get(target)
905        .and_then(Option::as_ref)
906        .and_then(|live| live.root_owned_lifecycle)
907        .map_or("unavailable", |lifecycle| lifecycle.label())
908}
909
910/// Apply or resume exactly one reviewed plan until it converges or returns a typed blocker.
911#[expect(
912    clippy::too_many_lines,
913    reason = "the sole effect driver keeps intent, reconciliation and terminal publication visibly ordered"
914)]
915pub fn apply<P>(
916    root: &Path,
917    desired: &crate::fleet_ensure::model::DesiredFleet,
918    desired_sha256: &str,
919    requested_fleet: &str,
920    reviewed_plan_sha256: &str,
921    platform: &mut P,
922) -> Result<FleetEnsureReport, EnsureWorkflowError<P::Error>>
923where
924    P: EnsurePlatform,
925{
926    validate_path_identity(desired, requested_fleet)?;
927    let paths = EnsurePaths::under(root, &desired.environment, requested_fleet);
928    let _lock = lock_operation(&paths)?;
929    let pending = crate::fleet_ensure::ops::reinstall::adoption::review(&paths)?;
930    let adopting = pending.is_some();
931    let retained_plan = verified_plan(match pending {
932        Some(plan) => plan,
933        None => retained_plan::read(&paths, &desired.environment, requested_fleet)?
934            .ok_or(EnsureWorkflowError::PlanMissing)?,
935    })?;
936    let retained_journal = if adopting {
937        None
938    } else {
939        retained_plan::journal(&paths, &desired.environment, requested_fleet)?
940    };
941    if retained_plan.plan_sha256 != reviewed_plan_sha256
942        && !retained_journal
943            .as_ref()
944            .is_some_and(|journal| funding::accepts(&retained_plan, journal, reviewed_plan_sha256))
945    {
946        return Err(EnsureWorkflowError::PlanDigestMismatch {
947            actual: retained_plan.plan_sha256,
948            expected: reviewed_plan_sha256.to_string(),
949        });
950    }
951    let mut state = read_state(&paths, requested_fleet)?;
952    let in_progress = retained_journal
953        .as_ref()
954        .is_some_and(|journal| journal.completion == FleetEnsureCompletion::InProgress);
955    if let Some(journal) = retained_journal.as_ref().filter(|_| in_progress) {
956        verify_journal(journal, &retained_plan, requested_fleet, &state)?;
957        compact_inline_plan(&paths, &retained_plan)?;
958    }
959    if !in_progress
960        && retained_plan.reinstall.is_none()
961        && retained_plan.desired_sha256 != desired_sha256
962    {
963        return Err(EnsureWorkflowError::DriftedBeforeApply);
964    }
965    let operation_desired = if in_progress || retained_plan.reinstall.is_some() {
966        if let Some(reviewed) = retained_plan.reviewed_desired.as_deref() {
967            let reviewed = reviewed.desired();
968            validate_path_identity(reviewed, requested_fleet)?;
969            if reviewed.environment != retained_plan.environment {
970                return Err(EnsureWorkflowError::PlanIntegrity);
971            }
972            reviewed
973        } else if retained_plan.desired_sha256 == desired_sha256 {
974            desired
975        } else {
976            return Err(EnsureWorkflowError::RetainedDesiredUnavailable {
977                actual: desired_sha256.to_string(),
978                expected: retained_plan.desired_sha256.clone(),
979            });
980        }
981    } else {
982        if retained_plan.desired_sha256 != desired_sha256 {
983            return Err(EnsureWorkflowError::DriftedBeforeApply);
984        }
985        desired
986    };
987    if !retained_journal.as_ref().is_some_and(|journal| {
988        journal.completion == FleetEnsureCompletion::Converged
989            && journal.plan_sha256 == retained_plan.plan_sha256
990    }) {
991        validate_creation_fee_scope(
992            operation_desired,
993            &retained_plan.canisters,
994            &retained_plan.conservation.estate_funding_domains,
995        )?;
996    }
997    platform
998        .bind_reviewed_desired(operation_desired)
999        .map_err(EnsureWorkflowError::Platform)?;
1000    let operation_desired_sha256 = retained_plan.desired_sha256.as_str();
1001    continuation::verify_inputs(root, operation_desired, &retained_plan)?;
1002    if let Some(journal) = &retained_journal {
1003        continuation::verify_canonical(&retained_plan, journal, &state, platform)?;
1004        if reinstall_recovery_ready(&retained_plan, journal, &state)? {
1005            return Err(EnsureWorkflowError::SuccessorReviewRequired {
1006                reason:
1007                    crate::fleet_ensure::model::FleetEnsureSuccessorReviewReason::AdditionalEffect,
1008                review: None,
1009            });
1010        }
1011    }
1012    let completed_journal = retained_journal.as_ref().filter(|journal| {
1013        (journal.completion == FleetEnsureCompletion::Converged
1014            || (journal.completion == FleetEnsureCompletion::Prepared
1015                && retained_plan.scope == FleetEnsurePlanScope::ReinstallPreparation))
1016            && journal.plan_sha256 == retained_plan.plan_sha256
1017    });
1018    if retained_plan.scope == FleetEnsurePlanScope::ReinstallPreparation
1019        && let Some(journal) = completed_journal
1020    {
1021        let actual = reinstall::complete(&retained_plan, journal, &state, platform)?;
1022        return Ok(FleetEnsureReport {
1023            funding_review: None,
1024            actual_conservation: Some(actual),
1025            effects_applied: 0,
1026            plan: retained_plan,
1027            terminal: true,
1028        });
1029    }
1030    if retained_plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite
1031        && let Some(journal) = completed_journal
1032    {
1033        verify_journal(journal, &retained_plan, requested_fleet, &state)?;
1034        let actual = root_reinstall::complete(&retained_plan, journal, &state, platform)?;
1035        report_progress_state(
1036            platform,
1037            &retained_plan,
1038            journal,
1039            FleetEnsurePhase::Infrastructure,
1040            FleetEnsureProgressState::PrerequisiteComplete,
1041        );
1042        return Ok(FleetEnsureReport {
1043            funding_review: None,
1044            actual_conservation: Some(actual),
1045            effects_applied: 0,
1046            plan: retained_plan,
1047            terminal: true,
1048        });
1049    }
1050    if retained_plan.scope == FleetEnsurePlanScope::Full
1051        && let Some(journal) = completed_journal
1052    {
1053        verify_journal(journal, &retained_plan, requested_fleet, &state)?;
1054        let actual = continuation::replay(
1055            root,
1056            operation_desired,
1057            &retained_plan,
1058            journal,
1059            &state,
1060            platform,
1061        )?;
1062        report_progress(
1063            platform,
1064            &retained_plan,
1065            journal,
1066            FleetEnsurePhase::Complete,
1067        );
1068        return Ok(FleetEnsureReport {
1069            funding_review: None,
1070            actual_conservation: Some(actual),
1071            effects_applied: 0,
1072            plan: retained_plan,
1073            terminal: true,
1074        });
1075    }
1076    if retained_plan.scope == FleetEnsurePlanScope::RootStartPrerequisite
1077        && let Some(journal) = completed_journal
1078    {
1079        verify_terminal_root_start_replay(
1080            operation_desired,
1081            &retained_plan,
1082            journal,
1083            requested_fleet,
1084            platform,
1085            &state,
1086        )?;
1087        report_progress_state(
1088            platform,
1089            &retained_plan,
1090            journal,
1091            FleetEnsurePhase::Infrastructure,
1092            FleetEnsureProgressState::PrerequisiteComplete,
1093        );
1094        return Ok(FleetEnsureReport {
1095            funding_review: None,
1096            actual_conservation: None,
1097            effects_applied: applied_count(journal),
1098            plan: retained_plan,
1099            terminal: true,
1100        });
1101    }
1102    let mut journal = match retained_journal {
1103        Some(journal) if journal.completion == FleetEnsureCompletion::InProgress => {
1104            verify_journal(&journal, &retained_plan, requested_fleet, &state)?;
1105            journal
1106        }
1107        _ => {
1108            let (observation, initial_controlled_cycles) = verify_fresh_plan(
1109                root,
1110                operation_desired,
1111                operation_desired_sha256,
1112                requested_fleet,
1113                &retained_plan,
1114                platform,
1115                &state,
1116            )?;
1117            root_reinstall::verify_initial_funding(
1118                &retained_plan,
1119                observation.operator_cycles,
1120                platform,
1121            )?;
1122            let journal = FleetEnsureJournalRecord {
1123                funding_reviews: Vec::new(),
1124                successor_phases: Vec::new(),
1125                completion: FleetEnsureCompletion::InProgress,
1126                estate_funding_required: None,
1127                effects: Vec::new(),
1128                fleet: requested_fleet.to_string(),
1129                initial_controlled_cycles,
1130                initial_estate_funding_cycles_by_root: retained_plan
1131                    .conservation
1132                    .estate_funding_domains
1133                    .iter()
1134                    .map(|domain| {
1135                        (
1136                            domain.root.clone(),
1137                            domain.available_cycles.unwrap_or_default(),
1138                        )
1139                    })
1140                    .collect(),
1141                initial_operator_cycles: observation.operator_cycles,
1142                operation_id: retained_plan.operation_id.clone(),
1143                plan_sha256: retained_plan.plan_sha256.clone(),
1144                schema_version: FLEET_ENSURE_SCHEMA_VERSION,
1145                stalled_observations: 0,
1146            };
1147            // Cross the nonterminal boundary before any effect-owned state can be retained.
1148            if adopting {
1149                crate::fleet_ensure::ops::reinstall::adoption::adopt(
1150                    &paths,
1151                    &retained_plan,
1152                    &journal,
1153                )?;
1154            } else {
1155                write_journal(&paths, &journal)?;
1156            }
1157            journal
1158        }
1159    };
1160
1161    funding::resume(
1162        &paths,
1163        &retained_plan,
1164        &mut journal,
1165        &state,
1166        reviewed_plan_sha256,
1167        platform,
1168    )?;
1169    if journal
1170        .estate_funding_required
1171        .as_ref()
1172        .is_some_and(|pause| funding::covered(&journal, pause))
1173    {
1174        retain_estate_funding_pause(&paths, &mut journal, None)?;
1175    }
1176    let mut terminal_state;
1177    loop {
1178        let execution_actions = continuation::actions(&retained_plan, &journal)
1179            .into_iter()
1180            .cloned()
1181            .collect::<Vec<_>>();
1182        let actions = execution_actions.iter().collect::<Vec<_>>();
1183        let mut replayed_issued_commands = BTreeSet::new();
1184        let mut reported_phase = None;
1185        loop {
1186            let mut deferred_controller_observation = false;
1187            for (index, action) in actions.iter().enumerate() {
1188                let action_hash = action_sha256(action);
1189                let observation_policy = effect_observation_policy(operation_desired, action)?;
1190                let retained_effect = journal.effects.get(index);
1191                if retained_effect.is_none_or(|effect| effect.state != EffectState::Applied) {
1192                    let phase = action_progress_phase(action);
1193                    if reported_phase != Some(phase) {
1194                        report_progress(platform, &retained_plan, &journal, phase);
1195                        reported_phase = Some(phase);
1196                    }
1197                }
1198                if matches!(action, EnsureAction::FleetProtocol { .. })
1199                    && retained_effect.is_none_or(|effect| effect.state == EffectState::Intent)
1200                    && !prior_fleet_protocol_effect_started(&actions, &journal, index)
1201                {
1202                    let funding_observation = platform
1203                        .observe(&journal.operation_id, &state)
1204                        .map_err(EnsureWorkflowError::Platform)?;
1205                    let funded_plan = funding_plan(&retained_plan, &journal)?;
1206                    let required =
1207                        estate_funding_requirement(&funded_plan, &state, &funding_observation)?;
1208                    retain_estate_funding_pause(&paths, &mut journal, required.as_ref())?;
1209                    if let Some(required) = required {
1210                        report_progress_state(
1211                            platform,
1212                            &retained_plan,
1213                            &journal,
1214                            action_progress_phase(action),
1215                            FleetEnsureProgressState::FundingRequired,
1216                        );
1217                        return Err(EnsureWorkflowError::EstateFundingRequired(Box::new(
1218                            estate_funding_error(&required),
1219                        )));
1220                    }
1221                }
1222                let mut initial_observation = if journal.effects.len() <= index {
1223                    let prepared = prepare_effect(platform, &journal.operation_id, action, &state)
1224                        .map_err(EnsureWorkflowError::Platform)?;
1225                    journal.effects.push(prepared.record);
1226                    write_journal(&paths, &journal)?;
1227                    prepared.observation
1228                } else {
1229                    None
1230                };
1231                let observation_started = Instant::now();
1232                loop {
1233                    let record = journal
1234                        .effects
1235                        .get(index)
1236                        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1237                    if record.action_sha256 != action_hash {
1238                        return Err(EnsureWorkflowError::DriftedBeforeApply);
1239                    }
1240                    if !create_record_is_exact(action, record, &state) {
1241                        return Err(EnsureWorkflowError::JournalIntegrity);
1242                    }
1243                    if matches!(record.state, EffectState::Applied) {
1244                        let retained_create = journal
1245                            .effects
1246                            .get(index)
1247                            .ok_or(EnsureWorkflowError::JournalIntegrity)
1248                            .and_then(|record| {
1249                                retain_applied_create_authority(
1250                                    operation_desired,
1251                                    action,
1252                                    record,
1253                                    &mut state,
1254                                )
1255                            })?;
1256                        let retained_funding = journal.effects.get(index).is_some_and(|record| {
1257                            retain_applied_funding_cycles(&mut state, action, record)
1258                        });
1259                        if retained_create || retained_funding {
1260                            write_state(&paths, &state)?;
1261                        }
1262                        break;
1263                    }
1264
1265                    let observed = match initial_observation.take() {
1266                        Some(observed) => observed,
1267                        None => platform
1268                            .observe_effect(&journal.operation_id, action, record, &state)
1269                            .map_err(EnsureWorkflowError::Platform)?,
1270                    };
1271                    let source_cycles = if observed.post_cycles.is_some() {
1272                        observed.post_cycles
1273                    } else {
1274                        platform
1275                            .action_cycles(action, &state)
1276                            .map_err(EnsureWorkflowError::Platform)?
1277                    };
1278                    let destination_cycles = platform
1279                        .action_destination_cycles(action, &state)
1280                        .map_err(EnsureWorkflowError::Platform)?;
1281                    let replan_after_create_balance =
1282                        observed.retry == EffectRetry::ReplanRequiredAfterCreateBalanceDrift;
1283                    let retained_create_balance_is_exact = !replan_after_create_balance
1284                        || (journal.effects.len() == index + 1
1285                            && journal.effects[..index]
1286                                .iter()
1287                                .all(|effect| effect.state == EffectState::Applied)
1288                            && record.state == EffectState::Issued
1289                            && create_identity_is_exact(action.name(), record, &state)
1290                            && observed.post_cycles.is_some()
1291                            && observed.post_cycles == source_cycles);
1292                    let protocol_funding_required = if record.state == EffectState::Issued {
1293                        observed
1294                            .estate_funding_required
1295                            .as_ref()
1296                            .map(|funding| {
1297                                protocol_estate_funding_requirement(&retained_plan, &state, funding)
1298                            })
1299                            .transpose()?
1300                    } else {
1301                        None
1302                    };
1303                    if let Some(required) = protocol_funding_required.as_ref()
1304                        && !funding::covered(&journal, required)
1305                    {
1306                        retain_estate_funding_pause(&paths, &mut journal, Some(required))?;
1307                        report_progress_state(
1308                            platform,
1309                            &retained_plan,
1310                            &journal,
1311                            action_progress_phase(action),
1312                            FleetEnsureProgressState::FundingRequired,
1313                        );
1314                        return Err(EnsureWorkflowError::EstateFundingRequired(Box::new(
1315                            estate_funding_error(required),
1316                        )));
1317                    }
1318                    if journal
1319                        .estate_funding_required
1320                        .as_ref()
1321                        .is_some_and(|required| required.pending_creation_operation_id.is_some())
1322                    {
1323                        retain_estate_funding_pause(&paths, &mut journal, None)?;
1324                    }
1325                    let record = journal
1326                        .effects
1327                        .get_mut(index)
1328                        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1329                    merge_observed_cycles(record, source_cycles, destination_cycles);
1330                    if replan_after_create_balance {
1331                        if !retained_create_balance_is_exact {
1332                            return Err(EnsureWorkflowError::JournalIntegrity);
1333                        }
1334                        let EnsureAction::Create {
1335                            name,
1336                            requested_initial_cycles,
1337                            ..
1338                        } = action
1339                        else {
1340                            return Err(EnsureWorkflowError::JournalIntegrity);
1341                        };
1342                        let actual_cycles = record
1343                            .post_cycles
1344                            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1345                        record.progress_identity = Some(observed.progress_identity);
1346                        record.state = EffectState::Applied;
1347                        retain_created_canister_for_replan(
1348                            operation_desired,
1349                            name,
1350                            record,
1351                            &mut state,
1352                        )?;
1353                        journal.completion = FleetEnsureCompletion::ReplanRequired;
1354                        journal.stalled_observations = 0;
1355                        write_journal(&paths, &journal)?;
1356                        write_state(&paths, &state)?;
1357                        let configured_fee_cycles = operation_desired
1358                            .management_creation_fee_cycles
1359                            .parse::<Cycles>()
1360                            .map(|cycles| cycles.to_u128())
1361                            .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
1362                        return Err(EnsureWorkflowError::ReplanRequiredAfterCreateBalanceDrift {
1363                            actual_cycles,
1364                            canister: name.clone(),
1365                            configured_fee_cycles,
1366                            deficit_cycles: requested_initial_cycles.saturating_sub(actual_cycles),
1367                            requested_cycles: *requested_initial_cycles,
1368                        });
1369                    }
1370                    if observed.applied {
1371                        record.progress_identity = Some(observed.progress_identity);
1372                        record.state = EffectState::Applied;
1373                        journal.stalled_observations = 0;
1374                        write_journal(&paths, &journal)?;
1375                        let record = journal
1376                            .effects
1377                            .get(index)
1378                            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1379                        let retained_create = retain_applied_create_authority(
1380                            operation_desired,
1381                            action,
1382                            record,
1383                            &mut state,
1384                        )?;
1385                        let retained_funding =
1386                            retain_applied_funding_cycles(&mut state, action, record);
1387                        if retained_create || retained_funding {
1388                            write_state(&paths, &state)?;
1389                        }
1390                        // Emit once per durable transition, including a reconciled lost reply.
1391                        report_progress(
1392                            platform,
1393                            &retained_plan,
1394                            &journal,
1395                            action_progress_phase(action),
1396                        );
1397                        break;
1398                    }
1399
1400                    if observed.retry == EffectRetry::DeferUntilControllerObservation {
1401                        if !deferred_create_observation_is_exact(
1402                            operation_desired,
1403                            &actions,
1404                            index,
1405                            action,
1406                            record,
1407                            &state,
1408                        ) {
1409                            return Err(EnsureWorkflowError::JournalIntegrity);
1410                        }
1411                        let progress_identity = observed.progress_identity.clone();
1412                        if record.progress_identity.as_deref() == Some(&observed.progress_identity)
1413                        {
1414                            journal.stalled_observations =
1415                                journal.stalled_observations.saturating_add(1);
1416                        } else {
1417                            record.progress_identity = Some(observed.progress_identity);
1418                            journal.stalled_observations = 0;
1419                        }
1420                        write_journal(&paths, &journal)?;
1421                        if journal.stalled_observations
1422                            >= observation_policy.maximum_stalled_observations
1423                        {
1424                            return Err(EnsureWorkflowError::Stalled {
1425                                action: action.name().to_string(),
1426                                observations: journal.stalled_observations,
1427                                progress_identity,
1428                            });
1429                        }
1430                        deferred_controller_observation = true;
1431                        break;
1432                    }
1433
1434                    let maintenance_continuation =
1435                        observed.retry == EffectRetry::ContinuePoolMaintenance;
1436                    if matches!(record.state, EffectState::Intent) || maintenance_continuation {
1437                        if let Some(maximum_attempts) = action.fixture_publication_attempt_limit() {
1438                            if !reserve_fixture_publication_attempt(record, maximum_attempts) {
1439                                return Err(EnsureWorkflowError::FixturePublicationBound {
1440                                    action: action.name().to_string(),
1441                                    maximum_attempts,
1442                                });
1443                            }
1444                            // Reserve before the external call, even if its response never returns.
1445                            write_journal(&paths, &journal)?;
1446                        }
1447                        let record = journal
1448                            .effects
1449                            .get_mut(index)
1450                            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1451                        if let EnsureAction::FleetProtocol {
1452                            action: current, ..
1453                        } = action
1454                            && let CurrentFleetProtocolAction::MaintainPoolReadiness {
1455                                maximum_updates,
1456                                ..
1457                            } = current.as_ref()
1458                        {
1459                            if record.maintenance_attempts >= *maximum_updates {
1460                                return Err(EnsureWorkflowError::PoolMaintenanceBound {
1461                                    action: action.name().to_string(),
1462                                    maximum_updates: *maximum_updates,
1463                                });
1464                            }
1465                            if record.progress_identity.as_deref()
1466                                == Some(&observed.progress_identity)
1467                            {
1468                                journal.stalled_observations =
1469                                    journal.stalled_observations.saturating_add(1);
1470                            } else {
1471                                journal.stalled_observations = 0;
1472                            }
1473                            record.maintenance_attempts += 1;
1474                            let attempts = record.maintenance_attempts;
1475                            // A failed or lost call consumes its reviewed attempt across process restart.
1476                            write_journal(&paths, &journal)?;
1477                            if attempts > 1 {
1478                                platform
1479                                    .pace_effect_observation(action, journal.stalled_observations);
1480                            }
1481                        } else if maintenance_continuation {
1482                            return Err(EnsureWorkflowError::JournalIntegrity);
1483                        }
1484                        let record = journal
1485                            .effects
1486                            .get(index)
1487                            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1488                        if retained_plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite {
1489                            root_reinstall::verify_effect_authority(
1490                                &retained_plan,
1491                                action,
1492                                record.pre_cycles,
1493                                &state,
1494                                platform,
1495                            )?;
1496                        }
1497                        verify_store_activation_binding(&retained_plan, action, &state, platform)?;
1498                        reinstall::verify_effect_authority(
1499                            &retained_plan,
1500                            action,
1501                            &state,
1502                            platform,
1503                        )?;
1504                        let outcome =
1505                            match platform.apply(&journal.operation_id, action, record, &state) {
1506                                Ok(outcome) => outcome,
1507                                Err(source) => {
1508                                    let progress_identity = record
1509                                        .progress_identity
1510                                        .clone()
1511                                        .unwrap_or_else(|| "effect-call-failed".to_string());
1512                                    journal.stalled_observations =
1513                                        journal.stalled_observations.saturating_add(1);
1514                                    write_journal(&paths, &journal)?;
1515                                    if journal.stalled_observations
1516                                        >= observation_policy.maximum_stalled_observations
1517                                    {
1518                                        return Err(EnsureWorkflowError::Stalled {
1519                                            action: action.name().to_string(),
1520                                            observations: journal.stalled_observations,
1521                                            progress_identity,
1522                                        });
1523                                    }
1524                                    return Err(EnsureWorkflowError::Platform(source));
1525                                }
1526                            };
1527                        if !create_outcome_is_exact(action, record, &state, &outcome) {
1528                            return Err(EnsureWorkflowError::JournalIntegrity);
1529                        }
1530                        if let Some(created) = &outcome.created_principal {
1531                            state
1532                                .pending_principals
1533                                .insert(action.name().to_string(), created.clone());
1534                            write_state(&paths, &state)?;
1535                        }
1536                        let record = journal
1537                            .effects
1538                            .get_mut(index)
1539                            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1540                        record.created_principal = outcome.created_principal;
1541                        record.receipt = outcome.receipt;
1542                        record.post_cycles = outcome.post_cycles;
1543                        record.progress_identity = Some(observed.progress_identity);
1544                        record.state = EffectState::Issued;
1545                        // A successful maintenance call can report no progress while Root work is in flight.
1546                        if !maintenance_continuation {
1547                            journal.stalled_observations = 0;
1548                        }
1549                        write_journal(&paths, &journal)?;
1550                        continue;
1551                    }
1552
1553                    if observed.retry == EffectRetry::ReplayExactIssuedCommand
1554                        && !replayed_issued_commands.contains(&index)
1555                    {
1556                        if !matches!(
1557                            action,
1558                            EnsureAction::FleetProtocol { action, .. }
1559                                if matches!(
1560                                    action.as_ref(),
1561                                    crate::fleet_ensure::model::CurrentFleetProtocolAction::ProvisionComponents { .. }
1562                                )
1563                        ) {
1564                            return Err(EnsureWorkflowError::JournalIntegrity);
1565                        }
1566                        let retained_receipt = record.receipt.clone();
1567                        let outcome = platform
1568                            .apply(&journal.operation_id, action, record, &state)
1569                            .map_err(EnsureWorkflowError::Platform)?;
1570                        if outcome.created_principal.is_some()
1571                            || outcome.receipt != retained_receipt
1572                            || outcome.post_cycles.is_some()
1573                        {
1574                            return Err(EnsureWorkflowError::JournalIntegrity);
1575                        }
1576                        let record = journal
1577                            .effects
1578                            .get_mut(index)
1579                            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1580                        record.progress_identity = Some(observed.progress_identity);
1581                        journal.stalled_observations = 0;
1582                        write_journal(&paths, &journal)?;
1583                        replayed_issued_commands.insert(index);
1584                        continue;
1585                    }
1586
1587                    if record.progress_identity.as_deref() == Some(&observed.progress_identity) {
1588                        journal.stalled_observations =
1589                            journal.stalled_observations.saturating_add(1);
1590                    } else {
1591                        record.progress_identity = Some(observed.progress_identity);
1592                        journal.stalled_observations = 0;
1593                    }
1594                    let progress_identity = record
1595                        .progress_identity
1596                        .clone()
1597                        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1598                    write_journal(&paths, &journal)?;
1599                    if journal.stalled_observations
1600                        >= observation_policy.maximum_stalled_observations
1601                    {
1602                        if let Some((failure, origin)) = observed
1603                            .provisioning_failure
1604                            .and_then(|failure| failure.origin.map(|origin| (failure, origin)))
1605                        {
1606                            return Err(EnsureWorkflowError::ProvisioningRetryPending {
1607                                root: failure.fleet_subnet_root,
1608                                stage: origin.stage,
1609                                target: origin.target,
1610                                operation_id: origin.operation_id,
1611                                diagnostic_code: origin.diagnostic_code,
1612                                observations: journal.stalled_observations,
1613                                failed_at_ns: origin.failed_at_ns,
1614                            });
1615                        }
1616                        return Err(EnsureWorkflowError::Stalled {
1617                            action: action.name().to_string(),
1618                            observations: journal.stalled_observations,
1619                            progress_identity,
1620                        });
1621                    }
1622                    if observation_policy.paced {
1623                        report_progress_state(
1624                            platform,
1625                            &retained_plan,
1626                            &journal,
1627                            action_progress_phase(action),
1628                            FleetEnsureProgressState::AwaitingProgress {
1629                                elapsed_seconds: observation_started.elapsed().as_secs(),
1630                                provisioning: observed.provisioning_progress,
1631                            },
1632                        );
1633                        platform.pace_effect_observation(action, journal.stalled_observations);
1634                    }
1635                }
1636            }
1637            if journal.effects.len() == actions.len()
1638                && journal
1639                    .effects
1640                    .iter()
1641                    .all(|record| record.state == EffectState::Applied)
1642            {
1643                break;
1644            }
1645            if !deferred_controller_observation {
1646                return Err(EnsureWorkflowError::JournalIntegrity);
1647            }
1648        }
1649
1650        report_progress(
1651            platform,
1652            &retained_plan,
1653            &journal,
1654            FleetEnsurePhase::TerminalVerification,
1655        );
1656        if retained_plan.scope == FleetEnsurePlanScope::ReinstallPreparation {
1657            let actual = reinstall::complete(&retained_plan, &journal, &state, platform)?;
1658            journal.completion = FleetEnsureCompletion::Prepared;
1659            journal.stalled_observations = 0;
1660            write_journal(&paths, &journal)?;
1661            report_progress_state(
1662                platform,
1663                &retained_plan,
1664                &journal,
1665                FleetEnsurePhase::Infrastructure,
1666                FleetEnsureProgressState::PrerequisiteComplete,
1667            );
1668            return Ok(FleetEnsureReport {
1669                funding_review: None,
1670                actual_conservation: Some(actual),
1671                effects_applied: applied_count(&journal),
1672                plan: retained_plan,
1673                terminal: true,
1674            });
1675        }
1676        if retained_plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite {
1677            let actual = root_reinstall::complete(&retained_plan, &journal, &state, platform)?;
1678            publish_terminal_state(operation_desired, &retained_plan, &journal, &mut state)
1679                .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
1680            state.active_registry = None;
1681            write_state(&paths, &state)?;
1682            journal.completion = FleetEnsureCompletion::Converged;
1683            journal.stalled_observations = 0;
1684            write_journal(&paths, &journal)?;
1685            report_progress_state(
1686                platform,
1687                &retained_plan,
1688                &journal,
1689                FleetEnsurePhase::Infrastructure,
1690                FleetEnsureProgressState::PrerequisiteComplete,
1691            );
1692            return Ok(FleetEnsureReport {
1693                funding_review: None,
1694                actual_conservation: Some(actual),
1695                effects_applied: applied_count(&journal),
1696                plan: retained_plan,
1697                terminal: true,
1698            });
1699        }
1700        if retained_plan.scope == FleetEnsurePlanScope::RootStartPrerequisite {
1701            let actual_conservation = complete_root_start_prerequisite(
1702                operation_desired,
1703                &retained_plan,
1704                platform,
1705                &state,
1706                &journal,
1707            )?;
1708            journal.completion = FleetEnsureCompletion::Converged;
1709            journal.stalled_observations = 0;
1710            write_journal(&paths, &journal)?;
1711            report_progress_state(
1712                platform,
1713                &retained_plan,
1714                &journal,
1715                FleetEnsurePhase::Infrastructure,
1716                FleetEnsureProgressState::PrerequisiteComplete,
1717            );
1718            return Ok(FleetEnsureReport {
1719                funding_review: None,
1720                actual_conservation: Some(actual_conservation),
1721                effects_applied: applied_count(&journal),
1722                plan: retained_plan,
1723                terminal: true,
1724            });
1725        }
1726
1727        terminal_state = state.clone();
1728        publish_terminal_state(
1729            operation_desired,
1730            &retained_plan,
1731            &journal,
1732            &mut terminal_state,
1733        )
1734        .map_err(|TerminalStatePublicationError::JournalIntegrity| {
1735            EnsureWorkflowError::JournalIntegrity
1736        })?;
1737        if completed_infrastructure_reinstall(operation_desired, &retained_plan, &journal) {
1738            terminal_state.active_registry = None;
1739        }
1740        project_current_fleet_inventory(&terminal_state)?;
1741        let mut terminal_observation = platform
1742            .observe(&retained_plan.operation_id, &terminal_state)
1743            .map_err(EnsureWorkflowError::Platform)?;
1744        {
1745            let artifacts = resolve_desired_artifacts(root, operation_desired)?;
1746            let observation_started = Instant::now();
1747            let converged = loop {
1748                let protocol_actions = platform
1749                    .protocol_actions(&retained_plan.operation_id, &terminal_state)
1750                    .map_err(EnsureWorkflowError::Platform)?;
1751                match compile_plan(
1752                    operation_desired,
1753                    &artifacts,
1754                    &protocol_actions,
1755                    operation_desired_sha256,
1756                    requested_fleet,
1757                    &terminal_observation,
1758                    retained_plan.planned_at_time,
1759                    &retained_plan.operation_id,
1760                    None,
1761                ) {
1762                    Ok(plan) => {
1763                        if journal.stalled_observations != 0 {
1764                            journal.stalled_observations = 0;
1765                            write_journal(&paths, &journal)?;
1766                        }
1767                        break plan;
1768                    }
1769                    Err(EnsurePolicyError::PendingRootOwnedBalance { name }) => {
1770                        journal.stalled_observations =
1771                            journal.stalled_observations.saturating_add(1);
1772                        retain_observed_cycles(&mut state, &terminal_observation);
1773                        write_state(&paths, &state)?;
1774                        write_journal(&paths, &journal)?;
1775                        if journal.stalled_observations
1776                            >= operation_desired.maximum_stalled_observations
1777                        {
1778                            return Err(EnsureWorkflowError::RootOwnedObservationStalled {
1779                                last_lifecycle: root_owned_lifecycle_label(
1780                                    &terminal_observation,
1781                                    &name,
1782                                )
1783                                .to_string(),
1784                                observations: journal.stalled_observations,
1785                                target: name,
1786                            });
1787                        }
1788                        report_progress_state(
1789                            platform,
1790                            &retained_plan,
1791                            &journal,
1792                            FleetEnsurePhase::TerminalVerification,
1793                            FleetEnsureProgressState::AwaitingProgress {
1794                                elapsed_seconds: observation_started.elapsed().as_secs(),
1795                                provisioning: None,
1796                            },
1797                        );
1798                        platform.pace_root_owned_observation(&name, journal.stalled_observations);
1799                        terminal_observation = platform
1800                            .observe(&retained_plan.operation_id, &terminal_state)
1801                            .map_err(EnsureWorkflowError::Platform)?;
1802                    }
1803                    Err(error @ EnsurePolicyError::EstatePoolCapacity { .. }) => {
1804                        retain_observed_cycles(&mut terminal_state, &terminal_observation);
1805                        retain_completed_reinstalls(&mut terminal_state, &retained_plan, &journal);
1806                        write_state(&paths, &terminal_state)?;
1807                        journal.completion = FleetEnsureCompletion::ReplanRequired;
1808                        journal.stalled_observations = 0;
1809                        write_journal(&paths, &journal)?;
1810                        return Err(error.into());
1811                    }
1812                    Err(error) => return Err(error.into()),
1813                }
1814            };
1815            if converged
1816                .canisters
1817                .iter()
1818                .any(|canister| !canister.actions.is_empty())
1819                || !converged.protocol_actions.is_empty()
1820            {
1821                if retained_plan.continuation.is_some() {
1822                    let appended = continuation::append(
1823                        &paths,
1824                        operation_desired,
1825                        &retained_plan,
1826                        &mut journal,
1827                        &terminal_state,
1828                        &terminal_observation,
1829                        converged,
1830                        platform,
1831                    );
1832                    match appended {
1833                        Ok(()) => {
1834                            write_state(&paths, &terminal_state)?;
1835                            state = terminal_state.clone();
1836                            report_progress_state(
1837                                platform,
1838                                &retained_plan,
1839                                &journal,
1840                                FleetEnsurePhase::TerminalVerification,
1841                                FleetEnsureProgressState::PrerequisiteComplete,
1842                            );
1843                            continue;
1844                        }
1845                        Err(EnsureWorkflowError::SuccessorReviewRequired { reason, review }) => {
1846                            retain_observed_cycles(&mut terminal_state, &terminal_observation);
1847                            retain_completed_reinstalls(
1848                                &mut terminal_state,
1849                                &retained_plan,
1850                                &journal,
1851                            );
1852                            write_state(&paths, &terminal_state)?;
1853                            journal.completion = FleetEnsureCompletion::ReplanRequired;
1854                            write_journal(&paths, &journal)?;
1855                            report_progress_state(
1856                                platform,
1857                                &retained_plan,
1858                                &journal,
1859                                FleetEnsurePhase::TerminalVerification,
1860                                FleetEnsureProgressState::ReviewRequired {
1861                                    reason,
1862                                    review: review.clone(),
1863                                },
1864                            );
1865                            return Err(EnsureWorkflowError::SuccessorReviewRequired {
1866                                reason,
1867                                review,
1868                            });
1869                        }
1870                        Err(error) => return Err(error),
1871                    }
1872                }
1873                retain_observed_cycles(&mut terminal_state, &terminal_observation);
1874                retain_completed_reinstalls(&mut terminal_state, &retained_plan, &journal);
1875                write_state(&paths, &terminal_state)?;
1876                journal.completion = FleetEnsureCompletion::ReplanRequired;
1877                journal.stalled_observations = 0;
1878                write_journal(&paths, &journal)?;
1879                let reason =
1880                    crate::fleet_ensure::model::FleetEnsureSuccessorReviewReason::AdditionalEffect;
1881                let review = Some(Box::new(
1882                    crate::fleet_ensure::ops::recovery::review_details(&converged),
1883                ));
1884                report_progress_state(
1885                    platform,
1886                    &retained_plan,
1887                    &journal,
1888                    FleetEnsurePhase::TerminalVerification,
1889                    FleetEnsureProgressState::ReviewRequired {
1890                        reason,
1891                        review: review.clone(),
1892                    },
1893                );
1894                return Err(EnsureWorkflowError::SuccessorReviewRequired { reason, review });
1895            }
1896        }
1897        break;
1898    }
1899    let terminal_inventory_operation_id =
1900        completed_inventory_operation(&retained_plan, &journal, &terminal_state)?;
1901    let terminal_inventory = platform
1902        .terminal_inventory(terminal_inventory_operation_id, &terminal_state)
1903        .map_err(EnsureWorkflowError::Platform)?;
1904    let terminal_cycles = terminal_inventory.controlled_cycles_by_principal.clone();
1905    merge_terminal_inventory(&mut terminal_state, terminal_inventory)?;
1906    project_current_fleet_inventory(&terminal_state)?;
1907    let mut final_observation = platform
1908        .observe(&retained_plan.operation_id, &terminal_state)
1909        .map_err(EnsureWorkflowError::Platform)?;
1910    attach_terminal_cycles(&mut final_observation, terminal_cycles)?;
1911    reinstall::verify_terminal_estate(&retained_plan, &final_observation)?;
1912    reinstall::verify_terminal_authority(&retained_plan, &terminal_state, platform)?;
1913    let actual_conservation = verify_terminal_conservation(
1914        &retained_plan,
1915        &journal,
1916        &terminal_state,
1917        &final_observation,
1918    )?;
1919    terminal_state.completed_reinstall_action_sha256.clear();
1920    terminal_state.completed_reinstall_operation_id = None;
1921    terminal_state.completed_reinstalls.clear();
1922    // Publish the fully validated topology before making its terminal journal visible to backup.
1923    write_state(&paths, &terminal_state)?;
1924    journal.completion = FleetEnsureCompletion::Converged;
1925    journal.stalled_observations = 0;
1926    write_journal(&paths, &journal)?;
1927    report_progress(
1928        platform,
1929        &retained_plan,
1930        &journal,
1931        FleetEnsurePhase::Complete,
1932    );
1933    Ok(FleetEnsureReport {
1934        funding_review: None,
1935        actual_conservation: Some(actual_conservation),
1936        effects_applied: applied_count(&journal),
1937        plan: retained_plan,
1938        terminal: true,
1939    })
1940}
1941
1942fn action_progress_phase(action: &EnsureAction) -> FleetEnsurePhase {
1943    match action {
1944        EnsureAction::FleetProtocol { action, .. } => match action.as_ref() {
1945            CurrentFleetProtocolAction::ReconcilePoolAsset { .. } => {
1946                FleetEnsurePhase::ImportReconciliation
1947            }
1948            CurrentFleetProtocolAction::ProvisionComponents { .. } => {
1949                FleetEnsurePhase::WorkloadProvisioning
1950            }
1951            CurrentFleetProtocolAction::MaintainPoolReadiness { .. }
1952            | CurrentFleetProtocolAction::ObservePoolReadiness { .. } => {
1953                FleetEnsurePhase::PoolReadiness
1954            }
1955            _ => FleetEnsurePhase::ControlPlane,
1956        },
1957        _ => FleetEnsurePhase::Infrastructure,
1958    }
1959}
1960
1961fn report_progress<P: EnsurePlatform>(
1962    platform: &mut P,
1963    plan: &FleetEnsurePlan,
1964    journal: &FleetEnsureJournalRecord,
1965    phase: FleetEnsurePhase,
1966) {
1967    let state = if phase == FleetEnsurePhase::Complete {
1968        FleetEnsureProgressState::Complete
1969    } else {
1970        FleetEnsureProgressState::Advancing
1971    };
1972    report_progress_state(platform, plan, journal, phase, state);
1973}
1974
1975fn report_progress_state<P: EnsurePlatform>(
1976    platform: &mut P,
1977    plan: &FleetEnsurePlan,
1978    journal: &FleetEnsureJournalRecord,
1979    phase: FleetEnsurePhase,
1980    state: FleetEnsureProgressState,
1981) {
1982    platform.report_progress(FleetEnsureProgress {
1983        state,
1984        operation_id: plan.operation_id.clone(),
1985        plan_sha256: plan.plan_sha256.clone(),
1986        phase,
1987        applied_effects: applied_count(journal),
1988        reviewed_effects: continuation::actions(plan, journal).len()
1989            + journal
1990                .funding_reviews
1991                .iter()
1992                .filter(|review| review.effect.is_some())
1993                .count(),
1994    });
1995}
1996
1997const fn merge_observed_cycles(
1998    record: &mut EffectRecord,
1999    source_cycles: Option<u128>,
2000    destination_cycles: Option<u128>,
2001) {
2002    if let Some(cycles) = source_cycles {
2003        record.post_cycles = Some(cycles);
2004    }
2005    if let Some(cycles) = destination_cycles {
2006        record.destination_post_cycles = Some(cycles);
2007    }
2008}
2009
2010pub(super) fn retain_applied_create_authority<E>(
2011    desired: &crate::fleet_ensure::model::DesiredFleet,
2012    action: &EnsureAction,
2013    record: &EffectRecord,
2014    state: &mut FleetEnsureStateRecord,
2015) -> Result<bool, EnsureWorkflowError<E>>
2016where
2017    E: std::error::Error + 'static,
2018{
2019    let EnsureAction::Create {
2020        name,
2021        requested_initial_cycles,
2022        ..
2023    } = action
2024    else {
2025        return Ok(false);
2026    };
2027    if record.state != EffectState::Applied {
2028        return Err(EnsureWorkflowError::JournalIntegrity);
2029    }
2030    let configured = desired
2031        .canisters
2032        .iter()
2033        .find(|canister| {
2034            canister.name == *name
2035                && canister.presence == crate::fleet_ensure::model::DesiredPresence::Present
2036        })
2037        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2038    let principal = record
2039        .created_principal
2040        .as_deref()
2041        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2042    let cycles = record
2043        .post_cycles
2044        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2045    let maximum_observation_burn_cycles =
2046        crate::fleet_ensure::ops::maximum_creation_observation_burn(desired)
2047            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2048    if !create_identity_is_exact(name, record, state)
2049        || !create_balance_is_terminal(
2050            Some(cycles),
2051            *requested_initial_cycles,
2052            maximum_observation_burn_cycles,
2053        )
2054        || state
2055            .retained_cycles_by_principal
2056            .get(principal)
2057            .is_some_and(|retained| *retained != cycles)
2058    {
2059        return Err(EnsureWorkflowError::JournalIntegrity);
2060    }
2061    validate_fresh_pool_creation_balance(
2062        desired,
2063        configured,
2064        name,
2065        cycles,
2066        maximum_observation_burn_cycles,
2067        *requested_initial_cycles,
2068    )?;
2069
2070    let prior_topology = state.topology.get(name);
2071    let topology_identity_matches = prior_topology.is_some_and(|topology| {
2072        topology.kind == configured.kind
2073            && topology.parent == configured.parent
2074            && topology.protocol_binding == configured.protocol_binding
2075            && topology.role
2076                == configured
2077                    .protocol_binding
2078                    .as_ref()
2079                    .map(|binding| binding.role.to_string())
2080    });
2081    let topology = crate::fleet_ensure::model::FleetEnsureTopologyRecord {
2082        kind: configured.kind,
2083        module_hash: if topology_identity_matches {
2084            prior_topology.and_then(|topology| topology.module_hash.clone())
2085        } else {
2086            None
2087        },
2088        parent: configured.parent.clone(),
2089        protocol_binding: configured.protocol_binding.clone(),
2090        role: configured
2091            .protocol_binding
2092            .as_ref()
2093            .map(|binding| binding.role.to_string()),
2094    };
2095    let changed = state.retained_cycles_by_principal.get(principal) != Some(&cycles)
2096        || state.topology.get(name) != Some(&topology);
2097    state
2098        .retained_cycles_by_principal
2099        .insert(principal.to_string(), cycles);
2100    state.topology.insert(name.clone(), topology);
2101    Ok(changed)
2102}
2103
2104fn validate_fresh_pool_creation_balance<E>(
2105    desired: &crate::fleet_ensure::model::DesiredFleet,
2106    configured: &crate::fleet_ensure::model::DesiredCanister,
2107    name: &str,
2108    cycles: u128,
2109    maximum_observation_burn_cycles: u128,
2110    requested_initial_cycles: u128,
2111) -> Result<(), EnsureWorkflowError<E>>
2112where
2113    E: std::error::Error + 'static,
2114{
2115    if configured.kind == crate::fleet_ensure::model::DesiredCanisterKind::Pool
2116        && desired
2117            .bootstrap
2118            .as_ref()
2119            .is_some_and(|bootstrap| bootstrap.fresh_estate)
2120        && configured.principal.is_none()
2121        && configured.controllers.is_empty()
2122        && configured.parent.as_ref().is_some_and(|parent| {
2123            configured.controller_canisters.as_slice() == std::slice::from_ref(parent)
2124        })
2125    {
2126        let readiness_floor_cycles = configured
2127            .minimum_cycles
2128            .parse::<Cycles>()
2129            .map(|cycles| cycles.to_u128())
2130            .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
2131        let remaining_controller_burn_cycles = desired
2132            .maximum_update_burn_cycles
2133            .parse::<Cycles>()
2134            .map(|cycles| cycles.to_u128())
2135            .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
2136        let required_pre_finalization_balance_cycles = readiness_floor_cycles
2137            .checked_add(remaining_controller_burn_cycles)
2138            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2139        if cycles < required_pre_finalization_balance_cycles {
2140            return Err(EnsureWorkflowError::FreshPoolCreationUnderfunded(Box::new(
2141                FreshPoolCreationUnderfundedError {
2142                    canister: name.to_string(),
2143                    live_balance_cycles: cycles,
2144                    maximum_observation_burn_cycles,
2145                    pre_finalization_shortfall_cycles: required_pre_finalization_balance_cycles
2146                        - cycles,
2147                    readiness_floor_cycles,
2148                    readiness_shortfall_cycles: readiness_floor_cycles.saturating_sub(cycles),
2149                    remaining_controller_burn_cycles,
2150                    requested_creation_funding_cycles: requested_initial_cycles,
2151                    required_pre_finalization_balance_cycles,
2152                },
2153            )));
2154        }
2155    }
2156    Ok(())
2157}
2158
2159fn retain_applied_funding_cycles(
2160    state: &mut FleetEnsureStateRecord,
2161    action: &EnsureAction,
2162    record: &EffectRecord,
2163) -> bool {
2164    let EnsureAction::Fund { principal, .. } = action else {
2165        return false;
2166    };
2167    if record.state != EffectState::Applied || record.receipt.is_none() {
2168        return false;
2169    }
2170    let Some(cycles) = record.post_cycles else {
2171        return false;
2172    };
2173    let principal = principal.strip_prefix("created:").map_or_else(
2174        || Some(principal.as_str()),
2175        |name| state.pending_principals.get(name).map(String::as_str),
2176    );
2177    let Some(principal) = principal else {
2178        return false;
2179    };
2180    if state.retained_cycles_by_principal.get(principal) == Some(&cycles) {
2181        return false;
2182    }
2183    state
2184        .retained_cycles_by_principal
2185        .insert(principal.to_string(), cycles);
2186    true
2187}
2188
2189fn retain_created_canister_for_replan<E>(
2190    desired: &crate::fleet_ensure::model::DesiredFleet,
2191    name: &str,
2192    record: &EffectRecord,
2193    state: &mut FleetEnsureStateRecord,
2194) -> Result<(), EnsureWorkflowError<E>>
2195where
2196    E: std::error::Error + 'static,
2197{
2198    let configured = desired
2199        .canisters
2200        .iter()
2201        .find(|canister| {
2202            canister.name == name
2203                && canister.presence == crate::fleet_ensure::model::DesiredPresence::Present
2204        })
2205        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2206    let principal = record
2207        .created_principal
2208        .as_deref()
2209        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2210    let cycles = record
2211        .post_cycles
2212        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2213    if state.pending_principals.get(name).map(String::as_str) != Some(principal) {
2214        return Err(EnsureWorkflowError::JournalIntegrity);
2215    }
2216    state.pending_principals.remove(name);
2217    state
2218        .principals
2219        .insert(name.to_string(), principal.to_string());
2220    state
2221        .retained_cycles_by_principal
2222        .insert(principal.to_string(), cycles);
2223    state.topology.insert(
2224        name.to_string(),
2225        crate::fleet_ensure::model::FleetEnsureTopologyRecord {
2226            kind: configured.kind,
2227            module_hash: None,
2228            parent: configured.parent.clone(),
2229            protocol_binding: configured.protocol_binding.clone(),
2230            role: configured
2231                .protocol_binding
2232                .as_ref()
2233                .map(|binding| binding.role.to_string()),
2234        },
2235    );
2236    Ok(())
2237}
2238
2239fn create_record_is_exact(
2240    action: &EnsureAction,
2241    record: &EffectRecord,
2242    state: &FleetEnsureStateRecord,
2243) -> bool {
2244    let EnsureAction::Create {
2245        name,
2246        requested_initial_cycles,
2247        ..
2248    } = action
2249    else {
2250        return true;
2251    };
2252    match record.state {
2253        EffectState::Intent => {
2254            record.created_principal.is_none()
2255                && record.receipt.is_none()
2256                && record.post_cycles.is_none()
2257        }
2258        EffectState::Issued => {
2259            create_identity_is_exact(name, record, state)
2260                && record.post_cycles == Some(*requested_initial_cycles)
2261        }
2262        EffectState::Applied => create_identity_is_exact(name, record, state),
2263    }
2264}
2265
2266fn create_identity_is_exact(
2267    name: &str,
2268    record: &EffectRecord,
2269    state: &FleetEnsureStateRecord,
2270) -> bool {
2271    let Some(created_principal) = record
2272        .created_principal
2273        .as_deref()
2274        .filter(|value| !value.is_empty())
2275    else {
2276        return false;
2277    };
2278    if record.receipt.as_deref().is_none_or(str::is_empty) {
2279        return false;
2280    }
2281    state_create_principal_matches(name, created_principal, state, true)
2282}
2283
2284fn state_create_principal_matches(
2285    name: &str,
2286    created_principal: &str,
2287    state: &FleetEnsureStateRecord,
2288    require_retained: bool,
2289) -> bool {
2290    let pending = state.pending_principals.get(name).map(String::as_str);
2291    let terminal = state.principals.get(name).map(String::as_str);
2292    let retained = pending.is_some() || terminal.is_some();
2293    let pending_matches = pending.is_none_or(|principal| principal == created_principal);
2294    let terminal_matches = terminal.is_none_or(|principal| principal == created_principal);
2295    (!require_retained || retained) && pending_matches && terminal_matches
2296}
2297
2298fn create_outcome_is_exact(
2299    action: &EnsureAction,
2300    record: &EffectRecord,
2301    state: &FleetEnsureStateRecord,
2302    outcome: &crate::fleet_ensure::ops::EffectOutcome,
2303) -> bool {
2304    if matches!(action, EnsureAction::FundEstate { .. }) {
2305        return outcome.created_principal.is_none()
2306            && outcome.post_cycles.is_none()
2307            && outcome
2308                .receipt
2309                .as_deref()
2310                .is_some_and(|receipt| !receipt.is_empty());
2311    }
2312    let EnsureAction::Create {
2313        name,
2314        requested_initial_cycles,
2315        ..
2316    } = action
2317    else {
2318        return true;
2319    };
2320    let Some(created_principal) = outcome
2321        .created_principal
2322        .as_deref()
2323        .filter(|value| !value.is_empty())
2324    else {
2325        return false;
2326    };
2327    let Some(receipt) = outcome.receipt.as_deref().filter(|value| !value.is_empty()) else {
2328        return false;
2329    };
2330    if outcome.post_cycles != Some(*requested_initial_cycles) {
2331        return false;
2332    }
2333    let principal_matches = record
2334        .created_principal
2335        .as_deref()
2336        .is_none_or(|retained| retained == created_principal);
2337    let receipt_matches = record
2338        .receipt
2339        .as_deref()
2340        .is_none_or(|retained| retained == receipt);
2341    let state_matches = state_create_principal_matches(name, created_principal, state, false);
2342    principal_matches && receipt_matches && state_matches
2343}
2344
2345fn verify_fresh_plan<P>(
2346    root: &Path,
2347    desired: &crate::fleet_ensure::model::DesiredFleet,
2348    desired_sha256: &str,
2349    requested_fleet: &str,
2350    retained_plan: &FleetEnsurePlan,
2351    platform: &mut P,
2352    state: &FleetEnsureStateRecord,
2353) -> Result<(FleetObservation, u128), EnsureWorkflowError<P::Error>>
2354where
2355    P: EnsurePlatform,
2356{
2357    if retained_plan.reinstall.is_some() {
2358        return reinstall::verify_before_apply(root, desired, retained_plan, platform, state);
2359    }
2360    if retained_plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite {
2361        return root_reinstall::verify_before_apply(root, desired, retained_plan, platform, state);
2362    }
2363    if retained_plan.scope == FleetEnsurePlanScope::RootStartPrerequisite {
2364        let targets = reviewed_root_start_targets(retained_plan)?;
2365        let management = platform
2366            .observe_root_management(state, &targets)
2367            .map_err(EnsureWorkflowError::Platform)?
2368            .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2369        let current = recompile_root_start_prerequisite_plan(
2370            RootStartPlanInput {
2371                state,
2372                authority: retained_plan.root_start_authority.as_deref(),
2373                created_at_time: retained_plan.planned_at_time,
2374                desired,
2375                desired_sha256,
2376                observation: &management,
2377                requested_fleet,
2378            },
2379            &targets,
2380        )?;
2381        if !compatible_root_start_prerequisite(retained_plan, &current, desired) {
2382            return Err(EnsureWorkflowError::DriftedBeforeApply);
2383        }
2384        let observation = root_management_fleet_observation(&management, &targets)?;
2385        return Ok((observation, current.conservation.observed_controlled_cycles));
2386    }
2387    let mut observation = platform
2388        .observe(&retained_plan.operation_id, state)
2389        .map_err(EnsureWorkflowError::Platform)?;
2390    let terminal_inventory_operation_id =
2391        reviewed_terminal_inventory_operation(retained_plan, state)?;
2392    if let Some(operation_id) = terminal_inventory_operation_id {
2393        attach_terminal_inventory_cycles(operation_id, state, platform, &mut observation)?;
2394    }
2395    let artifacts = resolve_desired_artifacts(root, desired)?;
2396    let mut retained_observations = 0_u32;
2397    let mut current = loop {
2398        let protocol_actions = platform
2399            .protocol_actions(&retained_plan.operation_id, state)
2400            .map_err(EnsureWorkflowError::Platform)?;
2401        match compile_plan(
2402            desired,
2403            &artifacts,
2404            &protocol_actions,
2405            desired_sha256,
2406            requested_fleet,
2407            &observation,
2408            retained_plan.planned_at_time,
2409            &retained_plan.operation_id,
2410            None,
2411        ) {
2412            Ok(plan) => break plan,
2413            Err(EnsurePolicyError::PendingRootOwnedBalance { name }) => {
2414                retained_observations = retained_observations.saturating_add(1);
2415                if retained_observations >= desired.maximum_stalled_observations {
2416                    return Err(EnsureWorkflowError::RootOwnedObservationStalled {
2417                        last_lifecycle: root_owned_lifecycle_label(&observation, &name).to_string(),
2418                        observations: retained_observations,
2419                        target: name,
2420                    });
2421                }
2422                platform.pace_root_owned_observation(&name, retained_observations);
2423                observation = platform
2424                    .observe(&retained_plan.operation_id, state)
2425                    .map_err(EnsureWorkflowError::Platform)?;
2426                if let Some(operation_id) = terminal_inventory_operation_id {
2427                    attach_terminal_inventory_cycles(
2428                        operation_id,
2429                        state,
2430                        platform,
2431                        &mut observation,
2432                    )?;
2433                }
2434            }
2435            Err(error) => return Err(error.into()),
2436        }
2437    };
2438    if observation.operator_cycles < retained_plan.conservation.maximum_operator_debit_cycles {
2439        return Err(EnsureWorkflowError::InsufficientOperatorCycles {
2440            actual: observation.operator_cycles,
2441            required: retained_plan.conservation.maximum_operator_debit_cycles,
2442        });
2443    }
2444    bind_terminal_inventory_operation(
2445        &mut current,
2446        retained_plan.terminal_inventory_operation_id.clone(),
2447    );
2448    if current.plan_sha256 != retained_plan.plan_sha256
2449        && !compatible_after_bounded_observation(retained_plan, &current, desired, &observation)
2450    {
2451        return Err(EnsureWorkflowError::DriftedBeforeApply);
2452    }
2453    Ok((observation, current.conservation.observed_controlled_cycles))
2454}
2455
2456fn estate_funding_requirement<E>(
2457    plan: &FleetEnsurePlan,
2458    state: &FleetEnsureStateRecord,
2459    observation: &FleetObservation,
2460) -> Result<Option<EstateFundingRequiredRecord>, EnsureWorkflowError<E>>
2461where
2462    E: std::error::Error + 'static,
2463{
2464    for reviewed in &plan.conservation.estate_funding_domains {
2465        let observed = observation
2466            .estate_funding_domains
2467            .get(&reviewed.root)
2468            .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2469        if observed.cycles_ledger != reviewed.cycles_ledger {
2470            return Err(EnsureWorkflowError::PlanIntegrity);
2471        }
2472        let root_principal = observed
2473            .root_principal
2474            .as_deref()
2475            .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2476        if reviewed_estate_root_principal(reviewed, state) != Some(root_principal) {
2477            return Err(EnsureWorkflowError::PlanIntegrity);
2478        }
2479        let available_cycles = observed
2480            .balance_cycles
2481            .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2482        let reviewed_available = reviewed.available_cycles.unwrap_or_default();
2483        let maximum_reviewed_balance = reviewed_available
2484            .checked_add(reviewed.maximum_funding_cycles)
2485            .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2486        if available_cycles < reviewed_available || available_cycles > maximum_reviewed_balance {
2487            return Err(EnsureWorkflowError::DriftedBeforeApply);
2488        }
2489        if reviewed.required_creation_count == 0 {
2490            if available_cycles != reviewed_available {
2491                return Err(EnsureWorkflowError::DriftedBeforeApply);
2492            }
2493            continue;
2494        }
2495        if available_cycles < reviewed.maximum_creation_debit_cycles {
2496            return Ok(Some(EstateFundingRequiredRecord {
2497                attempt_count: None,
2498                available_cycles,
2499                creation_amount_cycles: reviewed.creation_amount_cycles,
2500                creation_execution_margin_cycles: reviewed.creation_execution_margin_cycles,
2501                readiness_floor_cycles: reviewed.readiness_floor_cycles,
2502                cycles_ledger: reviewed.cycles_ledger.clone(),
2503                ledger_fee_cycles: reviewed.ledger_fee_cycles,
2504                management_creation_fee_cycles: reviewed.management_creation_fee_cycles,
2505                maximum_creation_debit_cycles: reviewed.maximum_creation_debit_cycles,
2506                maximum_creation_fee_cycles: reviewed.maximum_creation_fee_cycles,
2507                operation_id: plan.operation_id.clone(),
2508                pending_creation_operation_id: None,
2509                plan_sha256: plan.plan_sha256.clone(),
2510                required_creation_count: reviewed.required_creation_count,
2511                root: reviewed.root.clone(),
2512                root_principal: root_principal.to_string(),
2513                last_attempt_at_ns: None,
2514                retry_at_ns: None,
2515                shortfall_cycles: reviewed
2516                    .maximum_creation_debit_cycles
2517                    .saturating_sub(available_cycles),
2518            }));
2519        }
2520    }
2521    Ok(None)
2522}
2523
2524fn protocol_estate_funding_requirement<E>(
2525    plan: &FleetEnsurePlan,
2526    state: &FleetEnsureStateRecord,
2527    funding: &canic_core::dto::component_provisioning::RootEstateFundingRequired,
2528) -> Result<EstateFundingRequiredRecord, EnsureWorkflowError<E>>
2529where
2530    E: std::error::Error + 'static,
2531{
2532    let root_principal = funding.root.to_text();
2533    let reviewed = plan
2534        .conservation
2535        .estate_funding_domains
2536        .iter()
2537        .find(|domain| reviewed_estate_root_principal(domain, state) == Some(&root_principal))
2538        .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2539    let creation_amount_cycles = funding.creation_amount.to_u128();
2540    let ledger_fee_cycles = funding.ledger_fee.to_u128();
2541    let management_creation_fee_cycles = funding.management_creation_fee.to_u128();
2542    let required_cycles = creation_amount_cycles
2543        .checked_add(ledger_fee_cycles)
2544        .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2545    let maximum_creation_fee_cycles = management_creation_fee_cycles
2546        .checked_add(ledger_fee_cycles)
2547        .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2548    let exact_reviewed_authority = reviewed.required_creation_count > 0
2549        && reviewed.cycles_ledger == funding.cycles_ledger.to_text()
2550        && reviewed.creation_amount_cycles == creation_amount_cycles
2551        && reviewed.creation_execution_margin_cycles == funding.execution_margin.to_u128()
2552        && reviewed.readiness_floor_cycles == funding.readiness_floor.to_u128()
2553        && reviewed.ledger_fee_cycles == ledger_fee_cycles
2554        && reviewed.management_creation_fee_cycles == management_creation_fee_cycles;
2555    let exact_runtime_arithmetic = funding.required.to_u128() == required_cycles
2556        && funding.available < funding.required
2557        && funding.shortfall.to_u128()
2558            == required_cycles.saturating_sub(funding.available.to_u128());
2559    if !exact_reviewed_authority || !exact_runtime_arithmetic {
2560        return Err(EnsureWorkflowError::PlanIntegrity);
2561    }
2562    Ok(EstateFundingRequiredRecord {
2563        attempt_count: Some(funding.attempt_count),
2564        available_cycles: funding.available.to_u128(),
2565        creation_amount_cycles,
2566        cycles_ledger: funding.cycles_ledger.to_text(),
2567        creation_execution_margin_cycles: funding.execution_margin.to_u128(),
2568        readiness_floor_cycles: funding.readiness_floor.to_u128(),
2569        ledger_fee_cycles,
2570        management_creation_fee_cycles,
2571        maximum_creation_debit_cycles: required_cycles,
2572        maximum_creation_fee_cycles,
2573        operation_id: plan.operation_id.clone(),
2574        pending_creation_operation_id: Some(canic_core::cdk::utils::hash::hex_bytes(
2575            funding.operation_id,
2576        )),
2577        plan_sha256: plan.plan_sha256.clone(),
2578        required_creation_count: 1,
2579        root: reviewed.root.clone(),
2580        root_principal,
2581        last_attempt_at_ns: funding.last_attempt_at_ns,
2582        retry_at_ns: Some(funding.retry_at_ns),
2583        shortfall_cycles: funding.shortfall.to_u128(),
2584    })
2585}
2586
2587fn estate_funding_error(required: &EstateFundingRequiredRecord) -> EstateFundingRequiredError {
2588    EstateFundingRequiredError {
2589        attempt_count: required.attempt_count,
2590        available_cycles: required.available_cycles,
2591        creation_amount_cycles: required.creation_amount_cycles,
2592        creation_execution_margin_cycles: required.creation_execution_margin_cycles,
2593        readiness_floor_cycles: required.readiness_floor_cycles,
2594        cycles_ledger: required.cycles_ledger.clone(),
2595        ledger_fee_cycles: required.ledger_fee_cycles,
2596        management_creation_fee_cycles: required.management_creation_fee_cycles,
2597        maximum_creation_debit_cycles: required.maximum_creation_debit_cycles,
2598        maximum_creation_fee_cycles: required.maximum_creation_fee_cycles,
2599        pending_creation_operation_id: required.pending_creation_operation_id.clone(),
2600        required_creation_count: required.required_creation_count,
2601        root: required.root.clone(),
2602        root_principal: required.root_principal.clone(),
2603        last_attempt_at_ns: required.last_attempt_at_ns,
2604        retry_at_ns: required.retry_at_ns,
2605        shortfall_cycles: required.shortfall_cycles,
2606    }
2607}
2608
2609fn retain_estate_funding_pause<E>(
2610    paths: &EnsurePaths,
2611    journal: &mut FleetEnsureJournalRecord,
2612    required: Option<&EstateFundingRequiredRecord>,
2613) -> Result<bool, EnsureWorkflowError<E>>
2614where
2615    E: std::error::Error + 'static,
2616{
2617    if journal.estate_funding_required.as_ref() == required {
2618        return Ok(false);
2619    }
2620    journal.estate_funding_required = required.cloned();
2621    write_journal(paths, journal)?;
2622    Ok(true)
2623}
2624
2625fn reviewed_estate_root_principal<'a>(
2626    domain: &'a crate::fleet_ensure::model::EstateFundingDomainPlan,
2627    state: &'a FleetEnsureStateRecord,
2628) -> Option<&'a str> {
2629    domain
2630        .root_principal
2631        .as_deref()
2632        .or_else(|| {
2633            state
2634                .pending_principals
2635                .get(&domain.root)
2636                .map(String::as_str)
2637        })
2638        .or_else(|| state.principals.get(&domain.root).map(String::as_str))
2639}
2640
2641fn prior_fleet_protocol_effect_started(
2642    actions: &[&EnsureAction],
2643    journal: &FleetEnsureJournalRecord,
2644    current_index: usize,
2645) -> bool {
2646    actions
2647        .iter()
2648        .take(current_index)
2649        .zip(&journal.effects)
2650        .any(|(action, effect)| {
2651            matches!(action, EnsureAction::FleetProtocol { .. })
2652                && matches!(effect.state, EffectState::Issued | EffectState::Applied)
2653        })
2654}
2655
2656fn completed_inventory_operation<'a, E>(
2657    plan: &'a FleetEnsurePlan,
2658    journal: &FleetEnsureJournalRecord,
2659    state: &FleetEnsureStateRecord,
2660) -> Result<&'a str, EnsureWorkflowError<E>>
2661where
2662    E: std::error::Error + 'static,
2663{
2664    if plan.protocol_actions.is_empty()
2665        && journal.successor_phases.is_empty()
2666        && state.active_registry.is_some()
2667    {
2668        reviewed_terminal_inventory_operation(plan, state)?
2669            .ok_or(EnsureWorkflowError::PlanIntegrity)
2670    } else {
2671        Ok(&plan.operation_id)
2672    }
2673}
2674
2675fn reviewed_terminal_inventory_operation<'a, E>(
2676    plan: &'a FleetEnsurePlan,
2677    state: &FleetEnsureStateRecord,
2678) -> Result<Option<&'a str>, EnsureWorkflowError<E>>
2679where
2680    E: std::error::Error + 'static,
2681{
2682    match (
2683        state.active_registry.is_some(),
2684        plan.terminal_inventory_operation_id.as_deref(),
2685    ) {
2686        (false, None) => Ok(None),
2687        (true, Some(operation_id)) if !operation_id.is_empty() => Ok(Some(operation_id)),
2688        _ => Err(EnsureWorkflowError::PlanIntegrity),
2689    }
2690}
2691
2692fn complete_root_start_prerequisite<P>(
2693    desired: &crate::fleet_ensure::model::DesiredFleet,
2694    retained_plan: &FleetEnsurePlan,
2695    platform: &mut P,
2696    state: &FleetEnsureStateRecord,
2697    journal: &FleetEnsureJournalRecord,
2698) -> Result<ActualCycleConservation, EnsureWorkflowError<P::Error>>
2699where
2700    P: EnsurePlatform,
2701{
2702    let targets = reviewed_root_start_targets(retained_plan)?;
2703    let management = platform
2704        .observe_root_management(state, &targets)
2705        .map_err(EnsureWorkflowError::Platform)?
2706        .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2707    let current = recompile_root_start_prerequisite_plan(
2708        RootStartPlanInput {
2709            state,
2710            authority: retained_plan.root_start_authority.as_deref(),
2711            created_at_time: retained_plan.planned_at_time,
2712            desired,
2713            desired_sha256: &retained_plan.desired_sha256,
2714            observation: &management,
2715            requested_fleet: &retained_plan.fleet,
2716        },
2717        &targets,
2718    )?;
2719    if !compatible_root_start_prerequisite(retained_plan, &current, desired) {
2720        return Err(EnsureWorkflowError::ConvergenceDrift);
2721    }
2722    if management.roots.iter().any(|(name, observed)| {
2723        targets.contains(name)
2724            && observed.live.status != crate::fleet_ensure::model::CanisterRuntimeStatus::Running
2725    }) {
2726        return Err(EnsureWorkflowError::ConvergenceDrift);
2727    }
2728    let terminal = root_management_fleet_observation(&management, &targets)?;
2729    verify_terminal_conservation(retained_plan, journal, state, &terminal)
2730}
2731
2732fn verify_terminal_root_start_replay<P>(
2733    desired: &crate::fleet_ensure::model::DesiredFleet,
2734    plan: &FleetEnsurePlan,
2735    journal: &FleetEnsureJournalRecord,
2736    requested_fleet: &str,
2737    platform: &mut P,
2738    state: &FleetEnsureStateRecord,
2739) -> Result<(), EnsureWorkflowError<P::Error>>
2740where
2741    P: EnsurePlatform,
2742{
2743    verify_journal(journal, plan, requested_fleet, state)?;
2744    let actions = ordered_actions(plan);
2745    if actions.len() != journal.effects.len()
2746        || journal
2747            .effects
2748            .iter()
2749            .any(|effect| effect.state != EffectState::Applied)
2750    {
2751        return Err(EnsureWorkflowError::JournalIntegrity);
2752    }
2753    for (action, record) in actions.into_iter().zip(&journal.effects) {
2754        if action_sha256(action) != record.action_sha256
2755            || !platform
2756                .observe_effect(&journal.operation_id, action, record, state)
2757                .map_err(EnsureWorkflowError::Platform)?
2758                .applied
2759        {
2760            return Err(EnsureWorkflowError::ConvergenceDrift);
2761        }
2762    }
2763    let targets = reviewed_root_start_targets(plan)?;
2764    let management = platform
2765        .observe_root_management(state, &targets)
2766        .map_err(EnsureWorkflowError::Platform)?
2767        .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2768    let current = recompile_root_start_prerequisite_plan(
2769        RootStartPlanInput {
2770            state,
2771            authority: plan.root_start_authority.as_deref(),
2772            created_at_time: plan.planned_at_time,
2773            desired,
2774            desired_sha256: &plan.desired_sha256,
2775            observation: &management,
2776            requested_fleet: &plan.fleet,
2777        },
2778        &targets,
2779    )?;
2780    if normalized_plan(plan) != normalized_plan(&current)
2781        || management.roots.iter().any(|(name, observed)| {
2782            targets.contains(name)
2783                && observed.live.status
2784                    != crate::fleet_ensure::model::CanisterRuntimeStatus::Running
2785        })
2786    {
2787        return Err(EnsureWorkflowError::ConvergenceDrift);
2788    }
2789    Ok(())
2790}
2791
2792fn reviewed_root_start_targets<E>(
2793    plan: &FleetEnsurePlan,
2794) -> Result<BTreeSet<String>, EnsureWorkflowError<E>>
2795where
2796    E: std::error::Error + 'static,
2797{
2798    if plan.scope != FleetEnsurePlanScope::RootStartPrerequisite
2799        || !plan.protocol_actions.is_empty()
2800        || plan.canisters.is_empty()
2801        || plan.conservation.maximum_new_funding_cycles != 0
2802        || plan.conservation.maximum_operator_debit_cycles != 0
2803        || plan.conservation.maximum_unavoidable_fee_cycles != 0
2804        || !plan.conservation.estate_funding_domains.is_empty()
2805        || plan.conservation.scheduled_transfer_cycles != 0
2806        || plan.conservation.observed_controlled_cycles
2807            != plan.conservation.retained_in_reused_canisters_cycles
2808        || plan
2809            .conservation
2810            .observed_controlled_cycles
2811            .checked_sub(plan.conservation.maximum_execution_burn_cycles)
2812            != Some(plan.conservation.expected_post_operation_cycles)
2813    {
2814        return Err(EnsureWorkflowError::PlanIntegrity);
2815    }
2816    let mut targets = BTreeSet::new();
2817    for canister in &plan.canisters {
2818        if canister.disposition != CanisterDisposition::Reuse
2819            || canister.principal.is_none()
2820            || !matches!(
2821                canister.actions.as_slice(),
2822                [EnsureAction::Start { name, principal }]
2823                    if name == &canister.name
2824                        && Some(principal) == canister.principal.as_ref()
2825            )
2826            || !targets.insert(canister.name.clone())
2827        {
2828            return Err(EnsureWorkflowError::PlanIntegrity);
2829        }
2830    }
2831    Ok(targets)
2832}
2833
2834fn root_management_fleet_observation<E>(
2835    management: &RootManagementObservation,
2836    targets: &BTreeSet<String>,
2837) -> Result<FleetObservation, EnsureWorkflowError<E>>
2838where
2839    E: std::error::Error + 'static,
2840{
2841    let canisters = targets
2842        .iter()
2843        .map(|name| {
2844            management
2845                .roots
2846                .get(name)
2847                .map(|observed| (name.clone(), Some(observed.live.clone())))
2848                .ok_or(EnsureWorkflowError::PlanIntegrity)
2849        })
2850        .collect::<Result<BTreeMap<_, _>, _>>()?;
2851    Ok(FleetObservation {
2852        additional_controlled_cycles: BTreeMap::new(),
2853        canisters,
2854        estate_funding_domains: BTreeMap::new(),
2855        ledger_fee_cycles: 0,
2856        operator_cycles: management.operator_cycles,
2857        protocol_ready: BTreeMap::new(),
2858    })
2859}
2860
2861fn compatible_root_start_prerequisite(
2862    retained: &FleetEnsurePlan,
2863    current: &FleetEnsurePlan,
2864    desired: &crate::fleet_ensure::model::DesiredFleet,
2865) -> bool {
2866    let Ok(maximum_movement) = desired
2867        .maximum_update_burn_cycles
2868        .parse::<Cycles>()
2869        .map(|cycles| cycles.to_u128())
2870    else {
2871        return false;
2872    };
2873    retained.canisters.len() == current.canisters.len()
2874        && retained
2875            .canisters
2876            .iter()
2877            .zip(&current.canisters)
2878            .all(|(retained, current)| {
2879                retained.observed_cycles.abs_diff(current.observed_cycles) <= maximum_movement
2880            })
2881        && normalized_plan(retained) == normalized_plan(current)
2882}
2883
2884fn attach_terminal_inventory_cycles<P>(
2885    operation_id: &str,
2886    state: &FleetEnsureStateRecord,
2887    platform: &mut P,
2888    observation: &mut FleetObservation,
2889) -> Result<(), EnsureWorkflowError<P::Error>>
2890where
2891    P: EnsurePlatform,
2892{
2893    let inventory = platform
2894        .terminal_inventory(operation_id, state)
2895        .map_err(EnsureWorkflowError::Platform)?;
2896    attach_terminal_cycles(observation, inventory.controlled_cycles_by_principal)
2897}
2898
2899pub(super) fn completed_infrastructure_reinstall(
2900    desired: &crate::fleet_ensure::model::DesiredFleet,
2901    plan: &FleetEnsurePlan,
2902    journal: &FleetEnsureJournalRecord,
2903) -> bool {
2904    if desired.protocol.is_none() {
2905        return false;
2906    }
2907    let infrastructure = desired.canisters.iter().filter(|canister| {
2908        canister.presence == crate::fleet_ensure::model::DesiredPresence::Present
2909            && matches!(
2910                canister.kind,
2911                crate::fleet_ensure::model::DesiredCanisterKind::Coordinator
2912                    | crate::fleet_ensure::model::DesiredCanisterKind::Root
2913                    | crate::fleet_ensure::model::DesiredCanisterKind::Store
2914            )
2915    });
2916    let mut count = 0_usize;
2917    for configured in infrastructure {
2918        count += 1;
2919        let Some(action) = plan
2920            .canisters
2921            .iter()
2922            .find(|canister| {
2923                canister.name == configured.name
2924                    && canister.disposition == CanisterDisposition::Reinstall
2925            })
2926            .and_then(|canister| {
2927                canister.actions.iter().find(|action| {
2928                    matches!(
2929                        action,
2930                        EnsureAction::Install {
2931                            mode: crate::fleet_ensure::model::InstallMode::Reinstall,
2932                            name,
2933                            ..
2934                        } if name == &configured.name
2935                    )
2936                })
2937            })
2938        else {
2939            return false;
2940        };
2941        let action_sha256 = action_sha256(action);
2942        if !journal.effects.iter().any(|effect| {
2943            effect.action_sha256 == action_sha256 && effect.state == EffectState::Applied
2944        }) {
2945            return false;
2946        }
2947    }
2948    count > 0
2949}
2950
2951fn compatible_after_bounded_observation(
2952    retained: &FleetEnsurePlan,
2953    current: &FleetEnsurePlan,
2954    desired: &crate::fleet_ensure::model::DesiredFleet,
2955    observation: &FleetObservation,
2956) -> bool {
2957    let Ok(maximum_observation_movement) = desired
2958        .maximum_observation_burn_cycles
2959        .parse::<Cycles>()
2960        .map(|cycles| cycles.to_u128())
2961    else {
2962        return false;
2963    };
2964    if retained.canisters.len() != current.canisters.len() {
2965        return false;
2966    }
2967    for (retained_canister, current_canister) in retained.canisters.iter().zip(&current.canisters) {
2968        let movement = retained_canister
2969            .observed_cycles
2970            .abs_diff(current_canister.observed_cycles);
2971        if movement > maximum_observation_movement {
2972            return false;
2973        }
2974    }
2975    if !retained_funding_remains_sufficient(retained, desired, observation) {
2976        return false;
2977    }
2978    normalized_plan(retained) == normalized_plan(current)
2979}
2980
2981fn retained_funding_remains_sufficient(
2982    retained: &FleetEnsurePlan,
2983    desired: &crate::fleet_ensure::model::DesiredFleet,
2984    observation: &FleetObservation,
2985) -> bool {
2986    retained
2987        .canisters
2988        .iter()
2989        .flat_map(|canister| &canister.actions)
2990        .filter_map(|action| match action {
2991            EnsureAction::Fund {
2992                amount,
2993                name,
2994                pool_funding,
2995                principal,
2996                ..
2997            } => Some((amount, name, pool_funding, principal)),
2998            _ => None,
2999        })
3000        .all(|(amount, name, pool_funding, principal)| {
3001            if let Some(root) = pool_funding {
3002                return observation
3003                    .estate_funding_domains
3004                    .values()
3005                    .find(|domain| domain.root_principal.as_ref() == Some(&root.root))
3006                    .and_then(|domain| domain.pool.as_ref())
3007                    .is_some_and(|pool| {
3008                        pool.assets.iter().any(|asset| {
3009                            asset.principal == *principal
3010                                && asset.lifecycle == root.lifecycle
3011                                && asset
3012                                    .cycles
3013                                    .checked_add(*amount)
3014                                    .is_some_and(|funded| funded >= pool.readiness_floor_cycles)
3015                        })
3016                    });
3017            }
3018            let Some(live) = observation.canisters.get(name).and_then(Option::as_ref) else {
3019                return false;
3020            };
3021            let Some(configured) = desired
3022                .canisters
3023                .iter()
3024                .find(|configured| configured.name == *name)
3025            else {
3026                return false;
3027            };
3028            let Ok(minimum) = configured
3029                .minimum_cycles
3030                .parse::<Cycles>()
3031                .map(|cycles| cycles.to_u128())
3032            else {
3033                return false;
3034            };
3035            live.cycles
3036                .checked_add(*amount)
3037                .is_some_and(|funded| funded >= minimum)
3038        })
3039}
3040
3041fn normalized_plan(plan: &FleetEnsurePlan) -> FleetEnsurePlan {
3042    let mut normalized = plan.clone();
3043    normalized.plan_sha256.clear();
3044    normalized.reviewed_desired = None;
3045    if let Some(review) = &mut normalized.recovery_review {
3046        // These estimates move with bounded observations; they grant no funding authority.
3047        review.continuation_reserve_cycles = 0;
3048        for funding in &mut review.known_pool_funding {
3049            funding.amount_cycles = 0;
3050            funding.funding_deficit_cycles = 0;
3051        }
3052    }
3053    normalized.conservation = CycleConservation {
3054        estate_funding_domains: plan
3055            .conservation
3056            .estate_funding_domains
3057            .iter()
3058            .cloned()
3059            .map(|mut domain| {
3060                domain.available_cycles = None;
3061                domain.maximum_funding_cycles = 0;
3062                domain.root_principal = None;
3063                domain.shortfall_cycles = 0;
3064                domain
3065            })
3066            .collect(),
3067        expected_post_operation_cycles: 0,
3068        maximum_execution_burn_cycles: 0,
3069        maximum_new_funding_cycles: 0,
3070        maximum_operator_debit_cycles: 0,
3071        maximum_unavoidable_fee_cycles: 0,
3072        observed_controlled_cycles: 0,
3073        retained_in_reused_canisters_cycles: 0,
3074        scheduled_transfer_cycles: 0,
3075    };
3076    for canister in &mut normalized.canisters {
3077        canister.observed_cycles = 0;
3078        for action in &mut canister.actions {
3079            if let EnsureAction::Fund {
3080                amount,
3081                funding_deficit_cycles,
3082                ..
3083            } = action
3084            {
3085                *amount = 0;
3086                *funding_deficit_cycles = 0;
3087            }
3088        }
3089    }
3090    normalized
3091}
3092
3093fn attach_terminal_cycles<E>(
3094    observation: &mut FleetObservation,
3095    additional: BTreeMap<String, u128>,
3096) -> Result<(), EnsureWorkflowError<E>>
3097where
3098    E: std::error::Error + 'static,
3099{
3100    let mut retained = BTreeMap::new();
3101    for (principal, cycles) in additional {
3102        let Some(configured) = observation
3103            .canisters
3104            .values_mut()
3105            .filter_map(Option::as_mut)
3106            .find(|canister| canister.principal == principal)
3107        else {
3108            retained.insert(principal, cycles);
3109            continue;
3110        };
3111        match configured.root_owned_lifecycle {
3112            Some(crate::fleet_ensure::model::RootOwnedCanisterLifecycle::Workload) => {
3113                // Both protected observations identify the same running workload, but
3114                // execution may burn cycles between them. Retain the conservative lower value.
3115                configured.cycles = configured.cycles.min(cycles);
3116            }
3117            Some(crate::fleet_ensure::model::RootOwnedCanisterLifecycle::Idle)
3118                if configured.cycles == cycles => {}
3119            Some(crate::fleet_ensure::model::RootOwnedCanisterLifecycle::Idle) => {
3120                return Err(EnsureWorkflowError::TerminalInventory(
3121                    TerminalInventoryError::IdleCycleObservationConflict {
3122                        canister: principal,
3123                        expected: configured.cycles,
3124                        observed: cycles,
3125                    },
3126                ));
3127            }
3128            _ => {
3129                return Err(EnsureWorkflowError::TerminalInventory(
3130                    TerminalInventoryError::CycleObservationLifecycleConflict {
3131                        canister: principal,
3132                    },
3133                ));
3134            }
3135        }
3136    }
3137    observation.additional_controlled_cycles = retained;
3138    Ok(())
3139}
3140
3141fn funding_plan<'a, E: std::error::Error + 'static>(
3142    plan: &'a FleetEnsurePlan,
3143    journal: &FleetEnsureJournalRecord,
3144) -> Result<std::borrow::Cow<'a, FleetEnsurePlan>, EnsureWorkflowError<E>> {
3145    if journal.funding_reviews.is_empty() {
3146        return Ok(std::borrow::Cow::Borrowed(plan));
3147    }
3148    let mut funded = plan.clone();
3149    let (amount, fee) = funding::totals::<E>(journal, None)?;
3150    let conservation = &mut funded.conservation;
3151    conservation.maximum_operator_debit_cycles = conservation
3152        .maximum_operator_debit_cycles
3153        .checked_add(amount)
3154        .and_then(|n| n.checked_add(fee))
3155        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3156    conservation.maximum_new_funding_cycles = conservation
3157        .maximum_new_funding_cycles
3158        .checked_add(amount)
3159        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3160    conservation.maximum_unavoidable_fee_cycles = conservation
3161        .maximum_unavoidable_fee_cycles
3162        .checked_add(fee)
3163        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3164    for domain in &mut conservation.estate_funding_domains {
3165        domain.maximum_funding_cycles = domain
3166            .maximum_funding_cycles
3167            .checked_add(funding::totals::<E>(journal, Some(&domain.root))?.0)
3168            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3169    }
3170    Ok(std::borrow::Cow::Owned(funded))
3171}
3172
3173fn verify_terminal_conservation<E>(
3174    plan: &FleetEnsurePlan,
3175    journal: &FleetEnsureJournalRecord,
3176    state: &FleetEnsureStateRecord,
3177    terminal: &FleetObservation,
3178) -> Result<ActualCycleConservation, EnsureWorkflowError<E>>
3179where
3180    E: std::error::Error + 'static,
3181{
3182    let funded = funding_plan(plan, journal)?;
3183    let conservation = &funded.conservation;
3184    let final_controlled_cycles = controlled_cycles(terminal)?;
3185    let operator_debit_cycles = journal
3186        .initial_operator_cycles
3187        .checked_sub(terminal.operator_cycles)
3188        .ok_or_else(|| {
3189            EnsureWorkflowError::Conservation(
3190                "operator balance increased during apply; review a new plan".to_string(),
3191            )
3192        })?;
3193    if operator_debit_cycles > conservation.maximum_operator_debit_cycles {
3194        return Err(EnsureWorkflowError::Conservation(format!(
3195            "operator debit {operator_debit_cycles} exceeded reviewed maximum {}",
3196            conservation.maximum_operator_debit_cycles
3197        )));
3198    }
3199    let received_new_funding_cycles = operator_debit_cycles
3200        .checked_sub(conservation.maximum_unavoidable_fee_cycles)
3201        .ok_or_else(|| {
3202            EnsureWorkflowError::Conservation(
3203                "operator debit was below the exact reviewed fee total".to_string(),
3204            )
3205        })?;
3206    if received_new_funding_cycles > conservation.maximum_new_funding_cycles {
3207        return Err(EnsureWorkflowError::Conservation(format!(
3208            "received funding {received_new_funding_cycles} exceeded reviewed maximum {}",
3209            conservation.maximum_new_funding_cycles
3210        )));
3211    }
3212    let (estate_funding_cycles, exact_estate_creation_fee_cycles) =
3213        reconcile_estate_funding(plan, journal, state, terminal)?;
3214    let observed_settlement_credit_cycles = settlement::observed_credit(plan, journal)?;
3215    let available = journal
3216        .initial_controlled_cycles
3217        .checked_add(received_new_funding_cycles)
3218        .and_then(|cycles| cycles.checked_add(observed_settlement_credit_cycles))
3219        .ok_or_else(|| {
3220            EnsureWorkflowError::Conservation(
3221                "terminal controlled-cycle arithmetic overflowed".to_string(),
3222            )
3223        })?;
3224    let after_estate_fees = available
3225        .checked_sub(exact_estate_creation_fee_cycles)
3226        .ok_or_else(|| {
3227            EnsureWorkflowError::Conservation(
3228                "terminal controlled cycles cannot cover reviewed estate creation fees".to_string(),
3229            )
3230        })?;
3231    let measured_execution_burn_cycles = after_estate_fees
3232        .checked_sub(final_controlled_cycles)
3233        .ok_or_else(|| {
3234            EnsureWorkflowError::Conservation(
3235                "terminal estate exceeds starting cycles, received funding and bounded observed credits"
3236                    .to_string(),
3237            )
3238        })?;
3239    if measured_execution_burn_cycles > conservation.maximum_execution_burn_cycles {
3240        return Err(EnsureWorkflowError::Conservation(format!(
3241            "measured execution burn {measured_execution_burn_cycles} exceeded reviewed maximum {}",
3242            conservation.maximum_execution_burn_cycles
3243        )));
3244    }
3245    Ok(ActualCycleConservation {
3246        estate_funding_cycles,
3247        exact_estate_creation_fee_cycles,
3248        exact_unavoidable_fee_cycles: conservation.maximum_unavoidable_fee_cycles,
3249        final_controlled_cycles,
3250        measured_execution_burn_cycles,
3251        observed_starting_cycles: journal.initial_controlled_cycles,
3252        observed_settlement_credit_cycles,
3253        operator_debit_cycles,
3254        received_new_funding_cycles,
3255    })
3256}
3257
3258fn resolved_initial_pool_assets<E>(
3259    domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3260    state: &FleetEnsureStateRecord,
3261) -> Result<BTreeSet<String>, EnsureWorkflowError<E>>
3262where
3263    E: std::error::Error + 'static,
3264{
3265    domain
3266        .initial_pool_assets
3267        .iter()
3268        .map(|identity| {
3269            identity.strip_prefix("created:").map_or_else(
3270                || Ok(identity.clone()),
3271                |name| {
3272                    state
3273                        .pending_principals
3274                        .get(name)
3275                        .or_else(|| state.principals.get(name))
3276                        .cloned()
3277                        .ok_or(EnsureWorkflowError::JournalIntegrity)
3278                },
3279            )
3280        })
3281        .collect()
3282}
3283
3284fn exact_estate_creation_costs<E>(
3285    state: &FleetEnsureStateRecord,
3286    terminal: &FleetObservation,
3287    domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3288) -> Result<(u128, u128), EnsureWorkflowError<E>>
3289where
3290    E: std::error::Error + 'static,
3291{
3292    let observed = terminal
3293        .estate_funding_domains
3294        .get(&domain.root)
3295        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3296    let Some(pool) = observed.pool.as_ref() else {
3297        return if domain.required_creation_count == 0 {
3298            Ok((0, 0))
3299        } else {
3300            Err(EnsureWorkflowError::JournalIntegrity)
3301        };
3302    };
3303    if pool.pending_creation.is_some() {
3304        return Err(EnsureWorkflowError::JournalIntegrity);
3305    }
3306
3307    estate_creation_costs(state, terminal, domain)
3308}
3309
3310fn estate_creation_costs<E: std::error::Error + 'static>(
3311    state: &FleetEnsureStateRecord,
3312    observation: &FleetObservation,
3313    domain: &EstateFundingDomainPlan,
3314) -> Result<(u128, u128), EnsureWorkflowError<E>> {
3315    let pool = observation
3316        .estate_funding_domains
3317        .get(&domain.root)
3318        .and_then(|domain| domain.pool.as_ref())
3319        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3320    let initial = resolved_initial_pool_assets(domain, state)?;
3321    let terminal_principals = pool
3322        .assets
3323        .iter()
3324        .map(|asset| asset.principal.clone())
3325        .collect::<BTreeSet<_>>();
3326    if terminal_principals.len() != pool.assets.len() || !initial.is_subset(&terminal_principals) {
3327        return Err(EnsureWorkflowError::JournalIntegrity);
3328    }
3329    let created = pool
3330        .assets
3331        .iter()
3332        .filter(|asset| !initial.contains(&asset.principal))
3333        .collect::<Vec<_>>();
3334    let creation_count_is_bounded = created.len()
3335        <= usize::try_from(domain.required_creation_count)
3336            .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
3337    // Root maintenance can spend its reviewed reserve without provisioning a new Workload.
3338    // The per-Root budget and exact receipts bind that autonomous creation authority.
3339    let pool_policy_is_exact = created.is_empty() || terminal_pool_policy_is_exact(pool, domain);
3340    if !creation_count_is_bounded || !pool_policy_is_exact {
3341        return Err(EnsureWorkflowError::JournalIntegrity);
3342    }
3343
3344    let mut operation_ids = BTreeSet::new();
3345    let mut block_indexes = BTreeSet::new();
3346    let mut creation_debit = 0_u128;
3347    let mut creation_fee = 0_u128;
3348    for asset in created {
3349        let (asset_debit, asset_fee) = exact_estate_creation_receipt_costs(
3350            asset,
3351            domain,
3352            &mut operation_ids,
3353            &mut block_indexes,
3354        )?;
3355        creation_debit = creation_debit.checked_add(asset_debit).ok_or_else(|| {
3356            EnsureWorkflowError::Conservation(
3357                "estate creation-debit arithmetic overflowed".to_string(),
3358            )
3359        })?;
3360        creation_fee = creation_fee.checked_add(asset_fee).ok_or_else(|| {
3361            EnsureWorkflowError::Conservation(
3362                "estate creation-fee arithmetic overflowed".to_string(),
3363            )
3364        })?;
3365    }
3366    if creation_debit > domain.maximum_creation_debit_cycles
3367        || creation_fee > domain.maximum_creation_fee_cycles
3368    {
3369        return Err(EnsureWorkflowError::JournalIntegrity);
3370    }
3371    Ok((creation_debit, creation_fee))
3372}
3373
3374const fn terminal_pool_policy_is_exact(
3375    pool: &crate::fleet_ensure::model::EstatePoolInventoryObservation,
3376    domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3377) -> bool {
3378    let expected_maximum_size = domain.pool_maximum_size;
3379    let expected_readiness_floor = domain.readiness_floor_cycles;
3380    let expected_execution_margin = domain.creation_execution_margin_cycles;
3381    pool.maximum_size == expected_maximum_size
3382        && pool.readiness_floor_cycles == expected_readiness_floor
3383        && pool.creation_execution_margin_cycles == expected_execution_margin
3384}
3385
3386#[derive(Eq, PartialEq)]
3387struct EstateCreationReceiptAuthority<'a> {
3388    creation_execution_margin_cycles: u128,
3389    cycles_ledger: &'a str,
3390    ledger_amount_cycles: u128,
3391    ledger_fee_cycles: u128,
3392    management_creation_fee_cycles: u128,
3393    readiness_floor_cycles: u128,
3394}
3395
3396fn observed_estate_creation_authority(
3397    receipt: &crate::fleet_ensure::model::EstatePoolCreationReceiptObservation,
3398) -> EstateCreationReceiptAuthority<'_> {
3399    EstateCreationReceiptAuthority {
3400        creation_execution_margin_cycles: receipt.creation_execution_margin_cycles,
3401        cycles_ledger: &receipt.cycles_ledger,
3402        ledger_amount_cycles: receipt.ledger_amount_cycles,
3403        ledger_fee_cycles: receipt.ledger_fee_cycles,
3404        management_creation_fee_cycles: receipt.management_creation_fee_cycles,
3405        readiness_floor_cycles: receipt.readiness_floor_cycles,
3406    }
3407}
3408
3409fn reviewed_estate_creation_authority(
3410    domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3411) -> EstateCreationReceiptAuthority<'_> {
3412    EstateCreationReceiptAuthority {
3413        creation_execution_margin_cycles: domain.creation_execution_margin_cycles,
3414        cycles_ledger: &domain.cycles_ledger,
3415        ledger_amount_cycles: domain.creation_amount_cycles,
3416        ledger_fee_cycles: domain.ledger_fee_cycles,
3417        management_creation_fee_cycles: domain.management_creation_fee_cycles,
3418        readiness_floor_cycles: domain.readiness_floor_cycles,
3419    }
3420}
3421
3422fn first_estate_creation_observation_is_exact(
3423    receipt: &crate::fleet_ensure::model::EstatePoolCreationReceiptObservation,
3424) -> bool {
3425    receipt
3426        .ledger_amount_cycles
3427        .checked_sub(receipt.management_creation_fee_cycles)
3428        .zip(receipt.first_observed_cycles)
3429        .is_some_and(|(funded_native_cycles, observed_cycles)| {
3430            observed_cycles >= receipt.readiness_floor_cycles
3431                && observed_cycles <= funded_native_cycles
3432                && funded_native_cycles - observed_cycles
3433                    <= receipt.creation_execution_margin_cycles
3434        })
3435}
3436
3437fn exact_estate_creation_receipt_costs<E>(
3438    asset: &crate::fleet_ensure::model::EstatePoolAssetObservation,
3439    domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3440    operation_ids: &mut BTreeSet<String>,
3441    block_indexes: &mut BTreeSet<u64>,
3442) -> Result<(u128, u128), EnsureWorkflowError<E>>
3443where
3444    E: std::error::Error + 'static,
3445{
3446    let receipt = asset
3447        .creation_receipt
3448        .as_ref()
3449        .filter(|_| asset.origin == crate::fleet_ensure::model::EstatePoolAssetOrigin::Created)
3450        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3451    let authority_is_exact =
3452        observed_estate_creation_authority(receipt) == reviewed_estate_creation_authority(domain);
3453    if !is_sha256_hex(&receipt.operation_id)
3454        || !authority_is_exact
3455        || !first_estate_creation_observation_is_exact(receipt)
3456        || !operation_ids.insert(receipt.operation_id.clone())
3457        || !block_indexes.insert(receipt.block_index)
3458    {
3459        return Err(EnsureWorkflowError::JournalIntegrity);
3460    }
3461    let debit = receipt
3462        .ledger_amount_cycles
3463        .checked_add(receipt.ledger_fee_cycles)
3464        .ok_or_else(|| {
3465            EnsureWorkflowError::Conservation(
3466                "estate creation-debit arithmetic overflowed".to_string(),
3467            )
3468        })?;
3469    let fee = receipt
3470        .management_creation_fee_cycles
3471        .checked_add(receipt.ledger_fee_cycles)
3472        .ok_or_else(|| {
3473            EnsureWorkflowError::Conservation(
3474                "estate creation-fee arithmetic overflowed".to_string(),
3475            )
3476        })?;
3477    Ok((debit, fee))
3478}
3479
3480fn reconcile_estate_funding<E>(
3481    plan: &FleetEnsurePlan,
3482    journal: &FleetEnsureJournalRecord,
3483    state: &FleetEnsureStateRecord,
3484    terminal: &FleetObservation,
3485) -> Result<(u128, u128), EnsureWorkflowError<E>>
3486where
3487    E: std::error::Error + 'static,
3488{
3489    plan.conservation
3490        .estate_funding_domains
3491        .iter()
3492        .try_fold((0_u128, 0_u128), |(funding_total, fee_total), domain| {
3493            let initial = journal
3494                .initial_estate_funding_cycles_by_root
3495                .get(&domain.root)
3496                .copied()
3497                .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3498            let observed = terminal
3499                .estate_funding_domains
3500                .get(&domain.root)
3501                .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3502            if observed.cycles_ledger != domain.cycles_ledger
3503                || reviewed_estate_root_principal(domain, state)
3504                    != observed.root_principal.as_deref()
3505            {
3506                return Err(EnsureWorkflowError::JournalIntegrity);
3507            }
3508            let terminal_balance = observed
3509                .balance_cycles
3510                .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3511            let (creation_debit, creation_fee) = exact_estate_creation_costs(
3512                state,
3513                terminal,
3514                domain,
3515            )?;
3516            let funding = applied_estate_funding_for_domain(plan, journal, state, domain)?
3517                .checked_add(funding::totals::<E>(journal, Some(&domain.root))?.0)
3518                .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3519            let expected_terminal_balance = initial
3520                .checked_add(funding)
3521                .and_then(|available| available.checked_sub(creation_debit))
3522                .ok_or_else(|| {
3523                    EnsureWorkflowError::Conservation(format!(
3524                        "Root {} funding-account balance cannot reconcile its reviewed funding and creation debit",
3525                        domain.root
3526                    ))
3527                })?;
3528            if terminal_balance != expected_terminal_balance {
3529                return Err(EnsureWorkflowError::Conservation(format!(
3530                    "Root {} funding-account balance {terminal_balance} differs from reviewed terminal balance {expected_terminal_balance}",
3531                    domain.root
3532                )));
3533            }
3534            Ok((
3535                funding_total.checked_add(funding).ok_or_else(|| {
3536                    EnsureWorkflowError::Conservation(
3537                        "estate funding arithmetic overflowed".to_string(),
3538                    )
3539                })?,
3540                fee_total
3541                    .checked_add(creation_fee)
3542                    .ok_or_else(|| {
3543                        EnsureWorkflowError::Conservation(
3544                            "estate creation-fee arithmetic overflowed".to_string(),
3545                        )
3546                    })?,
3547            ))
3548        })
3549}
3550
3551fn applied_estate_funding_for_domain<E>(
3552    plan: &FleetEnsurePlan,
3553    journal: &FleetEnsureJournalRecord,
3554    state: &FleetEnsureStateRecord,
3555    domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3556) -> Result<u128, EnsureWorkflowError<E>>
3557where
3558    E: std::error::Error + 'static,
3559{
3560    let action = plan
3561        .canisters
3562        .iter()
3563        .find(|canister| canister.name == domain.root)
3564        .into_iter()
3565        .flat_map(|canister| &canister.actions)
3566        .find(|action| matches!(action, EnsureAction::FundEstate { .. }));
3567    let Some(action) = action else {
3568        return if domain.maximum_funding_cycles == 0 {
3569            Ok(0)
3570        } else {
3571            Err(EnsureWorkflowError::JournalIntegrity)
3572        };
3573    };
3574    let EnsureAction::FundEstate {
3575        amount,
3576        expected_post_cycles,
3577        ledger,
3578        ledger_fee_cycles,
3579        principal,
3580        ..
3581    } = action
3582    else {
3583        unreachable!("filtered estate-funding action")
3584    };
3585    let root_principal = principal.strip_prefix("created:").map_or_else(
3586        || Some(principal.as_str()),
3587        |name| {
3588            state
3589                .pending_principals
3590                .get(name)
3591                .or_else(|| state.principals.get(name))
3592                .map(String::as_str)
3593        },
3594    );
3595    let expected_destination = domain
3596        .available_cycles
3597        .unwrap_or_default()
3598        .checked_add(domain.shortfall_cycles)
3599        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3600    let action_authority_is_exact = *amount == domain.shortfall_cycles
3601        && *expected_post_cycles == expected_destination
3602        && ledger == &domain.cycles_ledger
3603        && *ledger_fee_cycles == domain.ledger_fee_cycles
3604        && root_principal == reviewed_estate_root_principal(domain, state);
3605    let effect = journal
3606        .effects
3607        .iter()
3608        .find(|effect| effect.action_sha256 == action_sha256(action));
3609    let Some(effect) = effect.filter(|effect| {
3610        effect.state == EffectState::Applied
3611            && effect
3612                .receipt
3613                .as_deref()
3614                .is_some_and(|receipt| !receipt.is_empty())
3615    }) else {
3616        return Err(EnsureWorkflowError::JournalIntegrity);
3617    };
3618    let balance_evidence_is_exact = crate::fleet_ensure::ops::estate_funding_applied(
3619        crate::fleet_ensure::ops::EstateFundingObservation {
3620            amount: *amount,
3621            destination_after: effect.destination_post_cycles.unwrap_or_default(),
3622            destination_before: effect.destination_pre_cycles,
3623            expected_destination_after: *expected_post_cycles,
3624            ledger_fee_cycles: *ledger_fee_cycles,
3625            source_after: effect.post_cycles.unwrap_or_default(),
3626            source_before: effect.pre_cycles,
3627        },
3628    );
3629    if !action_authority_is_exact || !balance_evidence_is_exact {
3630        return Err(EnsureWorkflowError::JournalIntegrity);
3631    }
3632    Ok(*amount)
3633}
3634
3635fn controlled_cycles<E>(observation: &FleetObservation) -> Result<u128, EnsureWorkflowError<E>>
3636where
3637    E: std::error::Error + 'static,
3638{
3639    observation
3640        .canisters
3641        .values()
3642        .filter_map(Option::as_ref)
3643        .try_fold(0_u128, |total, live| total.checked_add(live.cycles))
3644        .and_then(|total| {
3645            observation
3646                .additional_controlled_cycles
3647                .values()
3648                .try_fold(total, |subtotal, cycles| subtotal.checked_add(*cycles))
3649        })
3650        .and_then(|total| {
3651            observation
3652                .estate_funding_domains
3653                .values()
3654                .filter_map(|domain| domain.balance_cycles)
3655                .try_fold(total, u128::checked_add)
3656        })
3657        .ok_or_else(|| {
3658            EnsureWorkflowError::Conservation("controlled-cycle total overflowed".to_string())
3659        })
3660}
3661
3662fn verified_plan<E>(plan: FleetEnsurePlan) -> Result<FleetEnsurePlan, EnsureWorkflowError<E>>
3663where
3664    E: std::error::Error + 'static,
3665{
3666    if expected_plan_sha256(&plan) != plan.plan_sha256 {
3667        return Err(EnsureWorkflowError::PlanIntegrity);
3668    }
3669    Ok(plan)
3670}
3671
3672fn verify_journal<E>(
3673    journal: &FleetEnsureJournalRecord,
3674    plan: &FleetEnsurePlan,
3675    requested_fleet: &str,
3676    state: &FleetEnsureStateRecord,
3677) -> Result<(), EnsureWorkflowError<E>>
3678where
3679    E: std::error::Error + 'static,
3680{
3681    if journal.fleet != requested_fleet
3682        || journal.operation_id != plan.operation_id
3683        || journal.plan_sha256 != plan.plan_sha256
3684        || !initial_estate_funding_is_exact(journal, plan)
3685        || !estate_funding_pause_is_exact(journal, plan, state)
3686    {
3687        return Err(EnsureWorkflowError::JournalIntegrity);
3688    }
3689    funding::verify(plan, journal, state)?;
3690    continuation::verify_records(plan, journal)?;
3691    let actions = continuation::actions(plan, journal);
3692    let effect_count_matches = journal.effects.len() <= actions.len();
3693    let action_hashes_match = journal.effects.iter().zip(actions).all(|(effect, action)| {
3694        effect.action_sha256 == action_sha256(action)
3695            && maintenance_attempts_are_exact(effect, action)
3696            && publication_attempts_are_exact(effect, action)
3697    });
3698    if !(effect_count_matches && action_hashes_match) {
3699        return Err(EnsureWorkflowError::JournalIntegrity);
3700    }
3701    Ok(())
3702}
3703
3704fn publication_attempts_are_exact(effect: &EffectRecord, action: &EnsureAction) -> bool {
3705    match action.fixture_publication_attempt_limit() {
3706        Some(maximum) => maximum > 0 && effect.publication_attempts <= maximum,
3707        None => effect.publication_attempts == 0,
3708    }
3709}
3710
3711fn maintenance_attempts_are_exact(effect: &EffectRecord, action: &EnsureAction) -> bool {
3712    if let EnsureAction::FleetProtocol { action, .. } = action
3713        && let CurrentFleetProtocolAction::MaintainPoolReadiness {
3714            maximum_updates, ..
3715        } = action.as_ref()
3716    {
3717        effect.maintenance_attempts <= *maximum_updates
3718    } else {
3719        effect.maintenance_attempts == 0
3720    }
3721}
3722
3723fn initial_estate_funding_is_exact(
3724    journal: &FleetEnsureJournalRecord,
3725    plan: &FleetEnsurePlan,
3726) -> bool {
3727    journal.initial_estate_funding_cycles_by_root.len()
3728        == plan.conservation.estate_funding_domains.len()
3729        && plan
3730            .conservation
3731            .estate_funding_domains
3732            .iter()
3733            .all(|domain| {
3734                journal
3735                    .initial_estate_funding_cycles_by_root
3736                    .get(&domain.root)
3737                    .copied()
3738                    == Some(domain.available_cycles.unwrap_or_default())
3739            })
3740}
3741
3742fn estate_funding_pause_is_exact(
3743    journal: &FleetEnsureJournalRecord,
3744    plan: &FleetEnsurePlan,
3745    state: &FleetEnsureStateRecord,
3746) -> bool {
3747    let Some(required) = journal.estate_funding_required.as_ref() else {
3748        return true;
3749    };
3750    journal.completion == FleetEnsureCompletion::InProgress
3751        && estate_funding_record_is_exact(required, plan, state)
3752}
3753
3754fn estate_funding_record_is_exact(
3755    required: &EstateFundingRequiredRecord,
3756    plan: &FleetEnsurePlan,
3757    state: &FleetEnsureStateRecord,
3758) -> bool {
3759    let Some(reviewed) = plan
3760        .conservation
3761        .estate_funding_domains
3762        .iter()
3763        .find(|domain| domain.root == required.root)
3764    else {
3765        return false;
3766    };
3767    let operation_authority_is_exact =
3768        required.operation_id == plan.operation_id && required.plan_sha256 == plan.plan_sha256;
3769    let funding_authority_is_exact =
3770        EstateFundingPauseAuthority::from(required) == EstateFundingPauseAuthority::from(reviewed);
3771    let root_authority_is_exact =
3772        reviewed_estate_root_principal(reviewed, state) == Some(required.root_principal.as_str());
3773    let shortfall_is_exact = required.available_cycles < required.maximum_creation_debit_cycles
3774        && required.shortfall_cycles
3775            == required
3776                .maximum_creation_debit_cycles
3777                .saturating_sub(required.available_cycles);
3778    if !(operation_authority_is_exact
3779        && funding_authority_is_exact
3780        && root_authority_is_exact
3781        && shortfall_is_exact)
3782    {
3783        return false;
3784    }
3785    if required.pending_creation_operation_id.is_some() {
3786        current_creation_funding_pause_is_exact(required, reviewed)
3787    } else {
3788        planned_estate_funding_pause_is_exact(required, reviewed)
3789    }
3790}
3791
3792#[derive(Eq, PartialEq)]
3793struct EstateFundingPauseAuthority<'a> {
3794    creation_amount_cycles: u128,
3795    creation_execution_margin_cycles: u128,
3796    cycles_ledger: &'a str,
3797    ledger_fee_cycles: u128,
3798    management_creation_fee_cycles: u128,
3799    readiness_floor_cycles: u128,
3800    root: &'a str,
3801}
3802
3803impl<'a> From<&'a EstateFundingRequiredRecord> for EstateFundingPauseAuthority<'a> {
3804    fn from(required: &'a EstateFundingRequiredRecord) -> Self {
3805        Self {
3806            creation_amount_cycles: required.creation_amount_cycles,
3807            creation_execution_margin_cycles: required.creation_execution_margin_cycles,
3808            cycles_ledger: &required.cycles_ledger,
3809            ledger_fee_cycles: required.ledger_fee_cycles,
3810            management_creation_fee_cycles: required.management_creation_fee_cycles,
3811            readiness_floor_cycles: required.readiness_floor_cycles,
3812            root: &required.root,
3813        }
3814    }
3815}
3816
3817impl<'a> From<&'a EstateFundingDomainPlan> for EstateFundingPauseAuthority<'a> {
3818    fn from(reviewed: &'a EstateFundingDomainPlan) -> Self {
3819        Self {
3820            creation_amount_cycles: reviewed.creation_amount_cycles,
3821            creation_execution_margin_cycles: reviewed.creation_execution_margin_cycles,
3822            cycles_ledger: &reviewed.cycles_ledger,
3823            ledger_fee_cycles: reviewed.ledger_fee_cycles,
3824            management_creation_fee_cycles: reviewed.management_creation_fee_cycles,
3825            readiness_floor_cycles: reviewed.readiness_floor_cycles,
3826            root: &reviewed.root,
3827        }
3828    }
3829}
3830
3831fn current_creation_funding_pause_is_exact(
3832    required: &EstateFundingRequiredRecord,
3833    reviewed: &EstateFundingDomainPlan,
3834) -> bool {
3835    let operation_id_is_exact = required
3836        .pending_creation_operation_id
3837        .as_deref()
3838        .is_some_and(is_sha256_hex);
3839    let attempt_evidence_is_exact = required.attempt_count.is_some()
3840        && required
3841            .retry_at_ns
3842            .is_some_and(|retry_at_ns| retry_at_ns > 0)
3843        && required
3844            .last_attempt_at_ns
3845            .is_none_or(|attempted_at_ns| attempted_at_ns > 0);
3846    let single_creation_debit = required
3847        .creation_amount_cycles
3848        .checked_add(required.ledger_fee_cycles);
3849    let single_creation_fees = required
3850        .management_creation_fee_cycles
3851        .checked_add(required.ledger_fee_cycles);
3852    let debit_authority_is_exact = Some(required.maximum_creation_debit_cycles)
3853        == single_creation_debit
3854        && Some(required.maximum_creation_fee_cycles) == single_creation_fees;
3855    reviewed.required_creation_count > 0
3856        && required.required_creation_count == 1
3857        && operation_id_is_exact
3858        && attempt_evidence_is_exact
3859        && debit_authority_is_exact
3860}
3861
3862const fn planned_estate_funding_pause_is_exact(
3863    required: &EstateFundingRequiredRecord,
3864    reviewed: &EstateFundingDomainPlan,
3865) -> bool {
3866    let runtime_attempt_is_absent = required.attempt_count.is_none()
3867        && required.last_attempt_at_ns.is_none()
3868        && required.retry_at_ns.is_none();
3869    let reviewed_debit_is_exact = required.maximum_creation_debit_cycles
3870        == reviewed.maximum_creation_debit_cycles
3871        && required.maximum_creation_fee_cycles == reviewed.maximum_creation_fee_cycles
3872        && required.required_creation_count == reviewed.required_creation_count;
3873    runtime_attempt_is_absent && reviewed_debit_is_exact
3874}
3875
3876fn is_sha256_hex(value: &str) -> bool {
3877    value.len() == 64
3878        && value
3879            .bytes()
3880            .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
3881}
3882
3883#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3884enum TerminalStatePublicationError {
3885    JournalIntegrity,
3886}
3887
3888fn publish_terminal_state(
3889    desired: &crate::fleet_ensure::model::DesiredFleet,
3890    plan: &FleetEnsurePlan,
3891    journal: &FleetEnsureJournalRecord,
3892    state: &mut FleetEnsureStateRecord,
3893) -> Result<(), TerminalStatePublicationError> {
3894    let maximum_observation_burn_cycles =
3895        crate::fleet_ensure::ops::maximum_creation_observation_burn(desired)
3896            .ok_or(TerminalStatePublicationError::JournalIntegrity)?;
3897    let actions = continuation::actions(plan, journal);
3898    if actions.len() != journal.effects.len() {
3899        return Err(TerminalStatePublicationError::JournalIntegrity);
3900    }
3901    for (action, record) in actions.into_iter().zip(&journal.effects) {
3902        if action_sha256(action) != record.action_sha256 || record.state != EffectState::Applied {
3903            return Err(TerminalStatePublicationError::JournalIntegrity);
3904        }
3905        let EnsureAction::Create {
3906            name,
3907            requested_initial_cycles,
3908            ..
3909        } = action
3910        else {
3911            continue;
3912        };
3913        let Some(created_principal) = record.created_principal.as_deref() else {
3914            return Err(TerminalStatePublicationError::JournalIntegrity);
3915        };
3916        let Some(post_cycles) = record.post_cycles else {
3917            return Err(TerminalStatePublicationError::JournalIntegrity);
3918        };
3919        if !create_balance_is_terminal(
3920            Some(post_cycles),
3921            *requested_initial_cycles,
3922            maximum_observation_burn_cycles,
3923        ) || !create_identity_is_exact(name, record, state)
3924            || state
3925                .retained_cycles_by_principal
3926                .get(created_principal)
3927                .is_some_and(|retained| *retained != post_cycles)
3928        {
3929            return Err(TerminalStatePublicationError::JournalIntegrity);
3930        }
3931        state
3932            .retained_cycles_by_principal
3933            .insert(created_principal.to_string(), post_cycles);
3934    }
3935
3936    let prior_topology = state.topology.clone();
3937    for canister in &plan.canisters {
3938        if !desired
3939            .canisters
3940            .iter()
3941            .any(|configured| configured.name == canister.name)
3942        {
3943            // Dynamic pool assets remain in protected Root inventory. A funding-only
3944            // projection must not become a configured application identity.
3945            if !is_reviewed_dynamic_pool_funding(plan, canister) {
3946                return Err(TerminalStatePublicationError::JournalIntegrity);
3947            }
3948            continue;
3949        }
3950
3951        match canister.disposition {
3952            CanisterDisposition::Create | CanisterDisposition::Replace => {
3953                if let Some(principal) = state.pending_principals.remove(&canister.name) {
3954                    state.principals.insert(canister.name.clone(), principal);
3955                }
3956            }
3957            CanisterDisposition::Reinstall | CanisterDisposition::Reuse => {
3958                if let Some(principal) = state
3959                    .pending_principals
3960                    .remove(&canister.name)
3961                    .or_else(|| canister.principal.clone())
3962                {
3963                    state.principals.insert(canister.name.clone(), principal);
3964                }
3965            }
3966            CanisterDisposition::Delete => {
3967                state.principals.remove(&canister.name);
3968                state.pending_principals.remove(&canister.name);
3969            }
3970        }
3971    }
3972    state.topology = desired
3973        .canisters
3974        .iter()
3975        .filter(|canister| {
3976            canister.presence == crate::fleet_ensure::model::DesiredPresence::Present
3977        })
3978        .map(|canister| {
3979            (
3980                canister.name.clone(),
3981                crate::fleet_ensure::model::FleetEnsureTopologyRecord {
3982                    kind: canister.kind,
3983                    module_hash: projected_module_hash(plan, &prior_topology, canister),
3984                    parent: canister.parent.clone(),
3985                    protocol_binding: canister.protocol_binding.clone(),
3986                    role: canister
3987                        .protocol_binding
3988                        .as_ref()
3989                        .map(|binding| binding.role.to_string()),
3990                },
3991            )
3992        })
3993        .collect();
3994    retain_configured_principal_bindings(state);
3995    Ok(())
3996}
3997
3998fn is_reviewed_dynamic_pool_funding(
3999    plan: &FleetEnsurePlan,
4000    canister: &crate::fleet_ensure::model::CanisterPlan,
4001) -> bool {
4002    canister.disposition == CanisterDisposition::Reuse
4003        && !canister.actions.is_empty()
4004        && canister
4005            .actions
4006            .iter()
4007            .all(|action| is_reviewed_pool_funding(action, plan, canister.principal.as_deref()))
4008}
4009
4010fn is_reviewed_pool_funding(
4011    action: &EnsureAction,
4012    plan: &FleetEnsurePlan,
4013    expected_principal: Option<&str>,
4014) -> bool {
4015    let EnsureAction::Fund {
4016        pool_funding: Some(root),
4017        principal,
4018        ..
4019    } = action
4020    else {
4021        return false;
4022    };
4023    if Some(principal.as_str()) != expected_principal {
4024        return false;
4025    }
4026    plan.protocol_actions.iter().any(|candidate| {
4027        let EnsureAction::FleetProtocol {
4028            principal: target,
4029            action,
4030            ..
4031        } = candidate
4032        else {
4033            return false;
4034        };
4035        let CurrentFleetProtocolAction::ReconcilePoolAsset { request, .. } = action.as_ref() else {
4036            return false;
4037        };
4038        target == &root.root && request.canister_id.to_text() == *principal
4039    })
4040}
4041
4042fn projected_module_hash(
4043    plan: &FleetEnsurePlan,
4044    prior_topology: &BTreeMap<String, crate::fleet_ensure::model::FleetEnsureTopologyRecord>,
4045    canister: &crate::fleet_ensure::model::DesiredCanister,
4046) -> Option<String> {
4047    let installed = plan
4048        .canisters
4049        .iter()
4050        .find(|planned| planned.name == canister.name)
4051        .and_then(|planned| {
4052            planned
4053                .actions
4054                .iter()
4055                .rev()
4056                .find_map(|action| match action {
4057                    EnsureAction::Install { wasm_sha256, .. } => Some(wasm_sha256.clone()),
4058                    _ => None,
4059                })
4060        });
4061    installed.or_else(|| {
4062        prior_topology
4063            .get(&canister.name)
4064            .filter(|retained| retained.kind == canister.kind && retained.parent == canister.parent)
4065            .and_then(|retained| retained.module_hash.clone())
4066    })
4067}
4068
4069fn merge_terminal_inventory<E>(
4070    state: &mut FleetEnsureStateRecord,
4071    inventory: crate::fleet_ensure::ops::TerminalFleetInventory,
4072) -> Result<(), EnsureWorkflowError<E>>
4073where
4074    E: std::error::Error + 'static,
4075{
4076    let entries = inventory.entries;
4077    let mut names_by_principal = state
4078        .principals
4079        .iter()
4080        .map(|(name, principal)| (principal.clone(), name.clone()))
4081        .collect::<BTreeMap<_, _>>();
4082    if names_by_principal.len() != state.principals.len() {
4083        return Err(EnsureWorkflowError::TerminalInventory(
4084            TerminalInventoryError::DuplicateRetainedPrincipal,
4085        ));
4086    }
4087    for entry in &entries {
4088        names_by_principal
4089            .entry(entry.pid.clone())
4090            .or_insert_with(|| format!("observed:{}", entry.pid));
4091    }
4092    for entry in entries {
4093        let name = names_by_principal.get(&entry.pid).cloned().ok_or_else(|| {
4094            EnsureWorkflowError::TerminalInventory(TerminalInventoryError::LostPrincipalBinding {
4095                canister: entry.pid.clone(),
4096            })
4097        })?;
4098        let parent = entry
4099            .parent_pid
4100            .as_ref()
4101            .map(|principal| {
4102                names_by_principal.get(principal).cloned().ok_or_else(|| {
4103                    EnsureWorkflowError::TerminalInventory(TerminalInventoryError::UnknownParent {
4104                        canister: entry.pid.clone(),
4105                        parent: principal.clone(),
4106                    })
4107                })
4108            })
4109            .transpose()?;
4110        let existing = state.topology.get(&name);
4111        let kind = terminal_inventory_kind(existing, &entry);
4112        let parent_conflict = existing
4113            .and_then(|topology| topology.parent.as_ref())
4114            .zip(parent.as_ref())
4115            .filter(|(expected, actual)| expected != actual);
4116        if let Some((retained_parent, terminal_parent)) = parent_conflict
4117            && !verified_pool_to_component_parent_transition(existing, kind, &entry)
4118        {
4119            return Err(EnsureWorkflowError::TerminalInventory(
4120                TerminalInventoryError::RetainedParentConflict {
4121                    canister: entry.pid,
4122                    retained_parent: retained_parent.clone(),
4123                    terminal_parent: terminal_parent.clone(),
4124                },
4125            ));
4126        }
4127        state.principals.insert(name.clone(), entry.pid);
4128        state.topology.insert(
4129            name,
4130            crate::fleet_ensure::model::FleetEnsureTopologyRecord {
4131                kind,
4132                module_hash: entry.module_hash,
4133                parent,
4134                protocol_binding: entry.protocol_binding,
4135                role: entry.role,
4136            },
4137        );
4138    }
4139    state.active_registry = inventory.active_registry;
4140    Ok(())
4141}
4142
4143fn terminal_inventory_kind(
4144    existing: Option<&crate::fleet_ensure::model::FleetEnsureTopologyRecord>,
4145    entry: &crate::registry::RegistryEntry,
4146) -> crate::fleet_ensure::model::DesiredCanisterKind {
4147    existing.map_or_else(
4148        || {
4149            if entry.module_hash.is_some() {
4150                crate::fleet_ensure::model::DesiredCanisterKind::Component
4151            } else {
4152                crate::fleet_ensure::model::DesiredCanisterKind::Auxiliary
4153            }
4154        },
4155        |topology| {
4156            if topology.kind == crate::fleet_ensure::model::DesiredCanisterKind::Pool
4157                && entry.module_hash.is_some()
4158                && entry.protocol_binding.is_some()
4159            {
4160                crate::fleet_ensure::model::DesiredCanisterKind::Component
4161            } else {
4162                topology.kind
4163            }
4164        },
4165    )
4166}
4167
4168fn verified_pool_to_component_parent_transition(
4169    existing: Option<&crate::fleet_ensure::model::FleetEnsureTopologyRecord>,
4170    terminal_kind: crate::fleet_ensure::model::DesiredCanisterKind,
4171    entry: &crate::registry::RegistryEntry,
4172) -> bool {
4173    existing.is_some_and(|topology| {
4174        topology.kind == crate::fleet_ensure::model::DesiredCanisterKind::Pool
4175            && terminal_kind == crate::fleet_ensure::model::DesiredCanisterKind::Component
4176            && entry.module_hash.is_some()
4177            && entry.protocol_binding.is_some()
4178    })
4179}
4180
4181fn applied_count(journal: &FleetEnsureJournalRecord) -> u32 {
4182    u32::try_from(
4183        journal
4184            .effects
4185            .iter()
4186            .chain(
4187                journal
4188                    .funding_reviews
4189                    .iter()
4190                    .filter_map(|review| review.effect.as_ref()),
4191            )
4192            .filter(|effect| matches!(effect.state, EffectState::Applied))
4193            .count(),
4194    )
4195    .unwrap_or(u32::MAX)
4196}
4197
4198fn verify_store_activation_binding<P: EnsurePlatform>(
4199    plan: &FleetEnsurePlan,
4200    action: &EnsureAction,
4201    state: &FleetEnsureStateRecord,
4202    platform: &mut P,
4203) -> Result<(), EnsureWorkflowError<P::Error>> {
4204    use crate::fleet_ensure::model::DesiredCanisterInit;
4205    let EnsureAction::Install {
4206        canic_init: Some(DesiredCanisterInit::Store { root }),
4207        ..
4208    } = action
4209    else {
4210        return Ok(());
4211    };
4212    let root_will_be_installed = plan.canisters.iter().flat_map(|canister| &canister.actions)
4213        .any(|action| matches!(action,
4214            EnsureAction::Install { name, canic_init: Some(DesiredCanisterInit::Root { root: installed_root }), .. }
4215            if name == root && installed_root == root));
4216    if !root_will_be_installed {
4217        platform
4218            .require_retained_root_activation(&plan.operation_id, root, state)
4219            .map_err(EnsureWorkflowError::Platform)?;
4220    }
4221    Ok(())
4222}
4223
4224pub(super) fn ordered_actions(plan: &FleetEnsurePlan) -> Vec<&EnsureAction> {
4225    let mut actions = plan
4226        .canisters
4227        .iter()
4228        .flat_map(|canister| canister.actions.iter())
4229        .chain(plan.protocol_actions.iter())
4230        .collect::<Vec<_>>();
4231    if plan.scope == FleetEnsurePlanScope::ReinstallPreparation
4232        && plan
4233            .reinstall
4234            .as_ref()
4235            .is_some_and(|intent| intent.activation_reset.is_some())
4236    {
4237        // Preserve the compiler's Coordinator-first order, then settle every Root before restart.
4238        actions.sort_by_key(|action| matches!(action, EnsureAction::Start { .. }));
4239        return actions;
4240    }
4241    if plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite {
4242        return actions;
4243    }
4244    let temporary_pool_observation_finalizations = plan
4245        .reviewed_desired
4246        .as_deref()
4247        .map(|reviewed| {
4248            let desired = reviewed.desired();
4249            plan.canisters
4250                .iter()
4251                .filter_map(|canister_plan| {
4252                    let configured = desired.canisters.iter().find(|configured| {
4253                        configured.name == canister_plan.name
4254                            && configured.kind
4255                                == crate::fleet_ensure::model::DesiredCanisterKind::Pool
4256                    })?;
4257                    let temporary_create = desired
4258                        .bootstrap
4259                        .as_ref()
4260                        .is_some_and(|bootstrap| bootstrap.fresh_estate)
4261                        && configured.principal.is_none()
4262                        && configured.controllers.is_empty()
4263                        && configured.controller_canisters.len() == 1
4264                        && canister_plan.actions.iter().any(|action| {
4265                            matches!(
4266                                action,
4267                                EnsureAction::Create {
4268                                    controller_canisters,
4269                                    controllers,
4270                                    ..
4271                                } if controller_canisters == &configured.controller_canisters
4272                                    && controllers == std::slice::from_ref(&desired.operator)
4273                            )
4274                        });
4275                    temporary_create.then_some(canister_plan.name.as_str())
4276                })
4277                .collect::<BTreeSet<_>>()
4278        })
4279        .unwrap_or_default();
4280    actions.sort_by_key(|action| {
4281        if matches!(action, EnsureAction::SetControllers { name, .. } if temporary_pool_observation_finalizations.contains(name.as_str()))
4282        {
4283            6
4284        } else {
4285            action_order(action)
4286        }
4287    });
4288    actions
4289}
4290
4291fn deferred_create_observation_is_exact(
4292    desired: &crate::fleet_ensure::model::DesiredFleet,
4293    actions: &[&EnsureAction],
4294    index: usize,
4295    action: &EnsureAction,
4296    record: &EffectRecord,
4297    state: &FleetEnsureStateRecord,
4298) -> bool {
4299    let EnsureAction::Create {
4300        controller_canisters,
4301        controllers,
4302        name,
4303        ..
4304    } = action
4305    else {
4306        return false;
4307    };
4308    let Some(configured) = desired.canisters.iter().find(|canister| {
4309        canister.name == *name
4310            && canister.kind == crate::fleet_ensure::model::DesiredCanisterKind::Pool
4311    }) else {
4312        return false;
4313    };
4314    let Some(parent) = configured.parent.as_deref() else {
4315        return false;
4316    };
4317    let fresh_root_only_create = desired
4318        .bootstrap
4319        .as_ref()
4320        .is_some_and(|bootstrap| bootstrap.fresh_estate)
4321        && configured.principal.is_none()
4322        && configured.controllers.is_empty()
4323        && configured.controller_canisters.as_slice() == [parent]
4324        && controller_canisters == &configured.controller_canisters
4325        && controllers.is_empty();
4326    let exact_issued_create = record.state == EffectState::Issued
4327        && create_identity_is_exact(name, record, state)
4328        && record.post_cycles.is_some();
4329    let later_root_install = actions[index.saturating_add(1)..].iter().any(|candidate| {
4330        matches!(
4331            candidate,
4332            EnsureAction::Install {
4333                canic_init: Some(crate::fleet_ensure::model::DesiredCanisterInit::Root { root }),
4334                name: install_name,
4335                ..
4336            } if root == parent && install_name == parent
4337        )
4338    });
4339    let infrastructure_only = actions.iter().all(|candidate| {
4340        matches!(
4341            candidate,
4342            EnsureAction::Create { .. }
4343                | EnsureAction::FundEstate { .. }
4344                | EnsureAction::Install { .. }
4345                | EnsureAction::SetControllers { .. }
4346        )
4347    });
4348    fresh_root_only_create && exact_issued_create && later_root_install && infrastructure_only
4349}
4350
4351pub(super) const fn action_order(action: &EnsureAction) -> u8 {
4352    match action {
4353        EnsureAction::SealAuthority {
4354            authority_kind: crate::fleet_ensure::model::DesiredCanisterKind::Root,
4355            ..
4356        }
4357        | EnsureAction::Create { .. } => 0,
4358        EnsureAction::SealAuthority { .. }
4359        | EnsureAction::Fund { .. }
4360        | EnsureAction::FundEstate { .. } => 1,
4361        EnsureAction::Install {
4362            canic_init: Some(crate::fleet_ensure::model::DesiredCanisterInit::Coordinator),
4363            ..
4364        } => 2,
4365        EnsureAction::SetControllers { .. } => 3,
4366        EnsureAction::Install {
4367            canic_init: Some(crate::fleet_ensure::model::DesiredCanisterInit::Store { .. }),
4368            ..
4369        } => 4,
4370        EnsureAction::Install { .. } => 5,
4371        EnsureAction::Start { .. } => 6,
4372        EnsureAction::FleetProtocol { .. } | EnsureAction::Protocol { .. } => 7,
4373        EnsureAction::Transfer { .. } => 8,
4374        EnsureAction::Stop { .. } => 9,
4375        EnsureAction::Delete { .. } => 10,
4376    }
4377}
4378
4379#[cfg(test)]
4380mod tests {
4381    use super::*;
4382
4383    #[test]
4384    fn store_binding_requires_exact_retained_root_or_reviewed_root_install() {
4385        use crate::fleet_ensure::{
4386            model::{CanisterDisposition, CanisterPlan, DesiredCanisterInit, InstallMode},
4387            tests::MockPlatform,
4388        };
4389        let (state, _) = retained_evidence();
4390        let mut plan = estate_funding_plan();
4391        let desired = serde_json::from_value(serde_json::json!({
4392            "canisters": [], "cycles_ledger": "um5iw-rqaaa-aaaaq-qaaba-cai",
4393            "environment": "local", "fleet": "fleet", "ledger_fee_cycles": "0",
4394            "management_creation_fee_cycles": "0", "material_cycle_threshold": "1",
4395            "maximum_observation_burn_cycles": "1", "maximum_stalled_observations": 2,
4396            "maximum_update_burn_cycles": "2", "operator": "rrkah-fqaaa-aaaaa-aaaaq-cai",
4397            "schema_version": FLEET_ENSURE_SCHEMA_VERSION, "treasury": "root"
4398        }))
4399        .unwrap();
4400        let mut platform = MockPlatform::new(desired, []);
4401        platform.rejected_activation_root = Some("root".into());
4402        let store = EnsureAction::Install {
4403            canic_init: Some(DesiredCanisterInit::Store {
4404                root: "root".into(),
4405            }),
4406            reinstall_witness: None,
4407            init_arg: None,
4408            init_arg_sha256: None,
4409            init_candid: None,
4410            init_candid_sha256: None,
4411            mode: InstallMode::Install,
4412            name: "store".into(),
4413            principal: "store-principal".into(),
4414            wasm: "store.wasm".into(),
4415            wasm_sha256: "store-hash".into(),
4416        };
4417        assert!(matches!(
4418            verify_store_activation_binding(&plan, &store, &state, &mut platform),
4419            Err(EnsureWorkflowError::Platform(_))
4420        ));
4421        assert_eq!(
4422            platform.retained_activation_checks,
4423            [(plan.operation_id.clone(), "root".into())]
4424        );
4425        let mut root_install = store.clone();
4426        if let EnsureAction::Install {
4427            name, canic_init, ..
4428        } = &mut root_install
4429        {
4430            *name = "other-root".into();
4431            *canic_init = Some(DesiredCanisterInit::Root {
4432                root: "other-root".into(),
4433            });
4434        }
4435        plan.canisters.push(CanisterPlan {
4436            actions: vec![root_install],
4437            disposition: CanisterDisposition::Create,
4438            name: "root".into(),
4439            observed_cycles: 0,
4440            principal: None,
4441        });
4442        assert!(verify_store_activation_binding(&plan, &store, &state, &mut platform).is_err());
4443        if let EnsureAction::Install {
4444            name, canic_init, ..
4445        } = &mut plan.canisters.last_mut().unwrap().actions[0]
4446        {
4447            *name = "root".into();
4448            *canic_init = Some(DesiredCanisterInit::Root {
4449                root: "root".into(),
4450            });
4451        }
4452        let reads = platform.retained_activation_checks.len();
4453        verify_store_activation_binding(&plan, &store, &state, &mut platform).unwrap();
4454        assert_eq!(platform.retained_activation_checks.len(), reads);
4455        plan.canisters.pop();
4456        platform.rejected_activation_root = None;
4457        verify_store_activation_binding(&plan, &store, &state, &mut platform).unwrap();
4458        assert_eq!(platform.retained_activation_checks.len(), reads + 1);
4459    }
4460
4461    #[test]
4462    fn root_prerequisite_retains_activation_authority_across_desired_refresh() {
4463        let (mut state, mut journal) = retained_evidence();
4464        let mut plan = estate_funding_plan();
4465        plan.scope = FleetEnsurePlanScope::RootReinstallPrerequisite;
4466        plan.conservation.estate_funding_domains.clear();
4467        journal.effects.clear();
4468        journal.completion = FleetEnsureCompletion::Converged;
4469        let before = state.clone();
4470        assert!(matches!(
4471            reconcile_retained_planning_state::<std::io::Error>(
4472                &mut state,
4473                Some(&plan),
4474                Some(&journal),
4475                "fleet",
4476                "local",
4477                "refreshed-imports"
4478            ),
4479            Err(EnsureWorkflowError::RootActivationDesiredConflict { .. })
4480        ));
4481        assert_eq!(state, before);
4482        assert!(
4483            reconcile_retained_planning_state::<std::io::Error>(
4484                &mut state,
4485                Some(&plan),
4486                Some(&journal),
4487                "fleet",
4488                "local",
4489                "desired"
4490            )
4491            .unwrap()
4492            .is_some()
4493        );
4494    }
4495
4496    fn terminal_component_entry(canister: &str, parent: &str) -> crate::registry::RegistryEntry {
4497        crate::registry::RegistryEntry {
4498            pid: canister.to_string(),
4499            role: Some("managed_component".to_string()),
4500            parent_pid: Some(parent.to_string()),
4501            module_hash: Some("11".repeat(32)),
4502            protocol_binding: Some(crate::protocol_binding::RegistryProtocolBinding {
4503                release_identity: "0.110.test".to_string(),
4504                role: canic_core::ids::CanisterRole::from("managed_component"),
4505                capabilities: BTreeSet::new(),
4506                candid_sha256: [1; 32],
4507                protocol_profile_digest:
4508                    canic_core::role_contract::ProtocolProfileDigest::from_bytes([2; 32]),
4509            }),
4510        }
4511    }
4512
4513    pub(super) fn retained_evidence() -> (FleetEnsureStateRecord, FleetEnsureJournalRecord) {
4514        let state = FleetEnsureStateRecord {
4515            active_registry: None,
4516            completed_reinstall_action_sha256: BTreeMap::from([(
4517                "root".to_string(),
4518                "action".to_string(),
4519            )]),
4520            completed_reinstall_operation_id: Some("operation".to_string()),
4521            completed_reinstalls: BTreeMap::from([("root".to_string(), 7)]),
4522            fleet: "fleet".to_string(),
4523            pending_principals: BTreeMap::new(),
4524            principals: BTreeMap::new(),
4525            retained_cycles_by_principal: BTreeMap::new(),
4526            schema_version: crate::fleet_ensure::model::FLEET_ENSURE_SCHEMA_VERSION,
4527            topology: BTreeMap::new(),
4528        };
4529        let journal = FleetEnsureJournalRecord {
4530            funding_reviews: Vec::new(),
4531            successor_phases: Vec::new(),
4532            completion: FleetEnsureCompletion::ReplanRequired,
4533            estate_funding_required: None,
4534            effects: vec![crate::fleet_ensure::model::EffectRecord {
4535                publication_attempts: 0,
4536                maintenance_attempts: 0,
4537                action_sha256: "action".to_string(),
4538                created_principal: None,
4539                destination_post_cycles: None,
4540                destination_pre_cycles: None,
4541                post_cycles: None,
4542                pre_cycles: None,
4543                pre_canister_version: Some(7),
4544                progress_identity: None,
4545                receipt: None,
4546                state: EffectState::Applied,
4547            }],
4548            fleet: "fleet".to_string(),
4549            initial_controlled_cycles: 0,
4550            initial_estate_funding_cycles_by_root: BTreeMap::new(),
4551            initial_operator_cycles: 0,
4552            operation_id: "operation".to_string(),
4553            plan_sha256: "plan".to_string(),
4554            schema_version: crate::fleet_ensure::model::FLEET_ENSURE_SCHEMA_VERSION,
4555            stalled_observations: 0,
4556        };
4557        (state, journal)
4558    }
4559
4560    pub(super) fn estate_funding_plan() -> FleetEnsurePlan {
4561        FleetEnsurePlan {
4562            continuation: None,
4563            canisters: Vec::new(),
4564            conservation: CycleConservation {
4565                estate_funding_domains: vec![crate::fleet_ensure::model::EstateFundingDomainPlan {
4566                    allocated_workloads: 0,
4567                    available_cycles: Some(40),
4568                    available_pool_slots: 2,
4569                    creation_amount_cycles: 45,
4570                    creation_execution_margin_cycles: 5,
4571                    readiness_floor_cycles: 35,
4572                    cycles_ledger: "ledger".to_string(),
4573                    eligible_ready_pool_assets: 0,
4574                    initial_pool_assets: Vec::new(),
4575                    ledger_fee_cycles: 5,
4576                    management_creation_fee_cycles: 5,
4577                    maximum_creation_debit_cycles: 100,
4578                    maximum_creation_fee_cycles: 20,
4579                    maximum_funding_cycles: 60,
4580                    occupied_pool_assets: 0,
4581                    pending_creation_count: 0,
4582                    pending_creation: None,
4583                    planned_initial_workloads: 2,
4584                    pool_maximum_size: 2,
4585                    required_creation_count: 2,
4586                    root: "root".to_string(),
4587                    root_principal: Some("rrkah-fqaaa-aaaaa-aaaaq-cai".to_string()),
4588                    shortfall_cycles: 60,
4589                }],
4590                expected_post_operation_cycles: 80,
4591                maximum_execution_burn_cycles: 0,
4592                maximum_new_funding_cycles: 0,
4593                maximum_operator_debit_cycles: 0,
4594                maximum_unavoidable_fee_cycles: 0,
4595                observed_controlled_cycles: 40,
4596                retained_in_reused_canisters_cycles: 0,
4597                scheduled_transfer_cycles: 0,
4598            },
4599            desired_sha256: "desired".to_string(),
4600            environment: "local".to_string(),
4601            fleet: "fleet".to_string(),
4602            operation_id: "operation".to_string(),
4603            plan_sha256: "plan".to_string(),
4604            planned_at_time: 1,
4605            protocol_actions: Vec::new(),
4606            recovery_review: None,
4607            reinstall: None,
4608            root_reinstall_bindings: Vec::new(),
4609            root_start_authority: None,
4610            reviewed_desired: None,
4611            schema_version: FLEET_ENSURE_SCHEMA_VERSION,
4612            scope: FleetEnsurePlanScope::Full,
4613            terminal_inventory_operation_id: None,
4614        }
4615    }
4616
4617    pub(super) fn estate_funding_observation(balance_cycles: Option<u128>) -> FleetObservation {
4618        FleetObservation {
4619            additional_controlled_cycles: BTreeMap::new(),
4620            canisters: BTreeMap::new(),
4621            estate_funding_domains: BTreeMap::from([(
4622                "root".to_string(),
4623                crate::fleet_ensure::model::EstateFundingDomainObservation {
4624                    balance_cycles,
4625                    cycles_ledger: "ledger".to_string(),
4626                    pool: None,
4627                    root_principal: Some("rrkah-fqaaa-aaaaa-aaaaq-cai".to_string()),
4628                },
4629            )]),
4630            ledger_fee_cycles: 0,
4631            operator_cycles: 0,
4632            protocol_ready: BTreeMap::new(),
4633        }
4634    }
4635
4636    fn terminal_estate_creation_observation() -> FleetObservation {
4637        let mut observation = estate_funding_observation(Some(0));
4638        let receipt =
4639            |byte, block_index| crate::fleet_ensure::model::EstatePoolCreationReceiptObservation {
4640                block_index,
4641                operation_id: format!("{byte:02x}").repeat(32),
4642                cycles_ledger: "ledger".to_string(),
4643                ledger_amount_cycles: 45,
4644                ledger_fee_cycles: 5,
4645                readiness_floor_cycles: 35,
4646                creation_execution_margin_cycles: 5,
4647                management_creation_fee_cycles: 5,
4648                first_observed_cycles: Some(39),
4649            };
4650        observation
4651            .estate_funding_domains
4652            .get_mut("root")
4653            .expect("Root funding domain")
4654            .pool = Some(crate::fleet_ensure::model::EstatePoolInventoryObservation {
4655            assets: vec![
4656                crate::fleet_ensure::model::EstatePoolAssetObservation {
4657                    creation_receipt: Some(receipt(1, 10)),
4658                    cycles: 35,
4659                    lifecycle: crate::fleet_ensure::model::EstatePoolAssetLifecycle::Ready,
4660                    origin: crate::fleet_ensure::model::EstatePoolAssetOrigin::Created,
4661                    principal: "created-one".to_string(),
4662                },
4663                crate::fleet_ensure::model::EstatePoolAssetObservation {
4664                    creation_receipt: Some(receipt(2, 11)),
4665                    cycles: 35,
4666                    lifecycle: crate::fleet_ensure::model::EstatePoolAssetLifecycle::Workload,
4667                    origin: crate::fleet_ensure::model::EstatePoolAssetOrigin::Created,
4668                    principal: "created-two".to_string(),
4669                },
4670            ],
4671            maximum_size: 2,
4672            minimum_size: 0,
4673            pending_creation: None,
4674            readiness_floor_cycles: 35,
4675            creation_execution_margin_cycles: 5,
4676        });
4677        observation
4678    }
4679
4680    fn estate_creation_receipt(
4681        observation: &mut FleetObservation,
4682        asset_index: usize,
4683    ) -> &mut crate::fleet_ensure::model::EstatePoolCreationReceiptObservation {
4684        observation
4685            .estate_funding_domains
4686            .get_mut("root")
4687            .expect("Root domain")
4688            .pool
4689            .as_mut()
4690            .expect("pool")
4691            .assets[asset_index]
4692            .creation_receipt
4693            .as_mut()
4694            .expect("receipt")
4695    }
4696
4697    #[test]
4698    fn autonomous_reserve_creation_reconciles_without_host_provisioning() {
4699        let mut plan = estate_funding_plan();
4700        let domain = &mut plan.conservation.estate_funding_domains[0];
4701        domain.available_cycles = Some(100);
4702        domain.maximum_funding_cycles = 0;
4703        domain.shortfall_cycles = 0;
4704        let (state, mut journal) = retained_evidence();
4705        journal.effects.clear();
4706        journal
4707            .initial_estate_funding_cycles_by_root
4708            .insert("root".to_string(), 100);
4709        let terminal = terminal_estate_creation_observation();
4710        assert_eq!(
4711            reconcile_estate_funding::<std::io::Error>(&plan, &journal, &state, &terminal)
4712                .expect("reconcile exact Root-owned creation against the reviewed reserve"),
4713            (0, 20)
4714        );
4715    }
4716
4717    #[test]
4718    fn terminal_estate_creation_uses_exact_protected_receipts() {
4719        let plan = estate_funding_plan();
4720        let domain = &plan.conservation.estate_funding_domains[0];
4721        let (state, _) = retained_evidence();
4722        let exact = terminal_estate_creation_observation();
4723        assert_eq!(
4724            exact_estate_creation_costs::<std::io::Error>(&state, &exact, domain)
4725                .expect("reconcile exact creation receipts"),
4726            (100, 20)
4727        );
4728        for required_creation_count in [0, 1] {
4729            let mut insufficient = domain.clone();
4730            insufficient.required_creation_count = required_creation_count;
4731            assert!(matches!(
4732                exact_estate_creation_costs::<std::io::Error>(&state, &exact, &insufficient),
4733                Err(EnsureWorkflowError::JournalIntegrity)
4734            ));
4735        }
4736
4737        let mut drifted = Vec::new();
4738        let mut changed = exact.clone();
4739        estate_creation_receipt(&mut changed, 0).ledger_amount_cycles = 44;
4740        drifted.push(changed);
4741        let mut changed = exact.clone();
4742        estate_creation_receipt(&mut changed, 1).operation_id = "01".repeat(32);
4743        drifted.push(changed);
4744        let mut changed = exact.clone();
4745        estate_creation_receipt(&mut changed, 1).block_index = 10;
4746        drifted.push(changed);
4747        let mut changed = exact.clone();
4748        estate_creation_receipt(&mut changed, 0).first_observed_cycles = None;
4749        drifted.push(changed);
4750        let mut changed = exact.clone();
4751        estate_creation_receipt(&mut changed, 0).first_observed_cycles = Some(34);
4752        drifted.push(changed);
4753        let mut changed = exact.clone();
4754        estate_creation_receipt(&mut changed, 0).first_observed_cycles = Some(41);
4755        drifted.push(changed);
4756        let mut changed = exact;
4757        changed
4758            .estate_funding_domains
4759            .get_mut("root")
4760            .expect("Root domain")
4761            .pool
4762            .as_mut()
4763            .expect("pool")
4764            .assets[0]
4765            .origin = crate::fleet_ensure::model::EstatePoolAssetOrigin::Imported;
4766        drifted.push(changed);
4767        for observation in drifted {
4768            assert!(matches!(
4769                exact_estate_creation_costs::<std::io::Error>(&state, &observation, domain,),
4770                Err(EnsureWorkflowError::JournalIntegrity)
4771            ));
4772        }
4773    }
4774
4775    #[test]
4776    fn estate_funding_pause_is_exact_and_clears_only_after_sufficient_balance() {
4777        let plan = estate_funding_plan();
4778        let (state, mut journal) = retained_evidence();
4779        let required = estate_funding_requirement::<std::io::Error>(
4780            &plan,
4781            &state,
4782            &estate_funding_observation(Some(40)),
4783        )
4784        .expect("validate funding observation")
4785        .expect("underfunded estate pauses");
4786        assert_eq!(required.available_cycles, 40);
4787        assert_eq!(required.maximum_creation_debit_cycles, 100);
4788        assert_eq!(required.shortfall_cycles, 60);
4789        assert!(
4790            estate_funding_requirement::<std::io::Error>(
4791                &plan,
4792                &state,
4793                &estate_funding_observation(Some(100)),
4794            )
4795            .expect("validate sufficient funding observation")
4796            .is_none()
4797        );
4798
4799        journal.completion = FleetEnsureCompletion::InProgress;
4800        journal.effects.clear();
4801        journal.estate_funding_required = Some(required);
4802        journal.initial_estate_funding_cycles_by_root = BTreeMap::from([("root".to_string(), 40)]);
4803        assert!(initial_estate_funding_is_exact(&journal, &plan));
4804        assert!(estate_funding_pause_is_exact(&journal, &plan, &state));
4805
4806        journal
4807            .estate_funding_required
4808            .as_mut()
4809            .expect("funding pause")
4810            .shortfall_cycles = 59;
4811        assert!(!estate_funding_pause_is_exact(&journal, &plan, &state));
4812
4813        *journal
4814            .initial_estate_funding_cycles_by_root
4815            .get_mut("root")
4816            .expect("initial Root funding balance") = 41;
4817        assert!(!initial_estate_funding_is_exact(&journal, &plan));
4818    }
4819
4820    #[test]
4821    fn current_creation_funding_pause_is_bound_to_exact_runtime_evidence() {
4822        let mut plan = estate_funding_plan();
4823        plan.conservation.estate_funding_domains[0].cycles_ledger = "aaaaa-aa".to_string();
4824        let (state, mut journal) = retained_evidence();
4825        let funding = canic_core::dto::component_provisioning::RootEstateFundingRequired {
4826            available: Cycles::new(40),
4827            attempt_count: 2,
4828            creation_amount: Cycles::new(45),
4829            cycles_ledger: candid::Principal::from_text("aaaaa-aa").expect("Ledger Principal"),
4830            execution_margin: Cycles::new(5),
4831            last_attempt_at_ns: Some(10),
4832            ledger_fee: Cycles::new(5),
4833            management_creation_fee: Cycles::new(5),
4834            operation_id: [17; 32],
4835            readiness_floor: Cycles::new(35),
4836            required: Cycles::new(50),
4837            retry_at_ns: 20,
4838            root: candid::Principal::from_text("rrkah-fqaaa-aaaaa-aaaaq-cai")
4839                .expect("Root Principal"),
4840            shortfall: Cycles::new(10),
4841        };
4842        let required =
4843            protocol_estate_funding_requirement::<std::io::Error>(&plan, &state, &funding)
4844                .expect("bind runtime funding evidence");
4845        journal.completion = FleetEnsureCompletion::InProgress;
4846        journal.effects.clear();
4847        journal.estate_funding_required = Some(required);
4848        assert!(estate_funding_pause_is_exact(&journal, &plan, &state));
4849
4850        journal
4851            .estate_funding_required
4852            .as_mut()
4853            .expect("funding pause")
4854            .pending_creation_operation_id = Some("AA".repeat(32));
4855        assert!(!estate_funding_pause_is_exact(&journal, &plan, &state));
4856
4857        let mut drifted = funding;
4858        drifted.execution_margin = Cycles::new(6);
4859        assert!(matches!(
4860            protocol_estate_funding_requirement::<std::io::Error>(&plan, &state, &drifted),
4861            Err(EnsureWorkflowError::PlanIntegrity)
4862        ));
4863    }
4864
4865    #[test]
4866    fn estate_funding_observation_rejects_missing_or_mismatched_authority() {
4867        let plan = estate_funding_plan();
4868        let (state, _) = retained_evidence();
4869        let mut missing_balance = estate_funding_observation(None);
4870        assert!(matches!(
4871            estate_funding_requirement::<std::io::Error>(&plan, &state, &missing_balance),
4872            Err(EnsureWorkflowError::PlanIntegrity)
4873        ));
4874
4875        let observed = missing_balance
4876            .estate_funding_domains
4877            .get_mut("root")
4878            .expect("Root funding domain");
4879        observed.balance_cycles = Some(40);
4880        observed.root_principal = Some("aaaaa-aa".to_string());
4881        assert!(matches!(
4882            estate_funding_requirement::<std::io::Error>(&plan, &state, &missing_balance),
4883            Err(EnsureWorkflowError::PlanIntegrity)
4884        ));
4885
4886        for changed_balance in [39, 101] {
4887            assert!(matches!(
4888                estate_funding_requirement::<std::io::Error>(
4889                    &plan,
4890                    &state,
4891                    &estate_funding_observation(Some(changed_balance)),
4892                ),
4893                Err(EnsureWorkflowError::DriftedBeforeApply)
4894            ));
4895        }
4896    }
4897
4898    #[test]
4899    fn non_creation_protocol_completion_does_not_consume_estate_creation_authority() {
4900        let mut plan = estate_funding_plan();
4901        plan.conservation.estate_funding_domains[0].maximum_creation_debit_cycles = 0;
4902        plan.conservation.estate_funding_domains[0].maximum_creation_fee_cycles = 0;
4903        plan.conservation.estate_funding_domains[0].maximum_funding_cycles = 0;
4904        plan.conservation.estate_funding_domains[0].required_creation_count = 0;
4905        plan.conservation.estate_funding_domains[0].shortfall_cycles = 0;
4906        let action = EnsureAction::FleetProtocol {
4907            action: Box::new(
4908                crate::fleet_ensure::model::CurrentFleetProtocolAction::PrepareStoreChunkSet {
4909                    request: canic_control_plane::dto::template::TemplateChunkSetPrepareInput {
4910                        template_id: canic_control_plane::ids::TemplateId::from("root"),
4911                        version: canic_control_plane::ids::TemplateVersion::from("current"),
4912                        payload_hash: vec![1; 32],
4913                        payload_size_bytes: 1,
4914                        chunk_hashes: vec![vec![2; 32]],
4915                    },
4916                },
4917            ),
4918            candid: "store.did".to_string(),
4919            candid_sha256: "11".repeat(32),
4920            maximum_execution_burn_cycles: 0,
4921            name: "prepare-store-chunks".to_string(),
4922            principal: "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(),
4923        };
4924        let action_hash = action_sha256(&action);
4925        plan.protocol_actions.push(action);
4926        let (state, mut journal) = retained_evidence();
4927        journal.initial_estate_funding_cycles_by_root = BTreeMap::from([("root".to_string(), 40)]);
4928        journal.effects = vec![EffectRecord {
4929            publication_attempts: 0,
4930            maintenance_attempts: 0,
4931            action_sha256: action_hash,
4932            created_principal: None,
4933            destination_post_cycles: None,
4934            destination_pre_cycles: None,
4935            post_cycles: None,
4936            pre_cycles: None,
4937            pre_canister_version: None,
4938            progress_identity: Some("store chunks prepared".to_string()),
4939            receipt: Some("receipt".to_string()),
4940            state: EffectState::Applied,
4941        }];
4942
4943        assert_eq!(
4944            reconcile_estate_funding::<std::io::Error>(
4945                &plan,
4946                &journal,
4947                &state,
4948                &estate_funding_observation(Some(40)),
4949            )
4950            .expect("non-creation protocol preserves the Root Ledger balance"),
4951            (0, 0),
4952        );
4953    }
4954
4955    #[test]
4956    fn fresh_estate_funding_binds_the_created_root_from_durable_state() {
4957        let mut plan = estate_funding_plan();
4958        plan.conservation.estate_funding_domains[0].available_cycles = None;
4959        plan.conservation.estate_funding_domains[0].maximum_funding_cycles = 100;
4960        plan.conservation.estate_funding_domains[0].root_principal = None;
4961        let (mut state, _) = retained_evidence();
4962        state.pending_principals.insert(
4963            "root".to_string(),
4964            "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(),
4965        );
4966
4967        let required = estate_funding_requirement::<std::io::Error>(
4968            &plan,
4969            &state,
4970            &estate_funding_observation(Some(0)),
4971        )
4972        .expect("validate created Root funding authority")
4973        .expect("fresh underfunded estate pauses");
4974        assert_eq!(required.shortfall_cycles, 100);
4975        assert_eq!(required.root_principal, "rrkah-fqaaa-aaaaa-aaaaq-cai");
4976
4977        state
4978            .pending_principals
4979            .insert("root".to_string(), "aaaaa-aa".to_string());
4980        assert!(matches!(
4981            estate_funding_requirement::<std::io::Error>(
4982                &plan,
4983                &state,
4984                &estate_funding_observation(Some(0)),
4985            ),
4986            Err(EnsureWorkflowError::PlanIntegrity)
4987        ));
4988    }
4989
4990    #[test]
4991    fn unchanged_estate_funding_pause_does_not_rewrite_its_journal() {
4992        let root = crate::test_support::temp_dir("estate-funding-pause");
4993        let paths = EnsurePaths::under(&root, "local", "fleet");
4994        let plan = estate_funding_plan();
4995        let (state, mut journal) = retained_evidence();
4996        journal.completion = FleetEnsureCompletion::InProgress;
4997        journal.effects.clear();
4998        journal.initial_estate_funding_cycles_by_root = BTreeMap::from([("root".to_string(), 40)]);
4999        let required = estate_funding_requirement::<std::io::Error>(
5000            &plan,
5001            &state,
5002            &estate_funding_observation(Some(40)),
5003        )
5004        .expect("validate funding observation")
5005        .expect("underfunded estate pauses");
5006
5007        assert!(
5008            retain_estate_funding_pause::<std::io::Error>(&paths, &mut journal, Some(&required),)
5009                .expect("retain first funding pause")
5010        );
5011        let retained = std::fs::read(&paths.journal).expect("read first retained pause");
5012        assert!(
5013            !retain_estate_funding_pause::<std::io::Error>(&paths, &mut journal, Some(&required),)
5014                .expect("replay unchanged funding pause")
5015        );
5016        assert_eq!(
5017            std::fs::read(&paths.journal).expect("reread unchanged retained pause"),
5018            retained
5019        );
5020        assert!(
5021            retain_estate_funding_pause::<std::io::Error>(&paths, &mut journal, None)
5022                .expect("clear funded pause")
5023        );
5024        assert!(journal.estate_funding_required.is_none());
5025
5026        std::fs::remove_dir_all(root).expect("remove funding-pause fixture");
5027    }
5028
5029    #[test]
5030    fn estate_funding_is_rechecked_until_the_first_protocol_effect_starts() {
5031        let protocol_action = || EnsureAction::FleetProtocol {
5032            action: Box::new(
5033                crate::fleet_ensure::model::CurrentFleetProtocolAction::PrepareStoreChunkSet {
5034                    request: canic_control_plane::dto::template::TemplateChunkSetPrepareInput {
5035                        template_id: canic_control_plane::ids::TemplateId::from("root"),
5036                        version: canic_control_plane::ids::TemplateVersion::from("current"),
5037                        payload_hash: vec![1; 32],
5038                        payload_size_bytes: 1,
5039                        chunk_hashes: vec![vec![2; 32]],
5040                    },
5041                },
5042            ),
5043            candid: "store.did".to_string(),
5044            candid_sha256: "11".repeat(32),
5045            maximum_execution_burn_cycles: 1,
5046            name: "prepare".to_string(),
5047            principal: "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(),
5048        };
5049        let first = protocol_action();
5050        let second = protocol_action();
5051        let actions = [&first, &second];
5052        let (_, mut journal) = retained_evidence();
5053        journal.effects = vec![EffectRecord {
5054            publication_attempts: 0,
5055            maintenance_attempts: 0,
5056            action_sha256: action_sha256(&first),
5057            created_principal: None,
5058            destination_post_cycles: None,
5059            destination_pre_cycles: None,
5060            post_cycles: None,
5061            pre_cycles: None,
5062            pre_canister_version: None,
5063            progress_identity: None,
5064            receipt: None,
5065            state: EffectState::Intent,
5066        }];
5067        assert!(!prior_fleet_protocol_effect_started(&actions, &journal, 1));
5068
5069        journal.effects[0].state = EffectState::Issued;
5070        assert!(prior_fleet_protocol_effect_started(&actions, &journal, 1));
5071    }
5072
5073    #[test]
5074    fn configured_projection_publishes_unique_bindings_without_losing_dynamic_cycles() {
5075        let asset = "rrkah-fqaaa-aaaaa-aaaaq-cai";
5076        let dynamic = "ryjl3-tyaaa-aaaaa-aaaba-cai";
5077        for kind in ["pool", "component"] {
5078            let desired = serde_json::from_value(serde_json::json!({
5079                "canisters": [{
5080                    "controllers": [], "initial_cycles": "0", "kind": kind,
5081                    "minimum_cycles": "0", "name": "configured", "presence": "present",
5082                    "principal": asset, "replace": false, "subnet": "subnet"
5083                }],
5084                "cycles_ledger": "um5iw-rqaaa-aaaaq-qaaba-cai",
5085                "environment": "local", "fleet": "fleet", "ledger_fee_cycles": "0",
5086                "management_creation_fee_cycles": "0", "material_cycle_threshold": "1",
5087                "maximum_observation_burn_cycles": "1", "maximum_stalled_observations": 2,
5088                "maximum_update_burn_cycles": "2", "operator": asset,
5089                "schema_version": FLEET_ENSURE_SCHEMA_VERSION, "treasury": "configured"
5090            }))
5091            .expect("current desired projection");
5092            let (mut state, mut journal) = retained_evidence();
5093            journal.effects.clear();
5094            state.principals = BTreeMap::from([
5095                ("configured".to_string(), asset.to_string()),
5096                (format!("observed:{asset}"), asset.to_string()),
5097                (format!("observed:{dynamic}"), dynamic.to_string()),
5098            ]);
5099            state.retained_cycles_by_principal =
5100                BTreeMap::from([(asset.to_string(), 100), (dynamic.to_string(), 60)]);
5101            let cycles = state.retained_cycles_by_principal.clone();
5102            let journal_before = serde_json::to_vec(&journal).expect("journal before projection");
5103            let plan = estate_funding_plan();
5104
5105            publish_terminal_state(&desired, &plan, &journal, &mut state)
5106                .expect("publish current configured topology");
5107            assert_eq!(
5108                state.principals,
5109                BTreeMap::from([
5110                    ("configured".to_string(), asset.to_string()),
5111                    (format!("observed:{dynamic}"), dynamic.to_string()),
5112                ])
5113            );
5114            assert_eq!(state.retained_cycles_by_principal, cycles);
5115            let published = state.clone();
5116            publish_terminal_state(&desired, &plan, &journal, &mut state)
5117                .expect("repeat projection");
5118            assert_eq!(state, published);
5119            assert_eq!(
5120                serde_json::to_vec(&journal).expect("journal after projection"),
5121                journal_before
5122            );
5123        }
5124    }
5125
5126    #[test]
5127    fn configured_projection_preserves_duplicate_authority_rejection() {
5128        let asset = "rrkah-fqaaa-aaaaa-aaaaq-cai";
5129        for configured in [false, true] {
5130            let (mut state, _) = retained_evidence();
5131            state.principals = BTreeMap::from([
5132                ("first".to_string(), asset.to_string()),
5133                ("second".to_string(), asset.to_string()),
5134            ]);
5135            if configured {
5136                let topology = crate::fleet_ensure::model::FleetEnsureTopologyRecord {
5137                    kind: crate::fleet_ensure::model::DesiredCanisterKind::Pool,
5138                    module_hash: None,
5139                    parent: None,
5140                    protocol_binding: None,
5141                    role: None,
5142                };
5143                state.topology = BTreeMap::from([
5144                    ("first".to_string(), topology.clone()),
5145                    ("second".to_string(), topology),
5146                ]);
5147            }
5148            let before = state.clone();
5149            retain_configured_principal_bindings(&mut state);
5150            assert_eq!(state, before);
5151            for entries in [Vec::new(), vec![terminal_component_entry(asset, "parent")]] {
5152                assert!(matches!(
5153                    merge_terminal_inventory::<std::io::Error>(
5154                        &mut state,
5155                        crate::fleet_ensure::ops::TerminalFleetInventory {
5156                            entries,
5157                            ..Default::default()
5158                        }
5159                    ),
5160                    Err(EnsureWorkflowError::TerminalInventory(
5161                        TerminalInventoryError::DuplicateRetainedPrincipal
5162                    ))
5163                ));
5164                assert_eq!(state, before);
5165            }
5166        }
5167    }
5168
5169    #[test]
5170    fn terminal_inventory_allows_only_verified_pool_to_component_parent_transition() {
5171        let asset = "rrkah-fqaaa-aaaaa-aaaaq-cai";
5172        let old_root = "r7inp-6aaaa-aaaaa-aaabq-cai";
5173        let hub = "ryjl3-tyaaa-aaaaa-aaaba-cai";
5174        let state = || {
5175            let (mut state, _) = retained_evidence();
5176            state.principals = BTreeMap::from([
5177                ("asset".to_string(), asset.to_string()),
5178                ("hub".to_string(), hub.to_string()),
5179                ("root".to_string(), old_root.to_string()),
5180            ]);
5181            state.topology.insert(
5182                "asset".to_string(),
5183                crate::fleet_ensure::model::FleetEnsureTopologyRecord {
5184                    kind: crate::fleet_ensure::model::DesiredCanisterKind::Pool,
5185                    module_hash: None,
5186                    parent: Some("root".to_string()),
5187                    protocol_binding: None,
5188                    role: Some("canister_pool_asset".to_string()),
5189                },
5190            );
5191            state
5192        };
5193
5194        let mut promoted = state();
5195        merge_terminal_inventory::<std::io::Error>(
5196            &mut promoted,
5197            crate::fleet_ensure::ops::TerminalFleetInventory {
5198                active_registry: None,
5199                controlled_cycles_by_principal: BTreeMap::new(),
5200                entries: vec![terminal_component_entry(asset, hub)],
5201            },
5202        )
5203        .expect("verified pool workload assumes its terminal Component parent");
5204        let promoted = promoted.topology.get("asset").expect("promoted asset");
5205        assert_eq!(
5206            promoted.kind,
5207            crate::fleet_ensure::model::DesiredCanisterKind::Component
5208        );
5209        assert_eq!(promoted.parent.as_deref(), Some("hub"));
5210
5211        let mut already_component = state();
5212        already_component
5213            .topology
5214            .get_mut("asset")
5215            .expect("retained asset")
5216            .kind = crate::fleet_ensure::model::DesiredCanisterKind::Component;
5217        assert!(matches!(
5218            merge_terminal_inventory::<std::io::Error>(
5219                &mut already_component,
5220                crate::fleet_ensure::ops::TerminalFleetInventory {
5221                    active_registry: None,
5222                    controlled_cycles_by_principal: BTreeMap::new(),
5223                    entries: vec![terminal_component_entry(asset, hub)],
5224                },
5225            ),
5226            Err(EnsureWorkflowError::TerminalInventory(
5227                TerminalInventoryError::RetainedParentConflict {
5228                    canister,
5229                    retained_parent,
5230                    terminal_parent,
5231                }
5232            )) if canister == asset
5233                && retained_parent == "root"
5234                && terminal_parent == "hub"
5235        ));
5236
5237        for incomplete_entry in [
5238            crate::registry::RegistryEntry {
5239                module_hash: None,
5240                ..terminal_component_entry(asset, hub)
5241            },
5242            crate::registry::RegistryEntry {
5243                protocol_binding: None,
5244                ..terminal_component_entry(asset, hub)
5245            },
5246        ] {
5247            assert!(matches!(
5248                merge_terminal_inventory::<std::io::Error>(
5249                    &mut state(),
5250                    crate::fleet_ensure::ops::TerminalFleetInventory {
5251                        active_registry: None,
5252                        controlled_cycles_by_principal: BTreeMap::new(),
5253                        entries: vec![incomplete_entry],
5254                    },
5255                ),
5256                Err(EnsureWorkflowError::TerminalInventory(
5257                    TerminalInventoryError::RetainedParentConflict {
5258                        canister,
5259                        retained_parent,
5260                        terminal_parent,
5261                    }
5262                )) if canister == asset
5263                    && retained_parent == "root"
5264                    && terminal_parent == "hub"
5265            ));
5266        }
5267    }
5268
5269    #[test]
5270    fn reinstall_recovery_review_requires_complete_exact_install_evidence() {
5271        use crate::fleet_ensure::model::{CanisterPlan, FleetReinstallRecord, InstallMode};
5272        let (state, mut journal) = retained_evidence();
5273        let mut plan = estate_funding_plan();
5274        plan.conservation.estate_funding_domains.clear();
5275        plan.reinstall = Some(Box::new(FleetReinstallRecord {
5276            target_artifacts_sha256: Some("11".repeat(32)),
5277            source: None,
5278            activation_reset: None,
5279            operation_id: plan.operation_id.clone(),
5280            source_operation_id: "source-operation".into(),
5281            authorities: Vec::new(),
5282            assets: Vec::new(),
5283        }));
5284        let action = EnsureAction::Install {
5285            canic_init: None,
5286            reinstall_witness: None,
5287            init_arg: None,
5288            init_arg_sha256: None,
5289            init_candid: None,
5290            init_candid_sha256: None,
5291            mode: InstallMode::Reinstall,
5292            name: "root".into(),
5293            principal: "rrkah-fqaaa-aaaaa-aaaaq-cai".into(),
5294            wasm: "root.wasm".into(),
5295            wasm_sha256: "22".repeat(32),
5296        };
5297        journal.effects[0].action_sha256 = action_sha256(&action);
5298        plan.canisters.push(CanisterPlan {
5299            actions: vec![action],
5300            disposition: CanisterDisposition::Reuse,
5301            name: "root".into(),
5302            observed_cycles: 0,
5303            principal: Some("rrkah-fqaaa-aaaaa-aaaaq-cai".into()),
5304        });
5305        let admit = |journal: &FleetEnsureJournalRecord| {
5306            reinstall_recovery_ready::<std::io::Error>(&plan, journal, &state)
5307        };
5308        assert!(admit(&journal).unwrap());
5309        let mut issued = journal.clone();
5310        issued.effects[0].state = EffectState::Issued;
5311        let mut missing = journal.clone();
5312        missing.effects.clear();
5313        let mut wrong_action = journal.clone();
5314        wrong_action.effects[0].action_sha256 = "other".into();
5315        let mut no_version = journal.clone();
5316        no_version.effects[0].pre_canister_version = None;
5317        let mut wrong_operation = journal.clone();
5318        wrong_operation.operation_id = "other".into();
5319        for invalid in [issued, missing, wrong_action, no_version, wrong_operation] {
5320            assert!(matches!(
5321                admit(&invalid),
5322                Err(EnsureWorkflowError::JournalIntegrity)
5323            ));
5324        }
5325        for completion in [
5326            FleetEnsureCompletion::InProgress,
5327            FleetEnsureCompletion::Converged,
5328        ] {
5329            let mut other = journal.clone();
5330            other.completion = completion;
5331            assert!(!admit(&other).unwrap());
5332        }
5333    }
5334
5335    #[test]
5336    fn completed_reinstall_evidence_requires_exact_operation_action_and_version() {
5337        let (state, journal) = retained_evidence();
5338        assert!(retained_reinstall_operation_matches(
5339            &state,
5340            "fleet",
5341            "operation",
5342            &journal,
5343        ));
5344
5345        let mut wrong_action = journal.clone();
5346        wrong_action.effects[0].action_sha256 = "other".to_string();
5347        assert!(!retained_reinstall_operation_matches(
5348            &state,
5349            "fleet",
5350            "operation",
5351            &wrong_action,
5352        ));
5353
5354        let mut wrong_version = journal.clone();
5355        wrong_version.effects[0].pre_canister_version = Some(8);
5356        assert!(!retained_reinstall_operation_matches(
5357            &state,
5358            "fleet",
5359            "operation",
5360            &wrong_version,
5361        ));
5362
5363        for (fleet, operation) in [("other", "operation"), ("fleet", "other")] {
5364            assert!(!retained_reinstall_operation_matches(
5365                &state, fleet, operation, &journal,
5366            ));
5367        }
5368
5369        let mut incomplete = journal;
5370        incomplete.effects[0].state = EffectState::Issued;
5371        assert!(!retained_reinstall_operation_matches(
5372            &state,
5373            "fleet",
5374            "operation",
5375            &incomplete,
5376        ));
5377    }
5378
5379    #[test]
5380    fn fresh_fleet_alternate_desired_cannot_clear_completed_reinstall_evidence() {
5381        let (state, journal) = retained_evidence();
5382        assert!(retained_reinstall_desired_conflict(
5383            &state,
5384            "fleet",
5385            "operation",
5386            "reviewed-desired",
5387            "alternate-desired",
5388            &journal,
5389        ));
5390        assert!(!retained_reinstall_desired_conflict(
5391            &state,
5392            "fleet",
5393            "operation",
5394            "reviewed-desired",
5395            "reviewed-desired",
5396            &journal,
5397        ));
5398
5399        let mut terminal = journal;
5400        terminal.completion = FleetEnsureCompletion::Converged;
5401        assert!(!retained_reinstall_desired_conflict(
5402            &state,
5403            "fleet",
5404            "operation",
5405            "reviewed-desired",
5406            "alternate-desired",
5407            &terminal,
5408        ));
5409    }
5410
5411    fn cycle_observation(
5412        lifecycle: crate::fleet_ensure::model::RootOwnedCanisterLifecycle,
5413        cycles: u128,
5414    ) -> FleetObservation {
5415        FleetObservation {
5416            additional_controlled_cycles: BTreeMap::new(),
5417            canisters: BTreeMap::from([(
5418                "asset".to_string(),
5419                Some(crate::fleet_ensure::model::LiveCanister {
5420                    canister_version: Some(1),
5421                    controllers: Vec::new(),
5422                    cycles,
5423                    module_sha256: None,
5424                    principal: "controlled-principal".to_string(),
5425                    reinstall_required: false,
5426                    root_owned_lifecycle: Some(lifecycle),
5427                    status: crate::fleet_ensure::model::CanisterRuntimeStatus::Stopped,
5428                }),
5429            )]),
5430            estate_funding_domains: BTreeMap::new(),
5431            ledger_fee_cycles: 0,
5432            operator_cycles: 0,
5433            protocol_ready: BTreeMap::new(),
5434        }
5435    }
5436
5437    #[test]
5438    fn fresh_fleet_idle_cycle_duplicate_requires_exact_balance() {
5439        use crate::fleet_ensure::model::RootOwnedCanisterLifecycle;
5440
5441        let additional = BTreeMap::from([("controlled-principal".to_string(), 100)]);
5442        let mut idle = cycle_observation(RootOwnedCanisterLifecycle::Idle, 100);
5443        attach_terminal_cycles::<std::io::Error>(&mut idle, additional.clone())
5444            .expect("merge exact Idle observation");
5445        assert!(idle.additional_controlled_cycles.is_empty());
5446
5447        let mut conflicting_idle = cycle_observation(RootOwnedCanisterLifecycle::Idle, 99);
5448        assert!(matches!(
5449            attach_terminal_cycles::<std::io::Error>(&mut conflicting_idle, additional.clone()),
5450            Err(EnsureWorkflowError::TerminalInventory(
5451                TerminalInventoryError::IdleCycleObservationConflict {
5452                    canister,
5453                    expected: 99,
5454                    observed: 100,
5455                }
5456            )) if canister == "controlled-principal"
5457        ));
5458
5459        let mut workload = cycle_observation(RootOwnedCanisterLifecycle::Workload, 99);
5460        attach_terminal_cycles::<std::io::Error>(&mut workload, additional)
5461            .expect("merge conservative workload observation");
5462        assert_eq!(
5463            workload
5464                .canisters
5465                .get("asset")
5466                .and_then(Option::as_ref)
5467                .map(|live| live.cycles),
5468            Some(99),
5469        );
5470    }
5471}