Skip to main content

canic_host/fleet_ensure/workflow/
mod.rs

1//! Module: fleet_ensure::workflow
2//!
3//! Responsibility: plan and advance one durable idempotent Fleet convergence operation.
4//! Does not own: policy decisions, IC transport, or storage mechanics.
5//! Boundary: persists exact intent before each ops effect and reconciles it before any retry.
6
7mod continuation;
8mod funding;
9#[cfg(test)]
10mod funding_tests;
11mod reinstall;
12mod root_reinstall;
13
14use crate::fleet_ensure::{
15    dto::{FleetEnsurePhase, FleetEnsureProgress, FleetEnsureProgressState},
16    inventory::project_current_fleet_inventory,
17    model::{
18        ActualCycleConservation, CanisterDisposition, CurrentFleetProtocolAction,
19        CycleConservation, EffectRecord, EffectState, EnsureAction, EstateFundingDomainPlan,
20        EstateFundingRequiredRecord, FLEET_ENSURE_SCHEMA_VERSION, FleetEnsureCompletion,
21        FleetEnsureJournalRecord, FleetEnsurePlan, FleetEnsurePlanScope, FleetEnsureReport,
22        FleetEnsureStateRecord, FleetObservation, RootManagementObservation,
23        create_balance_is_terminal,
24    },
25    ops::{
26        EffectRetry, EnsurePaths, EnsurePlatform, EnsureStateError, action_sha256,
27        compact_inline_plan, lock_operation, read_journal, read_plan, read_root_start_authority,
28        read_state, resolve_desired_artifacts, write_journal, write_plan, write_state,
29    },
30    policy::{
31        EnsurePolicyError, RootStartPlanInput, compile_plan, compile_root_start_prerequisite_plan,
32        effect_observation_policy, expected_plan_sha256, operation_id,
33        recompile_root_start_prerequisite_plan, validate_path_identity, validate_path_labels,
34    },
35};
36use canic_core::cdk::types::Cycles;
37use std::{
38    collections::{BTreeMap, BTreeSet},
39    path::Path,
40};
41use thiserror::Error as ThisError;
42
43pub use reinstall::plan_reinstall;
44
45/// Exact no-effect evidence that a fresh pool cannot retain its readiness floor.
46
47#[derive(Debug, ThisError)]
48#[error(
49    "fresh pool canister {canister} requested {requested_creation_funding_cycles} creation cycles for readiness floor {readiness_floor_cycles}; its first live balance is {live_balance_cycles} cycles (readiness shortfall {readiness_shortfall_cycles}) and must retain {remaining_controller_burn_cycles} cycles for controller finalization (required pre-finalization balance {required_pre_finalization_balance_cycles}, pre-finalization shortfall {pre_finalization_shortfall_cycles}; maximum first-observation burn {maximum_observation_burn_cycles}); no controller or protocol action followed"
50)]
51pub struct FreshPoolCreationUnderfundedError {
52    pub canister: String,
53    pub live_balance_cycles: u128,
54    pub maximum_observation_burn_cycles: u128,
55    pub pre_finalization_shortfall_cycles: u128,
56    pub readiness_floor_cycles: u128,
57    pub readiness_shortfall_cycles: u128,
58    pub remaining_controller_burn_cycles: u128,
59    pub requested_creation_funding_cycles: u128,
60    pub required_pre_finalization_balance_cycles: u128,
61}
62
63/// Exact reviewed Root-account funding prerequisite that blocks protocol mutation.
64#[derive(Debug, ThisError)]
65#[error(
66    "Root {root} Cycles Ledger account {root_principal} on {cycles_ledger} has {available_cycles} cycles; {required_creation_count} autonomous creations require at most {maximum_creation_debit_cycles} cycles ({creation_amount_cycles} gross amount containing {readiness_floor_cycles} readiness floor, {management_creation_fee_cycles} management creation fee and {creation_execution_margin_cycles} execution margin, plus {ledger_fee_cycles} Ledger fee each; total reviewed fees {maximum_creation_fee_cycles}), shortfall {shortfall_cycles}; pending_creation={pending_creation_operation_id:?} attempts={attempt_count:?} last_attempt_at_ns={last_attempt_at_ns:?} retry_at_ns={retry_at_ns:?}; preserve this operation, repeat Fleet ensure without --apply and review funding_review, then apply its review_sha256 for the exact shortfall; do not fund the account out of band"
67)]
68pub struct EstateFundingRequiredError {
69    pub attempt_count: Option<u32>,
70    pub available_cycles: u128,
71    pub creation_amount_cycles: u128,
72    pub cycles_ledger: String,
73    pub creation_execution_margin_cycles: u128,
74    pub readiness_floor_cycles: u128,
75    pub ledger_fee_cycles: u128,
76    pub management_creation_fee_cycles: u128,
77    pub maximum_creation_debit_cycles: u128,
78    pub maximum_creation_fee_cycles: u128,
79    pub pending_creation_operation_id: Option<String>,
80    pub required_creation_count: u32,
81    pub root: String,
82    pub root_principal: String,
83    pub last_attempt_at_ns: Option<u64>,
84    pub retry_at_ns: Option<u64>,
85    pub shortfall_cycles: u128,
86}
87
88/// Exact terminal-inventory merge or cycle-observation invariant failure.
89#[derive(Debug, ThisError)]
90pub enum TerminalInventoryError {
91    #[error("one Principal is retained under more than one desired role")]
92    DuplicateRetainedPrincipal,
93
94    #[error("verified canister {canister} lost its Principal binding")]
95    LostPrincipalBinding { canister: String },
96
97    #[error("canister {canister} names unknown parent {parent}")]
98    UnknownParent { canister: String, parent: String },
99
100    #[error(
101        "canister {canister} retained parent {retained_parent} differs from terminal parent {terminal_parent}"
102    )]
103    RetainedParentConflict {
104        canister: String,
105        retained_parent: String,
106        terminal_parent: String,
107    },
108
109    #[error(
110        "terminal Root-owned Idle canister {canister} cycle observations differ: expected {expected}, observed {observed}"
111    )]
112    IdleCycleObservationConflict {
113        canister: String,
114        expected: u128,
115        observed: u128,
116    },
117
118    #[error(
119        "terminal cycle observation duplicates canister {canister} outside its exact Root-owned Idle or Workload lifecycle"
120    )]
121    CycleObservationLifecycleConflict { canister: String },
122}
123
124/// Fleet ensure planning or convergence failure.
125
126#[derive(Debug, ThisError)]
127pub enum EnsureWorkflowError<E>
128where
129    E: std::error::Error + 'static,
130{
131    #[error(
132        "activation {operation_id} has an Issued provisioning action after its Applied host prefix (source document {source_document_sha256}); preserve the plan and journal: recovery of this partial activation is not yet qualified"
133    )]
134    PartialActivationResetUnavailable {
135        operation_id: String,
136        source_document_sha256: String,
137    },
138
139    #[error(
140        "a new reinstall requires the unchanged, fully converged Fleet; preserve the retained plan and journal if activation is blocked, because partial-activation reset is not yet supported"
141    )]
142    ReinstallConflict,
143
144    #[error("reviewed Fleet plan digest changed before its first effect")]
145    DriftedBeforeApply,
146
147    #[error(
148        "in-progress Fleet plan requires exact desired input {expected}; supplied input is {actual}"
149    )]
150    RetainedDesiredUnavailable { actual: String, expected: String },
151
152    #[error(
153        "replan-required Fleet operation owns completed reinstall evidence for desired input {expected}; refusing alternate desired input {actual}"
154    )]
155    RetainedReinstallDesiredConflict { actual: String, expected: String },
156
157    #[error(
158        "Root prerequisite retains activation authority for desired input {expected}; refreshed input {actual} is refused before Store effects; preserve the retained plan and journal for recovery review"
159    )]
160    RootActivationDesiredConflict { actual: String, expected: String },
161
162    #[error("reviewed Fleet plan {expected} does not match retained plan {actual}")]
163    PlanDigestMismatch { actual: String, expected: String },
164
165    #[error("reviewed Fleet plan is missing; run `canic fleet ensure <fleet>` first")]
166    PlanMissing,
167
168    #[error("retained Fleet plan failed its content digest")]
169    PlanIntegrity,
170
171    #[error("retained Fleet journal does not match the reviewed Fleet operation")]
172    JournalIntegrity,
173
174    #[error(
175        "Fleet ensure action {action} made no progress for {observations} consecutive observations (last progress: {progress_identity}); operation remains resumable"
176    )]
177    Stalled {
178        action: String,
179        observations: u32,
180        progress_identity: String,
181    },
182
183    #[error("Root {root} provisioning has made no progress for {observations} observations: stage={stage:?}, target={target}, operation={operation}, diagnostic={diagnostic_code}, retry=Backoff, failed_at_ns={failed_at_ns}; the issued operation remains retained; resume the same reviewed apply after the dependency recovers", operation = canic_core::cdk::utils::hash::hex_bytes(.operation_id))]
184    ProvisioningRetryPending {
185        root: candid::Principal,
186        stage: canic_core::dto::component_provisioning::ProvisioningFailureStage,
187        target: candid::Principal,
188        operation_id: [u8; 32],
189        diagnostic_code: u16,
190        observations: u32,
191        failed_at_ns: u64,
192    },
193
194    #[error(
195        "Root-owned canister {target} remained {last_lifecycle} for {observations} consecutive passive observations; no command was reissued and the operation remains resumable"
196    )]
197    RootOwnedObservationStalled {
198        target: String,
199        observations: u32,
200        last_lifecycle: String,
201    },
202
203    #[error(
204        "selected Cycles Ledger account has {actual} cycles, below reviewed maximum debit {required}"
205    )]
206    InsufficientOperatorCycles { actual: u128, required: u128 },
207
208    #[error("terminal Fleet cycle conservation failed: {0}")]
209    Conservation(String),
210
211    #[error("terminal Fleet inventory failed exact current-authority validation: {0}")]
212    TerminalInventory(#[source] TerminalInventoryError),
213
214    #[error(
215        "reviewed effects completed, but live Fleet state still differs from desired state; review a new plan"
216    )]
217    ConvergenceDrift,
218
219    #[error(
220        "Root pool maintenance {action} exhausted its reviewed limit of {maximum_updates} updates; retained receipts remain authoritative"
221    )]
222    PoolMaintenanceBound {
223        action: String,
224        maximum_updates: u32,
225    },
226
227    #[error(
228        "created canister {canister} retained one exact Ledger creation receipt, but its first live balance is {actual_cycles} cycles instead of requested target {requested_cycles} (deficit {deficit_cycles}; configured management creation fee {configured_fee_cycles}); no controller or protocol action followed, and the immutable operation is now replan-required"
229    )]
230    ReplanRequiredAfterCreateBalanceDrift {
231        actual_cycles: u128,
232        canister: String,
233        configured_fee_cycles: u128,
234        deficit_cycles: u128,
235        requested_cycles: u128,
236    },
237
238    #[error("Fleet successor requires a new review: {reason:?}; {}", .review.as_deref().map(ToString::to_string).unwrap_or_default())]
239    SuccessorReviewRequired {
240        reason: crate::fleet_ensure::model::FleetEnsureSuccessorReviewReason,
241        review: Option<Box<crate::fleet_ensure::model::FleetSuccessorReview>>,
242    },
243
244    #[error(transparent)]
245    FreshPoolCreationUnderfunded(Box<FreshPoolCreationUnderfundedError>),
246
247    #[error(transparent)]
248    EstateFundingRequired(Box<EstateFundingRequiredError>),
249
250    #[error(transparent)]
251    Policy(#[from] EnsurePolicyError),
252
253    #[error(transparent)]
254    Inventory(#[from] crate::fleet_ensure::CurrentFleetInventoryError),
255
256    #[error("Fleet platform operation failed: {0}")]
257    Platform(#[source] E),
258
259    #[error(transparent)]
260    State(#[from] EnsureStateError),
261}
262
263/// Load and verify the sole immutable in-progress operation for one exact Fleet path.
264pub fn retained_in_progress_plan<E>(
265    root: &Path,
266    environment: &str,
267    requested_fleet: &str,
268) -> Result<Option<FleetEnsurePlan>, EnsureWorkflowError<E>>
269where
270    E: std::error::Error + 'static,
271{
272    validate_path_labels(environment, requested_fleet)?;
273    let paths = EnsurePaths::under(root, environment, requested_fleet);
274    let _lock = lock_operation(&paths)?;
275    if let Some(review) = crate::fleet_ensure::ops::reinstall::adoption::review(&paths)? {
276        if review.environment != environment || review.fleet != requested_fleet {
277            return Err(EnsureWorkflowError::PlanIntegrity);
278        }
279        return Ok(Some(review));
280    }
281    let Some(journal) = read_journal(&paths)? else {
282        return Ok(None);
283    };
284    if !matches!(
285        journal.completion,
286        FleetEnsureCompletion::InProgress | FleetEnsureCompletion::Prepared
287    ) {
288        return Ok(None);
289    }
290    let retained = verified_plan(read_plan(&paths)?.ok_or(EnsureWorkflowError::PlanMissing)?)?;
291    if retained.environment != environment || retained.fleet != requested_fleet {
292        return Err(EnsureWorkflowError::PlanIntegrity);
293    }
294    let state = read_state(&paths, requested_fleet)?;
295    let prepared_full = journal.completion == FleetEnsureCompletion::Prepared
296        && (retained.scope == FleetEnsurePlanScope::Full
297            || (retained.scope == FleetEnsurePlanScope::RootReinstallPrerequisite
298                && retained.reinstall.as_ref().is_some_and(|intent| {
299                    intent
300                        .activation_reset
301                        .as_ref()
302                        .is_some_and(|activation| activation.preparation.is_some())
303                })))
304        && retained.reinstall.as_ref().is_some_and(|intent| {
305            intent.operation_id == journal.operation_id
306                && intent.operation_id == retained.operation_id
307        });
308    if !prepared_full {
309        verify_journal(&journal, &retained, requested_fleet, &state)?;
310    }
311    Ok(Some(retained))
312}
313
314/// Build and retain one read-only plan from current desired state plus live observation.
315#[expect(
316    clippy::too_many_lines,
317    reason = "one locked read-only transaction reconciles retained authority, observation and the immutable plan"
318)]
319pub fn plan<P>(
320    root: &Path,
321    desired: &crate::fleet_ensure::model::DesiredFleet,
322    desired_sha256: &str,
323    requested_fleet: &str,
324    created_at_time: u64,
325    platform: &mut P,
326) -> Result<FleetEnsureReport, EnsureWorkflowError<P::Error>>
327where
328    P: EnsurePlatform,
329{
330    validate_path_identity(desired, requested_fleet)?;
331    let paths = EnsurePaths::under(root, &desired.environment, requested_fleet);
332    let _lock = lock_operation(&paths)?;
333    if let Some(review) = crate::fleet_ensure::ops::reinstall::adoption::review(&paths)? {
334        if review.desired_sha256 != desired_sha256 {
335            return Err(EnsureWorkflowError::DriftedBeforeApply);
336        }
337        return Ok(FleetEnsureReport {
338            funding_review: None,
339            actual_conservation: None,
340            effects_applied: 0,
341            plan: review,
342            terminal: false,
343        });
344    }
345    if let Some(mut journal) = read_journal(&paths)?
346        && journal.completion == FleetEnsureCompletion::InProgress
347    {
348        let retained = verified_plan(read_plan(&paths)?.ok_or(EnsureWorkflowError::PlanMissing)?)?;
349        let state = read_state(&paths, requested_fleet)?;
350        verify_journal(&journal, &retained, requested_fleet, &state)?;
351        if journal.estate_funding_required.is_some() {
352            if let Some(reviewed) = retained.reviewed_desired.as_deref() {
353                platform
354                    .bind_reviewed_desired(reviewed.desired())
355                    .map_err(EnsureWorkflowError::Platform)?;
356            } else if retained.desired_sha256 != desired_sha256 {
357                return Err(EnsureWorkflowError::RetainedDesiredUnavailable {
358                    actual: desired_sha256.to_string(),
359                    expected: retained.desired_sha256.clone(),
360                });
361            }
362        }
363        let funding_review = funding::prepare(
364            &paths,
365            &retained,
366            &mut journal,
367            &state,
368            created_at_time,
369            platform,
370        )?;
371        return Ok(FleetEnsureReport {
372            funding_review,
373            actual_conservation: None,
374            effects_applied: applied_count(&journal),
375            plan: retained,
376            terminal: false,
377        });
378    }
379    let mut state = read_state(&paths, requested_fleet)?;
380    let prior_plan = read_plan(&paths)?.map(verified_plan).transpose()?;
381    let prior_journal = read_journal(&paths)?;
382    if let Some(prior) = &prior_plan {
383        let activation_reset = prior
384            .reinstall
385            .as_ref()
386            .is_some_and(|intent| intent.activation_reset.is_some());
387        if activation_reset
388            && prior.scope == FleetEnsurePlanScope::RootReinstallPrerequisite
389            && let Some(journal) = prior_journal
390                .as_ref()
391                .filter(|journal| journal.completion == FleetEnsureCompletion::Converged)
392        {
393            verify_journal(journal, prior, requested_fleet, &state)?;
394            retain_completed_reinstalls(&mut state, prior, journal);
395            write_state(&paths, &state)?;
396        }
397        let unfinished_reset = prior.reinstall.is_some()
398            && !prior_journal.as_ref().is_some_and(|journal| {
399                journal.plan_sha256 == prior.plan_sha256
400                    && journal.completion == FleetEnsureCompletion::Converged
401            });
402        if prior.scope == FleetEnsurePlanScope::ReinstallPreparation
403            || unfinished_reset
404            || (activation_reset && prior.scope == FleetEnsurePlanScope::RootReinstallPrerequisite)
405        {
406            if prior.desired_sha256 != desired_sha256 {
407                return Err(EnsureWorkflowError::ReinstallConflict);
408            }
409            return reinstall::continue_plan(
410                root,
411                &paths,
412                desired,
413                prior,
414                prior_journal.as_ref(),
415                &state,
416                platform,
417            );
418        }
419    }
420    let matching_prior = reconcile_retained_planning_state(
421        &mut state,
422        prior_plan.as_ref(),
423        prior_journal.as_ref(),
424        requested_fleet,
425        &desired.environment,
426        desired_sha256,
427    )?;
428    if let Some(prior) = matching_prior {
429        retain_plan_cycles(&mut state, prior);
430        if let Some(journal) = &prior_journal {
431            retain_completed_reinstalls(&mut state, prior, journal);
432        }
433    }
434    let operation_id = matching_prior
435        .filter(|prior| prior.scope == FleetEnsurePlanScope::Full)
436        .map_or_else(
437            || operation_id(desired_sha256, &desired.environment, requested_fleet),
438            |prior| prior.operation_id.clone(),
439        );
440    if let Some(management) = platform
441        .observe_root_management(&state, &BTreeSet::new())
442        .map_err(EnsureWorkflowError::Platform)?
443    {
444        let root_start_authority = read_root_start_authority(&paths)?;
445        if let Some(plan) = compile_root_start_prerequisite_plan(RootStartPlanInput {
446            state: &state,
447            authority: root_start_authority.as_ref(),
448            created_at_time,
449            desired,
450            desired_sha256,
451            observation: &management,
452            requested_fleet,
453        })? {
454            write_plan(&paths, &plan)?;
455            return Ok(FleetEnsureReport {
456                funding_review: None,
457                actual_conservation: None,
458                effects_applied: 0,
459                plan,
460                terminal: false,
461            });
462        }
463        if let Some(plan) = crate::fleet_ensure::policy::root_reinstall::compile(
464            RootStartPlanInput {
465                state: &state,
466                authority: None,
467                created_at_time,
468                desired,
469                desired_sha256,
470                observation: &management,
471                requested_fleet,
472            },
473            &resolve_desired_artifacts(root, desired)?,
474        )? {
475            write_plan(&paths, &plan)?;
476            return Ok(FleetEnsureReport {
477                funding_review: None,
478                actual_conservation: None,
479                effects_applied: 0,
480                plan,
481                terminal: false,
482            });
483        }
484    }
485    let artifacts = resolve_desired_artifacts(root, desired)?;
486    let mut observation = platform
487        .observe(&operation_id, &state)
488        .map_err(EnsureWorkflowError::Platform)?;
489    retain_observed_cycles(&mut state, &observation);
490    // The prior Converged journal can remain visible until apply. Planning may refresh only
491    // backup-inert observation and prior-operation evidence, never successor topology authority.
492    write_state(&paths, &state)?;
493    let terminal_inventory_operation_id = current_terminal_inventory_operation(
494        &state,
495        prior_plan.as_ref(),
496        prior_journal.as_ref(),
497        requested_fleet,
498    )?;
499    if let Some(operation_id) = terminal_inventory_operation_id.as_deref() {
500        attach_terminal_inventory_cycles(operation_id, &state, platform, &mut observation)?;
501    }
502    let mut protocol_actions = platform
503        .protocol_actions(&operation_id, &state)
504        .map_err(EnsureWorkflowError::Platform)?;
505    let mut retained_observations = 0_u32;
506    let mut plan = loop {
507        match compile_plan(
508            desired,
509            &artifacts,
510            &protocol_actions,
511            desired_sha256,
512            requested_fleet,
513            &observation,
514            created_at_time,
515            &operation_id,
516            None,
517        ) {
518            Ok(plan) => break plan,
519            Err(EnsurePolicyError::PendingRootOwnedBalance { name }) => {
520                retained_observations = retained_observations.saturating_add(1);
521                if retained_observations >= desired.maximum_stalled_observations {
522                    return Err(EnsureWorkflowError::RootOwnedObservationStalled {
523                        last_lifecycle: root_owned_lifecycle_label(&observation, &name).to_string(),
524                        observations: retained_observations,
525                        target: name,
526                    });
527                }
528                platform.pace_root_owned_observation(&name, retained_observations);
529                observation = platform
530                    .observe(&operation_id, &state)
531                    .map_err(EnsureWorkflowError::Platform)?;
532                retain_observed_cycles(&mut state, &observation);
533                write_state(&paths, &state)?;
534                if let Some(operation_id) = terminal_inventory_operation_id.as_deref() {
535                    attach_terminal_inventory_cycles(
536                        operation_id,
537                        &state,
538                        platform,
539                        &mut observation,
540                    )?;
541                }
542                protocol_actions = platform
543                    .protocol_actions(&operation_id, &state)
544                    .map_err(EnsureWorkflowError::Platform)?;
545            }
546            Err(error) => return Err(error.into()),
547        }
548    };
549    for action in ordered_actions(&plan) {
550        verify_store_activation_binding(&plan, action, &state, platform)?;
551    }
552    bind_terminal_inventory_operation(&mut plan, terminal_inventory_operation_id);
553    write_plan(&paths, &plan)?;
554    Ok(FleetEnsureReport {
555        funding_review: None,
556        actual_conservation: None,
557        effects_applied: 0,
558        plan,
559        terminal: false,
560    })
561}
562
563fn current_terminal_inventory_operation<E>(
564    state: &FleetEnsureStateRecord,
565    prior_plan: Option<&FleetEnsurePlan>,
566    prior_journal: Option<&FleetEnsureJournalRecord>,
567    requested_fleet: &str,
568) -> Result<Option<String>, EnsureWorkflowError<E>>
569where
570    E: std::error::Error + 'static,
571{
572    if state.active_registry.is_none() {
573        return Ok(None);
574    }
575    let prior_plan = prior_plan.ok_or(EnsureWorkflowError::PlanMissing)?;
576    let prior_journal = prior_journal.ok_or(EnsureWorkflowError::JournalIntegrity)?;
577    verify_journal(prior_journal, prior_plan, requested_fleet, state)?;
578    if prior_journal.completion != FleetEnsureCompletion::Converged
579        || prior_plan.scope != FleetEnsurePlanScope::Full
580    {
581        return Err(EnsureWorkflowError::JournalIntegrity);
582    }
583    let operation_id = prior_plan
584        .terminal_inventory_operation_id
585        .as_ref()
586        .unwrap_or(&prior_plan.operation_id);
587    if operation_id.is_empty() {
588        return Err(EnsureWorkflowError::JournalIntegrity);
589    }
590    Ok(Some(operation_id.clone()))
591}
592
593fn bind_terminal_inventory_operation(plan: &mut FleetEnsurePlan, operation_id: Option<String>) {
594    plan.terminal_inventory_operation_id = operation_id;
595    plan.plan_sha256 = expected_plan_sha256(plan);
596}
597
598fn reconcile_retained_planning_state<'a, E>(
599    state: &mut FleetEnsureStateRecord,
600    prior_plan: Option<&'a FleetEnsurePlan>,
601    prior_journal: Option<&FleetEnsureJournalRecord>,
602    requested_fleet: &str,
603    desired_environment: &str,
604    desired_sha256: &str,
605) -> Result<Option<&'a FleetEnsurePlan>, EnsureWorkflowError<E>>
606where
607    E: std::error::Error + 'static,
608{
609    if let Some((prior, journal)) = prior_plan.zip(prior_journal)
610        && prior.fleet == requested_fleet
611        && prior.environment == desired_environment
612        && retained_reinstall_desired_conflict(
613            state,
614            &prior.fleet,
615            &prior.operation_id,
616            &prior.desired_sha256,
617            desired_sha256,
618            journal,
619        )
620    {
621        return Err(EnsureWorkflowError::RetainedReinstallDesiredConflict {
622            actual: desired_sha256.to_string(),
623            expected: prior.desired_sha256.clone(),
624        });
625    }
626    if let Some((prior, journal)) = prior_plan.zip(prior_journal)
627        && prior.scope == FleetEnsurePlanScope::RootReinstallPrerequisite
628        && prior.desired_sha256 != desired_sha256
629    {
630        verify_journal(journal, prior, requested_fleet, state)?;
631        return Err(EnsureWorkflowError::RootActivationDesiredConflict {
632            actual: desired_sha256.to_string(),
633            expected: prior.desired_sha256.clone(),
634        });
635    }
636    let matching_prior = prior_plan.filter(|prior| {
637        prior.fleet == requested_fleet
638            && prior.environment == desired_environment
639            && prior.desired_sha256 == desired_sha256
640    });
641    let retained_operation_matches =
642        matching_prior
643            .zip(prior_journal)
644            .is_some_and(|(prior, journal)| {
645                retained_reinstall_operation_matches(
646                    state,
647                    &prior.fleet,
648                    &prior.operation_id,
649                    journal,
650                )
651            });
652    if !retained_operation_matches {
653        state.completed_reinstall_action_sha256.clear();
654        state.completed_reinstall_operation_id = None;
655        state.completed_reinstalls.clear();
656    }
657    Ok(matching_prior)
658}
659
660fn retain_completed_reinstalls(
661    state: &mut FleetEnsureStateRecord,
662    plan: &FleetEnsurePlan,
663    journal: &FleetEnsureJournalRecord,
664) {
665    if journal.fleet != plan.fleet || journal.operation_id != plan.operation_id {
666        return;
667    }
668    for action in ordered_actions(plan) {
669        let EnsureAction::Install {
670            mode: crate::fleet_ensure::model::InstallMode::Reinstall,
671            name,
672            principal,
673            wasm_sha256,
674            ..
675        } = action
676        else {
677            continue;
678        };
679        let action_sha256 = action_sha256(action);
680        let Some(effect) = journal
681            .effects
682            .iter()
683            .find(|effect| effect.action_sha256 == action_sha256)
684        else {
685            continue;
686        };
687        if effect.state == EffectState::Applied
688            && let Some(pre_canister_version) = effect.pre_canister_version
689            && state
690                .principals
691                .get(name)
692                .is_some_and(|retained| retained == principal)
693        {
694            state.completed_reinstall_operation_id = Some(journal.operation_id.clone());
695            state
696                .completed_reinstall_action_sha256
697                .insert(name.clone(), action_sha256);
698            state
699                .completed_reinstalls
700                .insert(name.clone(), pre_canister_version);
701            if let Some(topology) = state.topology.get_mut(name) {
702                topology.module_hash = Some(wasm_sha256.clone());
703            }
704        }
705    }
706}
707
708fn completed_reinstall_evidence_matches(
709    state: &FleetEnsureStateRecord,
710    journal: &FleetEnsureJournalRecord,
711) -> bool {
712    !state.completed_reinstalls.is_empty()
713        && state
714            .completed_reinstalls
715            .iter()
716            .all(|(name, pre_version)| {
717                state
718                    .completed_reinstall_action_sha256
719                    .get(name)
720                    .and_then(|action_sha256| {
721                        journal
722                            .effects
723                            .iter()
724                            .find(|effect| effect.action_sha256 == *action_sha256)
725                    })
726                    .is_some_and(|effect| {
727                        effect.state == EffectState::Applied
728                            && effect.pre_canister_version == Some(*pre_version)
729                    })
730            })
731        && state.completed_reinstall_action_sha256.len() == state.completed_reinstalls.len()
732}
733
734fn retained_reinstall_operation_matches(
735    state: &FleetEnsureStateRecord,
736    prior_fleet: &str,
737    prior_operation_id: &str,
738    journal: &FleetEnsureJournalRecord,
739) -> bool {
740    journal.completion == FleetEnsureCompletion::ReplanRequired
741        && journal.fleet == prior_fleet
742        && journal.operation_id == prior_operation_id
743        && state.completed_reinstall_operation_id.as_deref() == Some(journal.operation_id.as_str())
744        && completed_reinstall_evidence_matches(state, journal)
745}
746
747fn retained_reinstall_desired_conflict(
748    state: &FleetEnsureStateRecord,
749    prior_fleet: &str,
750    prior_operation_id: &str,
751    prior_desired_sha256: &str,
752    supplied_desired_sha256: &str,
753    journal: &FleetEnsureJournalRecord,
754) -> bool {
755    prior_desired_sha256 != supplied_desired_sha256
756        && retained_reinstall_operation_matches(state, prior_fleet, prior_operation_id, journal)
757}
758
759fn retain_plan_cycles(state: &mut FleetEnsureStateRecord, plan: &FleetEnsurePlan) {
760    for canister in &plan.canisters {
761        if let Some(principal) = &canister.principal {
762            state
763                .retained_cycles_by_principal
764                .entry(principal.clone())
765                .or_insert(canister.observed_cycles);
766        }
767    }
768}
769
770fn retain_observed_cycles(state: &mut FleetEnsureStateRecord, observation: &FleetObservation) {
771    for live in observation.canisters.values().filter_map(Option::as_ref) {
772        state
773            .retained_cycles_by_principal
774            .insert(live.principal.clone(), live.cycles);
775    }
776    state
777        .retained_cycles_by_principal
778        .extend(observation.additional_controlled_cycles.clone());
779}
780
781fn root_owned_lifecycle_label(observation: &FleetObservation, target: &str) -> &'static str {
782    observation
783        .canisters
784        .get(target)
785        .and_then(Option::as_ref)
786        .and_then(|live| live.root_owned_lifecycle)
787        .map_or("unavailable", |lifecycle| lifecycle.label())
788}
789
790/// Apply or resume exactly one reviewed plan until it converges or returns a typed blocker.
791#[expect(
792    clippy::too_many_lines,
793    reason = "the sole effect driver keeps intent, reconciliation and terminal publication visibly ordered"
794)]
795pub fn apply<P>(
796    root: &Path,
797    desired: &crate::fleet_ensure::model::DesiredFleet,
798    desired_sha256: &str,
799    requested_fleet: &str,
800    reviewed_plan_sha256: &str,
801    platform: &mut P,
802) -> Result<FleetEnsureReport, EnsureWorkflowError<P::Error>>
803where
804    P: EnsurePlatform,
805{
806    validate_path_identity(desired, requested_fleet)?;
807    let paths = EnsurePaths::under(root, &desired.environment, requested_fleet);
808    let _lock = lock_operation(&paths)?;
809    let pending = crate::fleet_ensure::ops::reinstall::adoption::review(&paths)?;
810    let adopting = pending.is_some();
811    let retained_plan = verified_plan(match pending {
812        Some(plan) => plan,
813        None => read_plan(&paths)?.ok_or(EnsureWorkflowError::PlanMissing)?,
814    })?;
815    let retained_journal = if adopting {
816        None
817    } else {
818        read_journal(&paths)?
819    };
820    if retained_plan.plan_sha256 != reviewed_plan_sha256
821        && !retained_journal
822            .as_ref()
823            .is_some_and(|journal| funding::accepts(&retained_plan, journal, reviewed_plan_sha256))
824    {
825        return Err(EnsureWorkflowError::PlanDigestMismatch {
826            actual: retained_plan.plan_sha256,
827            expected: reviewed_plan_sha256.to_string(),
828        });
829    }
830    let mut state = read_state(&paths, requested_fleet)?;
831    let in_progress = retained_journal
832        .as_ref()
833        .is_some_and(|journal| journal.completion == FleetEnsureCompletion::InProgress);
834    if let Some(journal) = retained_journal.as_ref().filter(|_| in_progress) {
835        verify_journal(journal, &retained_plan, requested_fleet, &state)?;
836        compact_inline_plan(&paths, &retained_plan)?;
837    }
838    if !in_progress && retained_plan.desired_sha256 != desired_sha256 {
839        return Err(EnsureWorkflowError::DriftedBeforeApply);
840    }
841    let operation_desired = if in_progress || retained_plan.reinstall.is_some() {
842        if let Some(reviewed) = retained_plan.reviewed_desired.as_deref() {
843            let reviewed = reviewed.desired();
844            validate_path_identity(reviewed, requested_fleet)?;
845            if reviewed.environment != retained_plan.environment {
846                return Err(EnsureWorkflowError::PlanIntegrity);
847            }
848            reviewed
849        } else if retained_plan.desired_sha256 == desired_sha256 {
850            desired
851        } else {
852            return Err(EnsureWorkflowError::RetainedDesiredUnavailable {
853                actual: desired_sha256.to_string(),
854                expected: retained_plan.desired_sha256.clone(),
855            });
856        }
857    } else {
858        if retained_plan.desired_sha256 != desired_sha256 {
859            return Err(EnsureWorkflowError::DriftedBeforeApply);
860        }
861        desired
862    };
863    platform
864        .bind_reviewed_desired(operation_desired)
865        .map_err(EnsureWorkflowError::Platform)?;
866    let operation_desired_sha256 = retained_plan.desired_sha256.as_str();
867    continuation::verify_inputs(root, operation_desired, &retained_plan)?;
868    if let Some(journal) = &retained_journal {
869        continuation::verify_canonical(&retained_plan, journal, &state, platform)?;
870    }
871    let completed_journal = retained_journal.as_ref().filter(|journal| {
872        (journal.completion == FleetEnsureCompletion::Converged
873            || (journal.completion == FleetEnsureCompletion::Prepared
874                && retained_plan.scope == FleetEnsurePlanScope::ReinstallPreparation))
875            && journal.plan_sha256 == retained_plan.plan_sha256
876    });
877    if retained_plan.scope == FleetEnsurePlanScope::ReinstallPreparation
878        && let Some(journal) = completed_journal
879    {
880        let actual = reinstall::complete(&retained_plan, journal, &state, platform)?;
881        return Ok(FleetEnsureReport {
882            funding_review: None,
883            actual_conservation: Some(actual),
884            effects_applied: 0,
885            plan: retained_plan,
886            terminal: true,
887        });
888    }
889    if retained_plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite
890        && let Some(journal) = completed_journal
891    {
892        verify_journal(journal, &retained_plan, requested_fleet, &state)?;
893        let actual = root_reinstall::complete(&retained_plan, journal, &state, platform)?;
894        report_progress_state(
895            platform,
896            &retained_plan,
897            journal,
898            FleetEnsurePhase::Infrastructure,
899            FleetEnsureProgressState::PrerequisiteComplete,
900        );
901        return Ok(FleetEnsureReport {
902            funding_review: None,
903            actual_conservation: Some(actual),
904            effects_applied: 0,
905            plan: retained_plan,
906            terminal: true,
907        });
908    }
909    if retained_plan.scope == FleetEnsurePlanScope::Full
910        && let Some(journal) = completed_journal
911    {
912        verify_journal(journal, &retained_plan, requested_fleet, &state)?;
913        let actual = continuation::replay(
914            root,
915            operation_desired,
916            &retained_plan,
917            journal,
918            &state,
919            platform,
920        )?;
921        report_progress(
922            platform,
923            &retained_plan,
924            journal,
925            FleetEnsurePhase::Complete,
926        );
927        return Ok(FleetEnsureReport {
928            funding_review: None,
929            actual_conservation: Some(actual),
930            effects_applied: 0,
931            plan: retained_plan,
932            terminal: true,
933        });
934    }
935    if retained_plan.scope == FleetEnsurePlanScope::RootStartPrerequisite
936        && let Some(journal) = completed_journal
937    {
938        verify_terminal_root_start_replay(
939            operation_desired,
940            &retained_plan,
941            journal,
942            requested_fleet,
943            platform,
944            &state,
945        )?;
946        report_progress_state(
947            platform,
948            &retained_plan,
949            journal,
950            FleetEnsurePhase::Infrastructure,
951            FleetEnsureProgressState::PrerequisiteComplete,
952        );
953        return Ok(FleetEnsureReport {
954            funding_review: None,
955            actual_conservation: None,
956            effects_applied: applied_count(journal),
957            plan: retained_plan,
958            terminal: true,
959        });
960    }
961    let mut journal = match retained_journal {
962        Some(journal) if journal.completion == FleetEnsureCompletion::InProgress => {
963            verify_journal(&journal, &retained_plan, requested_fleet, &state)?;
964            journal
965        }
966        _ => {
967            let (observation, initial_controlled_cycles) = verify_fresh_plan(
968                root,
969                operation_desired,
970                operation_desired_sha256,
971                requested_fleet,
972                &retained_plan,
973                platform,
974                &state,
975            )?;
976            let journal = FleetEnsureJournalRecord {
977                funding_reviews: Vec::new(),
978                successor_phases: Vec::new(),
979                completion: FleetEnsureCompletion::InProgress,
980                estate_funding_required: None,
981                effects: Vec::new(),
982                fleet: requested_fleet.to_string(),
983                initial_controlled_cycles,
984                initial_estate_funding_cycles_by_root: retained_plan
985                    .conservation
986                    .estate_funding_domains
987                    .iter()
988                    .map(|domain| {
989                        (
990                            domain.root.clone(),
991                            domain.available_cycles.unwrap_or_default(),
992                        )
993                    })
994                    .collect(),
995                initial_operator_cycles: observation.operator_cycles,
996                operation_id: retained_plan.operation_id.clone(),
997                plan_sha256: retained_plan.plan_sha256.clone(),
998                schema_version: FLEET_ENSURE_SCHEMA_VERSION,
999                stalled_observations: 0,
1000            };
1001            // Cross the nonterminal boundary before any effect-owned state can be retained.
1002            if adopting {
1003                crate::fleet_ensure::ops::reinstall::adoption::adopt(
1004                    &paths,
1005                    &retained_plan,
1006                    &journal,
1007                )?;
1008            } else {
1009                write_journal(&paths, &journal)?;
1010            }
1011            journal
1012        }
1013    };
1014
1015    funding::resume(
1016        &paths,
1017        &retained_plan,
1018        &mut journal,
1019        &state,
1020        reviewed_plan_sha256,
1021        platform,
1022    )?;
1023    if journal
1024        .estate_funding_required
1025        .as_ref()
1026        .is_some_and(|pause| funding::covered(&journal, pause))
1027    {
1028        retain_estate_funding_pause(&paths, &mut journal, None)?;
1029    }
1030    let mut terminal_state;
1031    loop {
1032        let execution_actions = continuation::actions(&retained_plan, &journal)
1033            .into_iter()
1034            .cloned()
1035            .collect::<Vec<_>>();
1036        let actions = execution_actions.iter().collect::<Vec<_>>();
1037        let mut replayed_issued_commands = BTreeSet::new();
1038        let mut reported_phase = None;
1039        loop {
1040            let mut deferred_controller_observation = false;
1041            for (index, action) in actions.iter().enumerate() {
1042                let action_hash = action_sha256(action);
1043                let observation_policy = effect_observation_policy(operation_desired, action)?;
1044                let retained_effect = journal.effects.get(index);
1045                if retained_effect.is_none_or(|effect| effect.state != EffectState::Applied) {
1046                    let phase = action_progress_phase(action);
1047                    if reported_phase != Some(phase) {
1048                        report_progress(platform, &retained_plan, &journal, phase);
1049                        reported_phase = Some(phase);
1050                    }
1051                }
1052                if matches!(action, EnsureAction::FleetProtocol { .. })
1053                    && retained_effect.is_none_or(|effect| effect.state == EffectState::Intent)
1054                    && !prior_fleet_protocol_effect_started(&actions, &journal, index)
1055                {
1056                    let funding_observation = platform
1057                        .observe(&journal.operation_id, &state)
1058                        .map_err(EnsureWorkflowError::Platform)?;
1059                    let funded_plan = funding_plan(&retained_plan, &journal)?;
1060                    let required =
1061                        estate_funding_requirement(&funded_plan, &state, &funding_observation)?;
1062                    retain_estate_funding_pause(&paths, &mut journal, required.as_ref())?;
1063                    if let Some(required) = required {
1064                        report_progress_state(
1065                            platform,
1066                            &retained_plan,
1067                            &journal,
1068                            action_progress_phase(action),
1069                            FleetEnsureProgressState::FundingRequired,
1070                        );
1071                        return Err(EnsureWorkflowError::EstateFundingRequired(Box::new(
1072                            estate_funding_error(&required),
1073                        )));
1074                    }
1075                }
1076                if journal.effects.len() <= index {
1077                    let pre_cycles = platform
1078                        .action_cycles(action, &state)
1079                        .map_err(EnsureWorkflowError::Platform)?;
1080                    let destination_pre_cycles = platform
1081                        .action_destination_cycles(action, &state)
1082                        .map_err(EnsureWorkflowError::Platform)?;
1083                    let pre_canister_version = platform
1084                        .action_canister_version(action, &state)
1085                        .map_err(EnsureWorkflowError::Platform)?;
1086                    journal.effects.push(EffectRecord {
1087                        maintenance_attempts: 0,
1088                        action_sha256: action_hash.clone(),
1089                        created_principal: None,
1090                        destination_post_cycles: destination_pre_cycles,
1091                        destination_pre_cycles,
1092                        post_cycles: None,
1093                        pre_cycles,
1094                        pre_canister_version,
1095                        progress_identity: None,
1096                        receipt: None,
1097                        state: EffectState::Intent,
1098                    });
1099                    write_journal(&paths, &journal)?;
1100                }
1101                loop {
1102                    let record = journal
1103                        .effects
1104                        .get(index)
1105                        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1106                    if record.action_sha256 != action_hash {
1107                        return Err(EnsureWorkflowError::DriftedBeforeApply);
1108                    }
1109                    if !create_record_is_exact(action, record, &state) {
1110                        return Err(EnsureWorkflowError::JournalIntegrity);
1111                    }
1112                    if matches!(record.state, EffectState::Applied) {
1113                        let retained_create = journal
1114                            .effects
1115                            .get(index)
1116                            .ok_or(EnsureWorkflowError::JournalIntegrity)
1117                            .and_then(|record| {
1118                                retain_applied_create_authority(
1119                                    operation_desired,
1120                                    action,
1121                                    record,
1122                                    &mut state,
1123                                )
1124                            })?;
1125                        let retained_funding = journal.effects.get(index).is_some_and(|record| {
1126                            retain_applied_funding_cycles(&mut state, action, record)
1127                        });
1128                        if retained_create || retained_funding {
1129                            write_state(&paths, &state)?;
1130                        }
1131                        break;
1132                    }
1133
1134                    let observed = platform
1135                        .observe_effect(&journal.operation_id, action, record, &state)
1136                        .map_err(EnsureWorkflowError::Platform)?;
1137                    let source_cycles = if observed.post_cycles.is_some() {
1138                        observed.post_cycles
1139                    } else {
1140                        platform
1141                            .action_cycles(action, &state)
1142                            .map_err(EnsureWorkflowError::Platform)?
1143                    };
1144                    let destination_cycles = platform
1145                        .action_destination_cycles(action, &state)
1146                        .map_err(EnsureWorkflowError::Platform)?;
1147                    let replan_after_create_balance =
1148                        observed.retry == EffectRetry::ReplanRequiredAfterCreateBalanceDrift;
1149                    let retained_create_balance_is_exact = !replan_after_create_balance
1150                        || (journal.effects.len() == index + 1
1151                            && journal.effects[..index]
1152                                .iter()
1153                                .all(|effect| effect.state == EffectState::Applied)
1154                            && record.state == EffectState::Issued
1155                            && create_identity_is_exact(action.name(), record, &state)
1156                            && observed.post_cycles.is_some()
1157                            && observed.post_cycles == source_cycles);
1158                    let protocol_funding_required = if record.state == EffectState::Issued {
1159                        observed
1160                            .estate_funding_required
1161                            .as_ref()
1162                            .map(|funding| {
1163                                protocol_estate_funding_requirement(&retained_plan, &state, funding)
1164                            })
1165                            .transpose()?
1166                    } else {
1167                        None
1168                    };
1169                    if let Some(required) = protocol_funding_required.as_ref()
1170                        && !funding::covered(&journal, required)
1171                    {
1172                        retain_estate_funding_pause(&paths, &mut journal, Some(required))?;
1173                        report_progress_state(
1174                            platform,
1175                            &retained_plan,
1176                            &journal,
1177                            action_progress_phase(action),
1178                            FleetEnsureProgressState::FundingRequired,
1179                        );
1180                        return Err(EnsureWorkflowError::EstateFundingRequired(Box::new(
1181                            estate_funding_error(required),
1182                        )));
1183                    }
1184                    if journal
1185                        .estate_funding_required
1186                        .as_ref()
1187                        .is_some_and(|required| required.pending_creation_operation_id.is_some())
1188                    {
1189                        retain_estate_funding_pause(&paths, &mut journal, None)?;
1190                    }
1191                    let record = journal
1192                        .effects
1193                        .get_mut(index)
1194                        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1195                    merge_observed_cycles(record, source_cycles, destination_cycles);
1196                    if replan_after_create_balance {
1197                        if !retained_create_balance_is_exact {
1198                            return Err(EnsureWorkflowError::JournalIntegrity);
1199                        }
1200                        let EnsureAction::Create {
1201                            name,
1202                            requested_initial_cycles,
1203                            ..
1204                        } = action
1205                        else {
1206                            return Err(EnsureWorkflowError::JournalIntegrity);
1207                        };
1208                        let actual_cycles = record
1209                            .post_cycles
1210                            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1211                        record.progress_identity = Some(observed.progress_identity);
1212                        record.state = EffectState::Applied;
1213                        retain_created_canister_for_replan(
1214                            operation_desired,
1215                            name,
1216                            record,
1217                            &mut state,
1218                        )?;
1219                        journal.completion = FleetEnsureCompletion::ReplanRequired;
1220                        journal.stalled_observations = 0;
1221                        write_journal(&paths, &journal)?;
1222                        write_state(&paths, &state)?;
1223                        let configured_fee_cycles = operation_desired
1224                            .management_creation_fee_cycles
1225                            .parse::<Cycles>()
1226                            .map(|cycles| cycles.to_u128())
1227                            .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
1228                        return Err(EnsureWorkflowError::ReplanRequiredAfterCreateBalanceDrift {
1229                            actual_cycles,
1230                            canister: name.clone(),
1231                            configured_fee_cycles,
1232                            deficit_cycles: requested_initial_cycles.saturating_sub(actual_cycles),
1233                            requested_cycles: *requested_initial_cycles,
1234                        });
1235                    }
1236                    if observed.applied {
1237                        record.progress_identity = Some(observed.progress_identity);
1238                        record.state = EffectState::Applied;
1239                        journal.stalled_observations = 0;
1240                        write_journal(&paths, &journal)?;
1241                        let record = journal
1242                            .effects
1243                            .get(index)
1244                            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1245                        let retained_create = retain_applied_create_authority(
1246                            operation_desired,
1247                            action,
1248                            record,
1249                            &mut state,
1250                        )?;
1251                        let retained_funding =
1252                            retain_applied_funding_cycles(&mut state, action, record);
1253                        if retained_create || retained_funding {
1254                            write_state(&paths, &state)?;
1255                        }
1256                        break;
1257                    }
1258
1259                    if observed.retry == EffectRetry::DeferUntilControllerObservation {
1260                        if !deferred_create_observation_is_exact(
1261                            operation_desired,
1262                            &actions,
1263                            index,
1264                            action,
1265                            record,
1266                            &state,
1267                        ) {
1268                            return Err(EnsureWorkflowError::JournalIntegrity);
1269                        }
1270                        let progress_identity = observed.progress_identity.clone();
1271                        if record.progress_identity.as_deref() == Some(&observed.progress_identity)
1272                        {
1273                            journal.stalled_observations =
1274                                journal.stalled_observations.saturating_add(1);
1275                        } else {
1276                            record.progress_identity = Some(observed.progress_identity);
1277                            journal.stalled_observations = 0;
1278                        }
1279                        write_journal(&paths, &journal)?;
1280                        if journal.stalled_observations
1281                            >= observation_policy.maximum_stalled_observations
1282                        {
1283                            return Err(EnsureWorkflowError::Stalled {
1284                                action: action.name().to_string(),
1285                                observations: journal.stalled_observations,
1286                                progress_identity,
1287                            });
1288                        }
1289                        deferred_controller_observation = true;
1290                        break;
1291                    }
1292
1293                    let maintenance_continuation =
1294                        observed.retry == EffectRetry::ContinuePoolMaintenance;
1295                    if matches!(record.state, EffectState::Intent) || maintenance_continuation {
1296                        if let EnsureAction::FleetProtocol {
1297                            action: current, ..
1298                        } = action
1299                            && let CurrentFleetProtocolAction::MaintainPoolReadiness {
1300                                maximum_updates,
1301                                ..
1302                            } = current.as_ref()
1303                        {
1304                            if record.maintenance_attempts >= *maximum_updates {
1305                                return Err(EnsureWorkflowError::PoolMaintenanceBound {
1306                                    action: action.name().to_string(),
1307                                    maximum_updates: *maximum_updates,
1308                                });
1309                            }
1310                            if record.progress_identity.as_deref()
1311                                == Some(&observed.progress_identity)
1312                            {
1313                                journal.stalled_observations =
1314                                    journal.stalled_observations.saturating_add(1);
1315                            } else {
1316                                journal.stalled_observations = 0;
1317                            }
1318                            record.maintenance_attempts += 1;
1319                            let attempts = record.maintenance_attempts;
1320                            // A failed or lost call consumes its reviewed attempt across process restart.
1321                            write_journal(&paths, &journal)?;
1322                            if attempts > 1 {
1323                                platform
1324                                    .pace_effect_observation(action, journal.stalled_observations);
1325                            }
1326                        } else if maintenance_continuation {
1327                            return Err(EnsureWorkflowError::JournalIntegrity);
1328                        }
1329                        let record = journal
1330                            .effects
1331                            .get(index)
1332                            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1333                        if retained_plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite {
1334                            root_reinstall::verify_effect_authority(
1335                                &retained_plan,
1336                                action,
1337                                &state,
1338                                platform,
1339                            )?;
1340                        }
1341                        verify_store_activation_binding(&retained_plan, action, &state, platform)?;
1342                        reinstall::verify_effect_authority(
1343                            &retained_plan,
1344                            action,
1345                            &state,
1346                            platform,
1347                        )?;
1348                        let outcome =
1349                            match platform.apply(&journal.operation_id, action, record, &state) {
1350                                Ok(outcome) => outcome,
1351                                Err(source) => {
1352                                    let progress_identity = record
1353                                        .progress_identity
1354                                        .clone()
1355                                        .unwrap_or_else(|| "effect-call-failed".to_string());
1356                                    journal.stalled_observations =
1357                                        journal.stalled_observations.saturating_add(1);
1358                                    write_journal(&paths, &journal)?;
1359                                    if journal.stalled_observations
1360                                        >= observation_policy.maximum_stalled_observations
1361                                    {
1362                                        return Err(EnsureWorkflowError::Stalled {
1363                                            action: action.name().to_string(),
1364                                            observations: journal.stalled_observations,
1365                                            progress_identity,
1366                                        });
1367                                    }
1368                                    return Err(EnsureWorkflowError::Platform(source));
1369                                }
1370                            };
1371                        if !create_outcome_is_exact(action, record, &state, &outcome) {
1372                            return Err(EnsureWorkflowError::JournalIntegrity);
1373                        }
1374                        if let Some(created) = &outcome.created_principal {
1375                            state
1376                                .pending_principals
1377                                .insert(action.name().to_string(), created.clone());
1378                            write_state(&paths, &state)?;
1379                        }
1380                        let record = journal
1381                            .effects
1382                            .get_mut(index)
1383                            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1384                        record.created_principal = outcome.created_principal;
1385                        record.receipt = outcome.receipt;
1386                        record.post_cycles = outcome.post_cycles;
1387                        record.progress_identity = Some(observed.progress_identity);
1388                        record.state = EffectState::Issued;
1389                        // A successful maintenance call can report no progress while Root work is in flight.
1390                        if !maintenance_continuation {
1391                            journal.stalled_observations = 0;
1392                        }
1393                        write_journal(&paths, &journal)?;
1394                        continue;
1395                    }
1396
1397                    if observed.retry == EffectRetry::ReplayExactIssuedCommand
1398                        && !replayed_issued_commands.contains(&index)
1399                    {
1400                        if !matches!(
1401                            action,
1402                            EnsureAction::FleetProtocol { action, .. }
1403                                if matches!(
1404                                    action.as_ref(),
1405                                    crate::fleet_ensure::model::CurrentFleetProtocolAction::ProvisionComponents { .. }
1406                                )
1407                        ) {
1408                            return Err(EnsureWorkflowError::JournalIntegrity);
1409                        }
1410                        let retained_receipt = record.receipt.clone();
1411                        let outcome = platform
1412                            .apply(&journal.operation_id, action, record, &state)
1413                            .map_err(EnsureWorkflowError::Platform)?;
1414                        if outcome.created_principal.is_some()
1415                            || outcome.receipt != retained_receipt
1416                            || outcome.post_cycles.is_some()
1417                        {
1418                            return Err(EnsureWorkflowError::JournalIntegrity);
1419                        }
1420                        let record = journal
1421                            .effects
1422                            .get_mut(index)
1423                            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1424                        record.progress_identity = Some(observed.progress_identity);
1425                        journal.stalled_observations = 0;
1426                        write_journal(&paths, &journal)?;
1427                        replayed_issued_commands.insert(index);
1428                        continue;
1429                    }
1430
1431                    if record.progress_identity.as_deref() == Some(&observed.progress_identity) {
1432                        journal.stalled_observations =
1433                            journal.stalled_observations.saturating_add(1);
1434                    } else {
1435                        record.progress_identity = Some(observed.progress_identity);
1436                        journal.stalled_observations = 0;
1437                    }
1438                    let progress_identity = record
1439                        .progress_identity
1440                        .clone()
1441                        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1442                    write_journal(&paths, &journal)?;
1443                    if journal.stalled_observations
1444                        >= observation_policy.maximum_stalled_observations
1445                    {
1446                        if let Some((failure, origin)) = observed
1447                            .provisioning_failure
1448                            .and_then(|failure| failure.origin.map(|origin| (failure, origin)))
1449                        {
1450                            return Err(EnsureWorkflowError::ProvisioningRetryPending {
1451                                root: failure.fleet_subnet_root,
1452                                stage: origin.stage,
1453                                target: origin.target,
1454                                operation_id: origin.operation_id,
1455                                diagnostic_code: origin.diagnostic_code,
1456                                observations: journal.stalled_observations,
1457                                failed_at_ns: origin.failed_at_ns,
1458                            });
1459                        }
1460                        return Err(EnsureWorkflowError::Stalled {
1461                            action: action.name().to_string(),
1462                            observations: journal.stalled_observations,
1463                            progress_identity,
1464                        });
1465                    }
1466                    if observation_policy.paced {
1467                        report_progress_state(
1468                            platform,
1469                            &retained_plan,
1470                            &journal,
1471                            action_progress_phase(action),
1472                            FleetEnsureProgressState::AwaitingProgress,
1473                        );
1474                        platform.pace_effect_observation(action, journal.stalled_observations);
1475                    }
1476                }
1477            }
1478            if journal.effects.len() == actions.len()
1479                && journal
1480                    .effects
1481                    .iter()
1482                    .all(|record| record.state == EffectState::Applied)
1483            {
1484                break;
1485            }
1486            if !deferred_controller_observation {
1487                return Err(EnsureWorkflowError::JournalIntegrity);
1488            }
1489        }
1490
1491        report_progress(
1492            platform,
1493            &retained_plan,
1494            &journal,
1495            FleetEnsurePhase::TerminalVerification,
1496        );
1497        if retained_plan.scope == FleetEnsurePlanScope::ReinstallPreparation {
1498            let actual = reinstall::complete(&retained_plan, &journal, &state, platform)?;
1499            journal.completion = FleetEnsureCompletion::Prepared;
1500            journal.stalled_observations = 0;
1501            write_journal(&paths, &journal)?;
1502            report_progress_state(
1503                platform,
1504                &retained_plan,
1505                &journal,
1506                FleetEnsurePhase::Infrastructure,
1507                FleetEnsureProgressState::PrerequisiteComplete,
1508            );
1509            return Ok(FleetEnsureReport {
1510                funding_review: None,
1511                actual_conservation: Some(actual),
1512                effects_applied: applied_count(&journal),
1513                plan: retained_plan,
1514                terminal: true,
1515            });
1516        }
1517        if retained_plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite {
1518            let actual = root_reinstall::complete(&retained_plan, &journal, &state, platform)?;
1519            publish_terminal_state(operation_desired, &retained_plan, &journal, &mut state)
1520                .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
1521            state.active_registry = None;
1522            write_state(&paths, &state)?;
1523            journal.completion = FleetEnsureCompletion::Converged;
1524            journal.stalled_observations = 0;
1525            write_journal(&paths, &journal)?;
1526            report_progress_state(
1527                platform,
1528                &retained_plan,
1529                &journal,
1530                FleetEnsurePhase::Infrastructure,
1531                FleetEnsureProgressState::PrerequisiteComplete,
1532            );
1533            return Ok(FleetEnsureReport {
1534                funding_review: None,
1535                actual_conservation: Some(actual),
1536                effects_applied: applied_count(&journal),
1537                plan: retained_plan,
1538                terminal: true,
1539            });
1540        }
1541        if retained_plan.scope == FleetEnsurePlanScope::RootStartPrerequisite {
1542            let actual_conservation = complete_root_start_prerequisite(
1543                operation_desired,
1544                &retained_plan,
1545                platform,
1546                &state,
1547                &journal,
1548            )?;
1549            journal.completion = FleetEnsureCompletion::Converged;
1550            journal.stalled_observations = 0;
1551            write_journal(&paths, &journal)?;
1552            report_progress_state(
1553                platform,
1554                &retained_plan,
1555                &journal,
1556                FleetEnsurePhase::Infrastructure,
1557                FleetEnsureProgressState::PrerequisiteComplete,
1558            );
1559            return Ok(FleetEnsureReport {
1560                funding_review: None,
1561                actual_conservation: Some(actual_conservation),
1562                effects_applied: applied_count(&journal),
1563                plan: retained_plan,
1564                terminal: true,
1565            });
1566        }
1567
1568        terminal_state = state.clone();
1569        publish_terminal_state(
1570            operation_desired,
1571            &retained_plan,
1572            &journal,
1573            &mut terminal_state,
1574        )
1575        .map_err(|TerminalStatePublicationError::JournalIntegrity| {
1576            EnsureWorkflowError::JournalIntegrity
1577        })?;
1578        if completed_infrastructure_reinstall(operation_desired, &retained_plan, &journal) {
1579            terminal_state.active_registry = None;
1580        }
1581        project_current_fleet_inventory(&terminal_state)?;
1582        let mut terminal_observation = platform
1583            .observe(&retained_plan.operation_id, &terminal_state)
1584            .map_err(EnsureWorkflowError::Platform)?;
1585        {
1586            let artifacts = resolve_desired_artifacts(root, operation_desired)?;
1587            let converged = loop {
1588                let protocol_actions = platform
1589                    .protocol_actions(&retained_plan.operation_id, &terminal_state)
1590                    .map_err(EnsureWorkflowError::Platform)?;
1591                match compile_plan(
1592                    operation_desired,
1593                    &artifacts,
1594                    &protocol_actions,
1595                    operation_desired_sha256,
1596                    requested_fleet,
1597                    &terminal_observation,
1598                    retained_plan.planned_at_time,
1599                    &retained_plan.operation_id,
1600                    None,
1601                ) {
1602                    Ok(plan) => {
1603                        if journal.stalled_observations != 0 {
1604                            journal.stalled_observations = 0;
1605                            write_journal(&paths, &journal)?;
1606                        }
1607                        break plan;
1608                    }
1609                    Err(EnsurePolicyError::PendingRootOwnedBalance { name }) => {
1610                        journal.stalled_observations =
1611                            journal.stalled_observations.saturating_add(1);
1612                        retain_observed_cycles(&mut state, &terminal_observation);
1613                        write_state(&paths, &state)?;
1614                        write_journal(&paths, &journal)?;
1615                        if journal.stalled_observations
1616                            >= operation_desired.maximum_stalled_observations
1617                        {
1618                            return Err(EnsureWorkflowError::RootOwnedObservationStalled {
1619                                last_lifecycle: root_owned_lifecycle_label(
1620                                    &terminal_observation,
1621                                    &name,
1622                                )
1623                                .to_string(),
1624                                observations: journal.stalled_observations,
1625                                target: name,
1626                            });
1627                        }
1628                        report_progress_state(
1629                            platform,
1630                            &retained_plan,
1631                            &journal,
1632                            FleetEnsurePhase::TerminalVerification,
1633                            FleetEnsureProgressState::AwaitingProgress,
1634                        );
1635                        platform.pace_root_owned_observation(&name, journal.stalled_observations);
1636                        terminal_observation = platform
1637                            .observe(&retained_plan.operation_id, &terminal_state)
1638                            .map_err(EnsureWorkflowError::Platform)?;
1639                    }
1640                    Err(error @ EnsurePolicyError::EstatePoolCapacity { .. }) => {
1641                        retain_observed_cycles(&mut terminal_state, &terminal_observation);
1642                        retain_completed_reinstalls(&mut terminal_state, &retained_plan, &journal);
1643                        write_state(&paths, &terminal_state)?;
1644                        journal.completion = FleetEnsureCompletion::ReplanRequired;
1645                        journal.stalled_observations = 0;
1646                        write_journal(&paths, &journal)?;
1647                        return Err(error.into());
1648                    }
1649                    Err(error) => return Err(error.into()),
1650                }
1651            };
1652            if converged
1653                .canisters
1654                .iter()
1655                .any(|canister| !canister.actions.is_empty())
1656                || !converged.protocol_actions.is_empty()
1657            {
1658                if retained_plan.continuation.is_some() {
1659                    let appended = continuation::append(
1660                        &paths,
1661                        operation_desired,
1662                        &retained_plan,
1663                        &mut journal,
1664                        &terminal_state,
1665                        &terminal_observation,
1666                        converged,
1667                        platform,
1668                    );
1669                    match appended {
1670                        Ok(()) => {
1671                            write_state(&paths, &terminal_state)?;
1672                            state = terminal_state.clone();
1673                            report_progress_state(
1674                                platform,
1675                                &retained_plan,
1676                                &journal,
1677                                FleetEnsurePhase::TerminalVerification,
1678                                FleetEnsureProgressState::PrerequisiteComplete,
1679                            );
1680                            continue;
1681                        }
1682                        Err(EnsureWorkflowError::SuccessorReviewRequired { reason, review }) => {
1683                            retain_observed_cycles(&mut terminal_state, &terminal_observation);
1684                            write_state(&paths, &terminal_state)?;
1685                            journal.completion = FleetEnsureCompletion::ReplanRequired;
1686                            write_journal(&paths, &journal)?;
1687                            report_progress_state(
1688                                platform,
1689                                &retained_plan,
1690                                &journal,
1691                                FleetEnsurePhase::TerminalVerification,
1692                                FleetEnsureProgressState::ReviewRequired {
1693                                    reason,
1694                                    review: review.clone(),
1695                                },
1696                            );
1697                            return Err(EnsureWorkflowError::SuccessorReviewRequired {
1698                                reason,
1699                                review,
1700                            });
1701                        }
1702                        Err(error) => return Err(error),
1703                    }
1704                }
1705                retain_observed_cycles(&mut terminal_state, &terminal_observation);
1706                retain_completed_reinstalls(&mut terminal_state, &retained_plan, &journal);
1707                write_state(&paths, &terminal_state)?;
1708                journal.completion = FleetEnsureCompletion::ReplanRequired;
1709                journal.stalled_observations = 0;
1710                write_journal(&paths, &journal)?;
1711                let reason =
1712                    crate::fleet_ensure::model::FleetEnsureSuccessorReviewReason::AdditionalEffect;
1713                let review = Some(Box::new(
1714                    crate::fleet_ensure::ops::recovery::review_details(&converged),
1715                ));
1716                report_progress_state(
1717                    platform,
1718                    &retained_plan,
1719                    &journal,
1720                    FleetEnsurePhase::TerminalVerification,
1721                    FleetEnsureProgressState::ReviewRequired {
1722                        reason,
1723                        review: review.clone(),
1724                    },
1725                );
1726                return Err(EnsureWorkflowError::SuccessorReviewRequired { reason, review });
1727            }
1728        }
1729        break;
1730    }
1731    let terminal_inventory_operation_id =
1732        completed_inventory_operation(&retained_plan, &journal, &terminal_state)?;
1733    let terminal_inventory = platform
1734        .terminal_inventory(terminal_inventory_operation_id, &terminal_state)
1735        .map_err(EnsureWorkflowError::Platform)?;
1736    let terminal_cycles = terminal_inventory.controlled_cycles_by_principal.clone();
1737    merge_terminal_inventory(&mut terminal_state, terminal_inventory)?;
1738    project_current_fleet_inventory(&terminal_state)?;
1739    let mut final_observation = platform
1740        .observe(&retained_plan.operation_id, &terminal_state)
1741        .map_err(EnsureWorkflowError::Platform)?;
1742    attach_terminal_cycles(&mut final_observation, terminal_cycles)?;
1743    reinstall::verify_terminal_estate(&retained_plan, &final_observation)?;
1744    reinstall::verify_terminal_authority(&retained_plan, &terminal_state, platform)?;
1745    let actual_conservation = verify_terminal_conservation(
1746        &retained_plan,
1747        &journal,
1748        &terminal_state,
1749        &final_observation,
1750    )?;
1751    terminal_state.completed_reinstall_action_sha256.clear();
1752    terminal_state.completed_reinstall_operation_id = None;
1753    terminal_state.completed_reinstalls.clear();
1754    // Publish the fully validated topology before making its terminal journal visible to backup.
1755    write_state(&paths, &terminal_state)?;
1756    journal.completion = FleetEnsureCompletion::Converged;
1757    journal.stalled_observations = 0;
1758    write_journal(&paths, &journal)?;
1759    report_progress(
1760        platform,
1761        &retained_plan,
1762        &journal,
1763        FleetEnsurePhase::Complete,
1764    );
1765    Ok(FleetEnsureReport {
1766        funding_review: None,
1767        actual_conservation: Some(actual_conservation),
1768        effects_applied: applied_count(&journal),
1769        plan: retained_plan,
1770        terminal: true,
1771    })
1772}
1773
1774fn action_progress_phase(action: &EnsureAction) -> FleetEnsurePhase {
1775    match action {
1776        EnsureAction::FleetProtocol { action, .. } => match action.as_ref() {
1777            CurrentFleetProtocolAction::ReconcilePoolAsset { .. } => {
1778                FleetEnsurePhase::ImportReconciliation
1779            }
1780            CurrentFleetProtocolAction::ProvisionComponents { .. } => {
1781                FleetEnsurePhase::WorkloadProvisioning
1782            }
1783            CurrentFleetProtocolAction::MaintainPoolReadiness { .. }
1784            | CurrentFleetProtocolAction::ObservePoolReadiness { .. } => {
1785                FleetEnsurePhase::PoolReadiness
1786            }
1787            _ => FleetEnsurePhase::ControlPlane,
1788        },
1789        _ => FleetEnsurePhase::Infrastructure,
1790    }
1791}
1792
1793fn report_progress<P: EnsurePlatform>(
1794    platform: &mut P,
1795    plan: &FleetEnsurePlan,
1796    journal: &FleetEnsureJournalRecord,
1797    phase: FleetEnsurePhase,
1798) {
1799    let state = if phase == FleetEnsurePhase::Complete {
1800        FleetEnsureProgressState::Complete
1801    } else {
1802        FleetEnsureProgressState::Advancing
1803    };
1804    report_progress_state(platform, plan, journal, phase, state);
1805}
1806
1807fn report_progress_state<P: EnsurePlatform>(
1808    platform: &mut P,
1809    plan: &FleetEnsurePlan,
1810    journal: &FleetEnsureJournalRecord,
1811    phase: FleetEnsurePhase,
1812    state: FleetEnsureProgressState,
1813) {
1814    platform.report_progress(FleetEnsureProgress {
1815        state,
1816        operation_id: plan.operation_id.clone(),
1817        plan_sha256: plan.plan_sha256.clone(),
1818        phase,
1819        applied_effects: applied_count(journal),
1820        reviewed_effects: continuation::actions(plan, journal).len()
1821            + journal
1822                .funding_reviews
1823                .iter()
1824                .filter(|review| review.effect.is_some())
1825                .count(),
1826    });
1827}
1828
1829const fn merge_observed_cycles(
1830    record: &mut EffectRecord,
1831    source_cycles: Option<u128>,
1832    destination_cycles: Option<u128>,
1833) {
1834    if let Some(cycles) = source_cycles {
1835        record.post_cycles = Some(cycles);
1836    }
1837    if let Some(cycles) = destination_cycles {
1838        record.destination_post_cycles = Some(cycles);
1839    }
1840}
1841
1842pub(super) fn retain_applied_create_authority<E>(
1843    desired: &crate::fleet_ensure::model::DesiredFleet,
1844    action: &EnsureAction,
1845    record: &EffectRecord,
1846    state: &mut FleetEnsureStateRecord,
1847) -> Result<bool, EnsureWorkflowError<E>>
1848where
1849    E: std::error::Error + 'static,
1850{
1851    let EnsureAction::Create {
1852        name,
1853        requested_initial_cycles,
1854        ..
1855    } = action
1856    else {
1857        return Ok(false);
1858    };
1859    if record.state != EffectState::Applied {
1860        return Err(EnsureWorkflowError::JournalIntegrity);
1861    }
1862    let configured = desired
1863        .canisters
1864        .iter()
1865        .find(|canister| {
1866            canister.name == *name
1867                && canister.presence == crate::fleet_ensure::model::DesiredPresence::Present
1868        })
1869        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1870    let principal = record
1871        .created_principal
1872        .as_deref()
1873        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1874    let cycles = record
1875        .post_cycles
1876        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1877    let maximum_observation_burn_cycles =
1878        crate::fleet_ensure::ops::maximum_creation_observation_burn(desired)
1879            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1880    if !create_identity_is_exact(name, record, state)
1881        || !create_balance_is_terminal(
1882            Some(cycles),
1883            *requested_initial_cycles,
1884            maximum_observation_burn_cycles,
1885        )
1886        || state
1887            .retained_cycles_by_principal
1888            .get(principal)
1889            .is_some_and(|retained| *retained != cycles)
1890    {
1891        return Err(EnsureWorkflowError::JournalIntegrity);
1892    }
1893    validate_fresh_pool_creation_balance(
1894        desired,
1895        configured,
1896        name,
1897        cycles,
1898        maximum_observation_burn_cycles,
1899        *requested_initial_cycles,
1900    )?;
1901
1902    let prior_topology = state.topology.get(name);
1903    let topology_identity_matches = prior_topology.is_some_and(|topology| {
1904        topology.kind == configured.kind
1905            && topology.parent == configured.parent
1906            && topology.protocol_binding == configured.protocol_binding
1907            && topology.role
1908                == configured
1909                    .protocol_binding
1910                    .as_ref()
1911                    .map(|binding| binding.role.to_string())
1912    });
1913    let topology = crate::fleet_ensure::model::FleetEnsureTopologyRecord {
1914        kind: configured.kind,
1915        module_hash: if topology_identity_matches {
1916            prior_topology.and_then(|topology| topology.module_hash.clone())
1917        } else {
1918            None
1919        },
1920        parent: configured.parent.clone(),
1921        protocol_binding: configured.protocol_binding.clone(),
1922        role: configured
1923            .protocol_binding
1924            .as_ref()
1925            .map(|binding| binding.role.to_string()),
1926    };
1927    let changed = state.retained_cycles_by_principal.get(principal) != Some(&cycles)
1928        || state.topology.get(name) != Some(&topology);
1929    state
1930        .retained_cycles_by_principal
1931        .insert(principal.to_string(), cycles);
1932    state.topology.insert(name.clone(), topology);
1933    Ok(changed)
1934}
1935
1936fn validate_fresh_pool_creation_balance<E>(
1937    desired: &crate::fleet_ensure::model::DesiredFleet,
1938    configured: &crate::fleet_ensure::model::DesiredCanister,
1939    name: &str,
1940    cycles: u128,
1941    maximum_observation_burn_cycles: u128,
1942    requested_initial_cycles: u128,
1943) -> Result<(), EnsureWorkflowError<E>>
1944where
1945    E: std::error::Error + 'static,
1946{
1947    if configured.kind == crate::fleet_ensure::model::DesiredCanisterKind::Pool
1948        && desired
1949            .bootstrap
1950            .as_ref()
1951            .is_some_and(|bootstrap| bootstrap.fresh_estate)
1952        && configured.principal.is_none()
1953        && configured.controllers.is_empty()
1954        && configured.parent.as_ref().is_some_and(|parent| {
1955            configured.controller_canisters.as_slice() == std::slice::from_ref(parent)
1956        })
1957    {
1958        let readiness_floor_cycles = configured
1959            .minimum_cycles
1960            .parse::<Cycles>()
1961            .map(|cycles| cycles.to_u128())
1962            .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
1963        let remaining_controller_burn_cycles = desired
1964            .maximum_update_burn_cycles
1965            .parse::<Cycles>()
1966            .map(|cycles| cycles.to_u128())
1967            .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
1968        let required_pre_finalization_balance_cycles = readiness_floor_cycles
1969            .checked_add(remaining_controller_burn_cycles)
1970            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
1971        if cycles < required_pre_finalization_balance_cycles {
1972            return Err(EnsureWorkflowError::FreshPoolCreationUnderfunded(Box::new(
1973                FreshPoolCreationUnderfundedError {
1974                    canister: name.to_string(),
1975                    live_balance_cycles: cycles,
1976                    maximum_observation_burn_cycles,
1977                    pre_finalization_shortfall_cycles: required_pre_finalization_balance_cycles
1978                        - cycles,
1979                    readiness_floor_cycles,
1980                    readiness_shortfall_cycles: readiness_floor_cycles.saturating_sub(cycles),
1981                    remaining_controller_burn_cycles,
1982                    requested_creation_funding_cycles: requested_initial_cycles,
1983                    required_pre_finalization_balance_cycles,
1984                },
1985            )));
1986        }
1987    }
1988    Ok(())
1989}
1990
1991fn retain_applied_funding_cycles(
1992    state: &mut FleetEnsureStateRecord,
1993    action: &EnsureAction,
1994    record: &EffectRecord,
1995) -> bool {
1996    let EnsureAction::Fund { principal, .. } = action else {
1997        return false;
1998    };
1999    if record.state != EffectState::Applied || record.receipt.is_none() {
2000        return false;
2001    }
2002    let Some(cycles) = record.post_cycles else {
2003        return false;
2004    };
2005    let principal = principal.strip_prefix("created:").map_or_else(
2006        || Some(principal.as_str()),
2007        |name| state.pending_principals.get(name).map(String::as_str),
2008    );
2009    let Some(principal) = principal else {
2010        return false;
2011    };
2012    if state.retained_cycles_by_principal.get(principal) == Some(&cycles) {
2013        return false;
2014    }
2015    state
2016        .retained_cycles_by_principal
2017        .insert(principal.to_string(), cycles);
2018    true
2019}
2020
2021fn retain_created_canister_for_replan<E>(
2022    desired: &crate::fleet_ensure::model::DesiredFleet,
2023    name: &str,
2024    record: &EffectRecord,
2025    state: &mut FleetEnsureStateRecord,
2026) -> Result<(), EnsureWorkflowError<E>>
2027where
2028    E: std::error::Error + 'static,
2029{
2030    let configured = desired
2031        .canisters
2032        .iter()
2033        .find(|canister| {
2034            canister.name == name
2035                && canister.presence == crate::fleet_ensure::model::DesiredPresence::Present
2036        })
2037        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2038    let principal = record
2039        .created_principal
2040        .as_deref()
2041        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2042    let cycles = record
2043        .post_cycles
2044        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2045    if state.pending_principals.get(name).map(String::as_str) != Some(principal) {
2046        return Err(EnsureWorkflowError::JournalIntegrity);
2047    }
2048    state.pending_principals.remove(name);
2049    state
2050        .principals
2051        .insert(name.to_string(), principal.to_string());
2052    state
2053        .retained_cycles_by_principal
2054        .insert(principal.to_string(), cycles);
2055    state.topology.insert(
2056        name.to_string(),
2057        crate::fleet_ensure::model::FleetEnsureTopologyRecord {
2058            kind: configured.kind,
2059            module_hash: None,
2060            parent: configured.parent.clone(),
2061            protocol_binding: configured.protocol_binding.clone(),
2062            role: configured
2063                .protocol_binding
2064                .as_ref()
2065                .map(|binding| binding.role.to_string()),
2066        },
2067    );
2068    Ok(())
2069}
2070
2071fn create_record_is_exact(
2072    action: &EnsureAction,
2073    record: &EffectRecord,
2074    state: &FleetEnsureStateRecord,
2075) -> bool {
2076    let EnsureAction::Create {
2077        name,
2078        requested_initial_cycles,
2079        ..
2080    } = action
2081    else {
2082        return true;
2083    };
2084    match record.state {
2085        EffectState::Intent => {
2086            record.created_principal.is_none()
2087                && record.receipt.is_none()
2088                && record.post_cycles.is_none()
2089        }
2090        EffectState::Issued => {
2091            create_identity_is_exact(name, record, state)
2092                && record.post_cycles == Some(*requested_initial_cycles)
2093        }
2094        EffectState::Applied => create_identity_is_exact(name, record, state),
2095    }
2096}
2097
2098fn create_identity_is_exact(
2099    name: &str,
2100    record: &EffectRecord,
2101    state: &FleetEnsureStateRecord,
2102) -> bool {
2103    let Some(created_principal) = record
2104        .created_principal
2105        .as_deref()
2106        .filter(|value| !value.is_empty())
2107    else {
2108        return false;
2109    };
2110    if record.receipt.as_deref().is_none_or(str::is_empty) {
2111        return false;
2112    }
2113    state_create_principal_matches(name, created_principal, state, true)
2114}
2115
2116fn state_create_principal_matches(
2117    name: &str,
2118    created_principal: &str,
2119    state: &FleetEnsureStateRecord,
2120    require_retained: bool,
2121) -> bool {
2122    let pending = state.pending_principals.get(name).map(String::as_str);
2123    let terminal = state.principals.get(name).map(String::as_str);
2124    let retained = pending.is_some() || terminal.is_some();
2125    let pending_matches = pending.is_none_or(|principal| principal == created_principal);
2126    let terminal_matches = terminal.is_none_or(|principal| principal == created_principal);
2127    (!require_retained || retained) && pending_matches && terminal_matches
2128}
2129
2130fn create_outcome_is_exact(
2131    action: &EnsureAction,
2132    record: &EffectRecord,
2133    state: &FleetEnsureStateRecord,
2134    outcome: &crate::fleet_ensure::ops::EffectOutcome,
2135) -> bool {
2136    if matches!(action, EnsureAction::FundEstate { .. }) {
2137        return outcome.created_principal.is_none()
2138            && outcome.post_cycles.is_none()
2139            && outcome
2140                .receipt
2141                .as_deref()
2142                .is_some_and(|receipt| !receipt.is_empty());
2143    }
2144    let EnsureAction::Create {
2145        name,
2146        requested_initial_cycles,
2147        ..
2148    } = action
2149    else {
2150        return true;
2151    };
2152    let Some(created_principal) = outcome
2153        .created_principal
2154        .as_deref()
2155        .filter(|value| !value.is_empty())
2156    else {
2157        return false;
2158    };
2159    let Some(receipt) = outcome.receipt.as_deref().filter(|value| !value.is_empty()) else {
2160        return false;
2161    };
2162    if outcome.post_cycles != Some(*requested_initial_cycles) {
2163        return false;
2164    }
2165    let principal_matches = record
2166        .created_principal
2167        .as_deref()
2168        .is_none_or(|retained| retained == created_principal);
2169    let receipt_matches = record
2170        .receipt
2171        .as_deref()
2172        .is_none_or(|retained| retained == receipt);
2173    let state_matches = state_create_principal_matches(name, created_principal, state, false);
2174    principal_matches && receipt_matches && state_matches
2175}
2176
2177fn verify_fresh_plan<P>(
2178    root: &Path,
2179    desired: &crate::fleet_ensure::model::DesiredFleet,
2180    desired_sha256: &str,
2181    requested_fleet: &str,
2182    retained_plan: &FleetEnsurePlan,
2183    platform: &mut P,
2184    state: &FleetEnsureStateRecord,
2185) -> Result<(FleetObservation, u128), EnsureWorkflowError<P::Error>>
2186where
2187    P: EnsurePlatform,
2188{
2189    if retained_plan.reinstall.is_some() {
2190        return reinstall::verify_before_apply(root, desired, retained_plan, platform, state);
2191    }
2192    if retained_plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite {
2193        return root_reinstall::verify_before_apply(root, desired, retained_plan, platform, state);
2194    }
2195    if retained_plan.scope == FleetEnsurePlanScope::RootStartPrerequisite {
2196        let targets = reviewed_root_start_targets(retained_plan)?;
2197        let management = platform
2198            .observe_root_management(state, &targets)
2199            .map_err(EnsureWorkflowError::Platform)?
2200            .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2201        let current = recompile_root_start_prerequisite_plan(
2202            RootStartPlanInput {
2203                state,
2204                authority: retained_plan.root_start_authority.as_deref(),
2205                created_at_time: retained_plan.planned_at_time,
2206                desired,
2207                desired_sha256,
2208                observation: &management,
2209                requested_fleet,
2210            },
2211            &targets,
2212        )?;
2213        if !compatible_root_start_prerequisite(retained_plan, &current, desired) {
2214            return Err(EnsureWorkflowError::DriftedBeforeApply);
2215        }
2216        let observation = root_management_fleet_observation(&management, &targets)?;
2217        return Ok((observation, current.conservation.observed_controlled_cycles));
2218    }
2219    let mut observation = platform
2220        .observe(&retained_plan.operation_id, state)
2221        .map_err(EnsureWorkflowError::Platform)?;
2222    let terminal_inventory_operation_id =
2223        reviewed_terminal_inventory_operation(retained_plan, state)?;
2224    if let Some(operation_id) = terminal_inventory_operation_id {
2225        attach_terminal_inventory_cycles(operation_id, state, platform, &mut observation)?;
2226    }
2227    let artifacts = resolve_desired_artifacts(root, desired)?;
2228    let mut retained_observations = 0_u32;
2229    let mut current = loop {
2230        let protocol_actions = platform
2231            .protocol_actions(&retained_plan.operation_id, state)
2232            .map_err(EnsureWorkflowError::Platform)?;
2233        match compile_plan(
2234            desired,
2235            &artifacts,
2236            &protocol_actions,
2237            desired_sha256,
2238            requested_fleet,
2239            &observation,
2240            retained_plan.planned_at_time,
2241            &retained_plan.operation_id,
2242            None,
2243        ) {
2244            Ok(plan) => break plan,
2245            Err(EnsurePolicyError::PendingRootOwnedBalance { name }) => {
2246                retained_observations = retained_observations.saturating_add(1);
2247                if retained_observations >= desired.maximum_stalled_observations {
2248                    return Err(EnsureWorkflowError::RootOwnedObservationStalled {
2249                        last_lifecycle: root_owned_lifecycle_label(&observation, &name).to_string(),
2250                        observations: retained_observations,
2251                        target: name,
2252                    });
2253                }
2254                platform.pace_root_owned_observation(&name, retained_observations);
2255                observation = platform
2256                    .observe(&retained_plan.operation_id, state)
2257                    .map_err(EnsureWorkflowError::Platform)?;
2258                if let Some(operation_id) = terminal_inventory_operation_id {
2259                    attach_terminal_inventory_cycles(
2260                        operation_id,
2261                        state,
2262                        platform,
2263                        &mut observation,
2264                    )?;
2265                }
2266            }
2267            Err(error) => return Err(error.into()),
2268        }
2269    };
2270    if observation.operator_cycles < retained_plan.conservation.maximum_operator_debit_cycles {
2271        return Err(EnsureWorkflowError::InsufficientOperatorCycles {
2272            actual: observation.operator_cycles,
2273            required: retained_plan.conservation.maximum_operator_debit_cycles,
2274        });
2275    }
2276    bind_terminal_inventory_operation(
2277        &mut current,
2278        retained_plan.terminal_inventory_operation_id.clone(),
2279    );
2280    if current.plan_sha256 != retained_plan.plan_sha256
2281        && !compatible_after_bounded_observation(retained_plan, &current, desired, &observation)
2282    {
2283        return Err(EnsureWorkflowError::DriftedBeforeApply);
2284    }
2285    Ok((observation, current.conservation.observed_controlled_cycles))
2286}
2287
2288fn estate_funding_requirement<E>(
2289    plan: &FleetEnsurePlan,
2290    state: &FleetEnsureStateRecord,
2291    observation: &FleetObservation,
2292) -> Result<Option<EstateFundingRequiredRecord>, EnsureWorkflowError<E>>
2293where
2294    E: std::error::Error + 'static,
2295{
2296    for reviewed in &plan.conservation.estate_funding_domains {
2297        let observed = observation
2298            .estate_funding_domains
2299            .get(&reviewed.root)
2300            .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2301        if observed.cycles_ledger != reviewed.cycles_ledger {
2302            return Err(EnsureWorkflowError::PlanIntegrity);
2303        }
2304        let root_principal = observed
2305            .root_principal
2306            .as_deref()
2307            .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2308        if reviewed_estate_root_principal(reviewed, state) != Some(root_principal) {
2309            return Err(EnsureWorkflowError::PlanIntegrity);
2310        }
2311        let available_cycles = observed
2312            .balance_cycles
2313            .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2314        let reviewed_available = reviewed.available_cycles.unwrap_or_default();
2315        let maximum_reviewed_balance = reviewed_available
2316            .checked_add(reviewed.maximum_funding_cycles)
2317            .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2318        if available_cycles < reviewed_available || available_cycles > maximum_reviewed_balance {
2319            return Err(EnsureWorkflowError::DriftedBeforeApply);
2320        }
2321        if reviewed.required_creation_count == 0 {
2322            if available_cycles != reviewed_available {
2323                return Err(EnsureWorkflowError::DriftedBeforeApply);
2324            }
2325            continue;
2326        }
2327        if available_cycles < reviewed.maximum_creation_debit_cycles {
2328            return Ok(Some(EstateFundingRequiredRecord {
2329                attempt_count: None,
2330                available_cycles,
2331                creation_amount_cycles: reviewed.creation_amount_cycles,
2332                creation_execution_margin_cycles: reviewed.creation_execution_margin_cycles,
2333                readiness_floor_cycles: reviewed.readiness_floor_cycles,
2334                cycles_ledger: reviewed.cycles_ledger.clone(),
2335                ledger_fee_cycles: reviewed.ledger_fee_cycles,
2336                management_creation_fee_cycles: reviewed.management_creation_fee_cycles,
2337                maximum_creation_debit_cycles: reviewed.maximum_creation_debit_cycles,
2338                maximum_creation_fee_cycles: reviewed.maximum_creation_fee_cycles,
2339                operation_id: plan.operation_id.clone(),
2340                pending_creation_operation_id: None,
2341                plan_sha256: plan.plan_sha256.clone(),
2342                required_creation_count: reviewed.required_creation_count,
2343                root: reviewed.root.clone(),
2344                root_principal: root_principal.to_string(),
2345                last_attempt_at_ns: None,
2346                retry_at_ns: None,
2347                shortfall_cycles: reviewed
2348                    .maximum_creation_debit_cycles
2349                    .saturating_sub(available_cycles),
2350            }));
2351        }
2352    }
2353    Ok(None)
2354}
2355
2356fn protocol_estate_funding_requirement<E>(
2357    plan: &FleetEnsurePlan,
2358    state: &FleetEnsureStateRecord,
2359    funding: &canic_core::dto::component_provisioning::RootEstateFundingRequired,
2360) -> Result<EstateFundingRequiredRecord, EnsureWorkflowError<E>>
2361where
2362    E: std::error::Error + 'static,
2363{
2364    let root_principal = funding.root.to_text();
2365    let reviewed = plan
2366        .conservation
2367        .estate_funding_domains
2368        .iter()
2369        .find(|domain| reviewed_estate_root_principal(domain, state) == Some(&root_principal))
2370        .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2371    let creation_amount_cycles = funding.creation_amount.to_u128();
2372    let ledger_fee_cycles = funding.ledger_fee.to_u128();
2373    let management_creation_fee_cycles = funding.management_creation_fee.to_u128();
2374    let required_cycles = creation_amount_cycles
2375        .checked_add(ledger_fee_cycles)
2376        .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2377    let maximum_creation_fee_cycles = management_creation_fee_cycles
2378        .checked_add(ledger_fee_cycles)
2379        .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2380    let exact_reviewed_authority = reviewed.required_creation_count > 0
2381        && reviewed.cycles_ledger == funding.cycles_ledger.to_text()
2382        && reviewed.creation_amount_cycles == creation_amount_cycles
2383        && reviewed.creation_execution_margin_cycles == funding.execution_margin.to_u128()
2384        && reviewed.readiness_floor_cycles == funding.readiness_floor.to_u128()
2385        && reviewed.ledger_fee_cycles == ledger_fee_cycles
2386        && reviewed.management_creation_fee_cycles == management_creation_fee_cycles;
2387    let exact_runtime_arithmetic = funding.required.to_u128() == required_cycles
2388        && funding.available < funding.required
2389        && funding.shortfall.to_u128()
2390            == required_cycles.saturating_sub(funding.available.to_u128());
2391    if !exact_reviewed_authority || !exact_runtime_arithmetic {
2392        return Err(EnsureWorkflowError::PlanIntegrity);
2393    }
2394    Ok(EstateFundingRequiredRecord {
2395        attempt_count: Some(funding.attempt_count),
2396        available_cycles: funding.available.to_u128(),
2397        creation_amount_cycles,
2398        cycles_ledger: funding.cycles_ledger.to_text(),
2399        creation_execution_margin_cycles: funding.execution_margin.to_u128(),
2400        readiness_floor_cycles: funding.readiness_floor.to_u128(),
2401        ledger_fee_cycles,
2402        management_creation_fee_cycles,
2403        maximum_creation_debit_cycles: required_cycles,
2404        maximum_creation_fee_cycles,
2405        operation_id: plan.operation_id.clone(),
2406        pending_creation_operation_id: Some(canic_core::cdk::utils::hash::hex_bytes(
2407            funding.operation_id,
2408        )),
2409        plan_sha256: plan.plan_sha256.clone(),
2410        required_creation_count: 1,
2411        root: reviewed.root.clone(),
2412        root_principal,
2413        last_attempt_at_ns: funding.last_attempt_at_ns,
2414        retry_at_ns: Some(funding.retry_at_ns),
2415        shortfall_cycles: funding.shortfall.to_u128(),
2416    })
2417}
2418
2419fn estate_funding_error(required: &EstateFundingRequiredRecord) -> EstateFundingRequiredError {
2420    EstateFundingRequiredError {
2421        attempt_count: required.attempt_count,
2422        available_cycles: required.available_cycles,
2423        creation_amount_cycles: required.creation_amount_cycles,
2424        creation_execution_margin_cycles: required.creation_execution_margin_cycles,
2425        readiness_floor_cycles: required.readiness_floor_cycles,
2426        cycles_ledger: required.cycles_ledger.clone(),
2427        ledger_fee_cycles: required.ledger_fee_cycles,
2428        management_creation_fee_cycles: required.management_creation_fee_cycles,
2429        maximum_creation_debit_cycles: required.maximum_creation_debit_cycles,
2430        maximum_creation_fee_cycles: required.maximum_creation_fee_cycles,
2431        pending_creation_operation_id: required.pending_creation_operation_id.clone(),
2432        required_creation_count: required.required_creation_count,
2433        root: required.root.clone(),
2434        root_principal: required.root_principal.clone(),
2435        last_attempt_at_ns: required.last_attempt_at_ns,
2436        retry_at_ns: required.retry_at_ns,
2437        shortfall_cycles: required.shortfall_cycles,
2438    }
2439}
2440
2441fn retain_estate_funding_pause<E>(
2442    paths: &EnsurePaths,
2443    journal: &mut FleetEnsureJournalRecord,
2444    required: Option<&EstateFundingRequiredRecord>,
2445) -> Result<bool, EnsureWorkflowError<E>>
2446where
2447    E: std::error::Error + 'static,
2448{
2449    if journal.estate_funding_required.as_ref() == required {
2450        return Ok(false);
2451    }
2452    journal.estate_funding_required = required.cloned();
2453    write_journal(paths, journal)?;
2454    Ok(true)
2455}
2456
2457fn reviewed_estate_root_principal<'a>(
2458    domain: &'a crate::fleet_ensure::model::EstateFundingDomainPlan,
2459    state: &'a FleetEnsureStateRecord,
2460) -> Option<&'a str> {
2461    domain
2462        .root_principal
2463        .as_deref()
2464        .or_else(|| {
2465            state
2466                .pending_principals
2467                .get(&domain.root)
2468                .map(String::as_str)
2469        })
2470        .or_else(|| state.principals.get(&domain.root).map(String::as_str))
2471}
2472
2473fn prior_fleet_protocol_effect_started(
2474    actions: &[&EnsureAction],
2475    journal: &FleetEnsureJournalRecord,
2476    current_index: usize,
2477) -> bool {
2478    actions
2479        .iter()
2480        .take(current_index)
2481        .zip(&journal.effects)
2482        .any(|(action, effect)| {
2483            matches!(action, EnsureAction::FleetProtocol { .. })
2484                && matches!(effect.state, EffectState::Issued | EffectState::Applied)
2485        })
2486}
2487
2488fn completed_inventory_operation<'a, E>(
2489    plan: &'a FleetEnsurePlan,
2490    journal: &FleetEnsureJournalRecord,
2491    state: &FleetEnsureStateRecord,
2492) -> Result<&'a str, EnsureWorkflowError<E>>
2493where
2494    E: std::error::Error + 'static,
2495{
2496    if plan.protocol_actions.is_empty()
2497        && journal.successor_phases.is_empty()
2498        && state.active_registry.is_some()
2499    {
2500        reviewed_terminal_inventory_operation(plan, state)?
2501            .ok_or(EnsureWorkflowError::PlanIntegrity)
2502    } else {
2503        Ok(&plan.operation_id)
2504    }
2505}
2506
2507fn reviewed_terminal_inventory_operation<'a, E>(
2508    plan: &'a FleetEnsurePlan,
2509    state: &FleetEnsureStateRecord,
2510) -> Result<Option<&'a str>, EnsureWorkflowError<E>>
2511where
2512    E: std::error::Error + 'static,
2513{
2514    match (
2515        state.active_registry.is_some(),
2516        plan.terminal_inventory_operation_id.as_deref(),
2517    ) {
2518        (false, None) => Ok(None),
2519        (true, Some(operation_id)) if !operation_id.is_empty() => Ok(Some(operation_id)),
2520        _ => Err(EnsureWorkflowError::PlanIntegrity),
2521    }
2522}
2523
2524fn complete_root_start_prerequisite<P>(
2525    desired: &crate::fleet_ensure::model::DesiredFleet,
2526    retained_plan: &FleetEnsurePlan,
2527    platform: &mut P,
2528    state: &FleetEnsureStateRecord,
2529    journal: &FleetEnsureJournalRecord,
2530) -> Result<ActualCycleConservation, EnsureWorkflowError<P::Error>>
2531where
2532    P: EnsurePlatform,
2533{
2534    let targets = reviewed_root_start_targets(retained_plan)?;
2535    let management = platform
2536        .observe_root_management(state, &targets)
2537        .map_err(EnsureWorkflowError::Platform)?
2538        .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2539    let current = recompile_root_start_prerequisite_plan(
2540        RootStartPlanInput {
2541            state,
2542            authority: retained_plan.root_start_authority.as_deref(),
2543            created_at_time: retained_plan.planned_at_time,
2544            desired,
2545            desired_sha256: &retained_plan.desired_sha256,
2546            observation: &management,
2547            requested_fleet: &retained_plan.fleet,
2548        },
2549        &targets,
2550    )?;
2551    if !compatible_root_start_prerequisite(retained_plan, &current, desired) {
2552        return Err(EnsureWorkflowError::ConvergenceDrift);
2553    }
2554    if management.roots.iter().any(|(name, observed)| {
2555        targets.contains(name)
2556            && observed.live.status != crate::fleet_ensure::model::CanisterRuntimeStatus::Running
2557    }) {
2558        return Err(EnsureWorkflowError::ConvergenceDrift);
2559    }
2560    let terminal = root_management_fleet_observation(&management, &targets)?;
2561    verify_terminal_conservation(retained_plan, journal, state, &terminal)
2562}
2563
2564fn verify_terminal_root_start_replay<P>(
2565    desired: &crate::fleet_ensure::model::DesiredFleet,
2566    plan: &FleetEnsurePlan,
2567    journal: &FleetEnsureJournalRecord,
2568    requested_fleet: &str,
2569    platform: &mut P,
2570    state: &FleetEnsureStateRecord,
2571) -> Result<(), EnsureWorkflowError<P::Error>>
2572where
2573    P: EnsurePlatform,
2574{
2575    verify_journal(journal, plan, requested_fleet, state)?;
2576    let actions = ordered_actions(plan);
2577    if actions.len() != journal.effects.len()
2578        || journal
2579            .effects
2580            .iter()
2581            .any(|effect| effect.state != EffectState::Applied)
2582    {
2583        return Err(EnsureWorkflowError::JournalIntegrity);
2584    }
2585    for (action, record) in actions.into_iter().zip(&journal.effects) {
2586        if action_sha256(action) != record.action_sha256
2587            || !platform
2588                .observe_effect(&journal.operation_id, action, record, state)
2589                .map_err(EnsureWorkflowError::Platform)?
2590                .applied
2591        {
2592            return Err(EnsureWorkflowError::ConvergenceDrift);
2593        }
2594    }
2595    let targets = reviewed_root_start_targets(plan)?;
2596    let management = platform
2597        .observe_root_management(state, &targets)
2598        .map_err(EnsureWorkflowError::Platform)?
2599        .ok_or(EnsureWorkflowError::PlanIntegrity)?;
2600    let current = recompile_root_start_prerequisite_plan(
2601        RootStartPlanInput {
2602            state,
2603            authority: plan.root_start_authority.as_deref(),
2604            created_at_time: plan.planned_at_time,
2605            desired,
2606            desired_sha256: &plan.desired_sha256,
2607            observation: &management,
2608            requested_fleet: &plan.fleet,
2609        },
2610        &targets,
2611    )?;
2612    if normalized_plan(plan) != normalized_plan(&current)
2613        || management.roots.iter().any(|(name, observed)| {
2614            targets.contains(name)
2615                && observed.live.status
2616                    != crate::fleet_ensure::model::CanisterRuntimeStatus::Running
2617        })
2618    {
2619        return Err(EnsureWorkflowError::ConvergenceDrift);
2620    }
2621    Ok(())
2622}
2623
2624fn reviewed_root_start_targets<E>(
2625    plan: &FleetEnsurePlan,
2626) -> Result<BTreeSet<String>, EnsureWorkflowError<E>>
2627where
2628    E: std::error::Error + 'static,
2629{
2630    if plan.scope != FleetEnsurePlanScope::RootStartPrerequisite
2631        || !plan.protocol_actions.is_empty()
2632        || plan.canisters.is_empty()
2633        || plan.conservation.maximum_new_funding_cycles != 0
2634        || plan.conservation.maximum_operator_debit_cycles != 0
2635        || plan.conservation.maximum_unavoidable_fee_cycles != 0
2636        || !plan.conservation.estate_funding_domains.is_empty()
2637        || plan.conservation.scheduled_transfer_cycles != 0
2638        || plan.conservation.observed_controlled_cycles
2639            != plan.conservation.retained_in_reused_canisters_cycles
2640        || plan
2641            .conservation
2642            .observed_controlled_cycles
2643            .checked_sub(plan.conservation.maximum_execution_burn_cycles)
2644            != Some(plan.conservation.expected_post_operation_cycles)
2645    {
2646        return Err(EnsureWorkflowError::PlanIntegrity);
2647    }
2648    let mut targets = BTreeSet::new();
2649    for canister in &plan.canisters {
2650        if canister.disposition != CanisterDisposition::Reuse
2651            || canister.principal.is_none()
2652            || !matches!(
2653                canister.actions.as_slice(),
2654                [EnsureAction::Start { name, principal }]
2655                    if name == &canister.name
2656                        && Some(principal) == canister.principal.as_ref()
2657            )
2658            || !targets.insert(canister.name.clone())
2659        {
2660            return Err(EnsureWorkflowError::PlanIntegrity);
2661        }
2662    }
2663    Ok(targets)
2664}
2665
2666fn root_management_fleet_observation<E>(
2667    management: &RootManagementObservation,
2668    targets: &BTreeSet<String>,
2669) -> Result<FleetObservation, EnsureWorkflowError<E>>
2670where
2671    E: std::error::Error + 'static,
2672{
2673    let canisters = targets
2674        .iter()
2675        .map(|name| {
2676            management
2677                .roots
2678                .get(name)
2679                .map(|observed| (name.clone(), Some(observed.live.clone())))
2680                .ok_or(EnsureWorkflowError::PlanIntegrity)
2681        })
2682        .collect::<Result<BTreeMap<_, _>, _>>()?;
2683    Ok(FleetObservation {
2684        additional_controlled_cycles: BTreeMap::new(),
2685        canisters,
2686        estate_funding_domains: BTreeMap::new(),
2687        ledger_fee_cycles: 0,
2688        operator_cycles: management.operator_cycles,
2689        protocol_ready: BTreeMap::new(),
2690    })
2691}
2692
2693fn compatible_root_start_prerequisite(
2694    retained: &FleetEnsurePlan,
2695    current: &FleetEnsurePlan,
2696    desired: &crate::fleet_ensure::model::DesiredFleet,
2697) -> bool {
2698    let Ok(maximum_movement) = desired
2699        .maximum_update_burn_cycles
2700        .parse::<Cycles>()
2701        .map(|cycles| cycles.to_u128())
2702    else {
2703        return false;
2704    };
2705    retained.canisters.len() == current.canisters.len()
2706        && retained
2707            .canisters
2708            .iter()
2709            .zip(&current.canisters)
2710            .all(|(retained, current)| {
2711                retained.observed_cycles.abs_diff(current.observed_cycles) <= maximum_movement
2712            })
2713        && normalized_plan(retained) == normalized_plan(current)
2714}
2715
2716fn attach_terminal_inventory_cycles<P>(
2717    operation_id: &str,
2718    state: &FleetEnsureStateRecord,
2719    platform: &mut P,
2720    observation: &mut FleetObservation,
2721) -> Result<(), EnsureWorkflowError<P::Error>>
2722where
2723    P: EnsurePlatform,
2724{
2725    let inventory = platform
2726        .terminal_inventory(operation_id, state)
2727        .map_err(EnsureWorkflowError::Platform)?;
2728    attach_terminal_cycles(observation, inventory.controlled_cycles_by_principal)
2729}
2730
2731pub(super) fn completed_infrastructure_reinstall(
2732    desired: &crate::fleet_ensure::model::DesiredFleet,
2733    plan: &FleetEnsurePlan,
2734    journal: &FleetEnsureJournalRecord,
2735) -> bool {
2736    if desired.protocol.is_none() {
2737        return false;
2738    }
2739    let infrastructure = desired.canisters.iter().filter(|canister| {
2740        canister.presence == crate::fleet_ensure::model::DesiredPresence::Present
2741            && matches!(
2742                canister.kind,
2743                crate::fleet_ensure::model::DesiredCanisterKind::Coordinator
2744                    | crate::fleet_ensure::model::DesiredCanisterKind::Root
2745                    | crate::fleet_ensure::model::DesiredCanisterKind::Store
2746            )
2747    });
2748    let mut count = 0_usize;
2749    for configured in infrastructure {
2750        count += 1;
2751        let Some(action) = plan
2752            .canisters
2753            .iter()
2754            .find(|canister| {
2755                canister.name == configured.name
2756                    && canister.disposition == CanisterDisposition::Reinstall
2757            })
2758            .and_then(|canister| {
2759                canister.actions.iter().find(|action| {
2760                    matches!(
2761                        action,
2762                        EnsureAction::Install {
2763                            mode: crate::fleet_ensure::model::InstallMode::Reinstall,
2764                            name,
2765                            ..
2766                        } if name == &configured.name
2767                    )
2768                })
2769            })
2770        else {
2771            return false;
2772        };
2773        let action_sha256 = action_sha256(action);
2774        if !journal.effects.iter().any(|effect| {
2775            effect.action_sha256 == action_sha256 && effect.state == EffectState::Applied
2776        }) {
2777            return false;
2778        }
2779    }
2780    count > 0
2781}
2782
2783fn compatible_after_bounded_observation(
2784    retained: &FleetEnsurePlan,
2785    current: &FleetEnsurePlan,
2786    desired: &crate::fleet_ensure::model::DesiredFleet,
2787    observation: &FleetObservation,
2788) -> bool {
2789    let Ok(maximum_observation_movement) = desired
2790        .maximum_observation_burn_cycles
2791        .parse::<Cycles>()
2792        .map(|cycles| cycles.to_u128())
2793    else {
2794        return false;
2795    };
2796    if retained.canisters.len() != current.canisters.len() {
2797        return false;
2798    }
2799    for (retained_canister, current_canister) in retained.canisters.iter().zip(&current.canisters) {
2800        let movement = retained_canister
2801            .observed_cycles
2802            .abs_diff(current_canister.observed_cycles);
2803        if movement > maximum_observation_movement {
2804            return false;
2805        }
2806    }
2807    if !retained_funding_remains_sufficient(retained, desired, observation) {
2808        return false;
2809    }
2810    normalized_plan(retained) == normalized_plan(current)
2811}
2812
2813fn retained_funding_remains_sufficient(
2814    retained: &FleetEnsurePlan,
2815    desired: &crate::fleet_ensure::model::DesiredFleet,
2816    observation: &FleetObservation,
2817) -> bool {
2818    retained
2819        .canisters
2820        .iter()
2821        .flat_map(|canister| &canister.actions)
2822        .filter_map(|action| match action {
2823            EnsureAction::Fund {
2824                amount,
2825                name,
2826                pool_funding,
2827                principal,
2828                ..
2829            } => Some((amount, name, pool_funding, principal)),
2830            _ => None,
2831        })
2832        .all(|(amount, name, pool_funding, principal)| {
2833            if let Some(root) = pool_funding {
2834                return observation
2835                    .estate_funding_domains
2836                    .values()
2837                    .find(|domain| domain.root_principal.as_ref() == Some(&root.root))
2838                    .and_then(|domain| domain.pool.as_ref())
2839                    .is_some_and(|pool| {
2840                        pool.assets.iter().any(|asset| {
2841                            asset.principal == *principal
2842                                && asset.lifecycle == root.lifecycle
2843                                && asset
2844                                    .cycles
2845                                    .checked_add(*amount)
2846                                    .is_some_and(|funded| funded >= pool.readiness_floor_cycles)
2847                        })
2848                    });
2849            }
2850            let Some(live) = observation.canisters.get(name).and_then(Option::as_ref) else {
2851                return false;
2852            };
2853            let Some(configured) = desired
2854                .canisters
2855                .iter()
2856                .find(|configured| configured.name == *name)
2857            else {
2858                return false;
2859            };
2860            let Ok(minimum) = configured
2861                .minimum_cycles
2862                .parse::<Cycles>()
2863                .map(|cycles| cycles.to_u128())
2864            else {
2865                return false;
2866            };
2867            live.cycles
2868                .checked_add(*amount)
2869                .is_some_and(|funded| funded >= minimum)
2870        })
2871}
2872
2873fn normalized_plan(plan: &FleetEnsurePlan) -> FleetEnsurePlan {
2874    let mut normalized = plan.clone();
2875    normalized.plan_sha256.clear();
2876    normalized.reviewed_desired = None;
2877    if let Some(review) = &mut normalized.recovery_review {
2878        // These estimates move with bounded observations; they grant no funding authority.
2879        review.continuation_reserve_cycles = 0;
2880        for funding in &mut review.known_pool_funding {
2881            funding.amount_cycles = 0;
2882            funding.funding_deficit_cycles = 0;
2883        }
2884    }
2885    normalized.conservation = CycleConservation {
2886        estate_funding_domains: plan
2887            .conservation
2888            .estate_funding_domains
2889            .iter()
2890            .cloned()
2891            .map(|mut domain| {
2892                domain.available_cycles = None;
2893                domain.maximum_funding_cycles = 0;
2894                domain.root_principal = None;
2895                domain.shortfall_cycles = 0;
2896                domain
2897            })
2898            .collect(),
2899        expected_post_operation_cycles: 0,
2900        maximum_execution_burn_cycles: 0,
2901        maximum_new_funding_cycles: 0,
2902        maximum_operator_debit_cycles: 0,
2903        maximum_unavoidable_fee_cycles: 0,
2904        observed_controlled_cycles: 0,
2905        retained_in_reused_canisters_cycles: 0,
2906        scheduled_transfer_cycles: 0,
2907    };
2908    for canister in &mut normalized.canisters {
2909        canister.observed_cycles = 0;
2910        for action in &mut canister.actions {
2911            if let EnsureAction::Fund {
2912                amount,
2913                funding_deficit_cycles,
2914                ..
2915            } = action
2916            {
2917                *amount = 0;
2918                *funding_deficit_cycles = 0;
2919            }
2920        }
2921    }
2922    normalized
2923}
2924
2925fn attach_terminal_cycles<E>(
2926    observation: &mut FleetObservation,
2927    additional: BTreeMap<String, u128>,
2928) -> Result<(), EnsureWorkflowError<E>>
2929where
2930    E: std::error::Error + 'static,
2931{
2932    let mut retained = BTreeMap::new();
2933    for (principal, cycles) in additional {
2934        let Some(configured) = observation
2935            .canisters
2936            .values_mut()
2937            .filter_map(Option::as_mut)
2938            .find(|canister| canister.principal == principal)
2939        else {
2940            retained.insert(principal, cycles);
2941            continue;
2942        };
2943        match configured.root_owned_lifecycle {
2944            Some(crate::fleet_ensure::model::RootOwnedCanisterLifecycle::Workload) => {
2945                // Both protected observations identify the same running workload, but
2946                // execution may burn cycles between them. Retain the conservative lower value.
2947                configured.cycles = configured.cycles.min(cycles);
2948            }
2949            Some(crate::fleet_ensure::model::RootOwnedCanisterLifecycle::Idle)
2950                if configured.cycles == cycles => {}
2951            Some(crate::fleet_ensure::model::RootOwnedCanisterLifecycle::Idle) => {
2952                return Err(EnsureWorkflowError::TerminalInventory(
2953                    TerminalInventoryError::IdleCycleObservationConflict {
2954                        canister: principal,
2955                        expected: configured.cycles,
2956                        observed: cycles,
2957                    },
2958                ));
2959            }
2960            _ => {
2961                return Err(EnsureWorkflowError::TerminalInventory(
2962                    TerminalInventoryError::CycleObservationLifecycleConflict {
2963                        canister: principal,
2964                    },
2965                ));
2966            }
2967        }
2968    }
2969    observation.additional_controlled_cycles = retained;
2970    Ok(())
2971}
2972
2973fn funding_plan<'a, E: std::error::Error + 'static>(
2974    plan: &'a FleetEnsurePlan,
2975    journal: &FleetEnsureJournalRecord,
2976) -> Result<std::borrow::Cow<'a, FleetEnsurePlan>, EnsureWorkflowError<E>> {
2977    if journal.funding_reviews.is_empty() {
2978        return Ok(std::borrow::Cow::Borrowed(plan));
2979    }
2980    let mut funded = plan.clone();
2981    let (amount, fee) = funding::totals::<E>(journal, None)?;
2982    let conservation = &mut funded.conservation;
2983    conservation.maximum_operator_debit_cycles = conservation
2984        .maximum_operator_debit_cycles
2985        .checked_add(amount)
2986        .and_then(|n| n.checked_add(fee))
2987        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2988    conservation.maximum_new_funding_cycles = conservation
2989        .maximum_new_funding_cycles
2990        .checked_add(amount)
2991        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2992    conservation.maximum_unavoidable_fee_cycles = conservation
2993        .maximum_unavoidable_fee_cycles
2994        .checked_add(fee)
2995        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
2996    for domain in &mut conservation.estate_funding_domains {
2997        domain.maximum_funding_cycles = domain
2998            .maximum_funding_cycles
2999            .checked_add(funding::totals::<E>(journal, Some(&domain.root))?.0)
3000            .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3001    }
3002    Ok(std::borrow::Cow::Owned(funded))
3003}
3004
3005fn verify_terminal_conservation<E>(
3006    plan: &FleetEnsurePlan,
3007    journal: &FleetEnsureJournalRecord,
3008    state: &FleetEnsureStateRecord,
3009    terminal: &FleetObservation,
3010) -> Result<ActualCycleConservation, EnsureWorkflowError<E>>
3011where
3012    E: std::error::Error + 'static,
3013{
3014    let funded = funding_plan(plan, journal)?;
3015    let conservation = &funded.conservation;
3016    let final_controlled_cycles = controlled_cycles(terminal)?;
3017    let operator_debit_cycles = journal
3018        .initial_operator_cycles
3019        .checked_sub(terminal.operator_cycles)
3020        .ok_or_else(|| {
3021            EnsureWorkflowError::Conservation(
3022                "operator balance increased during apply; review a new plan".to_string(),
3023            )
3024        })?;
3025    if operator_debit_cycles > conservation.maximum_operator_debit_cycles {
3026        return Err(EnsureWorkflowError::Conservation(format!(
3027            "operator debit {operator_debit_cycles} exceeded reviewed maximum {}",
3028            conservation.maximum_operator_debit_cycles
3029        )));
3030    }
3031    let received_new_funding_cycles = operator_debit_cycles
3032        .checked_sub(conservation.maximum_unavoidable_fee_cycles)
3033        .ok_or_else(|| {
3034            EnsureWorkflowError::Conservation(
3035                "operator debit was below the exact reviewed fee total".to_string(),
3036            )
3037        })?;
3038    if received_new_funding_cycles > conservation.maximum_new_funding_cycles {
3039        return Err(EnsureWorkflowError::Conservation(format!(
3040            "received funding {received_new_funding_cycles} exceeded reviewed maximum {}",
3041            conservation.maximum_new_funding_cycles
3042        )));
3043    }
3044    let (estate_funding_cycles, exact_estate_creation_fee_cycles) =
3045        reconcile_estate_funding(plan, journal, state, terminal)?;
3046    let available = journal
3047        .initial_controlled_cycles
3048        .checked_add(received_new_funding_cycles)
3049        .ok_or_else(|| {
3050            EnsureWorkflowError::Conservation(
3051                "terminal controlled-cycle arithmetic overflowed".to_string(),
3052            )
3053        })?;
3054    let after_estate_fees = available
3055        .checked_sub(exact_estate_creation_fee_cycles)
3056        .ok_or_else(|| {
3057            EnsureWorkflowError::Conservation(
3058                "terminal controlled cycles cannot cover reviewed estate creation fees".to_string(),
3059            )
3060        })?;
3061    let measured_execution_burn_cycles = after_estate_fees
3062        .checked_sub(final_controlled_cycles)
3063        .ok_or_else(|| {
3064            EnsureWorkflowError::Conservation(
3065                "terminal estate contains more cycles than starting estate plus received funding"
3066                    .to_string(),
3067            )
3068        })?;
3069    if measured_execution_burn_cycles > conservation.maximum_execution_burn_cycles {
3070        return Err(EnsureWorkflowError::Conservation(format!(
3071            "measured execution burn {measured_execution_burn_cycles} exceeded reviewed maximum {}",
3072            conservation.maximum_execution_burn_cycles
3073        )));
3074    }
3075    Ok(ActualCycleConservation {
3076        estate_funding_cycles,
3077        exact_estate_creation_fee_cycles,
3078        exact_unavoidable_fee_cycles: conservation.maximum_unavoidable_fee_cycles,
3079        final_controlled_cycles,
3080        measured_execution_burn_cycles,
3081        observed_starting_cycles: journal.initial_controlled_cycles,
3082        operator_debit_cycles,
3083        received_new_funding_cycles,
3084    })
3085}
3086
3087fn resolved_initial_pool_assets<E>(
3088    domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3089    state: &FleetEnsureStateRecord,
3090) -> Result<BTreeSet<String>, EnsureWorkflowError<E>>
3091where
3092    E: std::error::Error + 'static,
3093{
3094    domain
3095        .initial_pool_assets
3096        .iter()
3097        .map(|identity| {
3098            identity.strip_prefix("created:").map_or_else(
3099                || Ok(identity.clone()),
3100                |name| {
3101                    state
3102                        .pending_principals
3103                        .get(name)
3104                        .or_else(|| state.principals.get(name))
3105                        .cloned()
3106                        .ok_or(EnsureWorkflowError::JournalIntegrity)
3107                },
3108            )
3109        })
3110        .collect()
3111}
3112
3113fn exact_estate_creation_costs<E>(
3114    state: &FleetEnsureStateRecord,
3115    terminal: &FleetObservation,
3116    domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3117) -> Result<(u128, u128), EnsureWorkflowError<E>>
3118where
3119    E: std::error::Error + 'static,
3120{
3121    let observed = terminal
3122        .estate_funding_domains
3123        .get(&domain.root)
3124        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3125    let Some(pool) = observed.pool.as_ref() else {
3126        return if domain.required_creation_count == 0 {
3127            Ok((0, 0))
3128        } else {
3129            Err(EnsureWorkflowError::JournalIntegrity)
3130        };
3131    };
3132    if pool.pending_creation.is_some() {
3133        return Err(EnsureWorkflowError::JournalIntegrity);
3134    }
3135
3136    estate_creation_costs(state, terminal, domain)
3137}
3138
3139fn estate_creation_costs<E: std::error::Error + 'static>(
3140    state: &FleetEnsureStateRecord,
3141    observation: &FleetObservation,
3142    domain: &EstateFundingDomainPlan,
3143) -> Result<(u128, u128), EnsureWorkflowError<E>> {
3144    let pool = observation
3145        .estate_funding_domains
3146        .get(&domain.root)
3147        .and_then(|domain| domain.pool.as_ref())
3148        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3149    let initial = resolved_initial_pool_assets(domain, state)?;
3150    let terminal_principals = pool
3151        .assets
3152        .iter()
3153        .map(|asset| asset.principal.clone())
3154        .collect::<BTreeSet<_>>();
3155    if terminal_principals.len() != pool.assets.len() || !initial.is_subset(&terminal_principals) {
3156        return Err(EnsureWorkflowError::JournalIntegrity);
3157    }
3158    let created = pool
3159        .assets
3160        .iter()
3161        .filter(|asset| !initial.contains(&asset.principal))
3162        .collect::<Vec<_>>();
3163    let creation_count_is_bounded = created.len()
3164        <= usize::try_from(domain.required_creation_count)
3165            .map_err(|_| EnsureWorkflowError::JournalIntegrity)?;
3166    // Root maintenance can spend its reviewed reserve without provisioning a new Workload.
3167    // The per-Root budget and exact receipts bind that autonomous creation authority.
3168    let pool_policy_is_exact = created.is_empty() || terminal_pool_policy_is_exact(pool, domain);
3169    if !creation_count_is_bounded || !pool_policy_is_exact {
3170        return Err(EnsureWorkflowError::JournalIntegrity);
3171    }
3172
3173    let mut operation_ids = BTreeSet::new();
3174    let mut block_indexes = BTreeSet::new();
3175    let mut creation_debit = 0_u128;
3176    let mut creation_fee = 0_u128;
3177    for asset in created {
3178        let (asset_debit, asset_fee) = exact_estate_creation_receipt_costs(
3179            asset,
3180            domain,
3181            &mut operation_ids,
3182            &mut block_indexes,
3183        )?;
3184        creation_debit = creation_debit.checked_add(asset_debit).ok_or_else(|| {
3185            EnsureWorkflowError::Conservation(
3186                "estate creation-debit arithmetic overflowed".to_string(),
3187            )
3188        })?;
3189        creation_fee = creation_fee.checked_add(asset_fee).ok_or_else(|| {
3190            EnsureWorkflowError::Conservation(
3191                "estate creation-fee arithmetic overflowed".to_string(),
3192            )
3193        })?;
3194    }
3195    if creation_debit > domain.maximum_creation_debit_cycles
3196        || creation_fee > domain.maximum_creation_fee_cycles
3197    {
3198        return Err(EnsureWorkflowError::JournalIntegrity);
3199    }
3200    Ok((creation_debit, creation_fee))
3201}
3202
3203const fn terminal_pool_policy_is_exact(
3204    pool: &crate::fleet_ensure::model::EstatePoolInventoryObservation,
3205    domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3206) -> bool {
3207    let expected_maximum_size = domain.pool_maximum_size;
3208    let expected_readiness_floor = domain.readiness_floor_cycles;
3209    let expected_execution_margin = domain.creation_execution_margin_cycles;
3210    pool.maximum_size == expected_maximum_size
3211        && pool.readiness_floor_cycles == expected_readiness_floor
3212        && pool.creation_execution_margin_cycles == expected_execution_margin
3213}
3214
3215#[derive(Eq, PartialEq)]
3216struct EstateCreationReceiptAuthority<'a> {
3217    creation_execution_margin_cycles: u128,
3218    cycles_ledger: &'a str,
3219    ledger_amount_cycles: u128,
3220    ledger_fee_cycles: u128,
3221    management_creation_fee_cycles: u128,
3222    readiness_floor_cycles: u128,
3223}
3224
3225fn observed_estate_creation_authority(
3226    receipt: &crate::fleet_ensure::model::EstatePoolCreationReceiptObservation,
3227) -> EstateCreationReceiptAuthority<'_> {
3228    EstateCreationReceiptAuthority {
3229        creation_execution_margin_cycles: receipt.creation_execution_margin_cycles,
3230        cycles_ledger: &receipt.cycles_ledger,
3231        ledger_amount_cycles: receipt.ledger_amount_cycles,
3232        ledger_fee_cycles: receipt.ledger_fee_cycles,
3233        management_creation_fee_cycles: receipt.management_creation_fee_cycles,
3234        readiness_floor_cycles: receipt.readiness_floor_cycles,
3235    }
3236}
3237
3238fn reviewed_estate_creation_authority(
3239    domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3240) -> EstateCreationReceiptAuthority<'_> {
3241    EstateCreationReceiptAuthority {
3242        creation_execution_margin_cycles: domain.creation_execution_margin_cycles,
3243        cycles_ledger: &domain.cycles_ledger,
3244        ledger_amount_cycles: domain.creation_amount_cycles,
3245        ledger_fee_cycles: domain.ledger_fee_cycles,
3246        management_creation_fee_cycles: domain.management_creation_fee_cycles,
3247        readiness_floor_cycles: domain.readiness_floor_cycles,
3248    }
3249}
3250
3251fn first_estate_creation_observation_is_exact(
3252    receipt: &crate::fleet_ensure::model::EstatePoolCreationReceiptObservation,
3253) -> bool {
3254    receipt
3255        .ledger_amount_cycles
3256        .checked_sub(receipt.management_creation_fee_cycles)
3257        .zip(receipt.first_observed_cycles)
3258        .is_some_and(|(funded_native_cycles, observed_cycles)| {
3259            observed_cycles >= receipt.readiness_floor_cycles
3260                && observed_cycles <= funded_native_cycles
3261                && funded_native_cycles - observed_cycles
3262                    <= receipt.creation_execution_margin_cycles
3263        })
3264}
3265
3266fn exact_estate_creation_receipt_costs<E>(
3267    asset: &crate::fleet_ensure::model::EstatePoolAssetObservation,
3268    domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3269    operation_ids: &mut BTreeSet<String>,
3270    block_indexes: &mut BTreeSet<u64>,
3271) -> Result<(u128, u128), EnsureWorkflowError<E>>
3272where
3273    E: std::error::Error + 'static,
3274{
3275    let receipt = asset
3276        .creation_receipt
3277        .as_ref()
3278        .filter(|_| asset.origin == crate::fleet_ensure::model::EstatePoolAssetOrigin::Created)
3279        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3280    let authority_is_exact =
3281        observed_estate_creation_authority(receipt) == reviewed_estate_creation_authority(domain);
3282    if !is_sha256_hex(&receipt.operation_id)
3283        || !authority_is_exact
3284        || !first_estate_creation_observation_is_exact(receipt)
3285        || !operation_ids.insert(receipt.operation_id.clone())
3286        || !block_indexes.insert(receipt.block_index)
3287    {
3288        return Err(EnsureWorkflowError::JournalIntegrity);
3289    }
3290    let debit = receipt
3291        .ledger_amount_cycles
3292        .checked_add(receipt.ledger_fee_cycles)
3293        .ok_or_else(|| {
3294            EnsureWorkflowError::Conservation(
3295                "estate creation-debit arithmetic overflowed".to_string(),
3296            )
3297        })?;
3298    let fee = receipt
3299        .management_creation_fee_cycles
3300        .checked_add(receipt.ledger_fee_cycles)
3301        .ok_or_else(|| {
3302            EnsureWorkflowError::Conservation(
3303                "estate creation-fee arithmetic overflowed".to_string(),
3304            )
3305        })?;
3306    Ok((debit, fee))
3307}
3308
3309fn reconcile_estate_funding<E>(
3310    plan: &FleetEnsurePlan,
3311    journal: &FleetEnsureJournalRecord,
3312    state: &FleetEnsureStateRecord,
3313    terminal: &FleetObservation,
3314) -> Result<(u128, u128), EnsureWorkflowError<E>>
3315where
3316    E: std::error::Error + 'static,
3317{
3318    plan.conservation
3319        .estate_funding_domains
3320        .iter()
3321        .try_fold((0_u128, 0_u128), |(funding_total, fee_total), domain| {
3322            let initial = journal
3323                .initial_estate_funding_cycles_by_root
3324                .get(&domain.root)
3325                .copied()
3326                .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3327            let observed = terminal
3328                .estate_funding_domains
3329                .get(&domain.root)
3330                .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3331            if observed.cycles_ledger != domain.cycles_ledger
3332                || reviewed_estate_root_principal(domain, state)
3333                    != observed.root_principal.as_deref()
3334            {
3335                return Err(EnsureWorkflowError::JournalIntegrity);
3336            }
3337            let terminal_balance = observed
3338                .balance_cycles
3339                .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3340            let (creation_debit, creation_fee) = exact_estate_creation_costs(
3341                state,
3342                terminal,
3343                domain,
3344            )?;
3345            let funding = applied_estate_funding_for_domain(plan, journal, state, domain)?
3346                .checked_add(funding::totals::<E>(journal, Some(&domain.root))?.0)
3347                .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3348            let expected_terminal_balance = initial
3349                .checked_add(funding)
3350                .and_then(|available| available.checked_sub(creation_debit))
3351                .ok_or_else(|| {
3352                    EnsureWorkflowError::Conservation(format!(
3353                        "Root {} funding-account balance cannot reconcile its reviewed funding and creation debit",
3354                        domain.root
3355                    ))
3356                })?;
3357            if terminal_balance != expected_terminal_balance {
3358                return Err(EnsureWorkflowError::Conservation(format!(
3359                    "Root {} funding-account balance {terminal_balance} differs from reviewed terminal balance {expected_terminal_balance}",
3360                    domain.root
3361                )));
3362            }
3363            Ok((
3364                funding_total.checked_add(funding).ok_or_else(|| {
3365                    EnsureWorkflowError::Conservation(
3366                        "estate funding arithmetic overflowed".to_string(),
3367                    )
3368                })?,
3369                fee_total
3370                    .checked_add(creation_fee)
3371                    .ok_or_else(|| {
3372                        EnsureWorkflowError::Conservation(
3373                            "estate creation-fee arithmetic overflowed".to_string(),
3374                        )
3375                    })?,
3376            ))
3377        })
3378}
3379
3380fn applied_estate_funding_for_domain<E>(
3381    plan: &FleetEnsurePlan,
3382    journal: &FleetEnsureJournalRecord,
3383    state: &FleetEnsureStateRecord,
3384    domain: &crate::fleet_ensure::model::EstateFundingDomainPlan,
3385) -> Result<u128, EnsureWorkflowError<E>>
3386where
3387    E: std::error::Error + 'static,
3388{
3389    let action = plan
3390        .canisters
3391        .iter()
3392        .find(|canister| canister.name == domain.root)
3393        .into_iter()
3394        .flat_map(|canister| &canister.actions)
3395        .find(|action| matches!(action, EnsureAction::FundEstate { .. }));
3396    let Some(action) = action else {
3397        return if domain.maximum_funding_cycles == 0 {
3398            Ok(0)
3399        } else {
3400            Err(EnsureWorkflowError::JournalIntegrity)
3401        };
3402    };
3403    let EnsureAction::FundEstate {
3404        amount,
3405        expected_post_cycles,
3406        ledger,
3407        ledger_fee_cycles,
3408        principal,
3409        ..
3410    } = action
3411    else {
3412        unreachable!("filtered estate-funding action")
3413    };
3414    let root_principal = principal.strip_prefix("created:").map_or_else(
3415        || Some(principal.as_str()),
3416        |name| {
3417            state
3418                .pending_principals
3419                .get(name)
3420                .or_else(|| state.principals.get(name))
3421                .map(String::as_str)
3422        },
3423    );
3424    let expected_destination = domain
3425        .available_cycles
3426        .unwrap_or_default()
3427        .checked_add(domain.shortfall_cycles)
3428        .ok_or(EnsureWorkflowError::JournalIntegrity)?;
3429    let action_authority_is_exact = *amount == domain.shortfall_cycles
3430        && *expected_post_cycles == expected_destination
3431        && ledger == &domain.cycles_ledger
3432        && *ledger_fee_cycles == domain.ledger_fee_cycles
3433        && root_principal == reviewed_estate_root_principal(domain, state);
3434    let effect = journal
3435        .effects
3436        .iter()
3437        .find(|effect| effect.action_sha256 == action_sha256(action));
3438    let Some(effect) = effect.filter(|effect| {
3439        effect.state == EffectState::Applied
3440            && effect
3441                .receipt
3442                .as_deref()
3443                .is_some_and(|receipt| !receipt.is_empty())
3444    }) else {
3445        return Err(EnsureWorkflowError::JournalIntegrity);
3446    };
3447    let balance_evidence_is_exact = crate::fleet_ensure::ops::estate_funding_applied(
3448        crate::fleet_ensure::ops::EstateFundingObservation {
3449            amount: *amount,
3450            destination_after: effect.destination_post_cycles.unwrap_or_default(),
3451            destination_before: effect.destination_pre_cycles,
3452            expected_destination_after: *expected_post_cycles,
3453            ledger_fee_cycles: *ledger_fee_cycles,
3454            source_after: effect.post_cycles.unwrap_or_default(),
3455            source_before: effect.pre_cycles,
3456        },
3457    );
3458    if !action_authority_is_exact || !balance_evidence_is_exact {
3459        return Err(EnsureWorkflowError::JournalIntegrity);
3460    }
3461    Ok(*amount)
3462}
3463
3464fn controlled_cycles<E>(observation: &FleetObservation) -> Result<u128, EnsureWorkflowError<E>>
3465where
3466    E: std::error::Error + 'static,
3467{
3468    observation
3469        .canisters
3470        .values()
3471        .filter_map(Option::as_ref)
3472        .try_fold(0_u128, |total, live| total.checked_add(live.cycles))
3473        .and_then(|total| {
3474            observation
3475                .additional_controlled_cycles
3476                .values()
3477                .try_fold(total, |subtotal, cycles| subtotal.checked_add(*cycles))
3478        })
3479        .and_then(|total| {
3480            observation
3481                .estate_funding_domains
3482                .values()
3483                .filter_map(|domain| domain.balance_cycles)
3484                .try_fold(total, u128::checked_add)
3485        })
3486        .ok_or_else(|| {
3487            EnsureWorkflowError::Conservation("controlled-cycle total overflowed".to_string())
3488        })
3489}
3490
3491fn verified_plan<E>(plan: FleetEnsurePlan) -> Result<FleetEnsurePlan, EnsureWorkflowError<E>>
3492where
3493    E: std::error::Error + 'static,
3494{
3495    if expected_plan_sha256(&plan) != plan.plan_sha256 {
3496        return Err(EnsureWorkflowError::PlanIntegrity);
3497    }
3498    Ok(plan)
3499}
3500
3501fn verify_journal<E>(
3502    journal: &FleetEnsureJournalRecord,
3503    plan: &FleetEnsurePlan,
3504    requested_fleet: &str,
3505    state: &FleetEnsureStateRecord,
3506) -> Result<(), EnsureWorkflowError<E>>
3507where
3508    E: std::error::Error + 'static,
3509{
3510    if journal.fleet != requested_fleet
3511        || journal.operation_id != plan.operation_id
3512        || journal.plan_sha256 != plan.plan_sha256
3513        || !initial_estate_funding_is_exact(journal, plan)
3514        || !estate_funding_pause_is_exact(journal, plan, state)
3515    {
3516        return Err(EnsureWorkflowError::JournalIntegrity);
3517    }
3518    funding::verify(plan, journal, state)?;
3519    continuation::verify_records(plan, journal)?;
3520    let actions = continuation::actions(plan, journal);
3521    let effect_count_matches = journal.effects.len() <= actions.len();
3522    let action_hashes_match = journal.effects.iter().zip(actions).all(|(effect, action)| {
3523        effect.action_sha256 == action_sha256(action)
3524            && maintenance_attempts_are_exact(effect, action)
3525    });
3526    if !(effect_count_matches && action_hashes_match) {
3527        return Err(EnsureWorkflowError::JournalIntegrity);
3528    }
3529    Ok(())
3530}
3531
3532fn maintenance_attempts_are_exact(effect: &EffectRecord, action: &EnsureAction) -> bool {
3533    if let EnsureAction::FleetProtocol { action, .. } = action
3534        && let CurrentFleetProtocolAction::MaintainPoolReadiness {
3535            maximum_updates, ..
3536        } = action.as_ref()
3537    {
3538        effect.maintenance_attempts <= *maximum_updates
3539    } else {
3540        effect.maintenance_attempts == 0
3541    }
3542}
3543
3544fn initial_estate_funding_is_exact(
3545    journal: &FleetEnsureJournalRecord,
3546    plan: &FleetEnsurePlan,
3547) -> bool {
3548    journal.initial_estate_funding_cycles_by_root.len()
3549        == plan.conservation.estate_funding_domains.len()
3550        && plan
3551            .conservation
3552            .estate_funding_domains
3553            .iter()
3554            .all(|domain| {
3555                journal
3556                    .initial_estate_funding_cycles_by_root
3557                    .get(&domain.root)
3558                    .copied()
3559                    == Some(domain.available_cycles.unwrap_or_default())
3560            })
3561}
3562
3563fn estate_funding_pause_is_exact(
3564    journal: &FleetEnsureJournalRecord,
3565    plan: &FleetEnsurePlan,
3566    state: &FleetEnsureStateRecord,
3567) -> bool {
3568    let Some(required) = journal.estate_funding_required.as_ref() else {
3569        return true;
3570    };
3571    journal.completion == FleetEnsureCompletion::InProgress
3572        && estate_funding_record_is_exact(required, plan, state)
3573}
3574
3575fn estate_funding_record_is_exact(
3576    required: &EstateFundingRequiredRecord,
3577    plan: &FleetEnsurePlan,
3578    state: &FleetEnsureStateRecord,
3579) -> bool {
3580    let Some(reviewed) = plan
3581        .conservation
3582        .estate_funding_domains
3583        .iter()
3584        .find(|domain| domain.root == required.root)
3585    else {
3586        return false;
3587    };
3588    let operation_authority_is_exact =
3589        required.operation_id == plan.operation_id && required.plan_sha256 == plan.plan_sha256;
3590    let funding_authority_is_exact =
3591        EstateFundingPauseAuthority::from(required) == EstateFundingPauseAuthority::from(reviewed);
3592    let root_authority_is_exact =
3593        reviewed_estate_root_principal(reviewed, state) == Some(required.root_principal.as_str());
3594    let shortfall_is_exact = required.available_cycles < required.maximum_creation_debit_cycles
3595        && required.shortfall_cycles
3596            == required
3597                .maximum_creation_debit_cycles
3598                .saturating_sub(required.available_cycles);
3599    if !(operation_authority_is_exact
3600        && funding_authority_is_exact
3601        && root_authority_is_exact
3602        && shortfall_is_exact)
3603    {
3604        return false;
3605    }
3606    if required.pending_creation_operation_id.is_some() {
3607        current_creation_funding_pause_is_exact(required, reviewed)
3608    } else {
3609        planned_estate_funding_pause_is_exact(required, reviewed)
3610    }
3611}
3612
3613#[derive(Eq, PartialEq)]
3614struct EstateFundingPauseAuthority<'a> {
3615    creation_amount_cycles: u128,
3616    creation_execution_margin_cycles: u128,
3617    cycles_ledger: &'a str,
3618    ledger_fee_cycles: u128,
3619    management_creation_fee_cycles: u128,
3620    readiness_floor_cycles: u128,
3621    root: &'a str,
3622}
3623
3624impl<'a> From<&'a EstateFundingRequiredRecord> for EstateFundingPauseAuthority<'a> {
3625    fn from(required: &'a EstateFundingRequiredRecord) -> Self {
3626        Self {
3627            creation_amount_cycles: required.creation_amount_cycles,
3628            creation_execution_margin_cycles: required.creation_execution_margin_cycles,
3629            cycles_ledger: &required.cycles_ledger,
3630            ledger_fee_cycles: required.ledger_fee_cycles,
3631            management_creation_fee_cycles: required.management_creation_fee_cycles,
3632            readiness_floor_cycles: required.readiness_floor_cycles,
3633            root: &required.root,
3634        }
3635    }
3636}
3637
3638impl<'a> From<&'a EstateFundingDomainPlan> for EstateFundingPauseAuthority<'a> {
3639    fn from(reviewed: &'a EstateFundingDomainPlan) -> Self {
3640        Self {
3641            creation_amount_cycles: reviewed.creation_amount_cycles,
3642            creation_execution_margin_cycles: reviewed.creation_execution_margin_cycles,
3643            cycles_ledger: &reviewed.cycles_ledger,
3644            ledger_fee_cycles: reviewed.ledger_fee_cycles,
3645            management_creation_fee_cycles: reviewed.management_creation_fee_cycles,
3646            readiness_floor_cycles: reviewed.readiness_floor_cycles,
3647            root: &reviewed.root,
3648        }
3649    }
3650}
3651
3652fn current_creation_funding_pause_is_exact(
3653    required: &EstateFundingRequiredRecord,
3654    reviewed: &EstateFundingDomainPlan,
3655) -> bool {
3656    let operation_id_is_exact = required
3657        .pending_creation_operation_id
3658        .as_deref()
3659        .is_some_and(is_sha256_hex);
3660    let attempt_evidence_is_exact = required.attempt_count.is_some()
3661        && required
3662            .retry_at_ns
3663            .is_some_and(|retry_at_ns| retry_at_ns > 0)
3664        && required
3665            .last_attempt_at_ns
3666            .is_none_or(|attempted_at_ns| attempted_at_ns > 0);
3667    let single_creation_debit = required
3668        .creation_amount_cycles
3669        .checked_add(required.ledger_fee_cycles);
3670    let single_creation_fees = required
3671        .management_creation_fee_cycles
3672        .checked_add(required.ledger_fee_cycles);
3673    let debit_authority_is_exact = Some(required.maximum_creation_debit_cycles)
3674        == single_creation_debit
3675        && Some(required.maximum_creation_fee_cycles) == single_creation_fees;
3676    reviewed.required_creation_count > 0
3677        && required.required_creation_count == 1
3678        && operation_id_is_exact
3679        && attempt_evidence_is_exact
3680        && debit_authority_is_exact
3681}
3682
3683const fn planned_estate_funding_pause_is_exact(
3684    required: &EstateFundingRequiredRecord,
3685    reviewed: &EstateFundingDomainPlan,
3686) -> bool {
3687    let runtime_attempt_is_absent = required.attempt_count.is_none()
3688        && required.last_attempt_at_ns.is_none()
3689        && required.retry_at_ns.is_none();
3690    let reviewed_debit_is_exact = required.maximum_creation_debit_cycles
3691        == reviewed.maximum_creation_debit_cycles
3692        && required.maximum_creation_fee_cycles == reviewed.maximum_creation_fee_cycles
3693        && required.required_creation_count == reviewed.required_creation_count;
3694    runtime_attempt_is_absent && reviewed_debit_is_exact
3695}
3696
3697fn is_sha256_hex(value: &str) -> bool {
3698    value.len() == 64
3699        && value
3700            .bytes()
3701            .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
3702}
3703
3704#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3705enum TerminalStatePublicationError {
3706    JournalIntegrity,
3707}
3708
3709fn publish_terminal_state(
3710    desired: &crate::fleet_ensure::model::DesiredFleet,
3711    plan: &FleetEnsurePlan,
3712    journal: &FleetEnsureJournalRecord,
3713    state: &mut FleetEnsureStateRecord,
3714) -> Result<(), TerminalStatePublicationError> {
3715    let maximum_observation_burn_cycles =
3716        crate::fleet_ensure::ops::maximum_creation_observation_burn(desired)
3717            .ok_or(TerminalStatePublicationError::JournalIntegrity)?;
3718    let actions = continuation::actions(plan, journal);
3719    if actions.len() != journal.effects.len() {
3720        return Err(TerminalStatePublicationError::JournalIntegrity);
3721    }
3722    for (action, record) in actions.into_iter().zip(&journal.effects) {
3723        if action_sha256(action) != record.action_sha256 || record.state != EffectState::Applied {
3724            return Err(TerminalStatePublicationError::JournalIntegrity);
3725        }
3726        let EnsureAction::Create {
3727            name,
3728            requested_initial_cycles,
3729            ..
3730        } = action
3731        else {
3732            continue;
3733        };
3734        let Some(created_principal) = record.created_principal.as_deref() else {
3735            return Err(TerminalStatePublicationError::JournalIntegrity);
3736        };
3737        let Some(post_cycles) = record.post_cycles else {
3738            return Err(TerminalStatePublicationError::JournalIntegrity);
3739        };
3740        if !create_balance_is_terminal(
3741            Some(post_cycles),
3742            *requested_initial_cycles,
3743            maximum_observation_burn_cycles,
3744        ) || !create_identity_is_exact(name, record, state)
3745            || state
3746                .retained_cycles_by_principal
3747                .get(created_principal)
3748                .is_some_and(|retained| *retained != post_cycles)
3749        {
3750            return Err(TerminalStatePublicationError::JournalIntegrity);
3751        }
3752        state
3753            .retained_cycles_by_principal
3754            .insert(created_principal.to_string(), post_cycles);
3755    }
3756
3757    let prior_topology = state.topology.clone();
3758    for canister in &plan.canisters {
3759        if !desired
3760            .canisters
3761            .iter()
3762            .any(|configured| configured.name == canister.name)
3763        {
3764            // Dynamic pool assets remain in protected Root inventory. A funding-only
3765            // projection must not become a configured application identity.
3766            if !is_reviewed_dynamic_pool_funding(plan, canister) {
3767                return Err(TerminalStatePublicationError::JournalIntegrity);
3768            }
3769            continue;
3770        }
3771
3772        match canister.disposition {
3773            CanisterDisposition::Create | CanisterDisposition::Replace => {
3774                if let Some(principal) = state.pending_principals.remove(&canister.name) {
3775                    state.principals.insert(canister.name.clone(), principal);
3776                }
3777            }
3778            CanisterDisposition::Reinstall | CanisterDisposition::Reuse => {
3779                if let Some(principal) = state
3780                    .pending_principals
3781                    .remove(&canister.name)
3782                    .or_else(|| canister.principal.clone())
3783                {
3784                    state.principals.insert(canister.name.clone(), principal);
3785                }
3786            }
3787            CanisterDisposition::Delete => {
3788                state.principals.remove(&canister.name);
3789                state.pending_principals.remove(&canister.name);
3790            }
3791        }
3792    }
3793    state.topology = desired
3794        .canisters
3795        .iter()
3796        .filter(|canister| {
3797            canister.presence == crate::fleet_ensure::model::DesiredPresence::Present
3798        })
3799        .map(|canister| {
3800            (
3801                canister.name.clone(),
3802                crate::fleet_ensure::model::FleetEnsureTopologyRecord {
3803                    kind: canister.kind,
3804                    module_hash: projected_module_hash(plan, &prior_topology, canister),
3805                    parent: canister.parent.clone(),
3806                    protocol_binding: canister.protocol_binding.clone(),
3807                    role: canister
3808                        .protocol_binding
3809                        .as_ref()
3810                        .map(|binding| binding.role.to_string()),
3811                },
3812            )
3813        })
3814        .collect();
3815    Ok(())
3816}
3817
3818fn is_reviewed_dynamic_pool_funding(
3819    plan: &FleetEnsurePlan,
3820    canister: &crate::fleet_ensure::model::CanisterPlan,
3821) -> bool {
3822    canister.disposition == CanisterDisposition::Reuse
3823        && !canister.actions.is_empty()
3824        && canister
3825            .actions
3826            .iter()
3827            .all(|action| is_reviewed_pool_funding(action, plan, canister.principal.as_deref()))
3828}
3829
3830fn is_reviewed_pool_funding(
3831    action: &EnsureAction,
3832    plan: &FleetEnsurePlan,
3833    expected_principal: Option<&str>,
3834) -> bool {
3835    let EnsureAction::Fund {
3836        pool_funding: Some(root),
3837        principal,
3838        ..
3839    } = action
3840    else {
3841        return false;
3842    };
3843    if Some(principal.as_str()) != expected_principal {
3844        return false;
3845    }
3846    plan.protocol_actions.iter().any(|candidate| {
3847        let EnsureAction::FleetProtocol {
3848            principal: target,
3849            action,
3850            ..
3851        } = candidate
3852        else {
3853            return false;
3854        };
3855        let CurrentFleetProtocolAction::ReconcilePoolAsset { request, .. } = action.as_ref() else {
3856            return false;
3857        };
3858        target == &root.root && request.canister_id.to_text() == *principal
3859    })
3860}
3861
3862fn projected_module_hash(
3863    plan: &FleetEnsurePlan,
3864    prior_topology: &BTreeMap<String, crate::fleet_ensure::model::FleetEnsureTopologyRecord>,
3865    canister: &crate::fleet_ensure::model::DesiredCanister,
3866) -> Option<String> {
3867    let installed = plan
3868        .canisters
3869        .iter()
3870        .find(|planned| planned.name == canister.name)
3871        .and_then(|planned| {
3872            planned
3873                .actions
3874                .iter()
3875                .rev()
3876                .find_map(|action| match action {
3877                    EnsureAction::Install { wasm_sha256, .. } => Some(wasm_sha256.clone()),
3878                    _ => None,
3879                })
3880        });
3881    installed.or_else(|| {
3882        prior_topology
3883            .get(&canister.name)
3884            .filter(|retained| retained.kind == canister.kind && retained.parent == canister.parent)
3885            .and_then(|retained| retained.module_hash.clone())
3886    })
3887}
3888
3889fn merge_terminal_inventory<E>(
3890    state: &mut FleetEnsureStateRecord,
3891    inventory: crate::fleet_ensure::ops::TerminalFleetInventory,
3892) -> Result<(), EnsureWorkflowError<E>>
3893where
3894    E: std::error::Error + 'static,
3895{
3896    if inventory.entries.is_empty() {
3897        state.active_registry = inventory.active_registry;
3898        return Ok(());
3899    }
3900    let entries = inventory.entries;
3901    let mut names_by_principal = state
3902        .principals
3903        .iter()
3904        .map(|(name, principal)| (principal.clone(), name.clone()))
3905        .collect::<BTreeMap<_, _>>();
3906    if names_by_principal.len() != state.principals.len() {
3907        return Err(EnsureWorkflowError::TerminalInventory(
3908            TerminalInventoryError::DuplicateRetainedPrincipal,
3909        ));
3910    }
3911    for entry in &entries {
3912        names_by_principal
3913            .entry(entry.pid.clone())
3914            .or_insert_with(|| format!("observed:{}", entry.pid));
3915    }
3916    for entry in entries {
3917        let name = names_by_principal.get(&entry.pid).cloned().ok_or_else(|| {
3918            EnsureWorkflowError::TerminalInventory(TerminalInventoryError::LostPrincipalBinding {
3919                canister: entry.pid.clone(),
3920            })
3921        })?;
3922        let parent = entry
3923            .parent_pid
3924            .as_ref()
3925            .map(|principal| {
3926                names_by_principal.get(principal).cloned().ok_or_else(|| {
3927                    EnsureWorkflowError::TerminalInventory(TerminalInventoryError::UnknownParent {
3928                        canister: entry.pid.clone(),
3929                        parent: principal.clone(),
3930                    })
3931                })
3932            })
3933            .transpose()?;
3934        let existing = state.topology.get(&name);
3935        let kind = terminal_inventory_kind(existing, &entry);
3936        let parent_conflict = existing
3937            .and_then(|topology| topology.parent.as_ref())
3938            .zip(parent.as_ref())
3939            .filter(|(expected, actual)| expected != actual);
3940        if let Some((retained_parent, terminal_parent)) = parent_conflict
3941            && !verified_pool_to_component_parent_transition(existing, kind, &entry)
3942        {
3943            return Err(EnsureWorkflowError::TerminalInventory(
3944                TerminalInventoryError::RetainedParentConflict {
3945                    canister: entry.pid,
3946                    retained_parent: retained_parent.clone(),
3947                    terminal_parent: terminal_parent.clone(),
3948                },
3949            ));
3950        }
3951        state.principals.insert(name.clone(), entry.pid);
3952        state.topology.insert(
3953            name,
3954            crate::fleet_ensure::model::FleetEnsureTopologyRecord {
3955                kind,
3956                module_hash: entry.module_hash,
3957                parent,
3958                protocol_binding: entry.protocol_binding,
3959                role: entry.role,
3960            },
3961        );
3962    }
3963    state.active_registry = inventory.active_registry;
3964    Ok(())
3965}
3966
3967fn terminal_inventory_kind(
3968    existing: Option<&crate::fleet_ensure::model::FleetEnsureTopologyRecord>,
3969    entry: &crate::registry::RegistryEntry,
3970) -> crate::fleet_ensure::model::DesiredCanisterKind {
3971    existing.map_or_else(
3972        || {
3973            if entry.module_hash.is_some() {
3974                crate::fleet_ensure::model::DesiredCanisterKind::Component
3975            } else {
3976                crate::fleet_ensure::model::DesiredCanisterKind::Auxiliary
3977            }
3978        },
3979        |topology| {
3980            if topology.kind == crate::fleet_ensure::model::DesiredCanisterKind::Pool
3981                && entry.module_hash.is_some()
3982                && entry.protocol_binding.is_some()
3983            {
3984                crate::fleet_ensure::model::DesiredCanisterKind::Component
3985            } else {
3986                topology.kind
3987            }
3988        },
3989    )
3990}
3991
3992fn verified_pool_to_component_parent_transition(
3993    existing: Option<&crate::fleet_ensure::model::FleetEnsureTopologyRecord>,
3994    terminal_kind: crate::fleet_ensure::model::DesiredCanisterKind,
3995    entry: &crate::registry::RegistryEntry,
3996) -> bool {
3997    existing.is_some_and(|topology| {
3998        topology.kind == crate::fleet_ensure::model::DesiredCanisterKind::Pool
3999            && terminal_kind == crate::fleet_ensure::model::DesiredCanisterKind::Component
4000            && entry.module_hash.is_some()
4001            && entry.protocol_binding.is_some()
4002    })
4003}
4004
4005fn applied_count(journal: &FleetEnsureJournalRecord) -> u32 {
4006    u32::try_from(
4007        journal
4008            .effects
4009            .iter()
4010            .chain(
4011                journal
4012                    .funding_reviews
4013                    .iter()
4014                    .filter_map(|review| review.effect.as_ref()),
4015            )
4016            .filter(|effect| matches!(effect.state, EffectState::Applied))
4017            .count(),
4018    )
4019    .unwrap_or(u32::MAX)
4020}
4021
4022fn verify_store_activation_binding<P: EnsurePlatform>(
4023    plan: &FleetEnsurePlan,
4024    action: &EnsureAction,
4025    state: &FleetEnsureStateRecord,
4026    platform: &mut P,
4027) -> Result<(), EnsureWorkflowError<P::Error>> {
4028    use crate::fleet_ensure::model::DesiredCanisterInit;
4029    let EnsureAction::Install {
4030        canic_init: Some(DesiredCanisterInit::Store { root }),
4031        ..
4032    } = action
4033    else {
4034        return Ok(());
4035    };
4036    let root_will_be_installed = plan.canisters.iter().flat_map(|canister| &canister.actions)
4037        .any(|action| matches!(action,
4038            EnsureAction::Install { name, canic_init: Some(DesiredCanisterInit::Root { root: installed_root }), .. }
4039            if name == root && installed_root == root));
4040    if !root_will_be_installed {
4041        platform
4042            .require_retained_root_activation(&plan.operation_id, root, state)
4043            .map_err(EnsureWorkflowError::Platform)?;
4044    }
4045    Ok(())
4046}
4047
4048pub(super) fn ordered_actions(plan: &FleetEnsurePlan) -> Vec<&EnsureAction> {
4049    let mut actions = plan
4050        .canisters
4051        .iter()
4052        .flat_map(|canister| canister.actions.iter())
4053        .chain(plan.protocol_actions.iter())
4054        .collect::<Vec<_>>();
4055    if plan.scope == FleetEnsurePlanScope::ReinstallPreparation
4056        && plan
4057            .reinstall
4058            .as_ref()
4059            .is_some_and(|intent| intent.activation_reset.is_some())
4060    {
4061        // Preserve the compiler's Coordinator-first order, then settle every Root before restart.
4062        actions.sort_by_key(|action| matches!(action, EnsureAction::Start { .. }));
4063        return actions;
4064    }
4065    if plan.scope == FleetEnsurePlanScope::RootReinstallPrerequisite {
4066        return actions;
4067    }
4068    let temporary_pool_observation_finalizations = plan
4069        .reviewed_desired
4070        .as_deref()
4071        .map(|reviewed| {
4072            let desired = reviewed.desired();
4073            plan.canisters
4074                .iter()
4075                .filter_map(|canister_plan| {
4076                    let configured = desired.canisters.iter().find(|configured| {
4077                        configured.name == canister_plan.name
4078                            && configured.kind
4079                                == crate::fleet_ensure::model::DesiredCanisterKind::Pool
4080                    })?;
4081                    let temporary_create = desired
4082                        .bootstrap
4083                        .as_ref()
4084                        .is_some_and(|bootstrap| bootstrap.fresh_estate)
4085                        && configured.principal.is_none()
4086                        && configured.controllers.is_empty()
4087                        && configured.controller_canisters.len() == 1
4088                        && canister_plan.actions.iter().any(|action| {
4089                            matches!(
4090                                action,
4091                                EnsureAction::Create {
4092                                    controller_canisters,
4093                                    controllers,
4094                                    ..
4095                                } if controller_canisters == &configured.controller_canisters
4096                                    && controllers == std::slice::from_ref(&desired.operator)
4097                            )
4098                        });
4099                    temporary_create.then_some(canister_plan.name.as_str())
4100                })
4101                .collect::<BTreeSet<_>>()
4102        })
4103        .unwrap_or_default();
4104    actions.sort_by_key(|action| {
4105        if matches!(action, EnsureAction::SetControllers { name, .. } if temporary_pool_observation_finalizations.contains(name.as_str()))
4106        {
4107            6
4108        } else {
4109            action_order(action)
4110        }
4111    });
4112    actions
4113}
4114
4115fn deferred_create_observation_is_exact(
4116    desired: &crate::fleet_ensure::model::DesiredFleet,
4117    actions: &[&EnsureAction],
4118    index: usize,
4119    action: &EnsureAction,
4120    record: &EffectRecord,
4121    state: &FleetEnsureStateRecord,
4122) -> bool {
4123    let EnsureAction::Create {
4124        controller_canisters,
4125        controllers,
4126        name,
4127        ..
4128    } = action
4129    else {
4130        return false;
4131    };
4132    let Some(configured) = desired.canisters.iter().find(|canister| {
4133        canister.name == *name
4134            && canister.kind == crate::fleet_ensure::model::DesiredCanisterKind::Pool
4135    }) else {
4136        return false;
4137    };
4138    let Some(parent) = configured.parent.as_deref() else {
4139        return false;
4140    };
4141    let fresh_root_only_create = desired
4142        .bootstrap
4143        .as_ref()
4144        .is_some_and(|bootstrap| bootstrap.fresh_estate)
4145        && configured.principal.is_none()
4146        && configured.controllers.is_empty()
4147        && configured.controller_canisters.as_slice() == [parent]
4148        && controller_canisters == &configured.controller_canisters
4149        && controllers.is_empty();
4150    let exact_issued_create = record.state == EffectState::Issued
4151        && create_identity_is_exact(name, record, state)
4152        && record.post_cycles.is_some();
4153    let later_root_install = actions[index.saturating_add(1)..].iter().any(|candidate| {
4154        matches!(
4155            candidate,
4156            EnsureAction::Install {
4157                canic_init: Some(crate::fleet_ensure::model::DesiredCanisterInit::Root { root }),
4158                name: install_name,
4159                ..
4160            } if root == parent && install_name == parent
4161        )
4162    });
4163    let infrastructure_only = actions.iter().all(|candidate| {
4164        matches!(
4165            candidate,
4166            EnsureAction::Create { .. }
4167                | EnsureAction::FundEstate { .. }
4168                | EnsureAction::Install { .. }
4169                | EnsureAction::SetControllers { .. }
4170        )
4171    });
4172    fresh_root_only_create && exact_issued_create && later_root_install && infrastructure_only
4173}
4174
4175pub(super) const fn action_order(action: &EnsureAction) -> u8 {
4176    match action {
4177        EnsureAction::SealAuthority {
4178            authority_kind: crate::fleet_ensure::model::DesiredCanisterKind::Root,
4179            ..
4180        }
4181        | EnsureAction::Create { .. } => 0,
4182        EnsureAction::SealAuthority { .. }
4183        | EnsureAction::Fund { .. }
4184        | EnsureAction::FundEstate { .. } => 1,
4185        EnsureAction::Install {
4186            canic_init: Some(crate::fleet_ensure::model::DesiredCanisterInit::Coordinator),
4187            ..
4188        } => 2,
4189        EnsureAction::SetControllers { .. } => 3,
4190        EnsureAction::Install {
4191            canic_init: Some(crate::fleet_ensure::model::DesiredCanisterInit::Store { .. }),
4192            ..
4193        } => 4,
4194        EnsureAction::Install { .. } => 5,
4195        EnsureAction::Start { .. } => 6,
4196        EnsureAction::FleetProtocol { .. } | EnsureAction::Protocol { .. } => 7,
4197        EnsureAction::Transfer { .. } => 8,
4198        EnsureAction::Stop { .. } => 9,
4199        EnsureAction::Delete { .. } => 10,
4200    }
4201}
4202
4203#[cfg(test)]
4204mod tests {
4205    use super::*;
4206
4207    #[test]
4208    fn store_binding_requires_exact_retained_root_or_reviewed_root_install() {
4209        use crate::fleet_ensure::{
4210            model::{CanisterDisposition, CanisterPlan, DesiredCanisterInit, InstallMode},
4211            tests::MockPlatform,
4212        };
4213        let (state, _) = retained_evidence();
4214        let mut plan = estate_funding_plan();
4215        let desired = serde_json::from_value(serde_json::json!({
4216            "canisters": [], "cycles_ledger": "um5iw-rqaaa-aaaaq-qaaba-cai",
4217            "environment": "local", "fleet": "fleet", "ledger_fee_cycles": "0",
4218            "management_creation_fee_cycles": "0", "material_cycle_threshold": "1",
4219            "maximum_observation_burn_cycles": "1", "maximum_stalled_observations": 2,
4220            "maximum_update_burn_cycles": "2", "operator": "rrkah-fqaaa-aaaaa-aaaaq-cai",
4221            "schema_version": FLEET_ENSURE_SCHEMA_VERSION, "treasury": "root"
4222        }))
4223        .unwrap();
4224        let mut platform = MockPlatform::new(desired, []);
4225        platform.rejected_activation_root = Some("root".into());
4226        let store = EnsureAction::Install {
4227            canic_init: Some(DesiredCanisterInit::Store {
4228                root: "root".into(),
4229            }),
4230            reinstall_witness: None,
4231            init_arg: None,
4232            init_arg_sha256: None,
4233            init_candid: None,
4234            init_candid_sha256: None,
4235            mode: InstallMode::Install,
4236            name: "store".into(),
4237            principal: "store-principal".into(),
4238            wasm: "store.wasm".into(),
4239            wasm_sha256: "store-hash".into(),
4240        };
4241        assert!(matches!(
4242            verify_store_activation_binding(&plan, &store, &state, &mut platform),
4243            Err(EnsureWorkflowError::Platform(_))
4244        ));
4245        assert_eq!(
4246            platform.retained_activation_checks,
4247            [(plan.operation_id.clone(), "root".into())]
4248        );
4249        let mut root_install = store.clone();
4250        if let EnsureAction::Install {
4251            name, canic_init, ..
4252        } = &mut root_install
4253        {
4254            *name = "other-root".into();
4255            *canic_init = Some(DesiredCanisterInit::Root {
4256                root: "other-root".into(),
4257            });
4258        }
4259        plan.canisters.push(CanisterPlan {
4260            actions: vec![root_install],
4261            disposition: CanisterDisposition::Create,
4262            name: "root".into(),
4263            observed_cycles: 0,
4264            principal: None,
4265        });
4266        assert!(verify_store_activation_binding(&plan, &store, &state, &mut platform).is_err());
4267        if let EnsureAction::Install {
4268            name, canic_init, ..
4269        } = &mut plan.canisters.last_mut().unwrap().actions[0]
4270        {
4271            *name = "root".into();
4272            *canic_init = Some(DesiredCanisterInit::Root {
4273                root: "root".into(),
4274            });
4275        }
4276        let reads = platform.retained_activation_checks.len();
4277        verify_store_activation_binding(&plan, &store, &state, &mut platform).unwrap();
4278        assert_eq!(platform.retained_activation_checks.len(), reads);
4279        plan.canisters.pop();
4280        platform.rejected_activation_root = None;
4281        verify_store_activation_binding(&plan, &store, &state, &mut platform).unwrap();
4282        assert_eq!(platform.retained_activation_checks.len(), reads + 1);
4283    }
4284
4285    #[test]
4286    fn root_prerequisite_retains_activation_authority_across_desired_refresh() {
4287        let (mut state, mut journal) = retained_evidence();
4288        let mut plan = estate_funding_plan();
4289        plan.scope = FleetEnsurePlanScope::RootReinstallPrerequisite;
4290        plan.conservation.estate_funding_domains.clear();
4291        journal.effects.clear();
4292        journal.completion = FleetEnsureCompletion::Converged;
4293        let before = state.clone();
4294        assert!(matches!(
4295            reconcile_retained_planning_state::<std::io::Error>(
4296                &mut state,
4297                Some(&plan),
4298                Some(&journal),
4299                "fleet",
4300                "local",
4301                "refreshed-imports"
4302            ),
4303            Err(EnsureWorkflowError::RootActivationDesiredConflict { .. })
4304        ));
4305        assert_eq!(state, before);
4306        assert!(
4307            reconcile_retained_planning_state::<std::io::Error>(
4308                &mut state,
4309                Some(&plan),
4310                Some(&journal),
4311                "fleet",
4312                "local",
4313                "desired"
4314            )
4315            .unwrap()
4316            .is_some()
4317        );
4318    }
4319
4320    fn terminal_component_entry(canister: &str, parent: &str) -> crate::registry::RegistryEntry {
4321        crate::registry::RegistryEntry {
4322            pid: canister.to_string(),
4323            role: Some("managed_component".to_string()),
4324            parent_pid: Some(parent.to_string()),
4325            module_hash: Some("11".repeat(32)),
4326            protocol_binding: Some(crate::protocol_binding::RegistryProtocolBinding {
4327                release_identity: "0.110.test".to_string(),
4328                role: canic_core::ids::CanisterRole::from("managed_component"),
4329                capabilities: BTreeSet::new(),
4330                candid_sha256: [1; 32],
4331                protocol_profile_digest:
4332                    canic_core::role_contract::ProtocolProfileDigest::from_bytes([2; 32]),
4333            }),
4334        }
4335    }
4336
4337    pub(super) fn retained_evidence() -> (FleetEnsureStateRecord, FleetEnsureJournalRecord) {
4338        let state = FleetEnsureStateRecord {
4339            active_registry: None,
4340            completed_reinstall_action_sha256: BTreeMap::from([(
4341                "root".to_string(),
4342                "action".to_string(),
4343            )]),
4344            completed_reinstall_operation_id: Some("operation".to_string()),
4345            completed_reinstalls: BTreeMap::from([("root".to_string(), 7)]),
4346            fleet: "fleet".to_string(),
4347            pending_principals: BTreeMap::new(),
4348            principals: BTreeMap::new(),
4349            retained_cycles_by_principal: BTreeMap::new(),
4350            schema_version: crate::fleet_ensure::model::FLEET_ENSURE_SCHEMA_VERSION,
4351            topology: BTreeMap::new(),
4352        };
4353        let journal = FleetEnsureJournalRecord {
4354            funding_reviews: Vec::new(),
4355            successor_phases: Vec::new(),
4356            completion: FleetEnsureCompletion::ReplanRequired,
4357            estate_funding_required: None,
4358            effects: vec![crate::fleet_ensure::model::EffectRecord {
4359                maintenance_attempts: 0,
4360                action_sha256: "action".to_string(),
4361                created_principal: None,
4362                destination_post_cycles: None,
4363                destination_pre_cycles: None,
4364                post_cycles: None,
4365                pre_cycles: None,
4366                pre_canister_version: Some(7),
4367                progress_identity: None,
4368                receipt: None,
4369                state: EffectState::Applied,
4370            }],
4371            fleet: "fleet".to_string(),
4372            initial_controlled_cycles: 0,
4373            initial_estate_funding_cycles_by_root: BTreeMap::new(),
4374            initial_operator_cycles: 0,
4375            operation_id: "operation".to_string(),
4376            plan_sha256: "plan".to_string(),
4377            schema_version: crate::fleet_ensure::model::FLEET_ENSURE_SCHEMA_VERSION,
4378            stalled_observations: 0,
4379        };
4380        (state, journal)
4381    }
4382
4383    pub(super) fn estate_funding_plan() -> FleetEnsurePlan {
4384        FleetEnsurePlan {
4385            continuation: None,
4386            canisters: Vec::new(),
4387            conservation: CycleConservation {
4388                estate_funding_domains: vec![crate::fleet_ensure::model::EstateFundingDomainPlan {
4389                    allocated_workloads: 0,
4390                    available_cycles: Some(40),
4391                    available_pool_slots: 2,
4392                    creation_amount_cycles: 45,
4393                    creation_execution_margin_cycles: 5,
4394                    readiness_floor_cycles: 35,
4395                    cycles_ledger: "ledger".to_string(),
4396                    eligible_ready_pool_assets: 0,
4397                    initial_pool_assets: Vec::new(),
4398                    ledger_fee_cycles: 5,
4399                    management_creation_fee_cycles: 5,
4400                    maximum_creation_debit_cycles: 100,
4401                    maximum_creation_fee_cycles: 20,
4402                    maximum_funding_cycles: 60,
4403                    occupied_pool_assets: 0,
4404                    pending_creation_count: 0,
4405                    pending_creation: None,
4406                    planned_initial_workloads: 2,
4407                    pool_maximum_size: 2,
4408                    required_creation_count: 2,
4409                    root: "root".to_string(),
4410                    root_principal: Some("rrkah-fqaaa-aaaaa-aaaaq-cai".to_string()),
4411                    shortfall_cycles: 60,
4412                }],
4413                expected_post_operation_cycles: 80,
4414                maximum_execution_burn_cycles: 0,
4415                maximum_new_funding_cycles: 0,
4416                maximum_operator_debit_cycles: 0,
4417                maximum_unavoidable_fee_cycles: 0,
4418                observed_controlled_cycles: 40,
4419                retained_in_reused_canisters_cycles: 0,
4420                scheduled_transfer_cycles: 0,
4421            },
4422            desired_sha256: "desired".to_string(),
4423            environment: "local".to_string(),
4424            fleet: "fleet".to_string(),
4425            operation_id: "operation".to_string(),
4426            plan_sha256: "plan".to_string(),
4427            planned_at_time: 1,
4428            protocol_actions: Vec::new(),
4429            recovery_review: None,
4430            reinstall: None,
4431            root_reinstall_bindings: Vec::new(),
4432            root_start_authority: None,
4433            reviewed_desired: None,
4434            schema_version: FLEET_ENSURE_SCHEMA_VERSION,
4435            scope: FleetEnsurePlanScope::Full,
4436            terminal_inventory_operation_id: None,
4437        }
4438    }
4439
4440    pub(super) fn estate_funding_observation(balance_cycles: Option<u128>) -> FleetObservation {
4441        FleetObservation {
4442            additional_controlled_cycles: BTreeMap::new(),
4443            canisters: BTreeMap::new(),
4444            estate_funding_domains: BTreeMap::from([(
4445                "root".to_string(),
4446                crate::fleet_ensure::model::EstateFundingDomainObservation {
4447                    balance_cycles,
4448                    cycles_ledger: "ledger".to_string(),
4449                    pool: None,
4450                    root_principal: Some("rrkah-fqaaa-aaaaa-aaaaq-cai".to_string()),
4451                },
4452            )]),
4453            ledger_fee_cycles: 0,
4454            operator_cycles: 0,
4455            protocol_ready: BTreeMap::new(),
4456        }
4457    }
4458
4459    fn terminal_estate_creation_observation() -> FleetObservation {
4460        let mut observation = estate_funding_observation(Some(0));
4461        let receipt =
4462            |byte, block_index| crate::fleet_ensure::model::EstatePoolCreationReceiptObservation {
4463                block_index,
4464                operation_id: format!("{byte:02x}").repeat(32),
4465                cycles_ledger: "ledger".to_string(),
4466                ledger_amount_cycles: 45,
4467                ledger_fee_cycles: 5,
4468                readiness_floor_cycles: 35,
4469                creation_execution_margin_cycles: 5,
4470                management_creation_fee_cycles: 5,
4471                first_observed_cycles: Some(39),
4472            };
4473        observation
4474            .estate_funding_domains
4475            .get_mut("root")
4476            .expect("Root funding domain")
4477            .pool = Some(crate::fleet_ensure::model::EstatePoolInventoryObservation {
4478            assets: vec![
4479                crate::fleet_ensure::model::EstatePoolAssetObservation {
4480                    creation_receipt: Some(receipt(1, 10)),
4481                    cycles: 35,
4482                    lifecycle: crate::fleet_ensure::model::EstatePoolAssetLifecycle::Ready,
4483                    origin: crate::fleet_ensure::model::EstatePoolAssetOrigin::Created,
4484                    principal: "created-one".to_string(),
4485                },
4486                crate::fleet_ensure::model::EstatePoolAssetObservation {
4487                    creation_receipt: Some(receipt(2, 11)),
4488                    cycles: 35,
4489                    lifecycle: crate::fleet_ensure::model::EstatePoolAssetLifecycle::Workload,
4490                    origin: crate::fleet_ensure::model::EstatePoolAssetOrigin::Created,
4491                    principal: "created-two".to_string(),
4492                },
4493            ],
4494            maximum_size: 2,
4495            minimum_size: 0,
4496            pending_creation: None,
4497            readiness_floor_cycles: 35,
4498            creation_execution_margin_cycles: 5,
4499        });
4500        observation
4501    }
4502
4503    fn estate_creation_receipt(
4504        observation: &mut FleetObservation,
4505        asset_index: usize,
4506    ) -> &mut crate::fleet_ensure::model::EstatePoolCreationReceiptObservation {
4507        observation
4508            .estate_funding_domains
4509            .get_mut("root")
4510            .expect("Root domain")
4511            .pool
4512            .as_mut()
4513            .expect("pool")
4514            .assets[asset_index]
4515            .creation_receipt
4516            .as_mut()
4517            .expect("receipt")
4518    }
4519
4520    #[test]
4521    fn autonomous_reserve_creation_reconciles_without_host_provisioning() {
4522        let mut plan = estate_funding_plan();
4523        let domain = &mut plan.conservation.estate_funding_domains[0];
4524        domain.available_cycles = Some(100);
4525        domain.maximum_funding_cycles = 0;
4526        domain.shortfall_cycles = 0;
4527        let (state, mut journal) = retained_evidence();
4528        journal.effects.clear();
4529        journal
4530            .initial_estate_funding_cycles_by_root
4531            .insert("root".to_string(), 100);
4532        let terminal = terminal_estate_creation_observation();
4533        assert_eq!(
4534            reconcile_estate_funding::<std::io::Error>(&plan, &journal, &state, &terminal)
4535                .expect("reconcile exact Root-owned creation against the reviewed reserve"),
4536            (0, 20)
4537        );
4538    }
4539
4540    #[test]
4541    fn terminal_estate_creation_uses_exact_protected_receipts() {
4542        let plan = estate_funding_plan();
4543        let domain = &plan.conservation.estate_funding_domains[0];
4544        let (state, _) = retained_evidence();
4545        let exact = terminal_estate_creation_observation();
4546        assert_eq!(
4547            exact_estate_creation_costs::<std::io::Error>(&state, &exact, domain)
4548                .expect("reconcile exact creation receipts"),
4549            (100, 20)
4550        );
4551        for required_creation_count in [0, 1] {
4552            let mut insufficient = domain.clone();
4553            insufficient.required_creation_count = required_creation_count;
4554            assert!(matches!(
4555                exact_estate_creation_costs::<std::io::Error>(&state, &exact, &insufficient),
4556                Err(EnsureWorkflowError::JournalIntegrity)
4557            ));
4558        }
4559
4560        let mut drifted = Vec::new();
4561        let mut changed = exact.clone();
4562        estate_creation_receipt(&mut changed, 0).ledger_amount_cycles = 44;
4563        drifted.push(changed);
4564        let mut changed = exact.clone();
4565        estate_creation_receipt(&mut changed, 1).operation_id = "01".repeat(32);
4566        drifted.push(changed);
4567        let mut changed = exact.clone();
4568        estate_creation_receipt(&mut changed, 1).block_index = 10;
4569        drifted.push(changed);
4570        let mut changed = exact.clone();
4571        estate_creation_receipt(&mut changed, 0).first_observed_cycles = None;
4572        drifted.push(changed);
4573        let mut changed = exact.clone();
4574        estate_creation_receipt(&mut changed, 0).first_observed_cycles = Some(34);
4575        drifted.push(changed);
4576        let mut changed = exact.clone();
4577        estate_creation_receipt(&mut changed, 0).first_observed_cycles = Some(41);
4578        drifted.push(changed);
4579        let mut changed = exact;
4580        changed
4581            .estate_funding_domains
4582            .get_mut("root")
4583            .expect("Root domain")
4584            .pool
4585            .as_mut()
4586            .expect("pool")
4587            .assets[0]
4588            .origin = crate::fleet_ensure::model::EstatePoolAssetOrigin::Imported;
4589        drifted.push(changed);
4590        for observation in drifted {
4591            assert!(matches!(
4592                exact_estate_creation_costs::<std::io::Error>(&state, &observation, domain,),
4593                Err(EnsureWorkflowError::JournalIntegrity)
4594            ));
4595        }
4596    }
4597
4598    #[test]
4599    fn estate_funding_pause_is_exact_and_clears_only_after_sufficient_balance() {
4600        let plan = estate_funding_plan();
4601        let (state, mut journal) = retained_evidence();
4602        let required = estate_funding_requirement::<std::io::Error>(
4603            &plan,
4604            &state,
4605            &estate_funding_observation(Some(40)),
4606        )
4607        .expect("validate funding observation")
4608        .expect("underfunded estate pauses");
4609        assert_eq!(required.available_cycles, 40);
4610        assert_eq!(required.maximum_creation_debit_cycles, 100);
4611        assert_eq!(required.shortfall_cycles, 60);
4612        assert!(
4613            estate_funding_requirement::<std::io::Error>(
4614                &plan,
4615                &state,
4616                &estate_funding_observation(Some(100)),
4617            )
4618            .expect("validate sufficient funding observation")
4619            .is_none()
4620        );
4621
4622        journal.completion = FleetEnsureCompletion::InProgress;
4623        journal.effects.clear();
4624        journal.estate_funding_required = Some(required);
4625        journal.initial_estate_funding_cycles_by_root = BTreeMap::from([("root".to_string(), 40)]);
4626        assert!(initial_estate_funding_is_exact(&journal, &plan));
4627        assert!(estate_funding_pause_is_exact(&journal, &plan, &state));
4628
4629        journal
4630            .estate_funding_required
4631            .as_mut()
4632            .expect("funding pause")
4633            .shortfall_cycles = 59;
4634        assert!(!estate_funding_pause_is_exact(&journal, &plan, &state));
4635
4636        *journal
4637            .initial_estate_funding_cycles_by_root
4638            .get_mut("root")
4639            .expect("initial Root funding balance") = 41;
4640        assert!(!initial_estate_funding_is_exact(&journal, &plan));
4641    }
4642
4643    #[test]
4644    fn current_creation_funding_pause_is_bound_to_exact_runtime_evidence() {
4645        let mut plan = estate_funding_plan();
4646        plan.conservation.estate_funding_domains[0].cycles_ledger = "aaaaa-aa".to_string();
4647        let (state, mut journal) = retained_evidence();
4648        let funding = canic_core::dto::component_provisioning::RootEstateFundingRequired {
4649            available: Cycles::new(40),
4650            attempt_count: 2,
4651            creation_amount: Cycles::new(45),
4652            cycles_ledger: candid::Principal::from_text("aaaaa-aa").expect("Ledger Principal"),
4653            execution_margin: Cycles::new(5),
4654            last_attempt_at_ns: Some(10),
4655            ledger_fee: Cycles::new(5),
4656            management_creation_fee: Cycles::new(5),
4657            operation_id: [17; 32],
4658            readiness_floor: Cycles::new(35),
4659            required: Cycles::new(50),
4660            retry_at_ns: 20,
4661            root: candid::Principal::from_text("rrkah-fqaaa-aaaaa-aaaaq-cai")
4662                .expect("Root Principal"),
4663            shortfall: Cycles::new(10),
4664        };
4665        let required =
4666            protocol_estate_funding_requirement::<std::io::Error>(&plan, &state, &funding)
4667                .expect("bind runtime funding evidence");
4668        journal.completion = FleetEnsureCompletion::InProgress;
4669        journal.effects.clear();
4670        journal.estate_funding_required = Some(required);
4671        assert!(estate_funding_pause_is_exact(&journal, &plan, &state));
4672
4673        journal
4674            .estate_funding_required
4675            .as_mut()
4676            .expect("funding pause")
4677            .pending_creation_operation_id = Some("AA".repeat(32));
4678        assert!(!estate_funding_pause_is_exact(&journal, &plan, &state));
4679
4680        let mut drifted = funding;
4681        drifted.execution_margin = Cycles::new(6);
4682        assert!(matches!(
4683            protocol_estate_funding_requirement::<std::io::Error>(&plan, &state, &drifted),
4684            Err(EnsureWorkflowError::PlanIntegrity)
4685        ));
4686    }
4687
4688    #[test]
4689    fn estate_funding_observation_rejects_missing_or_mismatched_authority() {
4690        let plan = estate_funding_plan();
4691        let (state, _) = retained_evidence();
4692        let mut missing_balance = estate_funding_observation(None);
4693        assert!(matches!(
4694            estate_funding_requirement::<std::io::Error>(&plan, &state, &missing_balance),
4695            Err(EnsureWorkflowError::PlanIntegrity)
4696        ));
4697
4698        let observed = missing_balance
4699            .estate_funding_domains
4700            .get_mut("root")
4701            .expect("Root funding domain");
4702        observed.balance_cycles = Some(40);
4703        observed.root_principal = Some("aaaaa-aa".to_string());
4704        assert!(matches!(
4705            estate_funding_requirement::<std::io::Error>(&plan, &state, &missing_balance),
4706            Err(EnsureWorkflowError::PlanIntegrity)
4707        ));
4708
4709        for changed_balance in [39, 101] {
4710            assert!(matches!(
4711                estate_funding_requirement::<std::io::Error>(
4712                    &plan,
4713                    &state,
4714                    &estate_funding_observation(Some(changed_balance)),
4715                ),
4716                Err(EnsureWorkflowError::DriftedBeforeApply)
4717            ));
4718        }
4719    }
4720
4721    #[test]
4722    fn non_creation_protocol_completion_does_not_consume_estate_creation_authority() {
4723        let mut plan = estate_funding_plan();
4724        plan.conservation.estate_funding_domains[0].maximum_creation_debit_cycles = 0;
4725        plan.conservation.estate_funding_domains[0].maximum_creation_fee_cycles = 0;
4726        plan.conservation.estate_funding_domains[0].maximum_funding_cycles = 0;
4727        plan.conservation.estate_funding_domains[0].required_creation_count = 0;
4728        plan.conservation.estate_funding_domains[0].shortfall_cycles = 0;
4729        let action = EnsureAction::FleetProtocol {
4730            action: Box::new(
4731                crate::fleet_ensure::model::CurrentFleetProtocolAction::PrepareStoreChunkSet {
4732                    request: canic_control_plane::dto::template::TemplateChunkSetPrepareInput {
4733                        template_id: canic_control_plane::ids::TemplateId::from("root"),
4734                        version: canic_control_plane::ids::TemplateVersion::from("current"),
4735                        payload_hash: vec![1; 32],
4736                        payload_size_bytes: 1,
4737                        chunk_hashes: vec![vec![2; 32]],
4738                    },
4739                },
4740            ),
4741            candid: "store.did".to_string(),
4742            candid_sha256: "11".repeat(32),
4743            maximum_execution_burn_cycles: 0,
4744            name: "prepare-store-chunks".to_string(),
4745            principal: "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(),
4746        };
4747        let action_hash = action_sha256(&action);
4748        plan.protocol_actions.push(action);
4749        let (state, mut journal) = retained_evidence();
4750        journal.initial_estate_funding_cycles_by_root = BTreeMap::from([("root".to_string(), 40)]);
4751        journal.effects = vec![EffectRecord {
4752            maintenance_attempts: 0,
4753            action_sha256: action_hash,
4754            created_principal: None,
4755            destination_post_cycles: None,
4756            destination_pre_cycles: None,
4757            post_cycles: None,
4758            pre_cycles: None,
4759            pre_canister_version: None,
4760            progress_identity: Some("store chunks prepared".to_string()),
4761            receipt: Some("receipt".to_string()),
4762            state: EffectState::Applied,
4763        }];
4764
4765        assert_eq!(
4766            reconcile_estate_funding::<std::io::Error>(
4767                &plan,
4768                &journal,
4769                &state,
4770                &estate_funding_observation(Some(40)),
4771            )
4772            .expect("non-creation protocol preserves the Root Ledger balance"),
4773            (0, 0),
4774        );
4775    }
4776
4777    #[test]
4778    fn fresh_estate_funding_binds_the_created_root_from_durable_state() {
4779        let mut plan = estate_funding_plan();
4780        plan.conservation.estate_funding_domains[0].available_cycles = None;
4781        plan.conservation.estate_funding_domains[0].maximum_funding_cycles = 100;
4782        plan.conservation.estate_funding_domains[0].root_principal = None;
4783        let (mut state, _) = retained_evidence();
4784        state.pending_principals.insert(
4785            "root".to_string(),
4786            "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(),
4787        );
4788
4789        let required = estate_funding_requirement::<std::io::Error>(
4790            &plan,
4791            &state,
4792            &estate_funding_observation(Some(0)),
4793        )
4794        .expect("validate created Root funding authority")
4795        .expect("fresh underfunded estate pauses");
4796        assert_eq!(required.shortfall_cycles, 100);
4797        assert_eq!(required.root_principal, "rrkah-fqaaa-aaaaa-aaaaq-cai");
4798
4799        state
4800            .pending_principals
4801            .insert("root".to_string(), "aaaaa-aa".to_string());
4802        assert!(matches!(
4803            estate_funding_requirement::<std::io::Error>(
4804                &plan,
4805                &state,
4806                &estate_funding_observation(Some(0)),
4807            ),
4808            Err(EnsureWorkflowError::PlanIntegrity)
4809        ));
4810    }
4811
4812    #[test]
4813    fn unchanged_estate_funding_pause_does_not_rewrite_its_journal() {
4814        let root = crate::test_support::temp_dir("estate-funding-pause");
4815        let paths = EnsurePaths::under(&root, "local", "fleet");
4816        let plan = estate_funding_plan();
4817        let (state, mut journal) = retained_evidence();
4818        journal.completion = FleetEnsureCompletion::InProgress;
4819        journal.effects.clear();
4820        journal.initial_estate_funding_cycles_by_root = BTreeMap::from([("root".to_string(), 40)]);
4821        let required = estate_funding_requirement::<std::io::Error>(
4822            &plan,
4823            &state,
4824            &estate_funding_observation(Some(40)),
4825        )
4826        .expect("validate funding observation")
4827        .expect("underfunded estate pauses");
4828
4829        assert!(
4830            retain_estate_funding_pause::<std::io::Error>(&paths, &mut journal, Some(&required),)
4831                .expect("retain first funding pause")
4832        );
4833        let retained = std::fs::read(&paths.journal).expect("read first retained pause");
4834        assert!(
4835            !retain_estate_funding_pause::<std::io::Error>(&paths, &mut journal, Some(&required),)
4836                .expect("replay unchanged funding pause")
4837        );
4838        assert_eq!(
4839            std::fs::read(&paths.journal).expect("reread unchanged retained pause"),
4840            retained
4841        );
4842        assert!(
4843            retain_estate_funding_pause::<std::io::Error>(&paths, &mut journal, None)
4844                .expect("clear funded pause")
4845        );
4846        assert!(journal.estate_funding_required.is_none());
4847
4848        std::fs::remove_dir_all(root).expect("remove funding-pause fixture");
4849    }
4850
4851    #[test]
4852    fn estate_funding_is_rechecked_until_the_first_protocol_effect_starts() {
4853        let protocol_action = || EnsureAction::FleetProtocol {
4854            action: Box::new(
4855                crate::fleet_ensure::model::CurrentFleetProtocolAction::PrepareStoreChunkSet {
4856                    request: canic_control_plane::dto::template::TemplateChunkSetPrepareInput {
4857                        template_id: canic_control_plane::ids::TemplateId::from("root"),
4858                        version: canic_control_plane::ids::TemplateVersion::from("current"),
4859                        payload_hash: vec![1; 32],
4860                        payload_size_bytes: 1,
4861                        chunk_hashes: vec![vec![2; 32]],
4862                    },
4863                },
4864            ),
4865            candid: "store.did".to_string(),
4866            candid_sha256: "11".repeat(32),
4867            maximum_execution_burn_cycles: 1,
4868            name: "prepare".to_string(),
4869            principal: "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(),
4870        };
4871        let first = protocol_action();
4872        let second = protocol_action();
4873        let actions = [&first, &second];
4874        let (_, mut journal) = retained_evidence();
4875        journal.effects = vec![EffectRecord {
4876            maintenance_attempts: 0,
4877            action_sha256: action_sha256(&first),
4878            created_principal: None,
4879            destination_post_cycles: None,
4880            destination_pre_cycles: None,
4881            post_cycles: None,
4882            pre_cycles: None,
4883            pre_canister_version: None,
4884            progress_identity: None,
4885            receipt: None,
4886            state: EffectState::Intent,
4887        }];
4888        assert!(!prior_fleet_protocol_effect_started(&actions, &journal, 1));
4889
4890        journal.effects[0].state = EffectState::Issued;
4891        assert!(prior_fleet_protocol_effect_started(&actions, &journal, 1));
4892    }
4893
4894    #[test]
4895    fn terminal_inventory_allows_only_verified_pool_to_component_parent_transition() {
4896        let asset = "rrkah-fqaaa-aaaaa-aaaaq-cai";
4897        let old_root = "r7inp-6aaaa-aaaaa-aaabq-cai";
4898        let hub = "ryjl3-tyaaa-aaaaa-aaaba-cai";
4899        let state = || {
4900            let (mut state, _) = retained_evidence();
4901            state.principals = BTreeMap::from([
4902                ("asset".to_string(), asset.to_string()),
4903                ("hub".to_string(), hub.to_string()),
4904                ("root".to_string(), old_root.to_string()),
4905            ]);
4906            state.topology.insert(
4907                "asset".to_string(),
4908                crate::fleet_ensure::model::FleetEnsureTopologyRecord {
4909                    kind: crate::fleet_ensure::model::DesiredCanisterKind::Pool,
4910                    module_hash: None,
4911                    parent: Some("root".to_string()),
4912                    protocol_binding: None,
4913                    role: Some("canister_pool_asset".to_string()),
4914                },
4915            );
4916            state
4917        };
4918
4919        let mut promoted = state();
4920        merge_terminal_inventory::<std::io::Error>(
4921            &mut promoted,
4922            crate::fleet_ensure::ops::TerminalFleetInventory {
4923                active_registry: None,
4924                controlled_cycles_by_principal: BTreeMap::new(),
4925                entries: vec![terminal_component_entry(asset, hub)],
4926            },
4927        )
4928        .expect("verified pool workload assumes its terminal Component parent");
4929        let promoted = promoted.topology.get("asset").expect("promoted asset");
4930        assert_eq!(
4931            promoted.kind,
4932            crate::fleet_ensure::model::DesiredCanisterKind::Component
4933        );
4934        assert_eq!(promoted.parent.as_deref(), Some("hub"));
4935
4936        let mut already_component = state();
4937        already_component
4938            .topology
4939            .get_mut("asset")
4940            .expect("retained asset")
4941            .kind = crate::fleet_ensure::model::DesiredCanisterKind::Component;
4942        assert!(matches!(
4943            merge_terminal_inventory::<std::io::Error>(
4944                &mut already_component,
4945                crate::fleet_ensure::ops::TerminalFleetInventory {
4946                    active_registry: None,
4947                    controlled_cycles_by_principal: BTreeMap::new(),
4948                    entries: vec![terminal_component_entry(asset, hub)],
4949                },
4950            ),
4951            Err(EnsureWorkflowError::TerminalInventory(
4952                TerminalInventoryError::RetainedParentConflict {
4953                    canister,
4954                    retained_parent,
4955                    terminal_parent,
4956                }
4957            )) if canister == asset
4958                && retained_parent == "root"
4959                && terminal_parent == "hub"
4960        ));
4961
4962        for incomplete_entry in [
4963            crate::registry::RegistryEntry {
4964                module_hash: None,
4965                ..terminal_component_entry(asset, hub)
4966            },
4967            crate::registry::RegistryEntry {
4968                protocol_binding: None,
4969                ..terminal_component_entry(asset, hub)
4970            },
4971        ] {
4972            assert!(matches!(
4973                merge_terminal_inventory::<std::io::Error>(
4974                    &mut state(),
4975                    crate::fleet_ensure::ops::TerminalFleetInventory {
4976                        active_registry: None,
4977                        controlled_cycles_by_principal: BTreeMap::new(),
4978                        entries: vec![incomplete_entry],
4979                    },
4980                ),
4981                Err(EnsureWorkflowError::TerminalInventory(
4982                    TerminalInventoryError::RetainedParentConflict {
4983                        canister,
4984                        retained_parent,
4985                        terminal_parent,
4986                    }
4987                )) if canister == asset
4988                    && retained_parent == "root"
4989                    && terminal_parent == "hub"
4990            ));
4991        }
4992    }
4993
4994    #[test]
4995    fn completed_reinstall_evidence_requires_exact_operation_action_and_version() {
4996        let (state, journal) = retained_evidence();
4997        assert!(retained_reinstall_operation_matches(
4998            &state,
4999            "fleet",
5000            "operation",
5001            &journal,
5002        ));
5003
5004        let mut wrong_action = journal.clone();
5005        wrong_action.effects[0].action_sha256 = "other".to_string();
5006        assert!(!retained_reinstall_operation_matches(
5007            &state,
5008            "fleet",
5009            "operation",
5010            &wrong_action,
5011        ));
5012
5013        let mut wrong_version = journal.clone();
5014        wrong_version.effects[0].pre_canister_version = Some(8);
5015        assert!(!retained_reinstall_operation_matches(
5016            &state,
5017            "fleet",
5018            "operation",
5019            &wrong_version,
5020        ));
5021
5022        for (fleet, operation) in [("other", "operation"), ("fleet", "other")] {
5023            assert!(!retained_reinstall_operation_matches(
5024                &state, fleet, operation, &journal,
5025            ));
5026        }
5027
5028        let mut incomplete = journal;
5029        incomplete.effects[0].state = EffectState::Issued;
5030        assert!(!retained_reinstall_operation_matches(
5031            &state,
5032            "fleet",
5033            "operation",
5034            &incomplete,
5035        ));
5036    }
5037
5038    #[test]
5039    fn fresh_fleet_alternate_desired_cannot_clear_completed_reinstall_evidence() {
5040        let (state, journal) = retained_evidence();
5041        assert!(retained_reinstall_desired_conflict(
5042            &state,
5043            "fleet",
5044            "operation",
5045            "reviewed-desired",
5046            "alternate-desired",
5047            &journal,
5048        ));
5049        assert!(!retained_reinstall_desired_conflict(
5050            &state,
5051            "fleet",
5052            "operation",
5053            "reviewed-desired",
5054            "reviewed-desired",
5055            &journal,
5056        ));
5057
5058        let mut terminal = journal;
5059        terminal.completion = FleetEnsureCompletion::Converged;
5060        assert!(!retained_reinstall_desired_conflict(
5061            &state,
5062            "fleet",
5063            "operation",
5064            "reviewed-desired",
5065            "alternate-desired",
5066            &terminal,
5067        ));
5068    }
5069
5070    fn cycle_observation(
5071        lifecycle: crate::fleet_ensure::model::RootOwnedCanisterLifecycle,
5072        cycles: u128,
5073    ) -> FleetObservation {
5074        FleetObservation {
5075            additional_controlled_cycles: BTreeMap::new(),
5076            canisters: BTreeMap::from([(
5077                "asset".to_string(),
5078                Some(crate::fleet_ensure::model::LiveCanister {
5079                    canister_version: Some(1),
5080                    controllers: Vec::new(),
5081                    cycles,
5082                    module_sha256: None,
5083                    principal: "controlled-principal".to_string(),
5084                    reinstall_required: false,
5085                    root_owned_lifecycle: Some(lifecycle),
5086                    status: crate::fleet_ensure::model::CanisterRuntimeStatus::Stopped,
5087                }),
5088            )]),
5089            estate_funding_domains: BTreeMap::new(),
5090            ledger_fee_cycles: 0,
5091            operator_cycles: 0,
5092            protocol_ready: BTreeMap::new(),
5093        }
5094    }
5095
5096    #[test]
5097    fn fresh_fleet_idle_cycle_duplicate_requires_exact_balance() {
5098        use crate::fleet_ensure::model::RootOwnedCanisterLifecycle;
5099
5100        let additional = BTreeMap::from([("controlled-principal".to_string(), 100)]);
5101        let mut idle = cycle_observation(RootOwnedCanisterLifecycle::Idle, 100);
5102        attach_terminal_cycles::<std::io::Error>(&mut idle, additional.clone())
5103            .expect("merge exact Idle observation");
5104        assert!(idle.additional_controlled_cycles.is_empty());
5105
5106        let mut conflicting_idle = cycle_observation(RootOwnedCanisterLifecycle::Idle, 99);
5107        assert!(matches!(
5108            attach_terminal_cycles::<std::io::Error>(&mut conflicting_idle, additional.clone()),
5109            Err(EnsureWorkflowError::TerminalInventory(
5110                TerminalInventoryError::IdleCycleObservationConflict {
5111                    canister,
5112                    expected: 99,
5113                    observed: 100,
5114                }
5115            )) if canister == "controlled-principal"
5116        ));
5117
5118        let mut workload = cycle_observation(RootOwnedCanisterLifecycle::Workload, 99);
5119        attach_terminal_cycles::<std::io::Error>(&mut workload, additional)
5120            .expect("merge conservative workload observation");
5121        assert_eq!(
5122            workload
5123                .canisters
5124                .get("asset")
5125                .and_then(Option::as_ref)
5126                .map(|live| live.cycles),
5127            Some(99),
5128        );
5129    }
5130}