canic_host/fleet_ensure/workflow/readiness/
mod.rs1#[cfg(test)]
6mod tests;
7
8use crate::{
9 fleet_ensure::{
10 model::FleetEnsureCompletion,
11 ops::{
12 EnsurePaths, EnsureStateError,
13 operator_mint::transport::{OperatorMintTransport, OperatorMintTransportError},
14 read_journal, read_plan,
15 },
16 policy::{EnsurePolicyError, validate_path_labels},
17 view::readiness::{
18 FleetReadiness, ReadinessBlocker, ReadinessConversionQuote, ReadinessUnresolved,
19 RetainedReadinessOperation,
20 },
21 },
22 icp::IcpCli,
23 network::{NetworkIdentityError, resolve_canonical_network_id_from_root},
24};
25use candid::Principal;
26use canic_core::ids::CanonicalNetworkId;
27use std::path::Path;
28use thiserror::Error;
29
30pub struct FleetReadinessRequest<'a> {
32 pub workspace: &'a Path,
33 pub environment: &'a str,
34 pub fleet: &'a str,
35 pub icp_executable: &'a str,
36 pub signing_identity: Option<&'a str>,
37 pub operator: Principal,
38 pub cycles_ledger: Principal,
39 pub estimated_required_cycles: Option<u128>,
40 pub desired: Option<&'a crate::fleet_ensure::dto::LoadedDesiredFleet>,
41 pub conversion: Option<ReadinessConversionRequest>,
42}
43
44#[derive(Clone, Copy)]
51pub struct ReadinessConversionRequest {
52 pub icp_ledger: Principal,
53 pub cmc: Principal,
54}
55
56#[derive(Debug, Error)]
58pub enum FleetReadinessError {
59 #[error(transparent)]
60 Policy(#[from] EnsurePolicyError),
61 #[error(transparent)]
62 State(#[from] EnsureStateError),
63 #[error(transparent)]
64 Network(#[from] NetworkIdentityError),
65 #[error(transparent)]
66 Transport(Box<OperatorMintTransportError>),
67 #[error("selected signer or network does not match readiness authority")]
68 AuthorityMismatch,
69 #[error("readiness observation clock is unavailable")]
70 Clock,
71 #[error(
72 "retained Fleet evidence is inconsistent; preserve it and use the existing recovery flow"
73 )]
74 RetainedEvidence,
75}
76
77impl From<OperatorMintTransportError> for FleetReadinessError {
78 fn from(error: OperatorMintTransportError) -> Self {
79 Self::Transport(Box::new(error))
80 }
81}
82
83pub fn inspect(request: &FleetReadinessRequest<'_>) -> Result<FleetReadiness, FleetReadinessError> {
85 let started = now_ms()?;
86 validate_path_labels(request.environment, request.fleet)?;
87 let paths = EnsurePaths::under(request.workspace, request.environment, request.fleet);
88 let retained_operation = retained(&paths, request)?;
89 let expected_network =
90 resolve_canonical_network_id_from_root(request.workspace, request.environment)?;
91 if let Some(selected) = request.desired {
92 let desired = &selected.desired;
93 crate::fleet_ensure::policy::validate_path_identity(desired, request.fleet)?;
94 let identity_matches = desired.environment == request.environment
95 && desired.operator == request.operator.to_text()
96 && desired.cycles_ledger == request.cycles_ledger.to_text();
97 let network_matches = desired
98 .bootstrap
99 .as_ref()
100 .is_none_or(|bootstrap| bootstrap.canonical_network_id == expected_network);
101 if !identity_matches || !network_matches {
102 return Err(FleetReadinessError::AuthorityMismatch);
103 }
104 crate::fleet_ensure::policy::validate_funding_policy(desired)?;
105 }
106 let icp = IcpCli::new(request.icp_executable, Some(request.environment.into()))
107 .with_identity(request.signing_identity)
108 .with_cwd(request.workspace);
109 let transport = OperatorMintTransport::from_icp(&icp)?;
110 verify_authority(
111 request.operator,
112 transport.operator()?,
113 &expected_network,
114 &transport.canonical_network_id()?,
115 )?;
116 let available = transport.operator_balance(request.cycles_ledger)?;
117 let mut report = report(
118 request,
119 expected_network.to_string(),
120 available,
121 retained_operation,
122 );
123 report.observed_at_unix_ms = started;
124 if let Some(selected) = request.desired {
125 report.funding =
126 crate::fleet_ensure::ops::readiness::roots(request.workspace, &selected.desired, &icp)?;
127 report.funding.desired_sha256 = Some(selected.sha256.clone());
128 if report
129 .funding
130 .roots
131 .iter()
132 .any(|root| root.floor_shortfall_cycles.is_some_and(|cycles| cycles > 0))
133 {
134 report.blockers.push(ReadinessBlocker::RootNativeShortfall);
135 }
136 if report
137 .funding
138 .roots
139 .iter()
140 .any(|root| root.unfunded_role.is_some())
141 {
142 report.blockers.push(ReadinessBlocker::StartupFundingPolicy);
143 }
144 }
145 if let Some(conversion) = request.conversion {
146 report.funding.unresolved.retain(|reason| *reason != crate::fleet_ensure::view::readiness::ReadinessUnresolved::ConversionNotRequested);
147 let quote =
148 transport.quote_blocking(conversion.icp_ledger, conversion.cmc, request.cycles_ledger);
149 apply_quote(&mut report, conversion, quote.ok(), now_ms()?);
150 }
151 report.completed_at_unix_ms = now_ms()?;
152 Ok(report)
153}
154
155fn retained(
156 paths: &EnsurePaths,
157 request: &FleetReadinessRequest<'_>,
158) -> Result<Option<RetainedReadinessOperation>, FleetReadinessError> {
159 let journal = match read_journal(paths) {
160 Ok(Some(journal)) => journal,
161 Ok(None) => return Ok(None),
162 Err(error @ EnsureStateError::Decode { .. }) => {
163 let source = crate::fleet_ensure::ops::reinstall::terminal::read(
164 paths,
165 request.environment,
166 request.fleet,
167 )
168 .map_err(|_| FleetReadinessError::State(error))?;
169 return Ok(Some(RetainedReadinessOperation {
170 operation_id: source.documents.operation_id,
171 plan_sha256: source.documents.plan_sha256,
172 completion: FleetEnsureCompletion::Converged,
173 terminal_review_required: true,
174 }));
175 }
176 Err(error) => return Err(error.into()),
177 };
178 if journal.fleet != request.fleet {
179 return Err(FleetReadinessError::RetainedEvidence);
180 }
181 if journal.completion != FleetEnsureCompletion::Converged {
182 let plan = read_plan(paths)?.ok_or(FleetReadinessError::RetainedEvidence)?;
183 if plan.plan_sha256 != journal.plan_sha256
184 || plan.operation_id != journal.operation_id
185 || plan.environment != request.environment
186 || plan.fleet != request.fleet
187 {
188 return Err(FleetReadinessError::RetainedEvidence);
189 }
190 }
191 Ok(Some(RetainedReadinessOperation {
192 operation_id: journal.operation_id,
193 plan_sha256: journal.plan_sha256,
194 completion: journal.completion,
195 terminal_review_required: false,
196 }))
197}
198
199fn report(
200 request: &FleetReadinessRequest<'_>,
201 network_identity: String,
202 available_cycles: u128,
203 retained_operation: Option<RetainedReadinessOperation>,
204) -> FleetReadiness {
205 let estimated_shortfall_cycles = request
206 .estimated_required_cycles
207 .map(|amount| amount.saturating_sub(available_cycles));
208 let mut blockers = Vec::new();
209 if retained_operation
210 .as_ref()
211 .is_some_and(|operation| operation.completion != FleetEnsureCompletion::Converged)
212 {
213 blockers.push(ReadinessBlocker::RetainedOperation);
214 }
215 if retained_operation
216 .as_ref()
217 .is_some_and(|operation| operation.terminal_review_required)
218 {
219 blockers.push(ReadinessBlocker::RetainedTerminalReview);
220 }
221 if estimated_shortfall_cycles.is_some_and(|shortfall| shortfall > 0) {
222 blockers.push(ReadinessBlocker::EstimatedFundingShortfall);
223 }
224 FleetReadiness {
225 environment: request.environment.into(),
226 fleet: request.fleet.into(),
227 operator: request.operator.to_text(),
228 cycles_ledger: request.cycles_ledger.to_text(),
229 network_identity,
230 available_cycles,
231 estimated_required_cycles: request.estimated_required_cycles,
232 estimated_shortfall_cycles,
233 retained_operation,
234 blockers,
235 observed_at_unix_ms: 0,
236 completed_at_unix_ms: 0,
237 funding: crate::fleet_ensure::ops::readiness::unknown(),
238 }
239}
240
241fn verify_authority(
242 expected_operator: Principal,
243 signer: Principal,
244 expected_network: &CanonicalNetworkId,
245 observed_network: &CanonicalNetworkId,
246) -> Result<(), FleetReadinessError> {
247 let valid_signer = expected_operator != Principal::anonymous() && signer == expected_operator;
248 if valid_signer && expected_network == observed_network {
249 Ok(())
250 } else {
251 Err(FleetReadinessError::AuthorityMismatch)
252 }
253}
254
255fn now_ms() -> Result<u64, FleetReadinessError> {
256 std::time::SystemTime::now()
257 .duration_since(std::time::UNIX_EPOCH)
258 .ok()
259 .and_then(|value| u64::try_from(value.as_millis()).ok())
260 .ok_or(FleetReadinessError::Clock)
261}
262
263fn apply_quote(
264 report: &mut FleetReadiness,
265 selected: ReadinessConversionRequest,
266 quote: Option<crate::fleet_ensure::view::operator_mint::OperatorMintRateQuote>,
267 observed_at_ms: u64,
268) {
269 let Some(rate) = quote.filter(|rate| {
270 rate.rate_timestamp_seconds <= observed_at_ms / 1000 && rate.xdr_permyriad_per_icp > 0
271 }) else {
272 report
273 .funding
274 .unresolved
275 .push(ReadinessUnresolved::ConversionObservationFailed);
276 return;
277 };
278 let amount = report.estimated_shortfall_cycles.and_then(|shortfall| {
279 if shortfall == 0 {
280 return Some(0);
281 }
282 crate::fleet_ensure::policy::operator_mint::quote::amount_e8s(
283 shortfall,
284 rate.estimated_deposit_fee_cycles,
285 rate.xdr_permyriad_per_icp,
286 rate.transfer_fee_e8s,
287 )
288 });
289 if amount.is_none() {
290 report
291 .funding
292 .unresolved
293 .push(ReadinessUnresolved::ConversionAmountUnavailable);
294 }
295 let debit = amount.and_then(|amount| {
296 if amount == 0 {
297 Some(0)
298 } else {
299 amount.checked_add(rate.transfer_fee_e8s)
300 }
301 });
302 report.funding.conversion = Some(ReadinessConversionQuote {
303 cmc: selected.cmc.to_text(),
304 icp_ledger: selected.icp_ledger.to_text(),
305 rate,
306 estimated_mint_e8s: amount,
307 estimated_total_icp_debit_e8s: debit,
308 });
309}