Skip to main content

xbp_deploy/
verifier.rs

1use std::sync::Arc;
2
3use async_trait::async_trait;
4use futures::future::join_all;
5use serde::{Deserialize, Serialize};
6use xbp_k8s::{K8sAdapter, KubeTarget};
7
8use crate::context::DeployContext;
9use crate::error::{DeployError, Result};
10use crate::lock::load_lock;
11use crate::providers::is_kubernetes_provider;
12use crate::types::DeployPlan;
13
14#[derive(Debug, Clone, Serialize, Deserialize)]
15pub struct VerifyResult {
16    pub ok: bool,
17    pub lines: Vec<String>,
18    pub drifts: Vec<String>,
19}
20
21#[async_trait]
22pub trait DeployVerifier: Send + Sync {
23    async fn verify(&self, ctx: &DeployContext, plan: &DeployPlan) -> Result<VerifyResult>;
24}
25
26pub struct DefaultDeployVerifier {
27    pub k8s: Arc<dyn K8sAdapter>,
28}
29
30#[async_trait]
31impl DeployVerifier for DefaultDeployVerifier {
32    async fn verify(&self, ctx: &DeployContext, plan: &DeployPlan) -> Result<VerifyResult> {
33        let mut lines: Vec<String> = Vec::new();
34        let mut drifts: Vec<String> = Vec::new();
35
36        // Only invoke kubectl verify when the plan actually includes k8s services.
37        let has_k8s = plan
38            .services
39            .iter()
40            .any(|s| is_kubernetes_provider(&s.provider))
41            || !plan.k8s_plan.services.is_empty();
42
43        if has_k8s {
44            let target: KubeTarget = KubeTarget {
45                context: plan.k8s_plan.context.clone(),
46                namespace: plan.k8s_plan.default_namespace.clone(),
47                kubeconfig: ctx
48                    .config
49                    .kubernetes
50                    .as_ref()
51                    .and_then(|k| k.kubeconfig.clone()),
52            };
53
54            match self.k8s.verify(&target, &plan.to_k8s_plan()).await {
55                Ok(vlines) => {
56                    for line in vlines {
57                        // kubectl verify historically logged rollout/pod failures as soft
58                        // notes only — that made `Result: ok` while the namespace/workload
59                        // was missing. Treat error-shaped lines as drifts.
60                        if k8s_verify_line_is_failure(&line) {
61                            drifts.push(line.clone());
62                        }
63                        lines.push(line);
64                    }
65                }
66                Err(e) => {
67                    drifts.push(format!("k8s verify: {e}"));
68                    lines.push(format!("k8s verify error: {e}"));
69                }
70            }
71        } else {
72            lines.push("kubernetes verify skipped (no k8s services)".into());
73        }
74
75        // Digest lock comparison
76        if let Some(lock) = load_lock(&ctx.config.lock_file)? {
77            if lock.hash != plan.hash && !plan.hash.is_empty() && !lock.hash.is_empty() {
78                drifts.push(format!(
79                    "plan hash drift: lock={} plan={}",
80                    lock.hash, plan.hash
81                ));
82            }
83            for svc in &plan.services {
84                if let Some(expected) = lock.images.get(&svc.name) {
85                    match &svc.digest {
86                        Some(d) if d != &expected.digest => {
87                            drifts.push(format!(
88                                "[{}] digest mismatch lock={} plan={}",
89                                svc.name, expected.digest, d
90                            ));
91                        }
92                        None => {
93                            drifts.push(format!(
94                                "[{}] missing plan digest (lock has {})",
95                                svc.name, expected.digest
96                            ));
97                        }
98                        _ => lines.push(format!("[{}] digest matches lock", svc.name)),
99                    }
100                }
101            }
102            for (name, img) in &lock.images {
103                if !plan.services.iter().any(|s| s.name == *name) {
104                    drifts.push(format!("[{name}] present in lock but not in plan"));
105                }
106                let _ = img;
107            }
108        } else {
109            lines.push("no deploy-lock.json (skip lock compare)".into());
110        }
111
112        // Parallel health
113        if !ctx.flags.skip_health {
114            let futs = plan.services.iter().flat_map(|svc| {
115                svc.deploy.health.iter().map(move |url| {
116                    let name = svc.name.clone();
117                    let url = url.clone();
118                    async move {
119                        let status = probe(&url).await;
120                        (name, url, status)
121                    }
122                })
123            });
124            for (name, url, status) in join_all(futs).await {
125                match status {
126                    Ok(code) if (200..400).contains(&code) => {
127                        lines.push(format!("[{name}] health {url} → {code}"));
128                    }
129                    Ok(code) => drifts.push(format!("[{name}] health {url} → {code}")),
130                    Err(e) => drifts.push(format!("[{name}] health {url}: {e}")),
131                }
132            }
133        }
134
135        let ok: bool = drifts.is_empty();
136        if !ok {
137            lines.push(format!("drift count: {}", drifts.len()));
138        }
139        Ok(VerifyResult { ok, lines, drifts })
140    }
141}
142
143async fn probe(url: &str) -> std::result::Result<u16, String> {
144    use tokio::process::Command;
145    let output: std::process::Output = Command::new("curl")
146        .args(["-sS", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "15", url])
147        .output()
148        .await
149        .map_err(|e| e.to_string())?;
150    if !output.status.success() {
151        return Err(String::from_utf8_lossy(&output.stderr).trim().to_string());
152    }
153    String::from_utf8_lossy(&output.stdout)
154        .trim()
155        .parse()
156        .map_err(|_| "invalid status".into())
157}
158
159/// Whether a kubectl verify log line indicates a real cluster failure.
160///
161/// Matches lines like:
162/// - `[athena] rollout: kubectl failed: Error from server (NotFound): namespaces "athena" not found`
163/// - `[athena] rollout: Error from server (NotFound): deployments.apps "athena" not found`
164/// - `[athena] pods: error: ...`
165pub fn k8s_verify_line_is_failure(line: &str) -> bool {
166    let trimmed = line.trim();
167    let body = trimmed
168        .split_once(']')
169        .map(|(_, rest)| rest.trim())
170        .unwrap_or(trimmed);
171    let lower = body.to_ascii_lowercase();
172
173    // Successful rollout status messages are not failures.
174    if lower.contains("successfully rolled out")
175        || lower.contains("rollout complete")
176        || lower.contains("deployment \"") && lower.contains("successfully")
177    {
178        return false;
179    }
180
181    // Deployment / namespace missing — always a hard failure (even outside rollout: prefix).
182    if lower.contains("not found")
183        && (lower.contains("deployment")
184            || lower.contains("deployments.apps")
185            || lower.contains("namespace")
186            || lower.contains("namespaces \""))
187    {
188        return true;
189    }
190
191    if body.starts_with("rollout:")
192        || body.starts_with("pods:")
193        || lower.contains("rollout:")
194        || lower.contains("kubectl failed")
195    {
196        if lower.contains("error")
197            || lower.contains("failed")
198            || lower.contains("not found")
199            || lower.contains("timed out")
200            || lower.contains("timeout")
201            || lower.contains("forbidden")
202            || lower.contains("unauthorized")
203            || lower.contains("connection refused")
204            || lower.contains("unable to connect")
205        {
206            return true;
207        }
208        // Empty pods list after a label query — soft signal only when selector was used
209        // and kubectl returned nothing useful.
210        if body.starts_with("pods:") {
211            let detail = body.trim_start_matches("pods:").trim();
212            if detail.is_empty() {
213                return false;
214            }
215        }
216    }
217    false
218}
219
220#[cfg(test)]
221mod tests {
222    use super::k8s_verify_line_is_failure;
223
224    #[test]
225    fn verify_treats_missing_namespace_rollout_as_failure() {
226        assert!(k8s_verify_line_is_failure(
227            r#"[athena] rollout: kubectl failed: Error from server (NotFound): namespaces "athena" not found"#
228        ));
229        assert!(k8s_verify_line_is_failure(
230            "[athena] rollout: Error from server (NotFound): deployments.apps \"athena\" not found"
231        ));
232        assert!(!k8s_verify_line_is_failure(
233            "[athena] deployment \"athena\" successfully rolled out"
234        ));
235        assert!(!k8s_verify_line_is_failure("[athena] pods: pod/athena-abc"));
236        assert!(!k8s_verify_line_is_failure("[athena] pods:"));
237        assert!(!k8s_verify_line_is_failure(
238            "[athena] health https://example.com/ → 200"
239        ));
240    }
241}
242
243/// Ensure verify surfaces as DeployError when ok=false if desired by caller.
244pub fn verify_to_result(v: VerifyResult) -> Result<VerifyResult> {
245    if v.ok {
246        Ok(v)
247    } else {
248        Err(DeployError::Verify(v.drifts.join("; ")))
249    }
250}