Skip to main content

kevy_cli/
doctor.rs

1//! `doctor` — run every table's `VERIFY` and turn the counters into an
2//! exit code.
3//!
4//! Lesson 8 of the migration playbook: *make `VERIFY` part of
5//! operations, not part of the migration.* The counters are fresh on
6//! every call and cheap enough for a cron; what was missing is the
7//! shell that turns them into something a cron can act on.
8//!
9//! The mapping is the lesson's own words, not a new opinion:
10//!
11//! * `drift` and `missing` **should be zero forever** — non-zero is a
12//!   failure;
13//! * non-zero `duplicates` on an ORDERPATH means **pagination needs a
14//!   bounded tie-break** — a warning about a design choice, not a
15//!   corruption;
16//! * `absent` / `excluded` / `coerce_failures` **name the rows each
17//!   exclusion cause claimed** — reported, never failed on, because
18//!   every one of them is a legitimate state.
19//!
20//! And one thing the lesson could not have known: `TABLE.VERIFY`
21//! answers `-INDEXBUILDING` while a backfill is still running. A cron
22//! that read that as a failure would page someone every time an index
23//! was declared, so it is its own outcome.
24
25use std::io;
26use std::process::ExitCode;
27
28use kevy_resp_client::{Reply, RespClient};
29
30/// What `doctor` concluded about one table.
31#[derive(Debug)]
32pub enum Health {
33    /// Every counter where it should be.
34    Ok,
35    /// `drift` or `missing` is non-zero — the index and the keyspace
36    /// disagree, which is the thing VERIFY exists to make falsifiable.
37    Drift {
38        /// Which counters were non-zero, with their values.
39        detail: String,
40    },
41    /// Non-zero `duplicates`: not corruption, but pagination over this
42    /// path needs a bounded tie-break or pages will repeat rows.
43    NeedsTieBreak {
44        /// How many duplicate order values were found.
45        duplicates: u64,
46    },
47    /// A backfill is still running. Not a verdict either way.
48    Building,
49}
50
51/// One table's name and what was concluded about it.
52#[derive(Debug)]
53pub struct TableHealth {
54    /// The declared table name.
55    pub name: String,
56    /// The verdict.
57    pub health: Health,
58    /// The counters worth showing whatever the verdict — the exclusion
59    /// causes, which are legitimate states rather than problems.
60    pub reported: String,
61}
62
63/// Pull `[field, value, …]` pairs out of a flat reply array.
64pub(crate) fn fields(items: &[Reply]) -> Vec<(String, String)> {
65    let bulks: Vec<String> = items
66        .iter()
67        .map(|r| match r {
68            Reply::Bulk(b) => String::from_utf8_lossy(b).into_owned(),
69            Reply::Int(i) => i.to_string(),
70            _ => String::new(),
71        })
72        .collect();
73    bulks.chunks(2).filter(|c| c.len() == 2).map(|c| (c[0].clone(), c[1].clone())).collect()
74}
75
76/// Every declared table's name, in declaration order.
77pub fn table_names(client: &mut RespClient) -> io::Result<Vec<String>> {
78    let Reply::Array(tables) = client.request_borrowed(&[b"TABLE.LIST"])? else {
79        return Ok(Vec::new());
80    };
81    Ok(tables
82        .iter()
83        .filter_map(|t| {
84            let Reply::Array(items) = t else { return None };
85            fields(items).into_iter().find(|(k, _)| k == "name").map(|(_, v)| v)
86        })
87        .collect())
88}
89
90/// Verify one table and read its counters against lesson 8's mapping.
91pub fn check_table(client: &mut RespClient, name: &str) -> io::Result<TableHealth> {
92    let reply = client.request_borrowed(&[b"TABLE.VERIFY", name.as_bytes()])?;
93    if let Reply::Error(e) = &reply {
94        let msg = String::from_utf8_lossy(e);
95        let health = if msg.starts_with("INDEXBUILDING") {
96            Health::Building
97        } else {
98            Health::Drift { detail: msg.into_owned() }
99        };
100        return Ok(TableHealth { name: name.to_string(), health, reported: String::new() });
101    }
102    // The reply is per-index groups plus a spot-check group; summing the
103    // counters across groups is the table-level answer.
104    let Reply::Array(groups) = reply else {
105        return Ok(TableHealth {
106            name: name.to_string(),
107            health: Health::Drift { detail: "unreadable VERIFY reply".into() },
108            reported: String::new(),
109        });
110    };
111    let mut sums: std::collections::BTreeMap<String, u64> = Default::default();
112    for g in &groups {
113        if let Reply::Array(items) = g {
114            for (k, v) in fields(items) {
115                if let Ok(n) = v.parse::<u64>() {
116                    *sums.entry(k).or_insert(0) += n;
117                }
118            }
119        }
120    }
121    let get = |k: &str| sums.get(k).copied().unwrap_or(0);
122    let reported = format!(
123        "rows {} · entries {} · absent {} · excluded {} · coerce_failures {}",
124        get("rows"),
125        get("entries"),
126        get("absent"),
127        get("excluded"),
128        get("coerce_failures")
129    );
130    Ok(TableHealth { name: name.to_string(), health: classify(&groups), reported })
131}
132
133/// Lesson 8's mapping, as one function so a test can state it without a
134/// server: zero-forever counters fail, duplicates warn, exclusion
135/// causes are reported and never fail.
136fn classify(groups: &[Reply]) -> Health {
137    let mut sums: std::collections::BTreeMap<String, u64> = Default::default();
138    for g in groups {
139        if let Reply::Array(items) = g {
140            for (k, v) in fields(items) {
141                if let Ok(n) = v.parse::<u64>() {
142                    *sums.entry(k).or_insert(0) += n;
143                }
144            }
145        }
146    }
147    let get = |k: &str| sums.get(k).copied().unwrap_or(0);
148    let (drift, missing, dups) = (get("drift"), get("missing"), get("duplicates"));
149    if drift > 0 || missing > 0 {
150        Health::Drift { detail: format!("drift {drift}, missing {missing}") }
151    } else if dups > 0 {
152        Health::NeedsTieBreak { duplicates: dups }
153    } else {
154        Health::Ok
155    }
156}
157
158/// Check every table and print one line each. Exit non-zero only on
159/// drift — a warning is information, and a cron that fails on
160/// information stops being read.
161pub fn run(client: &mut RespClient, warn_is_failure: bool) -> io::Result<ExitCode> {
162    let names = table_names(client)?;
163    if names.is_empty() {
164        println!("doctor: no tables declared — nothing to verify");
165        return Ok(ExitCode::SUCCESS);
166    }
167    let (mut bad, mut warned, mut building) = (0u32, 0u32, 0u32);
168    for name in &names {
169        let h = check_table(client, name)?;
170        match &h.health {
171            Health::Ok => println!("  OK       {name}  ({})", h.reported),
172            Health::Building => {
173                building += 1;
174                println!("  BUILDING {name}  — an index is still backfilling, not a verdict");
175            }
176            Health::NeedsTieBreak { duplicates } => {
177                warned += 1;
178                println!(
179                    "  WARN     {name}  duplicates {duplicates} — paging this path needs a \
180                     bounded tie-break or pages repeat rows  ({})",
181                    h.reported
182                );
183            }
184            Health::Drift { detail } => {
185                bad += 1;
186                println!("  DRIFT    {name}  {detail}  ({})", h.reported);
187            }
188        }
189    }
190    println!(
191        "doctor: {} table(s) — {bad} drifted, {warned} warned, {building} still building",
192        names.len()
193    );
194    Ok(if bad > 0 || (warn_is_failure && warned > 0) {
195        ExitCode::FAILURE
196    } else {
197        ExitCode::SUCCESS
198    })
199}
200
201/// `doctor [-h host] [-p port] [--warn-is-failure]`
202pub fn run_doctor_cli(args: &[String]) -> ExitCode {
203    let (mut host, mut port) = (crate::DEFAULT_HOST.to_string(), crate::DEFAULT_PORT);
204    let mut strict = false;
205    let mut i = 0;
206    while i < args.len() {
207        match args[i].as_str() {
208            "-h" if i + 1 < args.len() => {
209                host = args[i + 1].clone();
210                i += 2;
211            }
212            "-p" if i + 1 < args.len() => {
213                port = args[i + 1].parse().unwrap_or(crate::DEFAULT_PORT);
214                i += 2;
215            }
216            "--warn-is-failure" => {
217                strict = true;
218                i += 1;
219            }
220            _ => i += 1,
221        }
222    }
223    let mut client = match RespClient::connect(&host, port) {
224        Ok(c) => c,
225        Err(e) => {
226            eprintln!("kevy-cli: could not connect to {host}:{port}: {e}");
227            return ExitCode::FAILURE;
228        }
229    };
230    match run(&mut client, strict) {
231        Ok(code) => code,
232        Err(e) => {
233            eprintln!("kevy-cli doctor: {e}");
234            ExitCode::FAILURE
235        }
236    }
237}
238
239#[cfg(test)]
240mod tests {
241    use super::*;
242
243    fn arr(pairs: &[(&str, i64)]) -> Reply {
244        let mut items = Vec::new();
245        for (k, v) in pairs {
246            items.push(Reply::Bulk(k.as_bytes().to_vec()));
247            items.push(Reply::Bulk(v.to_string().into_bytes()));
248        }
249        Reply::Array(items)
250    }
251
252    /// The counters that must be zero forever are the ones that fail.
253    #[test]
254    fn drift_and_missing_are_the_failing_counters() {
255        for k in ["drift", "missing"] {
256            let sums = [("rows", 10), (k, 1)];
257            let groups = vec![arr(&sums)];
258            let health = classify(&groups);
259            assert!(matches!(health, Health::Drift { .. }), "{k} must fail");
260        }
261    }
262
263    /// Duplicates are a design signal, not corruption — the lesson says
264    /// it means pagination needs a bounded tie-break.
265    #[test]
266    fn duplicates_warn_rather_than_fail() {
267        let groups = vec![arr(&[("rows", 10), ("duplicates", 3), ("drift", 0)])];
268        assert!(matches!(classify(&groups), Health::NeedsTieBreak { duplicates: 3 }));
269    }
270
271    /// Every exclusion cause is a legitimate state. A doctor that failed
272    /// on them would be red on any table with a NULL column.
273    #[test]
274    fn exclusion_causes_never_fail() {
275        let groups =
276            vec![arr(&[("rows", 10), ("absent", 4), ("excluded", 2), ("coerce_failures", 1)])];
277        assert!(matches!(classify(&groups), Health::Ok));
278    }
279}