Skip to main content

kevy_cli/
lint.rs

1//! `lint` — the two questions worth asking about a shape, before and
2//! after the table exists.
3//!
4//! Lessons 1 and 6 of the migration playbook. They are one deliverable
5//! in the plan and two commands here, because they run at different
6//! moments and answer differently.
7//!
8//! **`lint overlap`** is lesson 1, and it is not the check the plan
9//! first described. That plan said to sample a candidate column and see
10//! whether it is single-valued — but a hash field holds one value by
11//! construction, so that check passes forever. The lesson says where
12//! the answer really lives: *"the answer is usually in your
13//! id-derivation or key-construction code, not in the row itself — a
14//! thread can live in several mailboxes."* The symptom of that **is**
15//! in the data, just not in the row: the same name appears under more
16//! than one owner. So this reads the family of owner-keyed collections
17//! and asks whether they intersect. They do ⇒ no column can carry that
18//! dimension, and a membership row is the shape.
19//!
20//! **`lint columns`** is lesson 6, and it can only run **after** the
21//! table is declared — it reads rows. Two columns whose values coincide
22//! on nearly every row are one column copied to get a second sort
23//! order; the answer is another ORDERPATH, which `IDX.ADVISE` names.
24//!
25//! The exit codes differ on purpose. Overlap is an **answer**: a column
26//! cannot carry a multi-valued dimension, so a script should stop.
27//! Coincidence is a **suspicion** — two columns may legitimately agree
28//! — so it reports and exits zero.
29
30use std::collections::BTreeMap;
31use std::io;
32use std::process::ExitCode;
33
34use kevy_resp_client::{Reply, RespClient};
35
36/// What the owner-keyed collections under a prefix look like together.
37#[derive(Debug)]
38pub struct Overlap {
39    /// How many owner collections were read.
40    pub owners: usize,
41    /// Distinct names across all of them.
42    pub names: usize,
43    /// Names that appear under more than one owner.
44    pub shared: usize,
45    /// A few of them, with the owners they appear under.
46    pub examples: Vec<(String, Vec<String>)>,
47    /// Keys under the prefix that are not collections at all — a
48    /// counter or a hash sitting beside the owner sets. Reported so a
49    /// prefix that matched the wrong family is visible.
50    pub skipped: usize,
51}
52
53/// Read every collection under `prefix` and see whether they intersect.
54pub fn overlap(client: &mut RespClient, prefix: &str) -> io::Result<Overlap> {
55    let keys = crate::collections::scan_prefix(client, prefix)?;
56    let mut owners_of: BTreeMap<Vec<u8>, Vec<String>> = BTreeMap::new();
57    let (mut owners, mut skipped) = (0usize, 0usize);
58    for k in &keys {
59        let owner = String::from_utf8_lossy(k).into_owned();
60        // Discovered, not named: a sidecar under the same prefix is a
61        // neighbour, not a failure — but it is counted and reported.
62        let Some(ms) = crate::collections::members_if_collection(client, &owner)? else {
63            skipped += 1;
64            continue;
65        };
66        owners += 1;
67        for m in ms {
68            owners_of.entry(m).or_default().push(owner.clone());
69        }
70    }
71    let mut o = tally(owners, &owners_of);
72    o.skipped = skipped;
73    Ok(o)
74}
75
76/// The question lesson 1 actually asks, as a function: does any name
77/// appear under more than one owner?
78fn tally(owners: usize, owners_of: &BTreeMap<Vec<u8>, Vec<String>>) -> Overlap {
79    let shared: Vec<_> = owners_of.iter().filter(|(_, o)| o.len() > 1).collect();
80    Overlap {
81        owners,
82        skipped: 0,
83        names: owners_of.len(),
84        shared: shared.len(),
85        examples: shared
86            .iter()
87            .take(5)
88            .map(|(n, o)| (String::from_utf8_lossy(n).into_owned(), (*o).clone()))
89            .collect(),
90    }
91}
92
93/// Two columns that agree on most of the rows they both appear in.
94#[derive(Debug)]
95pub struct Coincidence {
96    /// One column.
97    pub a: String,
98    /// The other.
99    pub b: String,
100    /// Rows where both are present and equal.
101    pub same: usize,
102    /// Rows where both are present.
103    pub compared: usize,
104}
105
106impl Coincidence {
107    /// How often they agreed, as a percentage of rows compared.
108    pub fn percent(&self) -> u32 {
109        (self.same * 100).checked_div(self.compared).unwrap_or(0) as u32
110    }
111}
112
113/// Sample rows under a prefix and find column pairs that nearly always
114/// carry the same value.
115pub fn column_pairs(
116    client: &mut RespClient,
117    prefix: &str,
118    sample: usize,
119    threshold: u32,
120) -> io::Result<(usize, Vec<Coincidence>)> {
121    let keys = crate::collections::scan_prefix(client, prefix)?;
122    let mut rows = Vec::new();
123    for k in keys.iter().take(sample) {
124        let row = hgetall(client, k)?;
125        if !row.is_empty() {
126            rows.push(row);
127        }
128    }
129    Ok((rows.len(), coincidences(&rows, threshold)))
130}
131
132/// Column pairs that agree on at least `threshold` percent of the rows
133/// where both are present, worst agreement last.
134fn coincidences(rows: &[BTreeMap<String, Vec<u8>>], threshold: u32) -> Vec<Coincidence> {
135    let mut pairs: BTreeMap<(String, String), (usize, usize)> = BTreeMap::new();
136    for row in rows {
137        let cols: Vec<&String> = row.keys().collect();
138        for (i, a) in cols.iter().enumerate() {
139            for b in &cols[i + 1..] {
140                let e = pairs.entry(((*a).clone(), (*b).clone())).or_insert((0, 0));
141                e.1 += 1;
142                if row[*a] == row[*b] {
143                    e.0 += 1;
144                }
145            }
146        }
147    }
148    let mut out: Vec<Coincidence> = pairs
149        .into_iter()
150        .map(|((a, b), (same, compared))| Coincidence { a, b, same, compared })
151        .filter(|c| c.percent() >= threshold)
152        .collect();
153    out.sort_by(|x, y| y.percent().cmp(&x.percent()).then(x.a.cmp(&y.a)));
154    out
155}
156
157fn hgetall(client: &mut RespClient, key: &[u8]) -> io::Result<BTreeMap<String, Vec<u8>>> {
158    let reply = client.request_borrowed(&[b"HGETALL", key])?;
159    let flat = crate::collections::bulks(reply);
160    Ok(flat
161        .chunks(2)
162        .filter(|c| c.len() == 2)
163        .map(|c| (String::from_utf8_lossy(&c[0]).into_owned(), c[1].clone()))
164        .collect())
165}
166
167/// The declared prefix of a table, from `TABLE.LIST`.
168fn table_prefix(client: &mut RespClient, table: &str) -> io::Result<String> {
169    let Reply::Array(tables) = client.request_borrowed(&[b"TABLE.LIST"])? else {
170        return Err(io::Error::other("TABLE.LIST did not answer with a list"));
171    };
172    for t in &tables {
173        let Reply::Array(items) = t else { continue };
174        let f = crate::doctor::fields(items);
175        let named = f.iter().any(|(k, v)| k == "name" && v == table);
176        if named && let Some((_, p)) = f.iter().find(|(k, _)| k == "prefix") {
177            return Ok(p.clone());
178        }
179    }
180    Err(io::Error::other(format!("no declared table named '{table}'")))
181}
182
183/// `lint overlap --prefix <p>` / `lint columns <table> [--sample N]
184/// [--threshold PCT]`
185pub fn run_lint_cli(args: &[String]) -> ExitCode {
186    let (mut host, mut port) = (crate::DEFAULT_HOST.to_string(), crate::DEFAULT_PORT);
187    let (mut prefix, mut table) = (String::new(), String::new());
188    let (mut sample, mut threshold) = (1000usize, 90u32);
189    let sub = args.first().cloned().unwrap_or_default();
190    let mut i = 1;
191    while i < args.len() {
192        let val = args.get(i + 1);
193        match (args[i].as_str(), val) {
194            ("-h", Some(v)) => host = v.clone(),
195            ("-p", Some(v)) => port = v.parse().unwrap_or(crate::DEFAULT_PORT),
196            ("--prefix", Some(v)) => prefix = v.clone(),
197            ("--sample", Some(v)) => sample = v.parse().unwrap_or(sample),
198            ("--threshold", Some(v)) => threshold = v.parse().unwrap_or(threshold),
199            (other, _) if !other.starts_with('-') && table.is_empty() => {
200                table = other.to_string();
201                i += 1;
202                continue;
203            }
204            _ => {
205                i += 1;
206                continue;
207            }
208        }
209        i += 2;
210    }
211    let mut client = match RespClient::connect(&host, port) {
212        Ok(c) => c,
213        Err(e) => {
214            eprintln!("kevy-cli lint: could not connect to {host}:{port}: {e}");
215            return ExitCode::FAILURE;
216        }
217    };
218    match sub.as_str() {
219        "overlap" => run_overlap(&mut client, &prefix),
220        "columns" => run_columns(&mut client, &table, sample, threshold),
221        other => {
222            eprintln!("kevy-cli lint: unknown subcommand '{other}'");
223            eprintln!("usage: kevy-cli lint overlap --prefix <p>");
224            eprintln!("       kevy-cli lint columns <table> [--sample N] [--threshold PCT]");
225            ExitCode::FAILURE
226        }
227    }
228}
229
230/// Overlap is an answer, not a hint: a column cannot carry a dimension
231/// that names more than one owner, so a non-empty intersection exits
232/// non-zero and a declaring script stops.
233fn run_overlap(client: &mut RespClient, prefix: &str) -> ExitCode {
234    if prefix.is_empty() {
235        eprintln!("kevy-cli lint overlap: --prefix names the family of owner keys");
236        return ExitCode::FAILURE;
237    }
238    let o = match overlap(client, prefix) {
239        Ok(o) => o,
240        Err(e) => {
241            eprintln!("kevy-cli lint overlap: {e}");
242            return ExitCode::FAILURE;
243        }
244    };
245    println!("{} owner(s) under {prefix}, {} distinct name(s)", o.owners, o.names);
246    if o.skipped > 0 {
247        println!("  ({} key(s) under this prefix are not collections and were skipped)", o.skipped);
248    }
249    if o.owners == 0 {
250        println!("no collection under {prefix} — is that the right prefix?");
251        return ExitCode::FAILURE;
252    }
253    if o.shared == 0 {
254        println!("no name appears under more than one owner — a column can carry this dimension");
255        return ExitCode::SUCCESS;
256    }
257    println!("{} name(s) appear under more than one owner:", o.shared);
258    for (name, owners) in &o.examples {
259        println!("  {name}  →  {}", owners.join(", "));
260    }
261    println!(
262        "this dimension is multi-valued, so no column can hold it — model a membership row \
263         per (owner, item) and let an ORDERPATH sort it"
264    );
265    ExitCode::FAILURE
266}
267
268/// Coincidence is a suspicion — two columns may legitimately agree —
269/// so this reports and exits zero whatever it finds.
270fn run_columns(client: &mut RespClient, table: &str, sample: usize, threshold: u32) -> ExitCode {
271    if table.is_empty() {
272        eprintln!("kevy-cli lint columns: name a declared table");
273        return ExitCode::FAILURE;
274    }
275    let prefix = match table_prefix(client, table) {
276        Ok(p) => p,
277        Err(e) => {
278            eprintln!("kevy-cli lint columns: {e}");
279            return ExitCode::FAILURE;
280        }
281    };
282    let (rows, found) = match column_pairs(client, &prefix, sample, threshold) {
283        Ok(r) => r,
284        Err(e) => {
285            eprintln!("kevy-cli lint columns: {e}");
286            return ExitCode::FAILURE;
287        }
288    };
289    println!("{table}: {rows} row(s) sampled under {prefix}");
290    if found.is_empty() {
291        println!("no two columns agree on {threshold}% or more of them");
292        return ExitCode::SUCCESS;
293    }
294    for c in &found {
295        println!("  {} and {} agree on {}% ({}/{})", c.a, c.b, c.percent(), c.same, c.compared);
296    }
297    println!(
298        "a column copied to get a second sort order is the shape lesson 6 warns about — \
299         the answer is another ORDERPATH; ask IDX.ADVISE which one"
300    );
301    ExitCode::SUCCESS
302}
303
304#[cfg(test)]
305mod tests {
306    use super::*;
307
308    fn under(pairs: &[(&str, &[&str])]) -> BTreeMap<Vec<u8>, Vec<String>> {
309        let mut m: BTreeMap<Vec<u8>, Vec<String>> = BTreeMap::new();
310        for (name, owners) in pairs {
311            m.insert(name.as_bytes().to_vec(), owners.iter().map(|o| o.to_string()).collect());
312        }
313        m
314    }
315
316    fn row(fields: &[(&str, &str)]) -> BTreeMap<String, Vec<u8>> {
317        fields.iter().map(|(k, v)| (k.to_string(), v.as_bytes().to_vec())).collect()
318    }
319
320    /// The mail system's case: a thread that lives in several
321    /// mailboxes. One name under two owners is the whole answer.
322    #[test]
323    fn a_name_under_two_owners_is_the_multi_valued_signal() {
324        let o = tally(2, &under(&[("t1", &["m:1"]), ("t2", &["m:1", "m:2"]), ("t3", &["m:2"])]));
325        assert_eq!((o.names, o.shared), (3, 1));
326        assert_eq!(o.examples[0].0, "t2");
327        assert_eq!(o.examples[0].1, ["m:1", "m:2"]);
328    }
329
330    /// Owners that share nothing mean a column *can* carry the
331    /// dimension — the answer this check exists to give when it is yes.
332    #[test]
333    fn disjoint_owners_leave_nothing_shared() {
334        let o = tally(2, &under(&[("x", &["a"]), ("y", &["b"])]));
335        assert_eq!(o.shared, 0);
336    }
337
338    /// Lesson 6's shape: one column copied to get a second sort order.
339    /// Drift in a few rows must not hide it, so the threshold is a
340    /// percentage rather than "always equal".
341    #[test]
342    fn a_copied_column_shows_up_below_perfect_agreement() {
343        let mut rows: Vec<_> =
344            (0..9).map(|i| row(&[("a", "1"), ("b", "1"), ("c", &format!("{i}"))])).collect();
345        rows.push(row(&[("a", "1"), ("b", "2"), ("c", "9")]));
346        let found = coincidences(&rows, 90);
347        let named: Vec<&str> = found.iter().map(|c| c.a.as_str()).collect();
348        assert_eq!(found.len(), 1, "only a/b agree enough, got {named:?}");
349        assert_eq!((found[0].a.as_str(), found[0].b.as_str()), ("a", "b"));
350        assert_eq!(found[0].percent(), 90);
351    }
352
353    /// Above what was actually found, nothing is reported — the
354    /// threshold is the caller's, not a fixed opinion.
355    #[test]
356    fn a_threshold_above_the_agreement_reports_nothing() {
357        let rows = vec![row(&[("a", "1"), ("b", "1")]), row(&[("a", "1"), ("b", "2")])];
358        assert!(coincidences(&rows, 60).is_empty());
359        assert_eq!(coincidences(&rows, 50).len(), 1);
360    }
361}