Skip to main content

kevy_cli/
lint.rs

1//! `lint` — the two questions worth asking about a shape, before and
2//! after the table exists.
3//!
4//! Lessons 1 and 6 of the migration playbook. They are one deliverable
5//! in the plan and two commands here, because they run at different
6//! moments and answer differently.
7//!
8//! **`lint overlap`** is lesson 1, and it is not the check the plan
9//! first described. That plan said to sample a candidate column and see
10//! whether it is single-valued — but a hash field holds one value by
11//! construction, so that check passes forever. The lesson says where
12//! the answer really lives: *"the answer is usually in your
13//! id-derivation or key-construction code, not in the row itself — a
14//! thread can live in several mailboxes."* The symptom of that **is**
15//! in the data, just not in the row: the same name appears under more
16//! than one owner. So this reads the family of owner-keyed collections
17//! and asks whether they intersect. They do ⇒ no column can carry that
18//! dimension, and a membership row is the shape.
19//!
20//! **`lint columns`** is lesson 6, and it can only run **after** the
21//! table is declared — it reads rows. Two columns whose values coincide
22//! on nearly every row are one column copied to get a second sort
23//! order; the answer is another ORDERPATH, which `IDX.ADVISE` names.
24//!
25//! The exit codes differ on purpose. Overlap is an **answer**: a column
26//! cannot carry a multi-valued dimension, so a script should stop.
27//! Coincidence is a **suspicion** — two columns may legitimately agree
28//! — so it reports and exits zero.
29//!
30//! ```
31//! use kevy_cli::lint::{column_pairs, overlap};
32//! # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
33//! let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
34//! // a thread can live in several mailboxes: no `mailbox` column can hold that
35//! client.request_borrowed(&[b"SADD", b"box:inbox", b"t1", b"t2"])?;
36//! client.request_borrowed(&[b"SADD", b"box:work", b"t2", b"t3"])?;
37//! assert_eq!(overlap(&mut client, "box:")?.shared, 1);
38//!
39//! // `updated` is `created` copied on two rows of three
40//! for (id, created, updated) in [("1", "10", "10"), ("2", "20", "20"), ("3", "30", "31")] {
41//!     client.request_borrowed(&[b"HSET", format!("post:{id}").as_bytes(),
42//!         b"created", created.as_bytes(), b"updated", updated.as_bytes()])?;
43//! }
44//! let (_, found) = column_pairs(&mut client, "post:", 100, 60)?;
45//! assert_eq!((found[0].a.as_str(), found[0].b.as_str()), ("created", "updated"));
46//! # Ok::<(), Box<dyn std::error::Error>>(())
47//! ```
48
49use std::collections::BTreeMap;
50use std::io;
51
52use crate::link::Link;
53
54mod command;
55pub(crate) use command::run_on;
56
57/// What the owner-keyed collections under a prefix look like together.
58///
59/// ```
60/// use kevy_cli::lint::overlap;
61/// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
62/// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
63/// client.request_borrowed(&[b"SADD", b"box:inbox", b"t1", b"t2"])?;
64/// client.request_borrowed(&[b"SADD", b"box:work", b"t2", b"t3"])?;
65/// let o = overlap(&mut client, "box:")?;
66/// assert_eq!((o.owners, o.names, o.shared, o.skipped), (2, 3, 1, 0));
67/// # Ok::<(), Box<dyn std::error::Error>>(())
68/// ```
69#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)]
70#[non_exhaustive]
71pub struct Overlap {
72    /// How many owner collections were read.
73    ///
74    /// ```
75    /// use kevy_cli::lint::overlap;
76    /// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
77    /// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
78    /// # client.request_borrowed(&[b"SADD", b"box:inbox", b"t1", b"t2"])?;
79    /// # client.request_borrowed(&[b"SADD", b"box:work", b"t2", b"t3"])?;
80    /// assert_eq!(overlap(&mut client, "box:")?.owners, 2);
81    /// # Ok::<(), Box<dyn std::error::Error>>(())
82    /// ```
83    pub owners: usize,
84    /// Distinct names across all of them.
85    ///
86    /// ```
87    /// use kevy_cli::lint::overlap;
88    /// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
89    /// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
90    /// # client.request_borrowed(&[b"SADD", b"box:inbox", b"t1", b"t2"])?;
91    /// # client.request_borrowed(&[b"SADD", b"box:work", b"t2", b"t3"])?;
92    /// assert_eq!(overlap(&mut client, "box:")?.names, 3, "t1, t2, t3");
93    /// # Ok::<(), Box<dyn std::error::Error>>(())
94    /// ```
95    pub names: usize,
96    /// Names that appear under more than one owner.
97    ///
98    /// ```
99    /// use kevy_cli::lint::overlap;
100    /// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
101    /// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
102    /// # client.request_borrowed(&[b"SADD", b"box:inbox", b"t1", b"t2"])?;
103    /// # client.request_borrowed(&[b"SADD", b"box:work", b"t2", b"t3"])?;
104    /// assert_eq!(overlap(&mut client, "box:")?.shared, 1, "only t2 has two owners");
105    /// # Ok::<(), Box<dyn std::error::Error>>(())
106    /// ```
107    pub shared: usize,
108    /// A few of them, with the owners they appear under.
109    ///
110    /// ```
111    /// use kevy_cli::lint::overlap;
112    /// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
113    /// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
114    /// # client.request_borrowed(&[b"SADD", b"box:inbox", b"t1", b"t2"])?;
115    /// # client.request_borrowed(&[b"SADD", b"box:work", b"t2", b"t3"])?;
116    /// let o = overlap(&mut client, "box:")?;
117    /// let mut owners = o.examples[0].1.clone();
118    /// owners.sort();
119    /// assert_eq!((o.examples[0].0.as_str(), owners), ("t2", vec!["box:inbox".to_string(), "box:work".into()]));
120    /// # Ok::<(), Box<dyn std::error::Error>>(())
121    /// ```
122    pub examples: Vec<(String, Vec<String>)>,
123    /// Keys under the prefix that are not collections at all — a
124    /// counter or a hash sitting beside the owner sets. Reported so a
125    /// prefix that matched the wrong family is visible.
126    ///
127    /// ```
128    /// use kevy_cli::lint::overlap;
129    /// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
130    /// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
131    /// # client.request_borrowed(&[b"SADD", b"box:inbox", b"t1", b"t2"])?;
132    /// # client.request_borrowed(&[b"SADD", b"box:work", b"t2", b"t3"])?;
133    /// client.request_borrowed(&[b"SET", b"box:count", b"2"])?; // a sidecar, not an owner
134    /// let o = overlap(&mut client, "box:")?;
135    /// assert_eq!((o.owners, o.skipped), (2, 1));
136    /// # Ok::<(), Box<dyn std::error::Error>>(())
137    /// ```
138    pub skipped: usize,
139}
140
141/// Read every collection under `prefix` and see whether they intersect.
142///
143/// ```
144/// use kevy_cli::lint::overlap;
145/// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
146/// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
147/// client.request_borrowed(&[b"SADD", b"box:a", b"t1"])?;
148/// client.request_borrowed(&[b"ZADD", b"box:b", b"1", b"t2"])?;
149/// assert_eq!(overlap(&mut client, "box:")?.shared, 0, "a column can carry this");
150/// # Ok::<(), Box<dyn std::error::Error>>(())
151/// ```
152pub fn overlap(client: &mut dyn Link, prefix: &str) -> io::Result<Overlap> {
153    let keys = crate::collections::scan_prefix(client, prefix)?;
154    let mut owners_of: BTreeMap<Vec<u8>, Vec<String>> = BTreeMap::new();
155    let (mut owners, mut skipped) = (0usize, 0usize);
156    for k in &keys {
157        let owner = String::from_utf8_lossy(k).into_owned();
158        // Discovered, not named: a sidecar under the same prefix is a
159        // neighbour, not a failure — but it is counted and reported.
160        let Some(ms) = crate::collections::members_if_collection(client, &owner)? else {
161            skipped += 1;
162            continue;
163        };
164        owners += 1;
165        for m in ms {
166            owners_of.entry(m).or_default().push(owner.clone());
167        }
168    }
169    let mut o = tally(owners, &owners_of);
170    o.skipped = skipped;
171    Ok(o)
172}
173
174/// The question lesson 1 actually asks, as a function: does any name
175/// appear under more than one owner?
176fn tally(owners: usize, owners_of: &BTreeMap<Vec<u8>, Vec<String>>) -> Overlap {
177    let shared: Vec<_> = owners_of.iter().filter(|(_, o)| o.len() > 1).collect();
178    Overlap {
179        owners,
180        skipped: 0,
181        names: owners_of.len(),
182        shared: shared.len(),
183        examples: shared
184            .iter()
185            .take(5)
186            .map(|(n, o)| (String::from_utf8_lossy(n).into_owned(), (*o).clone()))
187            .collect(),
188    }
189}
190
191/// Two columns that agree on most of the rows they both appear in.
192///
193/// ```
194/// use kevy_cli::lint::column_pairs;
195/// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
196/// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
197/// # for (id, created, updated) in [("1", "10", "10"), ("2", "20", "20"), ("3", "30", "31")] {
198/// #     client.request_borrowed(&[b"HSET", format!("post:{id}").as_bytes(),
199/// #         b"created", created.as_bytes(), b"updated", updated.as_bytes()])?;
200/// # }
201/// let (_, found) = column_pairs(&mut client, "post:", 100, 60)?;
202/// let c = &found[0];
203/// println!("{} and {} agree on {}% ({}/{})", c.a, c.b, c.percent(), c.same, c.compared);
204/// assert_eq!((c.same, c.compared), (2, 3));
205/// # Ok::<(), Box<dyn std::error::Error>>(())
206/// ```
207#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)]
208#[non_exhaustive]
209pub struct Coincidence {
210    /// One column.
211    ///
212    /// ```
213    /// use kevy_cli::lint::column_pairs;
214    /// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
215    /// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
216    /// # for (id, created, updated) in [("1", "10", "10"), ("2", "20", "20"), ("3", "30", "31")] {
217    /// #     client.request_borrowed(&[b"HSET", format!("post:{id}").as_bytes(),
218    /// #         b"created", created.as_bytes(), b"updated", updated.as_bytes()])?;
219    /// # }
220    /// let (_, found) = column_pairs(&mut client, "post:", 100, 60)?;
221    /// assert_eq!(found[0].a, "created");
222    /// # Ok::<(), Box<dyn std::error::Error>>(())
223    /// ```
224    pub a: String,
225    /// The other.
226    ///
227    /// ```
228    /// use kevy_cli::lint::column_pairs;
229    /// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
230    /// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
231    /// # for (id, created, updated) in [("1", "10", "10"), ("2", "20", "20"), ("3", "30", "31")] {
232    /// #     client.request_borrowed(&[b"HSET", format!("post:{id}").as_bytes(),
233    /// #         b"created", created.as_bytes(), b"updated", updated.as_bytes()])?;
234    /// # }
235    /// let (_, found) = column_pairs(&mut client, "post:", 100, 60)?;
236    /// assert_eq!(found[0].b, "updated");
237    /// # Ok::<(), Box<dyn std::error::Error>>(())
238    /// ```
239    pub b: String,
240    /// Rows where both are present and equal.
241    ///
242    /// ```
243    /// use kevy_cli::lint::column_pairs;
244    /// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
245    /// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
246    /// # for (id, created, updated) in [("1", "10", "10"), ("2", "20", "20"), ("3", "30", "31")] {
247    /// #     client.request_borrowed(&[b"HSET", format!("post:{id}").as_bytes(),
248    /// #         b"created", created.as_bytes(), b"updated", updated.as_bytes()])?;
249    /// # }
250    /// let (_, found) = column_pairs(&mut client, "post:", 100, 60)?;
251    /// assert_eq!(found[0].same, 2);
252    /// # Ok::<(), Box<dyn std::error::Error>>(())
253    /// ```
254    pub same: usize,
255    /// Rows where both are present.
256    ///
257    /// ```
258    /// use kevy_cli::lint::column_pairs;
259    /// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
260    /// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
261    /// # for (id, created, updated) in [("1", "10", "10"), ("2", "20", "20"), ("3", "30", "31")] {
262    /// #     client.request_borrowed(&[b"HSET", format!("post:{id}").as_bytes(),
263    /// #         b"created", created.as_bytes(), b"updated", updated.as_bytes()])?;
264    /// # }
265    /// let (_, found) = column_pairs(&mut client, "post:", 100, 60)?;
266    /// assert_eq!(found[0].compared, 3);
267    /// # Ok::<(), Box<dyn std::error::Error>>(())
268    /// ```
269    pub compared: usize,
270}
271
272impl Coincidence {
273    /// How often they agreed, as a percentage of rows compared.
274    ///
275    /// ```
276    /// use kevy_cli::lint::column_pairs;
277    /// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
278    /// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
279    /// # for (id, created, updated) in [("1", "10", "10"), ("2", "20", "20"), ("3", "30", "31")] {
280    /// #     client.request_borrowed(&[b"HSET", format!("post:{id}").as_bytes(),
281    /// #         b"created", created.as_bytes(), b"updated", updated.as_bytes()])?;
282    /// # }
283    /// let (_, found) = column_pairs(&mut client, "post:", 100, 60)?;
284    /// assert_eq!(found[0].percent(), 66, "2 of 3, rounded down");
285    /// # Ok::<(), Box<dyn std::error::Error>>(())
286    /// ```
287    pub fn percent(&self) -> u32 {
288        (self.same * 100).checked_div(self.compared).unwrap_or(0) as u32
289    }
290}
291
292/// Sample rows under a prefix and find column pairs that nearly always
293/// carry the same value.
294///
295/// ```
296/// use kevy_cli::lint::column_pairs;
297/// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
298/// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
299/// # for (id, created, updated) in [("1", "10", "10"), ("2", "20", "20"), ("3", "30", "31")] {
300/// #     client.request_borrowed(&[b"HSET", format!("post:{id}").as_bytes(),
301/// #         b"created", created.as_bytes(), b"updated", updated.as_bytes()])?;
302/// # }
303/// let (sampled, found) = column_pairs(&mut client, "post:", 100, 60)?;
304/// assert_eq!((sampled, found.len()), (3, 1));
305/// // raise the bar and the pair no longer qualifies
306/// assert!(column_pairs(&mut client, "post:", 100, 90)?.1.is_empty());
307/// # Ok::<(), Box<dyn std::error::Error>>(())
308/// ```
309pub fn column_pairs(
310    client: &mut dyn Link,
311    prefix: &str,
312    sample: usize,
313    threshold: u32,
314) -> io::Result<(usize, Vec<Coincidence>)> {
315    let keys = crate::collections::scan_prefix(client, prefix)?;
316    let mut rows = Vec::new();
317    for k in keys.iter().take(sample) {
318        let row = hgetall(client, k)?;
319        if !row.is_empty() {
320            rows.push(row);
321        }
322    }
323    Ok((rows.len(), coincidences(&rows, threshold)))
324}
325
326/// Column pairs that agree on at least `threshold` percent of the rows
327/// where both are present, worst agreement last.
328fn coincidences(rows: &[BTreeMap<String, Vec<u8>>], threshold: u32) -> Vec<Coincidence> {
329    let mut pairs: BTreeMap<(String, String), (usize, usize)> = BTreeMap::new();
330    for row in rows {
331        let cols: Vec<&String> = row.keys().collect();
332        for (i, a) in cols.iter().enumerate() {
333            for b in &cols[i + 1..] {
334                let e = pairs.entry(((*a).clone(), (*b).clone())).or_insert((0, 0));
335                e.1 += 1;
336                if row[*a] == row[*b] {
337                    e.0 += 1;
338                }
339            }
340        }
341    }
342    let mut out: Vec<Coincidence> = pairs
343        .into_iter()
344        .map(|((a, b), (same, compared))| Coincidence { a, b, same, compared })
345        .filter(|c| c.percent() >= threshold)
346        .collect();
347    out.sort_by(|x, y| y.percent().cmp(&x.percent()).then(x.a.cmp(&y.a)));
348    out
349}
350
351fn hgetall(client: &mut dyn Link, key: &[u8]) -> io::Result<BTreeMap<String, Vec<u8>>> {
352    let reply = client.request_borrowed(&[b"HGETALL", key])?;
353    let flat = crate::collections::bulks(reply);
354    Ok(flat
355        .chunks(2)
356        .filter(|c| c.len() == 2)
357        .map(|c| (String::from_utf8_lossy(&c[0]).into_owned(), c[1].clone()))
358        .collect())
359}
360
361#[cfg(test)]
362mod tests {
363    use super::*;
364
365    fn under(pairs: &[(&str, &[&str])]) -> BTreeMap<Vec<u8>, Vec<String>> {
366        let mut m: BTreeMap<Vec<u8>, Vec<String>> = BTreeMap::new();
367        for (name, owners) in pairs {
368            m.insert(name.as_bytes().to_vec(), owners.iter().map(|o| o.to_string()).collect());
369        }
370        m
371    }
372
373    fn row(fields: &[(&str, &str)]) -> BTreeMap<String, Vec<u8>> {
374        fields.iter().map(|(k, v)| (k.to_string(), v.as_bytes().to_vec())).collect()
375    }
376
377    /// The mail system's case: a thread that lives in several
378    /// mailboxes. One name under two owners is the whole answer.
379    #[test]
380    fn a_name_under_two_owners_is_the_multi_valued_signal() {
381        let o = tally(2, &under(&[("t1", &["m:1"]), ("t2", &["m:1", "m:2"]), ("t3", &["m:2"])]));
382        assert_eq!((o.names, o.shared), (3, 1));
383        assert_eq!(o.examples[0].0, "t2");
384        assert_eq!(o.examples[0].1, ["m:1", "m:2"]);
385    }
386
387    /// Owners that share nothing mean a column *can* carry the
388    /// dimension — the answer this check exists to give when it is yes.
389    #[test]
390    fn disjoint_owners_leave_nothing_shared() {
391        let o = tally(2, &under(&[("x", &["a"]), ("y", &["b"])]));
392        assert_eq!(o.shared, 0);
393    }
394
395    /// Lesson 6's shape: one column copied to get a second sort order.
396    /// Drift in a few rows must not hide it, so the threshold is a
397    /// percentage rather than "always equal".
398    #[test]
399    fn a_copied_column_shows_up_below_perfect_agreement() {
400        let mut rows: Vec<_> =
401            (0..9).map(|i| row(&[("a", "1"), ("b", "1"), ("c", &format!("{i}"))])).collect();
402        rows.push(row(&[("a", "1"), ("b", "2"), ("c", "9")]));
403        let found = coincidences(&rows, 90);
404        let named: Vec<&str> = found.iter().map(|c| c.a.as_str()).collect();
405        assert_eq!(found.len(), 1, "only a/b agree enough, got {named:?}");
406        assert_eq!((found[0].a.as_str(), found[0].b.as_str()), ("a", "b"));
407        assert_eq!(found[0].percent(), 90);
408    }
409
410    /// Above what was actually found, nothing is reported — the
411    /// threshold is the caller's, not a fixed opinion.
412    #[test]
413    fn a_threshold_above_the_agreement_reports_nothing() {
414        let rows = vec![row(&[("a", "1"), ("b", "1")]), row(&[("a", "1"), ("b", "2")])];
415        assert!(coincidences(&rows, 60).is_empty());
416        assert_eq!(coincidences(&rows, 50).len(), 1);
417    }
418}