kevy_cli/lint.rs
1//! `lint` — the two questions worth asking about a shape, before and
2//! after the table exists.
3//!
4//! Lessons 1 and 6 of the migration playbook. They are one deliverable
5//! in the plan and two commands here, because they run at different
6//! moments and answer differently.
7//!
8//! **`lint overlap`** is lesson 1, and it is not the check the plan
9//! first described. That plan said to sample a candidate column and see
10//! whether it is single-valued — but a hash field holds one value by
11//! construction, so that check passes forever. The lesson says where
12//! the answer really lives: *"the answer is usually in your
13//! id-derivation or key-construction code, not in the row itself — a
14//! thread can live in several mailboxes."* The symptom of that **is**
15//! in the data, just not in the row: the same name appears under more
16//! than one owner. So this reads the family of owner-keyed collections
17//! and asks whether they intersect. They do ⇒ no column can carry that
18//! dimension, and a membership row is the shape.
19//!
20//! **`lint columns`** is lesson 6, and it can only run **after** the
21//! table is declared — it reads rows. Two columns whose values coincide
22//! on nearly every row are one column copied to get a second sort
23//! order; the answer is another ORDERPATH, which `IDX.ADVISE` names.
24//!
25//! The exit codes differ on purpose. Overlap is an **answer**: a column
26//! cannot carry a multi-valued dimension, so a script should stop.
27//! Coincidence is a **suspicion** — two columns may legitimately agree
28//! — so it reports and exits zero.
29//!
30//! ```
31//! use kevy_cli::lint::{column_pairs, overlap};
32//! # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
33//! let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
34//! // a thread can live in several mailboxes: no `mailbox` column can hold that
35//! client.request_borrowed(&[b"SADD", b"box:inbox", b"t1", b"t2"])?;
36//! client.request_borrowed(&[b"SADD", b"box:work", b"t2", b"t3"])?;
37//! assert_eq!(overlap(&mut client, "box:")?.shared, 1);
38//!
39//! // `updated` is `created` copied on two rows of three
40//! for (id, created, updated) in [("1", "10", "10"), ("2", "20", "20"), ("3", "30", "31")] {
41//! client.request_borrowed(&[b"HSET", format!("post:{id}").as_bytes(),
42//! b"created", created.as_bytes(), b"updated", updated.as_bytes()])?;
43//! }
44//! let (_, found) = column_pairs(&mut client, "post:", 100, 60)?;
45//! assert_eq!((found[0].a.as_str(), found[0].b.as_str()), ("created", "updated"));
46//! # Ok::<(), Box<dyn std::error::Error>>(())
47//! ```
48
49use std::collections::BTreeMap;
50use std::io;
51
52use crate::link::Link;
53
54mod command;
55pub(crate) use command::run_on;
56
57/// What the owner-keyed collections under a prefix look like together.
58///
59/// ```
60/// use kevy_cli::lint::overlap;
61/// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
62/// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
63/// client.request_borrowed(&[b"SADD", b"box:inbox", b"t1", b"t2"])?;
64/// client.request_borrowed(&[b"SADD", b"box:work", b"t2", b"t3"])?;
65/// let o = overlap(&mut client, "box:")?;
66/// assert_eq!((o.owners, o.names, o.shared, o.skipped), (2, 3, 1, 0));
67/// # Ok::<(), Box<dyn std::error::Error>>(())
68/// ```
69#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)]
70#[non_exhaustive]
71pub struct Overlap {
72 /// How many owner collections were read.
73 ///
74 /// ```
75 /// use kevy_cli::lint::overlap;
76 /// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
77 /// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
78 /// # client.request_borrowed(&[b"SADD", b"box:inbox", b"t1", b"t2"])?;
79 /// # client.request_borrowed(&[b"SADD", b"box:work", b"t2", b"t3"])?;
80 /// assert_eq!(overlap(&mut client, "box:")?.owners, 2);
81 /// # Ok::<(), Box<dyn std::error::Error>>(())
82 /// ```
83 pub owners: usize,
84 /// Distinct names across all of them.
85 ///
86 /// ```
87 /// use kevy_cli::lint::overlap;
88 /// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
89 /// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
90 /// # client.request_borrowed(&[b"SADD", b"box:inbox", b"t1", b"t2"])?;
91 /// # client.request_borrowed(&[b"SADD", b"box:work", b"t2", b"t3"])?;
92 /// assert_eq!(overlap(&mut client, "box:")?.names, 3, "t1, t2, t3");
93 /// # Ok::<(), Box<dyn std::error::Error>>(())
94 /// ```
95 pub names: usize,
96 /// Names that appear under more than one owner.
97 ///
98 /// ```
99 /// use kevy_cli::lint::overlap;
100 /// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
101 /// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
102 /// # client.request_borrowed(&[b"SADD", b"box:inbox", b"t1", b"t2"])?;
103 /// # client.request_borrowed(&[b"SADD", b"box:work", b"t2", b"t3"])?;
104 /// assert_eq!(overlap(&mut client, "box:")?.shared, 1, "only t2 has two owners");
105 /// # Ok::<(), Box<dyn std::error::Error>>(())
106 /// ```
107 pub shared: usize,
108 /// A few of them, with the owners they appear under.
109 ///
110 /// ```
111 /// use kevy_cli::lint::overlap;
112 /// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
113 /// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
114 /// # client.request_borrowed(&[b"SADD", b"box:inbox", b"t1", b"t2"])?;
115 /// # client.request_borrowed(&[b"SADD", b"box:work", b"t2", b"t3"])?;
116 /// let o = overlap(&mut client, "box:")?;
117 /// let mut owners = o.examples[0].1.clone();
118 /// owners.sort();
119 /// assert_eq!((o.examples[0].0.as_str(), owners), ("t2", vec!["box:inbox".to_string(), "box:work".into()]));
120 /// # Ok::<(), Box<dyn std::error::Error>>(())
121 /// ```
122 pub examples: Vec<(String, Vec<String>)>,
123 /// Keys under the prefix that are not collections at all — a
124 /// counter or a hash sitting beside the owner sets. Reported so a
125 /// prefix that matched the wrong family is visible.
126 ///
127 /// ```
128 /// use kevy_cli::lint::overlap;
129 /// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
130 /// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
131 /// # client.request_borrowed(&[b"SADD", b"box:inbox", b"t1", b"t2"])?;
132 /// # client.request_borrowed(&[b"SADD", b"box:work", b"t2", b"t3"])?;
133 /// client.request_borrowed(&[b"SET", b"box:count", b"2"])?; // a sidecar, not an owner
134 /// let o = overlap(&mut client, "box:")?;
135 /// assert_eq!((o.owners, o.skipped), (2, 1));
136 /// # Ok::<(), Box<dyn std::error::Error>>(())
137 /// ```
138 pub skipped: usize,
139}
140
141/// Read every collection under `prefix` and see whether they intersect.
142///
143/// ```
144/// use kevy_cli::lint::overlap;
145/// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
146/// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
147/// client.request_borrowed(&[b"SADD", b"box:a", b"t1"])?;
148/// client.request_borrowed(&[b"ZADD", b"box:b", b"1", b"t2"])?;
149/// assert_eq!(overlap(&mut client, "box:")?.shared, 0, "a column can carry this");
150/// # Ok::<(), Box<dyn std::error::Error>>(())
151/// ```
152pub fn overlap(client: &mut dyn Link, prefix: &str) -> io::Result<Overlap> {
153 let keys = crate::collections::scan_prefix(client, prefix)?;
154 let mut owners_of: BTreeMap<Vec<u8>, Vec<String>> = BTreeMap::new();
155 let (mut owners, mut skipped) = (0usize, 0usize);
156 for k in &keys {
157 let owner = String::from_utf8_lossy(k).into_owned();
158 // Discovered, not named: a sidecar under the same prefix is a
159 // neighbour, not a failure — but it is counted and reported.
160 let Some(ms) = crate::collections::members_if_collection(client, &owner)? else {
161 skipped += 1;
162 continue;
163 };
164 owners += 1;
165 for m in ms {
166 owners_of.entry(m).or_default().push(owner.clone());
167 }
168 }
169 let mut o = tally(owners, &owners_of);
170 o.skipped = skipped;
171 Ok(o)
172}
173
174/// The question lesson 1 actually asks, as a function: does any name
175/// appear under more than one owner?
176fn tally(owners: usize, owners_of: &BTreeMap<Vec<u8>, Vec<String>>) -> Overlap {
177 let shared: Vec<_> = owners_of.iter().filter(|(_, o)| o.len() > 1).collect();
178 Overlap {
179 owners,
180 skipped: 0,
181 names: owners_of.len(),
182 shared: shared.len(),
183 examples: shared
184 .iter()
185 .take(5)
186 .map(|(n, o)| (String::from_utf8_lossy(n).into_owned(), (*o).clone()))
187 .collect(),
188 }
189}
190
191/// Two columns that agree on most of the rows they both appear in.
192///
193/// ```
194/// use kevy_cli::lint::column_pairs;
195/// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
196/// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
197/// # for (id, created, updated) in [("1", "10", "10"), ("2", "20", "20"), ("3", "30", "31")] {
198/// # client.request_borrowed(&[b"HSET", format!("post:{id}").as_bytes(),
199/// # b"created", created.as_bytes(), b"updated", updated.as_bytes()])?;
200/// # }
201/// let (_, found) = column_pairs(&mut client, "post:", 100, 60)?;
202/// let c = &found[0];
203/// println!("{} and {} agree on {}% ({}/{})", c.a, c.b, c.percent(), c.same, c.compared);
204/// assert_eq!((c.same, c.compared), (2, 3));
205/// # Ok::<(), Box<dyn std::error::Error>>(())
206/// ```
207#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)]
208#[non_exhaustive]
209pub struct Coincidence {
210 /// One column.
211 ///
212 /// ```
213 /// use kevy_cli::lint::column_pairs;
214 /// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
215 /// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
216 /// # for (id, created, updated) in [("1", "10", "10"), ("2", "20", "20"), ("3", "30", "31")] {
217 /// # client.request_borrowed(&[b"HSET", format!("post:{id}").as_bytes(),
218 /// # b"created", created.as_bytes(), b"updated", updated.as_bytes()])?;
219 /// # }
220 /// let (_, found) = column_pairs(&mut client, "post:", 100, 60)?;
221 /// assert_eq!(found[0].a, "created");
222 /// # Ok::<(), Box<dyn std::error::Error>>(())
223 /// ```
224 pub a: String,
225 /// The other.
226 ///
227 /// ```
228 /// use kevy_cli::lint::column_pairs;
229 /// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
230 /// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
231 /// # for (id, created, updated) in [("1", "10", "10"), ("2", "20", "20"), ("3", "30", "31")] {
232 /// # client.request_borrowed(&[b"HSET", format!("post:{id}").as_bytes(),
233 /// # b"created", created.as_bytes(), b"updated", updated.as_bytes()])?;
234 /// # }
235 /// let (_, found) = column_pairs(&mut client, "post:", 100, 60)?;
236 /// assert_eq!(found[0].b, "updated");
237 /// # Ok::<(), Box<dyn std::error::Error>>(())
238 /// ```
239 pub b: String,
240 /// Rows where both are present and equal.
241 ///
242 /// ```
243 /// use kevy_cli::lint::column_pairs;
244 /// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
245 /// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
246 /// # for (id, created, updated) in [("1", "10", "10"), ("2", "20", "20"), ("3", "30", "31")] {
247 /// # client.request_borrowed(&[b"HSET", format!("post:{id}").as_bytes(),
248 /// # b"created", created.as_bytes(), b"updated", updated.as_bytes()])?;
249 /// # }
250 /// let (_, found) = column_pairs(&mut client, "post:", 100, 60)?;
251 /// assert_eq!(found[0].same, 2);
252 /// # Ok::<(), Box<dyn std::error::Error>>(())
253 /// ```
254 pub same: usize,
255 /// Rows where both are present.
256 ///
257 /// ```
258 /// use kevy_cli::lint::column_pairs;
259 /// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
260 /// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
261 /// # for (id, created, updated) in [("1", "10", "10"), ("2", "20", "20"), ("3", "30", "31")] {
262 /// # client.request_borrowed(&[b"HSET", format!("post:{id}").as_bytes(),
263 /// # b"created", created.as_bytes(), b"updated", updated.as_bytes()])?;
264 /// # }
265 /// let (_, found) = column_pairs(&mut client, "post:", 100, 60)?;
266 /// assert_eq!(found[0].compared, 3);
267 /// # Ok::<(), Box<dyn std::error::Error>>(())
268 /// ```
269 pub compared: usize,
270}
271
272impl Coincidence {
273 /// How often they agreed, as a percentage of rows compared.
274 ///
275 /// ```
276 /// use kevy_cli::lint::column_pairs;
277 /// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
278 /// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
279 /// # for (id, created, updated) in [("1", "10", "10"), ("2", "20", "20"), ("3", "30", "31")] {
280 /// # client.request_borrowed(&[b"HSET", format!("post:{id}").as_bytes(),
281 /// # b"created", created.as_bytes(), b"updated", updated.as_bytes()])?;
282 /// # }
283 /// let (_, found) = column_pairs(&mut client, "post:", 100, 60)?;
284 /// assert_eq!(found[0].percent(), 66, "2 of 3, rounded down");
285 /// # Ok::<(), Box<dyn std::error::Error>>(())
286 /// ```
287 pub fn percent(&self) -> u32 {
288 (self.same * 100).checked_div(self.compared).unwrap_or(0) as u32
289 }
290}
291
292/// Sample rows under a prefix and find column pairs that nearly always
293/// carry the same value.
294///
295/// ```
296/// use kevy_cli::lint::column_pairs;
297/// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
298/// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
299/// # for (id, created, updated) in [("1", "10", "10"), ("2", "20", "20"), ("3", "30", "31")] {
300/// # client.request_borrowed(&[b"HSET", format!("post:{id}").as_bytes(),
301/// # b"created", created.as_bytes(), b"updated", updated.as_bytes()])?;
302/// # }
303/// let (sampled, found) = column_pairs(&mut client, "post:", 100, 60)?;
304/// assert_eq!((sampled, found.len()), (3, 1));
305/// // raise the bar and the pair no longer qualifies
306/// assert!(column_pairs(&mut client, "post:", 100, 90)?.1.is_empty());
307/// # Ok::<(), Box<dyn std::error::Error>>(())
308/// ```
309pub fn column_pairs(
310 client: &mut dyn Link,
311 prefix: &str,
312 sample: usize,
313 threshold: u32,
314) -> io::Result<(usize, Vec<Coincidence>)> {
315 let keys = crate::collections::scan_prefix(client, prefix)?;
316 let mut rows = Vec::new();
317 for k in keys.iter().take(sample) {
318 let row = hgetall(client, k)?;
319 if !row.is_empty() {
320 rows.push(row);
321 }
322 }
323 Ok((rows.len(), coincidences(&rows, threshold)))
324}
325
326/// Column pairs that agree on at least `threshold` percent of the rows
327/// where both are present, worst agreement last.
328fn coincidences(rows: &[BTreeMap<String, Vec<u8>>], threshold: u32) -> Vec<Coincidence> {
329 let mut pairs: BTreeMap<(String, String), (usize, usize)> = BTreeMap::new();
330 for row in rows {
331 let cols: Vec<&String> = row.keys().collect();
332 for (i, a) in cols.iter().enumerate() {
333 for b in &cols[i + 1..] {
334 let e = pairs.entry(((*a).clone(), (*b).clone())).or_insert((0, 0));
335 e.1 += 1;
336 if row[*a] == row[*b] {
337 e.0 += 1;
338 }
339 }
340 }
341 }
342 let mut out: Vec<Coincidence> = pairs
343 .into_iter()
344 .map(|((a, b), (same, compared))| Coincidence { a, b, same, compared })
345 .filter(|c| c.percent() >= threshold)
346 .collect();
347 out.sort_by(|x, y| y.percent().cmp(&x.percent()).then(x.a.cmp(&y.a)));
348 out
349}
350
351fn hgetall(client: &mut dyn Link, key: &[u8]) -> io::Result<BTreeMap<String, Vec<u8>>> {
352 let reply = client.request_borrowed(&[b"HGETALL", key])?;
353 let flat = crate::collections::bulks(reply);
354 Ok(flat
355 .chunks(2)
356 .filter(|c| c.len() == 2)
357 .map(|c| (String::from_utf8_lossy(&c[0]).into_owned(), c[1].clone()))
358 .collect())
359}
360
361#[cfg(test)]
362mod tests {
363 use super::*;
364
365 fn under(pairs: &[(&str, &[&str])]) -> BTreeMap<Vec<u8>, Vec<String>> {
366 let mut m: BTreeMap<Vec<u8>, Vec<String>> = BTreeMap::new();
367 for (name, owners) in pairs {
368 m.insert(name.as_bytes().to_vec(), owners.iter().map(|o| o.to_string()).collect());
369 }
370 m
371 }
372
373 fn row(fields: &[(&str, &str)]) -> BTreeMap<String, Vec<u8>> {
374 fields.iter().map(|(k, v)| (k.to_string(), v.as_bytes().to_vec())).collect()
375 }
376
377 /// The mail system's case: a thread that lives in several
378 /// mailboxes. One name under two owners is the whole answer.
379 #[test]
380 fn a_name_under_two_owners_is_the_multi_valued_signal() {
381 let o = tally(2, &under(&[("t1", &["m:1"]), ("t2", &["m:1", "m:2"]), ("t3", &["m:2"])]));
382 assert_eq!((o.names, o.shared), (3, 1));
383 assert_eq!(o.examples[0].0, "t2");
384 assert_eq!(o.examples[0].1, ["m:1", "m:2"]);
385 }
386
387 /// Owners that share nothing mean a column *can* carry the
388 /// dimension — the answer this check exists to give when it is yes.
389 #[test]
390 fn disjoint_owners_leave_nothing_shared() {
391 let o = tally(2, &under(&[("x", &["a"]), ("y", &["b"])]));
392 assert_eq!(o.shared, 0);
393 }
394
395 /// Lesson 6's shape: one column copied to get a second sort order.
396 /// Drift in a few rows must not hide it, so the threshold is a
397 /// percentage rather than "always equal".
398 #[test]
399 fn a_copied_column_shows_up_below_perfect_agreement() {
400 let mut rows: Vec<_> =
401 (0..9).map(|i| row(&[("a", "1"), ("b", "1"), ("c", &format!("{i}"))])).collect();
402 rows.push(row(&[("a", "1"), ("b", "2"), ("c", "9")]));
403 let found = coincidences(&rows, 90);
404 let named: Vec<&str> = found.iter().map(|c| c.a.as_str()).collect();
405 assert_eq!(found.len(), 1, "only a/b agree enough, got {named:?}");
406 assert_eq!((found[0].a.as_str(), found[0].b.as_str()), ("a", "b"));
407 assert_eq!(found[0].percent(), 90);
408 }
409
410 /// Above what was actually found, nothing is reported — the
411 /// threshold is the caller's, not a fixed opinion.
412 #[test]
413 fn a_threshold_above_the_agreement_reports_nothing() {
414 let rows = vec![row(&[("a", "1"), ("b", "1")]), row(&[("a", "1"), ("b", "2")])];
415 assert!(coincidences(&rows, 60).is_empty());
416 assert_eq!(coincidences(&rows, 50).len(), 1);
417 }
418}