Skip to main content

kevy_cli/
backfill_keys.rs

1//! `backfill-keys` — the union of every structure that can name an item.
2//!
3//! Lesson 3 of the migration playbook, and only the half a machine can
4//! do. The lesson splits itself: *"build the backfill key-set from the
5//! **union** of every structure that can name an item (old indexes, the
6//! primary keyspace scan, archives), then write rows from the
7//! authoritative record."* The union is mechanical. What the
8//! authoritative record is, and what a row looks like, is knowledge
9//! that lives in the application — a tool that guessed would write the
10//! wrong rows confidently.
11//!
12//! So this command produces the key-set and nothing else, and it splits
13//! its output the way a pipeline needs: **the names go to stdout**, one
14//! per line, ready to feed whatever writes the rows; **the accounting
15//! goes to stderr**, so redirecting the list does not lose it.
16//!
17//! The accounting is the point of doing this at all. Each source
18//! reports how many names *only it* contributed — and every non-zero
19//! number there is a row that backfilling from any single source would
20//! have missed. That is the 89 % / 76 % drift the lesson was paid for,
21//! measured on your own data instead of quoted from someone else's.
22
23// The progress report goes to stderr, and a stderr that has gone away
24// (a closed pipe, `| head`) is not a reason to abandon a backfill that
25// is otherwise succeeding. What matters is written to the store.
26#![expect(clippy::let_underscore_must_use, reason = "a report nobody is reading is not a failure")]
27
28use std::collections::BTreeSet;
29use std::io::{self, Write};
30use std::process::ExitCode;
31
32use kevy_resp_client::RespClient;
33
34/// Where a set of item names comes from.
35#[derive(Debug)]
36pub enum Source {
37    /// The members of a set, sorted set, or list key.
38    Index(String),
39    /// Every key in the keyspace under a prefix.
40    Prefix {
41        /// The prefix to scan.
42        prefix: String,
43        /// Keep the whole key rather than stripping the prefix.
44        ///
45        /// Stripping is the default because the names then line up with
46        /// the members of an index: `mail:123` under `mail:` becomes
47        /// `123`, which is what a sorted set of ids holds. Keeping the
48        /// prefix is right when the key *is* the name.
49        keep: bool,
50    },
51    /// One name per line, from a file (an archive listing, an export).
52    File(String),
53}
54
55impl Source {
56    /// How this source prints in the report.
57    pub fn label(&self) -> String {
58        match self {
59            Source::Index(k) => format!("index {k}"),
60            Source::Prefix { prefix, keep } => {
61                format!("prefix {prefix}{}", if *keep { " (whole keys)" } else { "" })
62            }
63            Source::File(p) => format!("file {p}"),
64        }
65    }
66}
67
68/// What one source contributed.
69#[derive(Debug)]
70pub struct SourceReport {
71    /// How the source was named on the command line.
72    pub label: String,
73    /// Names this source produced.
74    pub total: usize,
75    /// Names **no other source** produced. Non-zero means backfilling
76    /// from any single source would have missed these rows.
77    pub unique: usize,
78}
79
80/// The union, and where each name came from.
81#[derive(Debug)]
82pub struct Union {
83    /// Every name, first-seen order, deduplicated.
84    pub names: Vec<Vec<u8>>,
85    /// One entry per source, in the order they were given.
86    pub sources: Vec<SourceReport>,
87}
88
89/// Read every source and union their names.
90pub fn collect(client: &mut RespClient, sources: &[Source]) -> io::Result<Union> {
91    let mut per_source: Vec<BTreeSet<Vec<u8>>> = Vec::with_capacity(sources.len());
92    let mut names: Vec<Vec<u8>> = Vec::new();
93    let mut seen: BTreeSet<Vec<u8>> = BTreeSet::new();
94    for s in sources {
95        let got = read_source(client, s)?;
96        for n in &got {
97            if seen.insert(n.clone()) {
98                names.push(n.clone());
99            }
100        }
101        per_source.push(got.into_iter().collect());
102    }
103    let labels: Vec<String> = sources.iter().map(Source::label).collect();
104    Ok(Union { names, sources: account(&labels, &per_source) })
105}
106
107/// Who contributed what. A name is *unique* to a source when no other
108/// source produced it — which is the only number here worth reading,
109/// because each one is a row that backfilling from a single source
110/// would have missed.
111fn account(labels: &[String], per_source: &[BTreeSet<Vec<u8>>]) -> Vec<SourceReport> {
112    labels
113        .iter()
114        .zip(per_source)
115        .enumerate()
116        .map(|(i, (label, mine))| SourceReport {
117            label: label.clone(),
118            total: mine.len(),
119            unique: mine
120                .iter()
121                .filter(|n| !per_source.iter().enumerate().any(|(j, o)| j != i && o.contains(*n)))
122                .count(),
123        })
124        .collect()
125}
126
127fn read_source(client: &mut RespClient, s: &Source) -> io::Result<Vec<Vec<u8>>> {
128    match s {
129        Source::Index(key) => crate::collections::members(client, key),
130        Source::Prefix { prefix, keep } => read_prefix(client, prefix, *keep),
131        Source::File(path) => Ok(std::fs::read_to_string(path)?
132            .lines()
133            .map(str::trim)
134            .filter(|l| !l.is_empty())
135            .map(|l| l.as_bytes().to_vec())
136            .collect()),
137    }
138}
139
140/// Every key under a prefix, stripped unless the caller wants the key
141/// itself: stripped names line up with the members of an index, which
142/// is what makes the union meaningful.
143fn read_prefix(client: &mut RespClient, prefix: &str, keep: bool) -> io::Result<Vec<Vec<u8>>> {
144    Ok(crate::collections::scan_prefix(client, prefix)?
145        .into_iter()
146        .map(|k| if keep { k } else { k[prefix.len().min(k.len())..].to_vec() })
147        .collect())
148}
149
150/// The accounting, on stderr so redirecting the names keeps it.
151pub fn print_report(u: &Union) {
152    let e = io::stderr();
153    let mut e = e.lock();
154    let _ = writeln!(e, "{} name(s) in the union", u.names.len());
155    for s in &u.sources {
156        let _ = writeln!(e, "  {:<32} {} name(s), {} only here", s.label, s.total, s.unique);
157    }
158    let missed: usize = u.sources.iter().map(|s| s.unique).sum();
159    let _ = if missed == 0 && u.sources.len() > 1 {
160        writeln!(e, "every source named the same items — no source alone would have missed a row")
161    } else if u.sources.len() > 1 {
162        writeln!(
163            e,
164            "{missed} name(s) appear in only one source — backfilling from any single one \
165             would have missed them"
166        )
167    } else {
168        writeln!(e, "one source given; there is nothing to union it against")
169    };
170}
171
172/// The command line: host, port, and the sources in the order given.
173fn parse_args(args: &[String]) -> (String, u16, Vec<Source>) {
174    let (mut host, mut port) = (crate::DEFAULT_HOST.to_string(), crate::DEFAULT_PORT);
175    let (mut sources, mut keep) = (Vec::new(), false);
176    let mut i = 0;
177    while i < args.len() {
178        let has_val = i + 1 < args.len();
179        match args[i].as_str() {
180            "-h" if has_val => {
181                host = args[i + 1].clone();
182                i += 2;
183            }
184            "-p" if has_val => {
185                port = args[i + 1].parse().unwrap_or(crate::DEFAULT_PORT);
186                i += 2;
187            }
188            "--keep-prefix" => {
189                keep = true;
190                i += 1;
191            }
192            "--from-index" if has_val => {
193                sources.push(Source::Index(args[i + 1].clone()));
194                i += 2;
195            }
196            "--from-prefix" if has_val => {
197                sources.push(Source::Prefix { prefix: args[i + 1].clone(), keep: false });
198                i += 2;
199            }
200            "--from-file" if has_val => {
201                sources.push(Source::File(args[i + 1].clone()));
202                i += 2;
203            }
204            _ => i += 1,
205        }
206    }
207    if keep {
208        for s in &mut sources {
209            if let Source::Prefix { keep: k, .. } = s {
210                *k = true;
211            }
212        }
213    }
214    (host, port, sources)
215}
216
217/// `backfill-keys [-h host] [-p port] --from-index K --from-prefix P
218/// [--keep-prefix] --from-file F …`
219pub fn run_backfill_keys_cli(args: &[String]) -> ExitCode {
220    let (host, port, sources) = parse_args(args);
221    if sources.is_empty() {
222        eprintln!("kevy-cli backfill-keys: give at least one source");
223        eprintln!(
224            "usage: kevy-cli backfill-keys [-h host] [-p port] \
225             [--from-index <key>] [--from-prefix <p> [--keep-prefix]] [--from-file <path>] …"
226        );
227        return ExitCode::FAILURE;
228    }
229    emit(&host, port, &sources)
230}
231
232/// Names to stdout, accounting to stderr. A source that cannot be read
233/// is an error rather than an empty contribution — a silently empty
234/// source is exactly the hole this command exists to close.
235fn emit(host: &str, port: u16, sources: &[Source]) -> ExitCode {
236    let mut client = match RespClient::connect(host, port) {
237        Ok(c) => c,
238        Err(e) => {
239            eprintln!("kevy-cli backfill-keys: could not connect to {host}:{port}: {e}");
240            return ExitCode::FAILURE;
241        }
242    };
243    let u = match collect(&mut client, sources) {
244        Ok(u) => u,
245        Err(e) => {
246            eprintln!("kevy-cli backfill-keys: {e}");
247            return ExitCode::FAILURE;
248        }
249    };
250    let out = io::stdout();
251    let mut out = out.lock();
252    for n in &u.names {
253        let _ = out.write_all(n);
254        let _ = out.write_all(b"\n");
255    }
256    let _ = out.flush();
257    print_report(&u);
258    ExitCode::SUCCESS
259}
260
261#[cfg(test)]
262mod tests {
263    use super::*;
264
265    fn set(names: &[&str]) -> BTreeSet<Vec<u8>> {
266        names.iter().map(|n| n.as_bytes().to_vec()).collect()
267    }
268
269    fn labels(n: usize) -> Vec<String> {
270        (0..n).map(|i| format!("s{i}")).collect()
271    }
272
273    /// A name is unique to a source when no *other* source has it —
274    /// the number that says "backfilling from this one alone would
275    /// have missed these".
276    #[test]
277    fn unique_means_no_other_source_named_it() {
278        let sources =
279            [set(&["1", "2", "3"]), set(&["3", "4", "7"]), set(&["1", "2", "3", "4", "5"])];
280        let r = account(&labels(3), &sources);
281        assert_eq!((r[0].total, r[0].unique), (3, 0), "all of s0 is covered elsewhere");
282        assert_eq!((r[1].total, r[1].unique), (3, 1), "only s1 names 7");
283        assert_eq!((r[2].total, r[2].unique), (5, 1), "only s2 names 5");
284    }
285
286    /// Sources that agree contribute nothing unique — which is the
287    /// answer that means the drift this lesson warns about is absent.
288    #[test]
289    fn sources_that_agree_have_nothing_unique() {
290        let sources = [set(&["a", "b"]), set(&["b", "a"])];
291        for r in account(&labels(2), &sources) {
292            assert_eq!(r.unique, 0);
293        }
294    }
295
296    /// With one source there is nothing to be unique against, so every
297    /// name is — the report says so in words rather than letting the
298    /// number read as drift.
299    #[test]
300    fn a_lone_source_owns_everything_it_names() {
301        let r = account(&labels(1), &[set(&["a", "b"])]);
302        assert_eq!((r[0].total, r[0].unique), (2, 2));
303    }
304}