Skip to main content

kevy_cli/
migrate.rs

1//! Migration toolchain: `export` / `import`.
2//!
3//! Wire format = a RESP command stream of rebuild frames (SET / HSET /
4//! RPUSH / SADD / ZADD / PEXPIREAT) — bidirectionally compatible with
5//! `redis-cli --pipe`. Export walks SCAN cursors (per-key
6//! point-in-time; SCAN-class consistency). Import pipelines 512
7//! commands per batch with a fsynced progress file for `--resume`.
8//! Every key's frames start with DEL, so replay REBUILDS the key from
9//! scratch — genuinely idempotent for every type (RPUSH would
10//! otherwise append on re-import; the round-trip test caught exactly
11//! that). No cross-batch atomicity to lose.
12
13use std::fs::{File, OpenOptions};
14use std::io::{self, BufWriter, Read, Seek, SeekFrom, Write};
15use std::path::Path;
16
17use kevy_resp::{Reply, encode_command_borrowed};
18use kevy_resp_client::RespClient;
19
20const PIPELINE: usize = 512;
21
22/// Run `export` — walk the keyspace (optionally under `prefix`) and
23/// write rebuild frames to `out_path`. Returns exported key count.
24/// What an export did — including what it did NOT do. The skipped map
25/// is the half that used to be invisible: a type with no rebuild verb
26/// produced no frames, no error and no mention, so a migration could
27/// report success while leaving a whole type behind.
28#[derive(Debug)]
29pub struct Export {
30    /// Keys whose frames are in the file.
31    pub keys: u64,
32    /// Type name -> keys left out because nothing here rebuilds them.
33    pub skipped: std::collections::BTreeMap<Vec<u8>, u64>,
34}
35
36/// Walk the keyspace (optionally under `prefix`) and write rebuild
37/// frames to `out_path`, reporting both what went in and what could
38/// not.
39pub fn run_export(
40    client: &mut RespClient,
41    prefix: Option<&[u8]>,
42    out_path: &Path,
43) -> io::Result<Export> {
44    let mut out = BufWriter::new(File::create(out_path)?);
45    let mut cursor: Vec<u8> = b"0".to_vec();
46    let mut pattern = prefix.unwrap_or_default().to_vec();
47    pattern.push(b'*');
48    let mut n = 0u64;
49    // Types with no rebuild verb, counted by name. Silence here is how
50    // a migration loses a whole type and reports success.
51    let mut skipped: std::collections::BTreeMap<Vec<u8>, u64> = Default::default();
52    loop {
53        let reply =
54            client.request_borrowed(&[b"SCAN", &cursor, b"MATCH", &pattern, b"COUNT", b"512"])?;
55        let Reply::Array(items) = reply else {
56            return Err(io::Error::new(io::ErrorKind::InvalidData, "SCAN reply shape"));
57        };
58        let (Some(Reply::Bulk(next)), Some(Reply::Array(keys))) = (items.first(), items.get(1))
59        else {
60            return Err(io::Error::new(io::ErrorKind::InvalidData, "SCAN reply shape"));
61        };
62        let next = next.clone();
63        for k in keys {
64            let Reply::Bulk(key) = k else { continue };
65            let key = key.clone();
66            match export_key(client, &key, &mut out)? {
67                Some(None) => n += 1,
68                Some(Some(ty)) => *skipped.entry(ty).or_insert(0u64) += 1,
69                None => {}
70            }
71        }
72        cursor = next;
73        if cursor == b"0" {
74            break;
75        }
76    }
77    out.flush()?;
78    Ok(Export { keys: n, skipped })
79}
80
81/// Emit one key's rebuild frames. `Ok(None)` = nothing written; the
82/// type name comes back when the reason was "no rebuild verb", so the
83/// caller can report what it is leaving behind rather than count it as
84/// a key that happened to vanish.
85fn export_key(
86    client: &mut RespClient,
87    key: &[u8],
88    out: &mut impl Write,
89) -> io::Result<Option<Option<Vec<u8>>>> {
90    match rebuild_frames(client, key, key)? {
91        Rebuilt::Frames(frame) => {
92            out.write_all(&frame)?;
93            Ok(Some(None))
94        }
95        Rebuilt::Vanished => Ok(None),
96        Rebuilt::UnsupportedType(ty) => Ok(Some(Some(ty))),
97    }
98}
99
100/// Why a key produced no frames. The two reasons are not the same and
101/// were indistinguishable: a vanished key is a race the walk expects,
102/// an unsupported type is data the caller is about to leave behind.
103pub(crate) enum Rebuilt {
104    /// The rebuild frames, ready to write.
105    Frames(Vec<u8>),
106    /// The key was gone between SCAN and read — expected, uncounted.
107    Vanished,
108    /// The type has no rebuild verb here. Carries the type name so the
109    /// caller can tell someone rather than skip in silence.
110    UnsupportedType(Vec<u8>),
111}
112
113/// Read `key` and produce DEL+rebuild frames addressed to `dst`
114/// (`dst == key` for export; a re-prefixed name for copy-prefix —
115/// the server has no COPY verb, so copying IS read+rebuild).
116pub(crate) fn rebuild_frames(
117    client: &mut RespClient,
118    key: &[u8],
119    dst: &[u8],
120) -> io::Result<Rebuilt> {
121    let ty = match client.request_borrowed(&[b"TYPE", key])? {
122        Reply::Simple(t) => t,
123        _ => return Ok(Rebuilt::Vanished),
124    };
125    if ty == b"none" {
126        return Ok(Rebuilt::Vanished);
127    }
128    let mut frame = Vec::new();
129    // DEL first: replay rebuilds from scratch (idempotence for
130    // append-shaped verbs like RPUSH).
131    encode_command_borrowed(&mut frame, &[b"DEL", dst]);
132    match encode_body(client, key, dst, &ty, &mut frame)? {
133        Some(()) => {}
134        None => return Ok(Rebuilt::Vanished),
135    }
136    if frame.len() == encoded_del_len(dst) {
137        return Ok(Rebuilt::UnsupportedType(ty));
138    }
139    append_ttl_frame(client, key, dst, &mut frame)?;
140    Ok(Rebuilt::Frames(frame))
141}
142
143/// The `DEL <dst>` prologue's encoded length — how `rebuild_frames`
144/// tells "the body wrote nothing" from "the body wrote frames".
145fn encoded_del_len(dst: &[u8]) -> usize {
146    let mut probe = Vec::new();
147    encode_command_borrowed(&mut probe, &[b"DEL", dst]);
148    probe.len()
149}
150
151/// Append the type's rebuild verbs to `frame`. `None` = the key
152/// vanished mid-read; leaving `frame` untouched = no verb for this
153/// type, which the caller turns into `UnsupportedType`.
154fn encode_body(
155    client: &mut RespClient,
156    key: &[u8],
157    dst: &[u8],
158    ty: &[u8],
159    frame: &mut Vec<u8>,
160) -> io::Result<Option<()>> {
161    match ty {
162        b"string" => {
163            let Reply::Bulk(v) = client.request_borrowed(&[b"GET", key])? else {
164                return Ok(None);
165            };
166            encode_command_borrowed(frame, &[b"SET", dst, &v]);
167        }
168        b"hash" => {
169            let Some(items) = fetch_bulks(client, &[b"HGETALL", key])? else {
170                return Ok(None);
171            };
172            encode_multi(frame, b"HSET", dst, &items);
173        }
174        b"list" => {
175            let Some(vals) = fetch_bulks(client, &[b"LRANGE", key, b"0", b"-1"])? else {
176                return Ok(None);
177            };
178            encode_multi(frame, b"RPUSH", dst, &vals);
179        }
180        b"set" => {
181            let Some(ms) = fetch_bulks(client, &[b"SMEMBERS", key])? else {
182                return Ok(None);
183            };
184            encode_multi(frame, b"SADD", dst, &ms);
185        }
186        b"zset" => {
187            let zrange: &[&[u8]] = &[b"ZRANGE", key, b"0", b"-1", b"WITHSCORES"];
188            let Some(flat) = fetch_bulks(client, zrange)? else {
189                return Ok(None);
190            };
191            encode_zadd(frame, dst, &flat);
192        }
193        // Streams and anything added later: no rebuild verb here. The
194        // caller reports it by name — a migration that drops a type
195        // must say which one.
196        _ => return Ok(Some(())),
197    }
198    Ok(Some(()))
199}
200
201/// Issue `cmd` and unwrap its Array reply into bulk payloads.
202/// `None` when the reply isn't an array or the array is empty (the key
203/// vanished / changed type between TYPE and read).
204fn fetch_bulks(client: &mut RespClient, cmd: &[&[u8]]) -> io::Result<Option<Vec<Vec<u8>>>> {
205    let Reply::Array(items) = client.request_borrowed(cmd)? else {
206        return Ok(None);
207    };
208    if items.is_empty() {
209        return Ok(None);
210    }
211    Ok(Some(
212        items
213            .into_iter()
214            .filter_map(|r| if let Reply::Bulk(b) = r { Some(b) } else { None })
215            .collect(),
216    ))
217}
218
219/// Encode `<verb> <dst> <vals…>` onto `frame` (HSET / RPUSH / SADD).
220fn encode_multi(frame: &mut Vec<u8>, verb: &[u8], dst: &[u8], vals: &[Vec<u8>]) {
221    let mut argv: Vec<&[u8]> = vec![verb, dst];
222    argv.extend(vals.iter().map(Vec::as_slice));
223    encode_command_borrowed(frame, &argv);
224}
225
226/// Encode `ZADD <dst> score member …` onto `frame`.
227/// ZADD wants score member; ZRANGE gives member score.
228fn encode_zadd(frame: &mut Vec<u8>, dst: &[u8], flat: &[Vec<u8>]) {
229    let mut argv: Vec<&[u8]> = vec![b"ZADD", dst];
230    for pair in flat.chunks(2) {
231        if pair.len() == 2 {
232            argv.push(&pair[1]);
233            argv.push(&pair[0]);
234        }
235    }
236    encode_command_borrowed(frame, &argv);
237}
238
239/// TTL rides as an absolute PEXPIREAT follow-up.
240fn append_ttl_frame(
241    client: &mut RespClient,
242    key: &[u8],
243    dst: &[u8],
244    frame: &mut Vec<u8>,
245) -> io::Result<()> {
246    if let Reply::Int(ms) = client.request_borrowed(&[b"PTTL", key])?
247        && ms > 0
248    {
249        let now = std::time::SystemTime::now()
250            .duration_since(std::time::UNIX_EPOCH)
251            .map_err(io::Error::other)?
252            .as_millis() as i64;
253        encode_command_borrowed(frame, &[b"PEXPIREAT", dst, (now + ms).to_string().as_bytes()]);
254    }
255    Ok(())
256}
257
258/// Import stats.
259#[derive(Debug)]
260pub struct ImportReport {
261    /// Commands sent successfully.
262    pub sent: u64,
263    /// -ERR replies (counted, not fatal unless `strict`).
264    pub errors: u64,
265    /// Byte offset reached in the source file.
266    pub offset: u64,
267}
268
269/// Run `import` — stream `src` (a RESP command file) into the server,
270/// `PIPELINE` commands per batch. The progress file `<src>.progress`
271/// records the safely-applied byte offset after every batch (fsynced);
272/// `resume` starts there. Idempotent replay.
273pub fn run_import(
274    client: &mut RespClient,
275    src: &Path,
276    resume: bool,
277    strict: bool,
278) -> io::Result<ImportReport> {
279    let (mut progress, start) = open_progress(src, resume)?;
280    let mut f = File::open(src)?;
281    f.seek(SeekFrom::Start(start))?;
282    let mut pending: Vec<u8> = Vec::with_capacity(1 << 20);
283    let mut report = ImportReport { sent: 0, errors: 0, offset: start };
284    let mut chunk = vec![0u8; 1 << 20];
285    let mut batch_bytes = 0usize;
286    let mut batch_cmds = 0usize;
287    loop {
288        let n = f.read(&mut chunk)?;
289        if n == 0 {
290            break;
291        }
292        pending.extend_from_slice(&chunk[..n]);
293        // carve complete commands off `pending`
294        while let Some(used) = command_len(&pending[batch_bytes..]) {
295            batch_bytes += used;
296            batch_cmds += 1;
297            if batch_cmds == PIPELINE {
298                flush_batch(client, &pending[..batch_bytes], batch_cmds, strict, &mut report)?;
299                pending.drain(..batch_bytes);
300                write_progress(&mut progress, report.offset)?;
301                batch_bytes = 0;
302                batch_cmds = 0;
303            }
304        }
305    }
306    if batch_cmds > 0 {
307        flush_batch(client, &pending[..batch_bytes], batch_cmds, strict, &mut report)?;
308        write_progress(&mut progress, report.offset)?;
309    }
310    Ok(report)
311}
312
313fn flush_batch(
314    client: &mut RespClient,
315    raw: &[u8],
316    n: usize,
317    strict: bool,
318    report: &mut ImportReport,
319) -> io::Result<()> {
320    let replies = client.pipeline_raw(raw, n)?;
321    for r in replies {
322        if let Reply::Error(e) = r {
323            report.errors += 1;
324            if strict {
325                return Err(io::Error::new(
326                    io::ErrorKind::InvalidData,
327                    format!("server error (strict): {}", String::from_utf8_lossy(&e)),
328                ));
329            }
330        } else {
331            report.sent += 1;
332        }
333    }
334    report.offset += raw.len() as u64;
335    Ok(())
336}
337
338/// The progress file, opened, and the offset to start from.
339///
340/// The name is `<src>.progress`, APPENDED — the doc always said so, but
341/// `with_extension` turned dump.kevy into dump.progress, and everything
342/// written against the documented name (the migration drill's cleanup
343/// included) touched a file this code never read.
344///
345/// A fresh (non-resume) import resets the file immediately, before any
346/// batch. A completed earlier import leaves offset = EOF on disk; kill a
347/// fresh import before its first batch lands and that stale EOF
348/// survives — a later `--resume` then seeks to the end, imports nothing,
349/// and reports success. The migration drill hit exactly that window.
350fn open_progress(src: &Path, resume: bool) -> io::Result<(File, u64)> {
351    let path = {
352        let mut os = src.as_os_str().to_owned();
353        os.push(".progress");
354        std::path::PathBuf::from(os)
355    };
356    let mut start = 0u64;
357    if resume && let Ok(text) = std::fs::read_to_string(&path) {
358        start = text.trim().parse().unwrap_or(0);
359    }
360    let mut f = OpenOptions::new().create(true).truncate(false).write(true).open(&path)?;
361    if !resume {
362        write_progress(&mut f, 0)?;
363    }
364    Ok((f, start))
365}
366
367fn write_progress(f: &mut File, offset: u64) -> io::Result<()> {
368    f.set_len(0)?;
369    f.seek(SeekFrom::Start(0))?;
370    f.write_all(offset.to_string().as_bytes())?;
371    f.sync_data()
372}
373
374/// Crate-visible alias for [`command_len`] (bulk copy counts frames).
375pub(crate) fn command_len_pub(b: &[u8]) -> Option<usize> {
376    command_len(b)
377}
378
379/// Length of one complete RESP command at the head of `b`, or `None`.
380fn command_len(b: &[u8]) -> Option<usize> {
381    let mut pos = 0usize;
382    let line = take_line(b, &mut pos)?;
383    if line.first() != Some(&b'*') {
384        return None;
385    }
386    let n: usize = std::str::from_utf8(&line[1..]).ok()?.trim().parse().ok()?;
387    for _ in 0..n {
388        let hdr = take_line(b, &mut pos)?;
389        if hdr.first() != Some(&b'$') {
390            return None;
391        }
392        let len: usize = std::str::from_utf8(&hdr[1..]).ok()?.trim().parse().ok()?;
393        if b.len() < pos + len + 2 {
394            return None;
395        }
396        pos += len + 2;
397    }
398    Some(pos)
399}
400
401fn take_line<'b>(b: &'b [u8], pos: &mut usize) -> Option<&'b [u8]> {
402    let rest = &b[*pos..];
403    let idx = rest.windows(2).position(|w| w == b"\r\n")?;
404    let line = &rest[..idx];
405    *pos += idx + 2;
406    Some(line)
407}