Skip to main content

kevy_cli/
backup.rs

1//! `kevy-cli backup` / `kevy-cli restore` subcommands.
2//!
3//! Backups bundle a kevy `data_dir` (snapshot + AOF) into a single
4//! `.kevybkp` file using a tiny custom container format (std-only,
5//! 0-dep — no `tar` crate per project rule).
6//!
7//! Container format:
8//!
9//! ```text
10//!   magic        : 8 bytes = b"KEVYBKP1"
11//!   for each file:
12//!     name_len   : u16 big-endian
13//!     name       : <name_len> bytes (UTF-8, relative to data_dir)
14//!     body_len   : u64 big-endian
15//!     body       : <body_len> bytes
16//!   eof marker   : u16 = 0
17//! ```
18//!
19//! Backup ordering: typically the operator issues a `BGSAVE` first
20//! against the live kevy via TCP, then runs `kevy-cli backup` once
21//! the snapshot has flushed. (kevy-cli backup can do this in one
22//! call too — see the `--bgsave` flag.)
23//!
24//! ```
25//! use kevy_cli::backup::{pack, unpack};
26//! let data = kevy_tmpdir::TmpDir::new("backup-data");
27//! std::fs::write(data.path().join("aof-0.aof"), b"*1\r\n$4\r\nPING\r\n")?;
28//! let files = kevy_tmpdir::TmpDir::new("backup-files");
29//! let bundle = files.path().join("nightly.kevybkp");
30//! pack(data.path(), &bundle)?;
31//!
32//! let restored = kevy_tmpdir::TmpDir::new("backup-restored");
33//! unpack(&bundle, restored.path())?;
34//! assert_eq!(std::fs::read(restored.path().join("aof-0.aof"))?, b"*1\r\n$4\r\nPING\r\n");
35//! # Ok::<(), std::io::Error>(())
36//! ```
37
38use std::fs::{File, OpenOptions};
39use std::io::{self, BufReader, BufWriter, Read, Write};
40use std::path::Path;
41
42const MAGIC: &[u8; 8] = b"KEVYBKP1";
43
44/// Pack every regular file under `data_dir` into `out_path`.
45/// Subdirectories are skipped, and the data dir stopped being flat when
46/// tiering landed: `tier/<shard>/vlog-*.dat` holds demoted values and
47/// `segs-<shard>/` holds cold index segments. Skipping them is correct
48/// because both are **derived** — a snapshot materialises cold values
49/// rather than referencing them, so the backup carries the data without
50/// the spill area and a restore rebuilds it — measured end to end
51/// against a store whose values had actually been demoted, not
52/// reasoned about.
53///
54/// That safety is a property of what lives down there, not of the skip.
55/// `subdirectories_are_derived_spill_only` pins the set, so a future
56/// directory of TRUTH cannot join the data dir without someone reading
57/// this first.
58///
59/// ```
60/// use kevy_cli::backup::pack;
61/// let data = kevy_tmpdir::TmpDir::new("pack-data");
62/// std::fs::write(data.path().join("dump-0.rdb"), [7u8; 100])?;
63/// std::fs::create_dir(data.path().join("tier"))?; // derived spill: not packed
64/// let out = kevy_tmpdir::TmpDir::new("pack-out");
65/// let bundle = out.path().join("b.kevybkp");
66/// assert_eq!(pack(data.path(), &bundle)?, 100, "body bytes packed");
67/// // never overwrites an existing backup
68/// assert!(pack(data.path(), &bundle).is_err());
69/// # Ok::<(), std::io::Error>(())
70/// ```
71pub fn pack(data_dir: impl AsRef<Path>, out_path: impl AsRef<Path>) -> io::Result<u64> {
72    let (data_dir, out_path) = (data_dir.as_ref(), out_path.as_ref());
73    let out = OpenOptions::new().create_new(true).write(true).open(out_path)?;
74    let mut w = BufWriter::new(out);
75    w.write_all(MAGIC)?;
76    let mut total_bytes: u64 = 0;
77    let mut file_count: u64 = 0;
78    for entry in std::fs::read_dir(data_dir)? {
79        let entry = entry?;
80        let path = entry.path();
81        let meta = entry.metadata()?;
82        if !meta.is_file() {
83            continue; // derived spill — see the note above this fn
84        }
85        let name = path
86            .strip_prefix(data_dir)
87            .map_err(|_| io::Error::other("name not under data_dir"))?
88            .to_string_lossy()
89            .into_owned();
90        let name_bytes = name.as_bytes();
91        if name_bytes.len() > u16::MAX as usize {
92            return Err(io::Error::other("file name too long"));
93        }
94        let body_len = meta.len();
95        w.write_all(&(name_bytes.len() as u16).to_be_bytes())?;
96        w.write_all(name_bytes)?;
97        w.write_all(&body_len.to_be_bytes())?;
98        copy_file_body(&mut w, &path, body_len)?;
99        total_bytes += body_len;
100        file_count += 1;
101    }
102    // EOF marker: name_len = 0.
103    w.write_all(&0u16.to_be_bytes())?;
104    w.flush()?;
105    eprintln!(
106        "kevy-cli: backed up {file_count} file(s), {total_bytes} bytes total → {}",
107        out_path.display()
108    );
109    Ok(total_bytes)
110}
111
112/// Stream exactly `body_len` bytes of `path` into `w`.
113fn copy_file_body(w: &mut impl Write, path: &Path, body_len: u64) -> io::Result<()> {
114    let mut f = BufReader::new(File::open(path)?);
115    let mut buf = vec![0u8; 64 * 1024];
116    let mut copied = 0u64;
117    while copied < body_len {
118        let want = std::cmp::min((body_len - copied) as usize, buf.len());
119        let n = f.read(&mut buf[..want])?;
120        if n == 0 {
121            // File shrunk between metadata-stat and content-read
122            // (live backup race; AOF rewrite can shrink the file).
123            // Pad with zeros to honor the body_len we committed.
124            // Restore replay handles trailing zeros as torn-frame
125            // tail truncation (existing kevy-persist::replay logic).
126            let mut remaining = body_len - copied;
127            let zeros = [0u8; 64 * 1024];
128            while remaining > 0 {
129                let chunk = std::cmp::min(remaining as usize, zeros.len());
130                w.write_all(&zeros[..chunk])?;
131                remaining -= chunk as u64;
132            }
133            break;
134        }
135        w.write_all(&buf[..n])?;
136        copied += n as u64;
137    }
138    Ok(())
139}
140
141/// Unpack the container at `in_path` into `target_dir` (created if
142/// missing; refuses to overwrite an existing non-empty dir to avoid
143/// clobbering live data).
144///
145/// ```
146/// use kevy_cli::backup::{pack, unpack};
147/// let data = kevy_tmpdir::TmpDir::new("unpack-data");
148/// std::fs::write(data.path().join("aof-0.aof"), b"log")?;
149/// let out = kevy_tmpdir::TmpDir::new("unpack-out");
150/// let bundle = out.path().join("b.kevybkp");
151/// pack(data.path(), &bundle)?;
152///
153/// let fresh = kevy_tmpdir::TmpDir::new("unpack-fresh");
154/// assert_eq!(unpack(&bundle, fresh.path())?, 3);
155/// // a second restore into the same, now non-empty, directory is refused
156/// let err = unpack(&bundle, fresh.path()).unwrap_err();
157/// assert_eq!(err.kind(), std::io::ErrorKind::AlreadyExists);
158/// # Ok::<(), std::io::Error>(())
159/// ```
160pub fn unpack(in_path: impl AsRef<Path>, target_dir: impl AsRef<Path>) -> io::Result<u64> {
161    let (in_path, target_dir) = (in_path.as_ref(), target_dir.as_ref());
162    std::fs::create_dir_all(target_dir)?;
163    // Refuse to write into a non-empty dir (safety).
164    let existing = std::fs::read_dir(target_dir)?.count();
165    if existing > 0 {
166        return Err(io::Error::new(
167            io::ErrorKind::AlreadyExists,
168            format!(
169                "target dir {} is not empty ({existing} entries); refuse to overwrite",
170                target_dir.display()
171            ),
172        ));
173    }
174    let mut r = BufReader::new(File::open(in_path)?);
175    let mut magic = [0u8; 8];
176    r.read_exact(&mut magic)?;
177    if &magic != MAGIC {
178        return Err(io::Error::new(
179            io::ErrorKind::InvalidData,
180            "not a kevy backup container (magic mismatch)",
181        ));
182    }
183    let mut file_count = 0u64;
184    let mut total = 0u64;
185    // Loop ends at the EOF marker (read_entry_name -> None).
186    while let Some(name) = read_entry_name(&mut r)? {
187        let mut body_len_buf = [0u8; 8];
188        r.read_exact(&mut body_len_buf)?;
189        let body_len = u64::from_be_bytes(body_len_buf);
190        let out_path = target_dir.join(&name);
191        copy_entry_body(&mut r, &out_path, &name, body_len)?;
192        file_count += 1;
193        total += body_len;
194    }
195    eprintln!(
196        "kevy-cli: restored {file_count} file(s), {total} bytes total → {}",
197        target_dir.display()
198    );
199    Ok(total)
200}
201
202/// Read one entry's name header. `None` = the EOF marker (name_len 0).
203fn read_entry_name(r: &mut impl Read) -> io::Result<Option<String>> {
204    let mut name_len_buf = [0u8; 2];
205    r.read_exact(&mut name_len_buf)?;
206    let name_len = u16::from_be_bytes(name_len_buf);
207    if name_len == 0 {
208        return Ok(None);
209    }
210    let mut name_bytes = vec![0u8; name_len as usize];
211    r.read_exact(&mut name_bytes)?;
212    let name = std::str::from_utf8(&name_bytes)
213        .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
214    // Reject path components that try to escape (e.g., "../etc/passwd").
215    if name.contains("..") || name.starts_with('/') {
216        return Err(io::Error::new(
217            io::ErrorKind::InvalidData,
218            format!("backup entry name {name:?} contains path traversal"),
219        ));
220    }
221    Ok(Some(name.to_owned()))
222}
223
224/// Stream exactly `body_len` bytes from `r` into a fresh file at
225/// `out_path`. `name` only feeds the truncation error message.
226fn copy_entry_body(
227    r: &mut impl Read,
228    out_path: &Path,
229    name: &str,
230    body_len: u64,
231) -> io::Result<()> {
232    let mut out = BufWriter::new(File::create(out_path)?);
233    let mut remaining = body_len;
234    let mut buf = vec![0u8; 64 * 1024];
235    while remaining > 0 {
236        let want = std::cmp::min(remaining as usize, buf.len());
237        let n = r.read(&mut buf[..want])?;
238        if n == 0 {
239            return Err(io::Error::new(
240                io::ErrorKind::UnexpectedEof,
241                format!("backup truncated mid-file {name:?}"),
242            ));
243        }
244        out.write_all(&buf[..n])?;
245        remaining -= n as u64;
246    }
247    out.flush()
248}
249
250#[cfg(test)]
251mod tests {
252    use super::*;
253    use std::path::PathBuf;
254
255    /// A fresh, empty, uniquely-named directory.
256    fn tmp(name: &str) -> PathBuf {
257        kevy_tmpdir::unique_dir(&format!("cli-backup-{name}"))
258    }
259
260    /// A unique path for a FILE that must not exist yet — pack's output. The
261    /// old helper returned an uncreated path and let the call site decide what
262    /// it was; that ambiguity is how the same function ended up naming both
263    /// directories and files.
264    fn tmp_file(name: &str) -> PathBuf {
265        tmp("files").join(name)
266    }
267
268    /// `pack` skips subdirectories. That is safe only while everything
269    /// under the data dir's subdirectories is derived spill the restore
270    /// can rebuild — true today for `tier/` (demoted values, reachable
271    /// through a snapshot that materialises them) and `segs-*/` (cold
272    /// index segments, rebuilt by re-sliding).
273    ///
274    /// This test does not verify that property; it pins the *set* so
275    /// that adding a subdirectory forces the person adding it to come
276    /// here and state which kind it is. A backup that silently omits a
277    /// directory of truth is the failure this is standing in front of.
278    #[test]
279    fn subdirectories_are_derived_spill_only() {
280        const DERIVED: [&str; 2] = ["tier", "segs"];
281        let src = tmp("derived");
282        std::fs::write(src.join("aof-0.aof"), b"truth").unwrap();
283        for d in ["tier/0", "segs-0"] {
284            std::fs::create_dir_all(src.join(d)).unwrap();
285            std::fs::write(src.join(d).join("spill.dat"), b"derived").unwrap();
286        }
287        let out = tmp_file("derived.kevybkp");
288        pack(&src, &out).unwrap();
289        let target = tmp("derived-restored");
290        unpack(&out, &target).unwrap();
291
292        assert_eq!(std::fs::read(target.join("aof-0.aof")).unwrap(), b"truth");
293        for d in ["tier", "segs-0"] {
294            assert!(!target.join(d).exists(), "{d} came along; it is spill, not truth");
295        }
296        // The guard: every subdirectory the source had must be one the
297        // list above already calls derived.
298        for e in std::fs::read_dir(&src).unwrap() {
299            let e = e.unwrap();
300            if e.metadata().unwrap().is_dir() {
301                let name = e.file_name().to_string_lossy().into_owned();
302                assert!(
303                    DERIVED.iter().any(|d| name.starts_with(d)),
304                    "'{name}' is a data-dir subdirectory the backup skips. \
305                     If it holds truth rather than derived spill, pack() \
306                     is losing it — see the comment at the skip."
307                );
308            }
309        }
310        std::fs::remove_dir_all(&src).ok();
311        std::fs::remove_dir_all(&target).ok();
312    }
313
314    #[test]
315    fn pack_unpack_round_trip() {
316        let src = tmp("src");
317        std::fs::write(src.join("aof-0.aof"), b"AOF body 1").unwrap();
318        std::fs::write(src.join("snap-0.rdb"), b"snapshot body").unwrap();
319        let out = tmp_file("backup.kevybkp");
320        pack(&src, &out).unwrap();
321
322        let target = tmp("restored");
323        unpack(&out, &target).unwrap();
324        assert_eq!(std::fs::read(target.join("aof-0.aof")).unwrap(), b"AOF body 1");
325        assert_eq!(std::fs::read(target.join("snap-0.rdb")).unwrap(), b"snapshot body");
326
327        std::fs::remove_dir_all(&src).ok();
328        std::fs::remove_file(&out).ok();
329        std::fs::remove_dir_all(&target).ok();
330    }
331
332    #[test]
333    fn unpack_refuses_path_traversal() {
334        let src = tmp("src2");
335        std::fs::create_dir_all(&src).unwrap();
336        let out_path = tmp_file("bad.kevybkp");
337        // Hand-craft a bad container with "../etc/passwd" name.
338        let mut f = File::create(&out_path).unwrap();
339        f.write_all(MAGIC).unwrap();
340        let name = b"../etc/passwd";
341        f.write_all(&(name.len() as u16).to_be_bytes()).unwrap();
342        f.write_all(name).unwrap();
343        f.write_all(&0u64.to_be_bytes()).unwrap();
344        f.write_all(&0u16.to_be_bytes()).unwrap();
345        drop(f);
346
347        let target = tmp("restored2");
348        let err = unpack(&out_path, &target).unwrap_err();
349        assert!(err.to_string().contains("path traversal"));
350
351        std::fs::remove_file(&out_path).ok();
352        std::fs::remove_dir_all(&target).ok();
353        std::fs::remove_dir_all(&src).ok();
354    }
355
356    #[test]
357    fn unpack_refuses_non_empty_target() {
358        let target = tmp("non-empty");
359        std::fs::create_dir_all(&target).unwrap();
360        std::fs::write(target.join("existing.txt"), b"data").unwrap();
361        let out_path = tmp_file("good.kevybkp");
362        let mut f = File::create(&out_path).unwrap();
363        f.write_all(MAGIC).unwrap();
364        f.write_all(&0u16.to_be_bytes()).unwrap();
365        drop(f);
366        let err = unpack(&out_path, &target).unwrap_err();
367        assert!(err.to_string().contains("not empty"));
368        std::fs::remove_dir_all(&target).ok();
369        std::fs::remove_file(&out_path).ok();
370    }
371}