Skip to main content

kevy_cli/
bulk.rs

1//! Prefix bulk ops + diagnostics:
2//! `copy-prefix` / `delete-prefix` (token-bucket rate limit,
3//! `--dry-run`), `digest`, `diff`, `inspect`.
4//!
5//! ```
6//! use kevy_cli::bulk::{DeleteMode, run_copy_prefix, run_delete_prefix, run_digest};
7//! # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
8//! let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
9//! client.request_borrowed(&[b"SET", b"old:1", b"a"])?;
10//! client.request_borrowed(&[b"SADD", b"old:2", b"x", b"y"])?;
11//! // move a prefix: copy, prove it arrived, then drop the source
12//! run_copy_prefix(&mut client, b"old:", b"new:", 0)?;
13//! assert_eq!(run_digest(&mut client, b"new:")?.0, 2);
14//! assert_eq!(run_delete_prefix(&mut client, b"old:", 0, DeleteMode::Unlink)?, 2);
15//! assert_eq!(run_digest(&mut client, b"old:")?.0, 0);
16//! # Ok::<(), Box<dyn std::error::Error>>(())
17//! ```
18
19use std::io::{self, Write};
20use std::time::{Duration, Instant};
21
22use crate::link::Link;
23use kevy_resp::Reply;
24
25/// Token bucket: `rate` ops/second, starting EMPTY (strict pacing —
26/// a full-bucket start lets a small job burn its whole burst
27/// unthrottled, defeating the point of `--rate` for short sweeps).
28/// `rate == 0` = unlimited.
29///
30/// ```
31/// use kevy_cli::bulk::RateLimiter;
32/// use std::time::{Duration, Instant};
33///
34/// let mut limiter = RateLimiter::new(100); // 100 ops/s
35/// let start = Instant::now();
36/// for _ in 0..5 {
37///     limiter.take();
38/// }
39/// // the bucket starts empty, so even a short burst is paced
40/// assert!(start.elapsed() >= Duration::from_millis(40));
41/// ```
42#[derive(Debug)]
43pub struct RateLimiter {
44    rate: u64,
45    tokens: f64,
46    last: Instant,
47}
48
49impl RateLimiter {
50    /// New limiter at `rate` ops/s (0 = off).
51    ///
52    /// ```
53    /// use kevy_cli::bulk::RateLimiter;
54    /// let mut unlimited = RateLimiter::new(0);
55    /// let start = std::time::Instant::now();
56    /// for _ in 0..10_000 {
57    ///     unlimited.take(); // never blocks
58    /// }
59    /// assert!(start.elapsed() < std::time::Duration::from_secs(1));
60    /// ```
61    pub fn new(rate: u64) -> Self {
62        Self { rate, tokens: 0.0, last: Instant::now() }
63    }
64
65    /// Block until one op is admitted.
66    ///
67    /// ```
68    /// use kevy_cli::bulk::RateLimiter;
69    /// use std::time::{Duration, Instant};
70    ///
71    /// let mut limiter = RateLimiter::new(50);
72    /// let start = Instant::now();
73    /// limiter.take(); // the first op already waits for its token (~20 ms)
74    /// assert!(start.elapsed() >= Duration::from_millis(15));
75    /// ```
76    pub fn take(&mut self) {
77        if self.rate == 0 {
78            return;
79        }
80        loop {
81            let now = Instant::now();
82            self.tokens = (self.tokens
83                + now.duration_since(self.last).as_secs_f64() * self.rate as f64)
84                .min(self.rate as f64);
85            self.last = now;
86            if self.tokens >= 1.0 {
87                self.tokens -= 1.0;
88                return;
89            }
90            std::thread::sleep(Duration::from_millis(2));
91        }
92    }
93}
94
95fn scan_page(
96    client: &mut dyn Link,
97    cursor: &[u8],
98    pattern: &[u8],
99) -> io::Result<(Vec<u8>, Vec<Vec<u8>>)> {
100    let reply = client.request_borrowed(&[b"SCAN", cursor, b"MATCH", pattern, b"COUNT", b"512"])?;
101    let Reply::Array(items) = reply else {
102        return Err(io::Error::new(io::ErrorKind::InvalidData, "SCAN reply shape"));
103    };
104    let (Some(Reply::Bulk(next)), Some(Reply::Array(keys))) = (items.first(), items.get(1)) else {
105        return Err(io::Error::new(io::ErrorKind::InvalidData, "SCAN reply shape"));
106    };
107    let keys = keys
108        .iter()
109        .filter_map(|k| if let Reply::Bulk(b) = k { Some(b.clone()) } else { None })
110        .collect();
111    Ok((next.clone(), keys))
112}
113
114/// Whether `delete-prefix` deletes what it finds or only counts it.
115///
116/// ```
117/// use kevy_cli::bulk::DeleteMode;
118/// // `delete-prefix --dry-run`
119/// assert_ne!(DeleteMode::DryRun, DeleteMode::default());
120/// ```
121#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
122#[non_exhaustive]
123pub enum DeleteMode {
124    /// UNLINK every key found.
125    ///
126    /// ```
127    /// use kevy_cli::bulk::{DeleteMode, run_delete_prefix};
128    /// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
129    /// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
130    /// client.request_borrowed(&[b"SET", b"tmp:a", b"1"])?;
131    /// assert_eq!(run_delete_prefix(&mut client, b"tmp:", 0, DeleteMode::Unlink)?, 1);
132    /// let left = client.request_borrowed(&[b"EXISTS", b"tmp:a"])?;
133    /// assert_eq!(left, kevy_resp_client::Reply::Int(0));
134    /// # Ok::<(), Box<dyn std::error::Error>>(())
135    /// ```
136    #[default]
137    Unlink,
138    /// Count the keys that would be deleted and touch nothing
139    /// (`--dry-run`).
140    ///
141    /// ```
142    /// use kevy_cli::bulk::{DeleteMode, run_delete_prefix};
143    /// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
144    /// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
145    /// client.request_borrowed(&[b"SET", b"tmp:a", b"1"])?;
146    /// assert_eq!(run_delete_prefix(&mut client, b"tmp:", 0, DeleteMode::DryRun)?, 1);
147    /// let left = client.request_borrowed(&[b"EXISTS", b"tmp:a"])?;
148    /// assert_eq!(left, kevy_resp_client::Reply::Int(1), "a dry run touches nothing");
149    /// # Ok::<(), Box<dyn std::error::Error>>(())
150    /// ```
151    DryRun,
152}
153
154/// `delete-prefix`: SCAN + UNLINK, rate-limited. Returns the deleted
155/// count, or under [`DeleteMode::DryRun`] the count that would be.
156///
157/// ```
158/// use kevy_cli::bulk::{DeleteMode, run_delete_prefix};
159/// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
160/// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
161/// for k in ["tmp:1", "tmp:2", "keep:1"] {
162///     client.request_borrowed(&[b"SET", k.as_bytes(), b"v"])?;
163/// }
164/// let would = run_delete_prefix(&mut client, b"tmp:", 0, DeleteMode::DryRun)?;
165/// assert_eq!(would, 2);
166/// assert_eq!(run_delete_prefix(&mut client, b"tmp:", 1000, DeleteMode::Unlink)?, would);
167/// # Ok::<(), Box<dyn std::error::Error>>(())
168/// ```
169pub fn run_delete_prefix(
170    client: &mut dyn Link,
171    prefix: &[u8],
172    rate: u64,
173    mode: DeleteMode,
174) -> io::Result<u64> {
175    let mut pattern = prefix.to_vec();
176    pattern.push(b'*');
177    let mut cursor: Vec<u8> = b"0".to_vec();
178    let mut limiter = RateLimiter::new(rate);
179    let mut n = 0u64;
180    loop {
181        let (next, keys) = scan_page(client, &cursor, &pattern)?;
182        for key in &keys {
183            if mode == DeleteMode::DryRun {
184                n += 1;
185                continue;
186            }
187            limiter.take();
188            if let Reply::Int(d) = client.request_borrowed(&[b"UNLINK", key])? {
189                n += d as u64;
190            }
191        }
192        cursor = next;
193        if cursor == b"0" {
194            return Ok(n);
195        }
196    }
197}
198
199/// `copy-prefix`: SCAN src prefix, re-key under dst prefix via
200/// read+rebuild frames (the server has no COPY verb; TTL carried as
201/// absolute PEXPIREAT). Rate-limited per source key.
202///
203/// Returns what it copied **and what it could not** — same contract as
204/// `export`, for the same reason: the rebuild set does not cover every
205/// type, and a copy that quietly drops one is worse than a refusal.
206///
207/// ```
208/// use kevy_cli::bulk::run_copy_prefix;
209/// use kevy_resp_client::Reply;
210/// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
211/// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
212///
213/// client.request_borrowed(&[b"HSET", b"src:1", b"name", b"ada"])?;
214/// client.request_borrowed(&[b"SET", b"src:2", b"x"])?;
215/// client.request_borrowed(&[b"PEXPIRE", b"src:2", b"60000"])?;
216/// let copied = run_copy_prefix(&mut client, b"src:", b"dst:", 0)?;
217/// assert_eq!(copied.keys, 2);
218/// assert!(copied.skipped.is_empty(), "every type had a rebuild verb");
219/// let name = client.request_borrowed(&[b"HGET", b"dst:1", b"name"])?;
220/// assert_eq!(name, Reply::Bulk(b"ada".to_vec()));
221/// // the TTL travels with the key
222/// assert!(matches!(client.request_borrowed(&[b"PTTL", b"dst:2"])?, Reply::Int(ms) if ms > 0));
223/// # Ok::<(), Box<dyn std::error::Error>>(())
224/// ```
225pub fn run_copy_prefix(
226    client: &mut dyn Link,
227    src_prefix: &[u8],
228    dst_prefix: &[u8],
229    rate: u64,
230) -> io::Result<crate::migrate::Export> {
231    let mut pattern = src_prefix.to_vec();
232    pattern.push(b'*');
233    let mut cursor: Vec<u8> = b"0".to_vec();
234    let mut limiter = RateLimiter::new(rate);
235    let mut n = 0u64;
236    // Same skip as `export`, and it must be as loud: a copy that leaves
237    // a type behind and says "copied N keys" is the same silence.
238    let mut skipped: std::collections::BTreeMap<Vec<u8>, u64> = Default::default();
239    loop {
240        let (next, keys) = scan_page(client, &cursor, &pattern)?;
241        for key in &keys {
242            limiter.take();
243            let mut dst = dst_prefix.to_vec();
244            dst.extend_from_slice(&key[src_prefix.len()..]);
245            let frames = match crate::migrate::rebuild_frames(client, key, &dst)? {
246                crate::migrate::Rebuilt::Frames(f) => f,
247                crate::migrate::Rebuilt::Vanished => continue,
248                crate::migrate::Rebuilt::UnsupportedType(ty) => {
249                    *skipped.entry(ty).or_insert(0u64) += 1;
250                    continue;
251                }
252            };
253            let n_cmds = count_commands(&frames);
254            for r in client.pipeline_raw(&frames, n_cmds)? {
255                if let Reply::Error(e) = r {
256                    return Err(io::Error::new(
257                        io::ErrorKind::InvalidData,
258                        String::from_utf8_lossy(&e).into_owned(),
259                    ));
260                }
261            }
262            n += 1;
263        }
264        cursor = next;
265        if cursor == b"0" {
266            return Ok(crate::migrate::Export { keys: n, skipped });
267        }
268    }
269}
270
271/// Number of RESP commands in a frame buffer (top-level '*' headers).
272fn count_commands(mut b: &[u8]) -> usize {
273    let mut n = 0;
274    while let Some(len) = crate::migrate::command_len_pub(b) {
275        n += 1;
276        b = &b[len..];
277    }
278    n
279}
280
281/// `digest <prefix>` → (count, hex).
282///
283/// ```
284/// use kevy_cli::bulk::run_digest;
285/// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
286/// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
287/// client.request_borrowed(&[b"SET", b"cfg:a", b"1"])?;
288/// let (count, before) = run_digest(&mut client, b"cfg:")?;
289/// assert_eq!(count, 1);
290/// client.request_borrowed(&[b"SET", b"cfg:a", b"2"])?;
291/// let (_, after) = run_digest(&mut client, b"cfg:")?;
292/// assert_ne!(before, after, "any change to a value changes the digest");
293/// # Ok::<(), Box<dyn std::error::Error>>(())
294/// ```
295pub fn run_digest(client: &mut dyn Link, prefix: &[u8]) -> io::Result<(i64, String)> {
296    let r = client.request_borrowed(&[b"PREFIX.DIGEST", prefix])?;
297    let Reply::Array(items) = r else {
298        return Err(io::Error::new(io::ErrorKind::InvalidData, "PREFIX.DIGEST reply"));
299    };
300    let (Some(Reply::Int(n)), Some(Reply::Bulk(hex))) = (items.first(), items.get(1)) else {
301        return Err(io::Error::new(io::ErrorKind::InvalidData, "PREFIX.DIGEST reply"));
302    };
303    Ok((*n, String::from_utf8_lossy(hex).into_owned()))
304}
305
306/// `diff`: compare prefixes across two servers. Returns mismatching
307/// prefixes.
308///
309/// ```
310/// use kevy_cli::bulk::run_diff;
311/// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
312/// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
313/// # let port_b = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
314/// let mut replica = kevy_resp_client::RespClient::connect("127.0.0.1", port_b)?;
315/// for c in [&mut client, &mut replica] {
316///     c.request_borrowed(&[b"SET", b"user:1", b"ada"])?;
317///     c.request_borrowed(&[b"SET", b"order:1", b"open"])?;
318/// }
319/// replica.request_borrowed(&[b"SET", b"order:1", b"shipped"])?;
320///
321/// let mut out = Vec::new();
322/// let prefixes = [b"user:".to_vec(), b"order:".to_vec()];
323/// let bad = run_diff(&mut client, &mut replica, &prefixes, &mut out)?;
324/// assert_eq!(bad, vec![b"order:".to_vec()]);
325/// assert!(String::from_utf8(out)?.lines().next().is_some_and(|l| l.ends_with("OK")));
326/// # Ok::<(), Box<dyn std::error::Error>>(())
327/// ```
328pub fn run_diff(
329    a: &mut dyn Link,
330    b: &mut dyn Link,
331    prefixes: &[Vec<u8>],
332    mut out: impl Write,
333) -> io::Result<Vec<Vec<u8>>> {
334    let mut bad = Vec::new();
335    for p in prefixes {
336        let (na, da) = run_digest(a, p)?;
337        let (nb, db) = run_digest(b, p)?;
338        let ok = na == nb && da == db;
339        writeln!(
340            out,
341            "{}  A: {na} keys {da}  B: {nb} keys {db}  {}",
342            String::from_utf8_lossy(p),
343            if ok { "OK" } else { "MISMATCH" }
344        )?;
345        if !ok {
346            bad.push(p.clone());
347        }
348    }
349    Ok(bad)
350}
351
352/// `inspect <prefix>`: sample keys, type distribution, sizes.
353///
354/// ```
355/// use kevy_cli::bulk::run_inspect;
356/// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
357/// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
358/// client.request_borrowed(&[b"SET", b"s:1", b"a"])?;
359/// client.request_borrowed(&[b"SET", b"s:2", b"b"])?;
360/// client.request_borrowed(&[b"SADD", b"s:3", b"m"])?;
361/// let mut out = Vec::new();
362/// run_inspect(&mut client, b"s:", &mut out)?;
363/// let text = String::from_utf8(out)?;
364/// assert!(text.starts_with("prefix s:: 3 keys\n  string: 2\n  set: 1\n"), "{text}");
365/// # Ok::<(), Box<dyn std::error::Error>>(())
366/// ```
367pub fn run_inspect(client: &mut dyn Link, prefix: &[u8], mut out: impl Write) -> io::Result<()> {
368    let mut pattern = prefix.to_vec();
369    pattern.push(b'*');
370    let mut cursor: Vec<u8> = b"0".to_vec();
371    let mut total = 0u64;
372    let mut by_type: Vec<(String, u64)> = Vec::new();
373    let mut samples: Vec<String> = Vec::new();
374    loop {
375        let (next, keys) = scan_page(client, &cursor, &pattern)?;
376        for key in &keys {
377            total += 1;
378            if samples.len() < 8 {
379                samples.push(String::from_utf8_lossy(key).into_owned());
380            }
381            if let Reply::Simple(t) = client.request_borrowed(&[b"TYPE", key])? {
382                let t = String::from_utf8_lossy(&t).into_owned();
383                match by_type.iter_mut().find(|(n, _)| *n == t) {
384                    Some((_, c)) => *c += 1,
385                    None => by_type.push((t, 1)),
386                }
387            }
388        }
389        cursor = next;
390        if cursor == b"0" {
391            break;
392        }
393    }
394    writeln!(out, "prefix {}: {total} keys", String::from_utf8_lossy(prefix))?;
395    by_type.sort_by_key(|(_, c)| std::cmp::Reverse(*c));
396    for (t, c) in &by_type {
397        writeln!(out, "  {t}: {c}")?;
398    }
399    if !samples.is_empty() {
400        writeln!(out, "  samples: {}", samples.join(", "))?;
401    }
402    Ok(())
403}