kevy_cli/bulk.rs
1//! Prefix bulk ops + diagnostics:
2//! `copy-prefix` / `delete-prefix` (token-bucket rate limit,
3//! `--dry-run`), `digest`, `diff`, `inspect`.
4//!
5//! ```
6//! use kevy_cli::bulk::{DeleteMode, run_copy_prefix, run_delete_prefix, run_digest};
7//! # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
8//! let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
9//! client.request_borrowed(&[b"SET", b"old:1", b"a"])?;
10//! client.request_borrowed(&[b"SADD", b"old:2", b"x", b"y"])?;
11//! // move a prefix: copy, prove it arrived, then drop the source
12//! run_copy_prefix(&mut client, b"old:", b"new:", 0)?;
13//! assert_eq!(run_digest(&mut client, b"new:")?.0, 2);
14//! assert_eq!(run_delete_prefix(&mut client, b"old:", 0, DeleteMode::Unlink)?, 2);
15//! assert_eq!(run_digest(&mut client, b"old:")?.0, 0);
16//! # Ok::<(), Box<dyn std::error::Error>>(())
17//! ```
18
19use std::io::{self, Write};
20use std::time::{Duration, Instant};
21
22use crate::link::Link;
23use kevy_resp::Reply;
24
25/// Token bucket: `rate` ops/second, starting EMPTY (strict pacing —
26/// a full-bucket start lets a small job burn its whole burst
27/// unthrottled, defeating the point of `--rate` for short sweeps).
28/// `rate == 0` = unlimited.
29///
30/// ```
31/// use kevy_cli::bulk::RateLimiter;
32/// use std::time::{Duration, Instant};
33///
34/// let mut limiter = RateLimiter::new(100); // 100 ops/s
35/// let start = Instant::now();
36/// for _ in 0..5 {
37/// limiter.take();
38/// }
39/// // the bucket starts empty, so even a short burst is paced
40/// assert!(start.elapsed() >= Duration::from_millis(40));
41/// ```
42#[derive(Debug)]
43pub struct RateLimiter {
44 rate: u64,
45 tokens: f64,
46 last: Instant,
47}
48
49impl RateLimiter {
50 /// New limiter at `rate` ops/s (0 = off).
51 ///
52 /// ```
53 /// use kevy_cli::bulk::RateLimiter;
54 /// let mut unlimited = RateLimiter::new(0);
55 /// let start = std::time::Instant::now();
56 /// for _ in 0..10_000 {
57 /// unlimited.take(); // never blocks
58 /// }
59 /// assert!(start.elapsed() < std::time::Duration::from_secs(1));
60 /// ```
61 pub fn new(rate: u64) -> Self {
62 Self { rate, tokens: 0.0, last: Instant::now() }
63 }
64
65 /// Block until one op is admitted.
66 ///
67 /// ```
68 /// use kevy_cli::bulk::RateLimiter;
69 /// use std::time::{Duration, Instant};
70 ///
71 /// let mut limiter = RateLimiter::new(50);
72 /// let start = Instant::now();
73 /// limiter.take(); // the first op already waits for its token (~20 ms)
74 /// assert!(start.elapsed() >= Duration::from_millis(15));
75 /// ```
76 pub fn take(&mut self) {
77 if self.rate == 0 {
78 return;
79 }
80 loop {
81 let now = Instant::now();
82 self.tokens = (self.tokens
83 + now.duration_since(self.last).as_secs_f64() * self.rate as f64)
84 .min(self.rate as f64);
85 self.last = now;
86 if self.tokens >= 1.0 {
87 self.tokens -= 1.0;
88 return;
89 }
90 std::thread::sleep(Duration::from_millis(2));
91 }
92 }
93}
94
95fn scan_page(
96 client: &mut dyn Link,
97 cursor: &[u8],
98 pattern: &[u8],
99) -> io::Result<(Vec<u8>, Vec<Vec<u8>>)> {
100 let reply = client.request_borrowed(&[b"SCAN", cursor, b"MATCH", pattern, b"COUNT", b"512"])?;
101 let Reply::Array(items) = reply else {
102 return Err(io::Error::new(io::ErrorKind::InvalidData, "SCAN reply shape"));
103 };
104 let (Some(Reply::Bulk(next)), Some(Reply::Array(keys))) = (items.first(), items.get(1)) else {
105 return Err(io::Error::new(io::ErrorKind::InvalidData, "SCAN reply shape"));
106 };
107 let keys = keys
108 .iter()
109 .filter_map(|k| if let Reply::Bulk(b) = k { Some(b.clone()) } else { None })
110 .collect();
111 Ok((next.clone(), keys))
112}
113
114/// Whether `delete-prefix` deletes what it finds or only counts it.
115///
116/// ```
117/// use kevy_cli::bulk::DeleteMode;
118/// // `delete-prefix --dry-run`
119/// assert_ne!(DeleteMode::DryRun, DeleteMode::default());
120/// ```
121#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
122#[non_exhaustive]
123pub enum DeleteMode {
124 /// UNLINK every key found.
125 ///
126 /// ```
127 /// use kevy_cli::bulk::{DeleteMode, run_delete_prefix};
128 /// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
129 /// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
130 /// client.request_borrowed(&[b"SET", b"tmp:a", b"1"])?;
131 /// assert_eq!(run_delete_prefix(&mut client, b"tmp:", 0, DeleteMode::Unlink)?, 1);
132 /// let left = client.request_borrowed(&[b"EXISTS", b"tmp:a"])?;
133 /// assert_eq!(left, kevy_resp_client::Reply::Int(0));
134 /// # Ok::<(), Box<dyn std::error::Error>>(())
135 /// ```
136 #[default]
137 Unlink,
138 /// Count the keys that would be deleted and touch nothing
139 /// (`--dry-run`).
140 ///
141 /// ```
142 /// use kevy_cli::bulk::{DeleteMode, run_delete_prefix};
143 /// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
144 /// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
145 /// client.request_borrowed(&[b"SET", b"tmp:a", b"1"])?;
146 /// assert_eq!(run_delete_prefix(&mut client, b"tmp:", 0, DeleteMode::DryRun)?, 1);
147 /// let left = client.request_borrowed(&[b"EXISTS", b"tmp:a"])?;
148 /// assert_eq!(left, kevy_resp_client::Reply::Int(1), "a dry run touches nothing");
149 /// # Ok::<(), Box<dyn std::error::Error>>(())
150 /// ```
151 DryRun,
152}
153
154/// `delete-prefix`: SCAN + UNLINK, rate-limited. Returns the deleted
155/// count, or under [`DeleteMode::DryRun`] the count that would be.
156///
157/// ```
158/// use kevy_cli::bulk::{DeleteMode, run_delete_prefix};
159/// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
160/// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
161/// for k in ["tmp:1", "tmp:2", "keep:1"] {
162/// client.request_borrowed(&[b"SET", k.as_bytes(), b"v"])?;
163/// }
164/// let would = run_delete_prefix(&mut client, b"tmp:", 0, DeleteMode::DryRun)?;
165/// assert_eq!(would, 2);
166/// assert_eq!(run_delete_prefix(&mut client, b"tmp:", 1000, DeleteMode::Unlink)?, would);
167/// # Ok::<(), Box<dyn std::error::Error>>(())
168/// ```
169pub fn run_delete_prefix(
170 client: &mut dyn Link,
171 prefix: &[u8],
172 rate: u64,
173 mode: DeleteMode,
174) -> io::Result<u64> {
175 let mut pattern = prefix.to_vec();
176 pattern.push(b'*');
177 let mut cursor: Vec<u8> = b"0".to_vec();
178 let mut limiter = RateLimiter::new(rate);
179 let mut n = 0u64;
180 loop {
181 let (next, keys) = scan_page(client, &cursor, &pattern)?;
182 for key in &keys {
183 if mode == DeleteMode::DryRun {
184 n += 1;
185 continue;
186 }
187 limiter.take();
188 if let Reply::Int(d) = client.request_borrowed(&[b"UNLINK", key])? {
189 n += d as u64;
190 }
191 }
192 cursor = next;
193 if cursor == b"0" {
194 return Ok(n);
195 }
196 }
197}
198
199/// `copy-prefix`: SCAN src prefix, re-key under dst prefix via
200/// read+rebuild frames (the server has no COPY verb; TTL carried as
201/// absolute PEXPIREAT). Rate-limited per source key.
202///
203/// Returns what it copied **and what it could not** — same contract as
204/// `export`, for the same reason: the rebuild set does not cover every
205/// type, and a copy that quietly drops one is worse than a refusal.
206///
207/// ```
208/// use kevy_cli::bulk::run_copy_prefix;
209/// use kevy_resp_client::Reply;
210/// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
211/// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
212///
213/// client.request_borrowed(&[b"HSET", b"src:1", b"name", b"ada"])?;
214/// client.request_borrowed(&[b"SET", b"src:2", b"x"])?;
215/// client.request_borrowed(&[b"PEXPIRE", b"src:2", b"60000"])?;
216/// let copied = run_copy_prefix(&mut client, b"src:", b"dst:", 0)?;
217/// assert_eq!(copied.keys, 2);
218/// assert!(copied.skipped.is_empty(), "every type had a rebuild verb");
219/// let name = client.request_borrowed(&[b"HGET", b"dst:1", b"name"])?;
220/// assert_eq!(name, Reply::Bulk(b"ada".to_vec()));
221/// // the TTL travels with the key
222/// assert!(matches!(client.request_borrowed(&[b"PTTL", b"dst:2"])?, Reply::Int(ms) if ms > 0));
223/// # Ok::<(), Box<dyn std::error::Error>>(())
224/// ```
225pub fn run_copy_prefix(
226 client: &mut dyn Link,
227 src_prefix: &[u8],
228 dst_prefix: &[u8],
229 rate: u64,
230) -> io::Result<crate::migrate::Export> {
231 let mut pattern = src_prefix.to_vec();
232 pattern.push(b'*');
233 let mut cursor: Vec<u8> = b"0".to_vec();
234 let mut limiter = RateLimiter::new(rate);
235 let mut n = 0u64;
236 // Same skip as `export`, and it must be as loud: a copy that leaves
237 // a type behind and says "copied N keys" is the same silence.
238 let mut skipped: std::collections::BTreeMap<Vec<u8>, u64> = Default::default();
239 loop {
240 let (next, keys) = scan_page(client, &cursor, &pattern)?;
241 for key in &keys {
242 limiter.take();
243 let mut dst = dst_prefix.to_vec();
244 dst.extend_from_slice(&key[src_prefix.len()..]);
245 let frames = match crate::migrate::rebuild_frames(client, key, &dst)? {
246 crate::migrate::Rebuilt::Frames(f) => f,
247 crate::migrate::Rebuilt::Vanished => continue,
248 crate::migrate::Rebuilt::UnsupportedType(ty) => {
249 *skipped.entry(ty).or_insert(0u64) += 1;
250 continue;
251 }
252 };
253 let n_cmds = count_commands(&frames);
254 for r in client.pipeline_raw(&frames, n_cmds)? {
255 if let Reply::Error(e) = r {
256 return Err(io::Error::new(
257 io::ErrorKind::InvalidData,
258 String::from_utf8_lossy(&e).into_owned(),
259 ));
260 }
261 }
262 n += 1;
263 }
264 cursor = next;
265 if cursor == b"0" {
266 return Ok(crate::migrate::Export { keys: n, skipped });
267 }
268 }
269}
270
271/// Number of RESP commands in a frame buffer (top-level '*' headers).
272fn count_commands(mut b: &[u8]) -> usize {
273 let mut n = 0;
274 while let Some(len) = crate::migrate::command_len_pub(b) {
275 n += 1;
276 b = &b[len..];
277 }
278 n
279}
280
281/// `digest <prefix>` → (count, hex).
282///
283/// ```
284/// use kevy_cli::bulk::run_digest;
285/// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
286/// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
287/// client.request_borrowed(&[b"SET", b"cfg:a", b"1"])?;
288/// let (count, before) = run_digest(&mut client, b"cfg:")?;
289/// assert_eq!(count, 1);
290/// client.request_borrowed(&[b"SET", b"cfg:a", b"2"])?;
291/// let (_, after) = run_digest(&mut client, b"cfg:")?;
292/// assert_ne!(before, after, "any change to a value changes the digest");
293/// # Ok::<(), Box<dyn std::error::Error>>(())
294/// ```
295pub fn run_digest(client: &mut dyn Link, prefix: &[u8]) -> io::Result<(i64, String)> {
296 let r = client.request_borrowed(&[b"PREFIX.DIGEST", prefix])?;
297 let Reply::Array(items) = r else {
298 return Err(io::Error::new(io::ErrorKind::InvalidData, "PREFIX.DIGEST reply"));
299 };
300 let (Some(Reply::Int(n)), Some(Reply::Bulk(hex))) = (items.first(), items.get(1)) else {
301 return Err(io::Error::new(io::ErrorKind::InvalidData, "PREFIX.DIGEST reply"));
302 };
303 Ok((*n, String::from_utf8_lossy(hex).into_owned()))
304}
305
306/// `diff`: compare prefixes across two servers. Returns mismatching
307/// prefixes.
308///
309/// ```
310/// use kevy_cli::bulk::run_diff;
311/// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
312/// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
313/// # let port_b = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
314/// let mut replica = kevy_resp_client::RespClient::connect("127.0.0.1", port_b)?;
315/// for c in [&mut client, &mut replica] {
316/// c.request_borrowed(&[b"SET", b"user:1", b"ada"])?;
317/// c.request_borrowed(&[b"SET", b"order:1", b"open"])?;
318/// }
319/// replica.request_borrowed(&[b"SET", b"order:1", b"shipped"])?;
320///
321/// let mut out = Vec::new();
322/// let prefixes = [b"user:".to_vec(), b"order:".to_vec()];
323/// let bad = run_diff(&mut client, &mut replica, &prefixes, &mut out)?;
324/// assert_eq!(bad, vec![b"order:".to_vec()]);
325/// assert!(String::from_utf8(out)?.lines().next().is_some_and(|l| l.ends_with("OK")));
326/// # Ok::<(), Box<dyn std::error::Error>>(())
327/// ```
328pub fn run_diff(
329 a: &mut dyn Link,
330 b: &mut dyn Link,
331 prefixes: &[Vec<u8>],
332 mut out: impl Write,
333) -> io::Result<Vec<Vec<u8>>> {
334 let mut bad = Vec::new();
335 for p in prefixes {
336 let (na, da) = run_digest(a, p)?;
337 let (nb, db) = run_digest(b, p)?;
338 let ok = na == nb && da == db;
339 writeln!(
340 out,
341 "{} A: {na} keys {da} B: {nb} keys {db} {}",
342 String::from_utf8_lossy(p),
343 if ok { "OK" } else { "MISMATCH" }
344 )?;
345 if !ok {
346 bad.push(p.clone());
347 }
348 }
349 Ok(bad)
350}
351
352/// `inspect <prefix>`: sample keys, type distribution, sizes.
353///
354/// ```
355/// use kevy_cli::bulk::run_inspect;
356/// # let port = include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/doc_server/serve.rs"));
357/// let mut client = kevy_resp_client::RespClient::connect("127.0.0.1", port)?;
358/// client.request_borrowed(&[b"SET", b"s:1", b"a"])?;
359/// client.request_borrowed(&[b"SET", b"s:2", b"b"])?;
360/// client.request_borrowed(&[b"SADD", b"s:3", b"m"])?;
361/// let mut out = Vec::new();
362/// run_inspect(&mut client, b"s:", &mut out)?;
363/// let text = String::from_utf8(out)?;
364/// assert!(text.starts_with("prefix s:: 3 keys\n string: 2\n set: 1\n"), "{text}");
365/// # Ok::<(), Box<dyn std::error::Error>>(())
366/// ```
367pub fn run_inspect(client: &mut dyn Link, prefix: &[u8], mut out: impl Write) -> io::Result<()> {
368 let mut pattern = prefix.to_vec();
369 pattern.push(b'*');
370 let mut cursor: Vec<u8> = b"0".to_vec();
371 let mut total = 0u64;
372 let mut by_type: Vec<(String, u64)> = Vec::new();
373 let mut samples: Vec<String> = Vec::new();
374 loop {
375 let (next, keys) = scan_page(client, &cursor, &pattern)?;
376 for key in &keys {
377 total += 1;
378 if samples.len() < 8 {
379 samples.push(String::from_utf8_lossy(key).into_owned());
380 }
381 if let Reply::Simple(t) = client.request_borrowed(&[b"TYPE", key])? {
382 let t = String::from_utf8_lossy(&t).into_owned();
383 match by_type.iter_mut().find(|(n, _)| *n == t) {
384 Some((_, c)) => *c += 1,
385 None => by_type.push((t, 1)),
386 }
387 }
388 }
389 cursor = next;
390 if cursor == b"0" {
391 break;
392 }
393 }
394 writeln!(out, "prefix {}: {total} keys", String::from_utf8_lossy(prefix))?;
395 by_type.sort_by_key(|(_, c)| std::cmp::Reverse(*c));
396 for (t, c) in &by_type {
397 writeln!(out, " {t}: {c}")?;
398 }
399 if !samples.is_empty() {
400 writeln!(out, " samples: {}", samples.join(", "))?;
401 }
402 Ok(())
403}