1use std::fs::{File, OpenOptions};
14use std::io::{self, BufWriter, Read, Seek, SeekFrom, Write};
15use std::path::Path;
16
17use kevy_resp::{Reply, encode_command_borrowed};
18use kevy_resp_client::RespClient;
19
20const PIPELINE: usize = 512;
21
22#[derive(Debug)]
29pub struct Export {
30 pub keys: u64,
32 pub skipped: std::collections::BTreeMap<Vec<u8>, u64>,
34}
35
36pub fn run_export(
40 client: &mut RespClient,
41 prefix: Option<&[u8]>,
42 out_path: &Path,
43) -> io::Result<Export> {
44 let mut out = BufWriter::new(File::create(out_path)?);
45 let mut cursor: Vec<u8> = b"0".to_vec();
46 let mut pattern = prefix.unwrap_or_default().to_vec();
47 pattern.push(b'*');
48 let mut n = 0u64;
49 let mut skipped: std::collections::BTreeMap<Vec<u8>, u64> = Default::default();
52 loop {
53 let reply =
54 client.request_borrowed(&[b"SCAN", &cursor, b"MATCH", &pattern, b"COUNT", b"512"])?;
55 let Reply::Array(items) = reply else {
56 return Err(io::Error::new(io::ErrorKind::InvalidData, "SCAN reply shape"));
57 };
58 let (Some(Reply::Bulk(next)), Some(Reply::Array(keys))) = (items.first(), items.get(1))
59 else {
60 return Err(io::Error::new(io::ErrorKind::InvalidData, "SCAN reply shape"));
61 };
62 let next = next.clone();
63 for k in keys {
64 let Reply::Bulk(key) = k else { continue };
65 let key = key.clone();
66 match export_key(client, &key, &mut out)? {
67 Some(None) => n += 1,
68 Some(Some(ty)) => *skipped.entry(ty).or_insert(0u64) += 1,
69 None => {}
70 }
71 }
72 cursor = next;
73 if cursor == b"0" {
74 break;
75 }
76 }
77 out.flush()?;
78 Ok(Export { keys: n, skipped })
79}
80
81fn export_key(
86 client: &mut RespClient,
87 key: &[u8],
88 out: &mut impl Write,
89) -> io::Result<Option<Option<Vec<u8>>>> {
90 match rebuild_frames(client, key, key)? {
91 Rebuilt::Frames(frame) => {
92 out.write_all(&frame)?;
93 Ok(Some(None))
94 }
95 Rebuilt::Vanished => Ok(None),
96 Rebuilt::UnsupportedType(ty) => Ok(Some(Some(ty))),
97 }
98}
99
100pub(crate) enum Rebuilt {
104 Frames(Vec<u8>),
106 Vanished,
108 UnsupportedType(Vec<u8>),
111}
112
113pub(crate) fn rebuild_frames(
117 client: &mut RespClient,
118 key: &[u8],
119 dst: &[u8],
120) -> io::Result<Rebuilt> {
121 let ty = match client.request_borrowed(&[b"TYPE", key])? {
122 Reply::Simple(t) => t,
123 _ => return Ok(Rebuilt::Vanished),
124 };
125 if ty == b"none" {
126 return Ok(Rebuilt::Vanished);
127 }
128 let mut frame = Vec::new();
129 encode_command_borrowed(&mut frame, &[b"DEL", dst]);
132 match encode_body(client, key, dst, &ty, &mut frame)? {
133 Some(()) => {}
134 None => return Ok(Rebuilt::Vanished),
135 }
136 if frame.len() == encoded_del_len(dst) {
137 return Ok(Rebuilt::UnsupportedType(ty));
138 }
139 append_ttl_frame(client, key, dst, &mut frame)?;
140 Ok(Rebuilt::Frames(frame))
141}
142
143fn encoded_del_len(dst: &[u8]) -> usize {
146 let mut probe = Vec::new();
147 encode_command_borrowed(&mut probe, &[b"DEL", dst]);
148 probe.len()
149}
150
151fn encode_body(
155 client: &mut RespClient,
156 key: &[u8],
157 dst: &[u8],
158 ty: &[u8],
159 frame: &mut Vec<u8>,
160) -> io::Result<Option<()>> {
161 match ty {
162 b"string" => {
163 let Reply::Bulk(v) = client.request_borrowed(&[b"GET", key])? else {
164 return Ok(None);
165 };
166 encode_command_borrowed(frame, &[b"SET", dst, &v]);
167 }
168 b"hash" => {
169 let Some(items) = fetch_bulks(client, &[b"HGETALL", key])? else {
170 return Ok(None);
171 };
172 encode_multi(frame, b"HSET", dst, &items);
173 }
174 b"list" => {
175 let Some(vals) = fetch_bulks(client, &[b"LRANGE", key, b"0", b"-1"])? else {
176 return Ok(None);
177 };
178 encode_multi(frame, b"RPUSH", dst, &vals);
179 }
180 b"set" => {
181 let Some(ms) = fetch_bulks(client, &[b"SMEMBERS", key])? else {
182 return Ok(None);
183 };
184 encode_multi(frame, b"SADD", dst, &ms);
185 }
186 b"zset" => {
187 let zrange: &[&[u8]] = &[b"ZRANGE", key, b"0", b"-1", b"WITHSCORES"];
188 let Some(flat) = fetch_bulks(client, zrange)? else {
189 return Ok(None);
190 };
191 encode_zadd(frame, dst, &flat);
192 }
193 _ => return Ok(Some(())),
197 }
198 Ok(Some(()))
199}
200
201fn fetch_bulks(client: &mut RespClient, cmd: &[&[u8]]) -> io::Result<Option<Vec<Vec<u8>>>> {
205 let Reply::Array(items) = client.request_borrowed(cmd)? else {
206 return Ok(None);
207 };
208 if items.is_empty() {
209 return Ok(None);
210 }
211 Ok(Some(
212 items
213 .into_iter()
214 .filter_map(|r| if let Reply::Bulk(b) = r { Some(b) } else { None })
215 .collect(),
216 ))
217}
218
219fn encode_multi(frame: &mut Vec<u8>, verb: &[u8], dst: &[u8], vals: &[Vec<u8>]) {
221 let mut argv: Vec<&[u8]> = vec![verb, dst];
222 argv.extend(vals.iter().map(Vec::as_slice));
223 encode_command_borrowed(frame, &argv);
224}
225
226fn encode_zadd(frame: &mut Vec<u8>, dst: &[u8], flat: &[Vec<u8>]) {
229 let mut argv: Vec<&[u8]> = vec![b"ZADD", dst];
230 for pair in flat.chunks(2) {
231 if pair.len() == 2 {
232 argv.push(&pair[1]);
233 argv.push(&pair[0]);
234 }
235 }
236 encode_command_borrowed(frame, &argv);
237}
238
239fn append_ttl_frame(
241 client: &mut RespClient,
242 key: &[u8],
243 dst: &[u8],
244 frame: &mut Vec<u8>,
245) -> io::Result<()> {
246 if let Reply::Int(ms) = client.request_borrowed(&[b"PTTL", key])?
247 && ms > 0
248 {
249 let now = std::time::SystemTime::now()
250 .duration_since(std::time::UNIX_EPOCH)
251 .map_err(io::Error::other)?
252 .as_millis() as i64;
253 encode_command_borrowed(frame, &[b"PEXPIREAT", dst, (now + ms).to_string().as_bytes()]);
254 }
255 Ok(())
256}
257
258#[derive(Debug)]
260pub struct ImportReport {
261 pub sent: u64,
263 pub errors: u64,
265 pub offset: u64,
267}
268
269pub fn run_import(
274 client: &mut RespClient,
275 src: &Path,
276 resume: bool,
277 strict: bool,
278) -> io::Result<ImportReport> {
279 let (mut progress, start) = open_progress(src, resume)?;
280 let mut f = File::open(src)?;
281 f.seek(SeekFrom::Start(start))?;
282 let mut pending: Vec<u8> = Vec::with_capacity(1 << 20);
283 let mut report = ImportReport { sent: 0, errors: 0, offset: start };
284 let mut chunk = vec![0u8; 1 << 20];
285 let mut batch_bytes = 0usize;
286 let mut batch_cmds = 0usize;
287 loop {
288 let n = f.read(&mut chunk)?;
289 if n == 0 {
290 break;
291 }
292 pending.extend_from_slice(&chunk[..n]);
293 while let Some(used) = command_len(&pending[batch_bytes..]) {
295 batch_bytes += used;
296 batch_cmds += 1;
297 if batch_cmds == PIPELINE {
298 flush_batch(client, &pending[..batch_bytes], batch_cmds, strict, &mut report)?;
299 pending.drain(..batch_bytes);
300 write_progress(&mut progress, report.offset)?;
301 batch_bytes = 0;
302 batch_cmds = 0;
303 }
304 }
305 }
306 if batch_cmds > 0 {
307 flush_batch(client, &pending[..batch_bytes], batch_cmds, strict, &mut report)?;
308 write_progress(&mut progress, report.offset)?;
309 }
310 Ok(report)
311}
312
313fn flush_batch(
314 client: &mut RespClient,
315 raw: &[u8],
316 n: usize,
317 strict: bool,
318 report: &mut ImportReport,
319) -> io::Result<()> {
320 let replies = client.pipeline_raw(raw, n)?;
321 for r in replies {
322 if let Reply::Error(e) = r {
323 report.errors += 1;
324 if strict {
325 return Err(io::Error::new(
326 io::ErrorKind::InvalidData,
327 format!("server error (strict): {}", String::from_utf8_lossy(&e)),
328 ));
329 }
330 } else {
331 report.sent += 1;
332 }
333 }
334 report.offset += raw.len() as u64;
335 Ok(())
336}
337
338fn open_progress(src: &Path, resume: bool) -> io::Result<(File, u64)> {
351 let path = {
352 let mut os = src.as_os_str().to_owned();
353 os.push(".progress");
354 std::path::PathBuf::from(os)
355 };
356 let mut start = 0u64;
357 if resume && let Ok(text) = std::fs::read_to_string(&path) {
358 start = text.trim().parse().unwrap_or(0);
359 }
360 let mut f = OpenOptions::new().create(true).truncate(false).write(true).open(&path)?;
361 if !resume {
362 write_progress(&mut f, 0)?;
363 }
364 Ok((f, start))
365}
366
367fn write_progress(f: &mut File, offset: u64) -> io::Result<()> {
368 f.set_len(0)?;
369 f.seek(SeekFrom::Start(0))?;
370 f.write_all(offset.to_string().as_bytes())?;
371 f.sync_data()
372}
373
374pub(crate) fn command_len_pub(b: &[u8]) -> Option<usize> {
376 command_len(b)
377}
378
379fn command_len(b: &[u8]) -> Option<usize> {
381 let mut pos = 0usize;
382 let line = take_line(b, &mut pos)?;
383 if line.first() != Some(&b'*') {
384 return None;
385 }
386 let n: usize = std::str::from_utf8(&line[1..]).ok()?.trim().parse().ok()?;
387 for _ in 0..n {
388 let hdr = take_line(b, &mut pos)?;
389 if hdr.first() != Some(&b'$') {
390 return None;
391 }
392 let len: usize = std::str::from_utf8(&hdr[1..]).ok()?.trim().parse().ok()?;
393 if b.len() < pos + len + 2 {
394 return None;
395 }
396 pos += len + 2;
397 }
398 Some(pos)
399}
400
401fn take_line<'b>(b: &'b [u8], pos: &mut usize) -> Option<&'b [u8]> {
402 let rest = &b[*pos..];
403 let idx = rest.windows(2).position(|w| w == b"\r\n")?;
404 let line = &rest[..idx];
405 *pos += idx + 2;
406 Some(line)
407}