use std::fs::{File, OpenOptions};
use std::io::{self, BufWriter, Read, Seek, SeekFrom, Write};
use std::path::Path;
use crate::link::Link;
use kevy_resp::Reply;
mod rebuild;
use rebuild::export_key;
pub(crate) use rebuild::{Rebuilt, rebuild_frames};
const PIPELINE: usize = 512;
#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)]
#[non_exhaustive]
pub struct Export {
pub keys: u64,
pub skipped: std::collections::BTreeMap<Vec<u8>, u64>,
}
pub fn run_export(
client: &mut dyn Link,
prefix: Option<&[u8]>,
out_path: &Path,
) -> io::Result<Export> {
let mut out = BufWriter::new(File::create(out_path)?);
let mut cursor: Vec<u8> = b"0".to_vec();
let mut pattern = prefix.unwrap_or_default().to_vec();
pattern.push(b'*');
let mut n = 0u64;
let mut skipped: std::collections::BTreeMap<Vec<u8>, u64> = Default::default();
loop {
let reply =
client.request_borrowed(&[b"SCAN", &cursor, b"MATCH", &pattern, b"COUNT", b"512"])?;
let Reply::Array(items) = reply else {
return Err(io::Error::new(io::ErrorKind::InvalidData, "SCAN reply shape"));
};
let (Some(Reply::Bulk(next)), Some(Reply::Array(keys))) = (items.first(), items.get(1))
else {
return Err(io::Error::new(io::ErrorKind::InvalidData, "SCAN reply shape"));
};
let next = next.clone();
for k in keys {
let Reply::Bulk(key) = k else { continue };
let key = key.clone();
match export_key(client, &key, &mut out)? {
Some(None) => n += 1,
Some(Some(ty)) => *skipped.entry(ty).or_insert(0u64) += 1,
None => {}
}
}
cursor = next;
if cursor == b"0" {
break;
}
}
out.flush()?;
Ok(Export { keys: n, skipped })
}
#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)]
#[non_exhaustive]
pub struct ImportReport {
pub sent: u64,
pub errors: u64,
pub offset: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
#[non_exhaustive]
pub enum ImportStart {
#[default]
Fresh,
Resume,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
#[non_exhaustive]
pub enum OnErrorReply {
#[default]
Count,
Abort,
}
pub fn run_import(
client: &mut dyn Link,
src: &Path,
start: ImportStart,
on_error: OnErrorReply,
) -> io::Result<ImportReport> {
let mut f = File::open(src)?;
let (mut progress, start) = open_progress(src, start)?;
f.seek(SeekFrom::Start(start))?;
let mut pending: Vec<u8> = Vec::with_capacity(1 << 20);
let mut report = ImportReport { sent: 0, errors: 0, offset: start };
let mut chunk = vec![0u8; 1 << 20];
let mut batch_bytes = 0usize;
let mut batch_cmds = 0usize;
loop {
let n = f.read(&mut chunk)?;
if n == 0 {
break;
}
pending.extend_from_slice(&chunk[..n]);
while let Some(used) = command_len(&pending[batch_bytes..]) {
batch_bytes += used;
batch_cmds += 1;
if batch_cmds == PIPELINE {
flush_batch(client, &pending[..batch_bytes], batch_cmds, on_error, &mut report)?;
pending.drain(..batch_bytes);
write_progress(&mut progress, report.offset)?;
batch_bytes = 0;
batch_cmds = 0;
}
}
}
if batch_cmds > 0 {
flush_batch(client, &pending[..batch_bytes], batch_cmds, on_error, &mut report)?;
write_progress(&mut progress, report.offset)?;
}
Ok(report)
}
fn flush_batch(
client: &mut dyn Link,
raw: &[u8],
n: usize,
on_error: OnErrorReply,
report: &mut ImportReport,
) -> io::Result<()> {
let replies = client.pipeline_raw(raw, n)?;
for r in replies {
if let Reply::Error(e) = r {
report.errors += 1;
if on_error == OnErrorReply::Abort {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("server error (strict): {}", String::from_utf8_lossy(&e)),
));
}
} else {
report.sent += 1;
}
}
report.offset += raw.len() as u64;
Ok(())
}
pub(crate) fn open_progress(src: &Path, start: ImportStart) -> io::Result<(File, u64)> {
let resume = start == ImportStart::Resume;
let path = {
let mut os = src.as_os_str().to_owned();
os.push(".progress");
std::path::PathBuf::from(os)
};
let mut start = 0u64;
if resume && let Ok(text) = std::fs::read_to_string(&path) {
start = text.trim().parse().unwrap_or(0);
}
let mut f = OpenOptions::new().create(true).truncate(false).write(true).open(&path)?;
if !resume {
write_progress(&mut f, 0)?;
}
Ok((f, start))
}
pub(crate) fn write_progress(f: &mut File, offset: u64) -> io::Result<()> {
f.set_len(0)?;
f.seek(SeekFrom::Start(0))?;
f.write_all(offset.to_string().as_bytes())?;
f.sync_data()
}
pub(crate) fn command_len_pub(b: &[u8]) -> Option<usize> {
command_len(b)
}
fn command_len(b: &[u8]) -> Option<usize> {
let mut pos = 0usize;
let line = take_line(b, &mut pos)?;
if line.first() != Some(&b'*') {
return None;
}
let n: usize = std::str::from_utf8(&line[1..]).ok()?.trim().parse().ok()?;
for _ in 0..n {
let hdr = take_line(b, &mut pos)?;
if hdr.first() != Some(&b'$') {
return None;
}
let len: usize = std::str::from_utf8(&hdr[1..]).ok()?.trim().parse().ok()?;
if b.len() < pos + len + 2 {
return None;
}
pos += len + 2;
}
Some(pos)
}
fn take_line<'b>(b: &'b [u8], pos: &mut usize) -> Option<&'b [u8]> {
let rest = &b[*pos..];
let idx = rest.windows(2).position(|w| w == b"\r\n")?;
let line = &rest[..idx];
*pos += idx + 2;
Some(line)
}