use std::fs::File;
use std::io::{self, BufRead, BufReader, BufWriter, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use crate::plan::Operation;
pub const THRESHOLD: usize = 250_000;
const CHUNK: usize = 4096;
#[derive(Debug)]
struct TempPath(PathBuf);
impl Drop for TempPath {
fn drop(&mut self) {
let _ = std::fs::remove_file(&self.0);
}
}
#[derive(Debug)]
pub struct Writer {
inner: BufWriter<File>,
path: TempPath,
len: usize,
}
impl Writer {
pub fn create() -> Option<Writer> {
let dir = std::env::temp_dir();
let pid = std::process::id();
for attempt in 0..8u32 {
let path = dir.join(format!("po-plan-{pid}-{attempt}.jsonl"));
if let Ok(file) = File::options()
.read(true)
.write(true)
.create_new(true)
.open(&path)
{
return Some(Writer {
inner: BufWriter::new(file),
path: TempPath(path),
len: 0,
});
}
}
None
}
pub fn path(&self) -> &Path {
&self.path.0
}
pub fn push(&mut self, op: &Operation) -> io::Result<()> {
serde_json::to_writer(&mut self.inner, op)?;
self.inner.write_all(b"\n")?;
self.len += 1;
Ok(())
}
pub fn finish(self) -> io::Result<Spill> {
let file = self.inner.into_inner().map_err(io::Error::from)?;
Ok(Spill {
file,
path: self.path,
len: self.len,
})
}
}
#[derive(Debug)]
pub struct Spill {
file: File,
path: TempPath,
len: usize,
}
impl Spill {
pub fn len(&self) -> usize {
self.len
}
pub fn is_empty(&self) -> bool {
self.len == 0
}
pub fn path(&self) -> &Path {
&self.path.0
}
pub fn chunks(&self) -> io::Result<Chunks> {
let mut file = self.file.try_clone()?;
file.seek(SeekFrom::Start(0))?;
Ok(Chunks {
reader: BufReader::new(file),
line: String::new(),
read: 0,
len: self.len,
failed: false,
})
}
}
pub struct Chunks {
reader: BufReader<File>,
line: String,
read: usize,
len: usize,
failed: bool,
}
impl Chunks {
pub fn remaining(&self) -> usize {
self.len - self.read
}
}
impl Iterator for Chunks {
type Item = io::Result<Vec<Operation>>;
fn next(&mut self) -> Option<io::Result<Vec<Operation>>> {
if self.failed || self.read >= self.len {
return None;
}
let mut out = Vec::with_capacity(CHUNK.min(self.len - self.read));
while out.len() < CHUNK && self.read + out.len() < self.len {
self.line.clear();
let read = self.reader.read_line(&mut self.line);
let parsed = match read {
Ok(0) => break,
Ok(_) => serde_json::from_str(self.line.trim_end())
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)),
Err(e) => Err(e),
};
match parsed {
Ok(op) => out.push(op),
Err(e) => {
self.failed = true;
return Some(Err(e));
}
}
}
if out.is_empty() {
self.failed = true;
return Some(Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"the spilled plan is shorter than it should be",
)));
}
self.read += out.len();
Some(Ok(out))
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::plan::{Action, Reason};
fn op(n: usize) -> Operation {
Operation {
source_rel: PathBuf::from(format!("in/{n}.jpg")),
dest_rel: Some(PathBuf::from(format!("2026/{n}.jpg"))),
action: Action::Copy,
reason: Reason::Planned,
bytes: n as u64,
duplicate_of: None,
}
}
fn spill(count: usize) -> Spill {
let mut w = Writer::create().unwrap();
for i in 0..count {
w.push(&op(i)).unwrap();
}
w.finish().unwrap()
}
#[test]
fn round_trips_in_order() {
let s = spill(10);
let back: Vec<Operation> = s.chunks().unwrap().flat_map(|c| c.unwrap()).collect();
assert_eq!(back.len(), 10);
assert_eq!(back[7].source_rel, PathBuf::from("in/7.jpg"));
assert_eq!(back[7].bytes, 7);
assert_eq!(back[7].dest_rel, Some(PathBuf::from("2026/7.jpg")));
}
#[test]
fn splits_into_chunks() {
let s = spill(CHUNK + 5);
let sizes: Vec<usize> = s.chunks().unwrap().map(|c| c.unwrap().len()).collect();
assert_eq!(sizes, vec![CHUNK, 5]);
}
#[test]
fn empty_plan_yields_nothing() {
let s = spill(0);
assert!(s.is_empty());
assert_eq!(s.chunks().unwrap().count(), 0);
}
#[test]
fn truncation_is_an_error_not_a_short_read() {
let s = spill(4);
s.file.set_len(0).unwrap();
let mut chunks = s.chunks().unwrap();
assert!(chunks.next().unwrap().is_err());
assert_eq!(chunks.remaining(), 4);
assert!(chunks.next().is_none());
}
#[test]
fn the_file_goes_away_with_the_spill() {
let s = spill(1);
let path = s.path().to_path_buf();
assert!(path.exists());
drop(s);
assert!(!path.exists());
}
}