use crate::packet;
use crate::worker::Par2Worker;
use crate::SimdPath;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};
mod geometry;
pub use geometry::calculate_geometry;
#[derive(Debug, Clone)]
pub struct CreateOptions {
pub slice_size: Option<usize>,
pub slice_count: Option<usize>,
pub recovery_count: Option<usize>,
pub recovery_pct: u8,
pub memory_limit: usize,
pub threads: usize,
pub simd: SimdPath,
pub slice_unit: Option<usize>,
pub effort: f64,
}
impl Default for CreateOptions {
fn default() -> Self {
Self {
slice_size: None,
slice_count: None,
recovery_count: None,
recovery_pct: 10,
memory_limit: 1024 * 1024 * 1024, threads: 0, simd: SimdPath::Auto,
slice_unit: None,
effort: 1.0,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct MemoryLayout {
pub recovery_per_pass: usize,
pub slice_chunk: usize,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SliceWindow {
pub offset: usize,
pub len: usize,
}
#[derive(Debug, Default)]
pub struct IngestHashes {
pub hashes: Vec<crate::encoder::FileHashes>,
pub checksums: Vec<packet::SliceChecksum>,
}
pub fn plan_memory_layout(
slice_size: usize,
recovery_count: usize,
memory_limit: usize,
) -> MemoryLayout {
let slice_size = slice_size.max(32);
if recovery_count == 0 {
return MemoryLayout {
recovery_per_pass: 0,
slice_chunk: slice_size,
};
}
let limit = memory_limit.max(slice_size);
if recovery_count.saturating_mul(slice_size) <= limit {
return MemoryLayout {
recovery_per_pass: recovery_count,
slice_chunk: slice_size,
};
}
let raw = (limit / recovery_count).max(32);
let chunk = (raw / 32) * 32;
if chunk < slice_size && chunk >= 4096 {
return MemoryLayout {
recovery_per_pass: recovery_count,
slice_chunk: chunk,
};
}
let per = (limit / slice_size).max(1).min(recovery_count);
MemoryLayout {
recovery_per_pass: per,
slice_chunk: slice_size,
}
}
#[derive(Debug, Clone)]
pub struct InputFile {
pub path: PathBuf,
pub display_name: String,
pub size: u64,
}
pub fn sort_files_by_file_id(files: &mut Vec<InputFile>) -> Result<()> {
use std::io::Read;
let mut keyed: Vec<([u8; 16], InputFile)> = Vec::with_capacity(files.len());
for f in files.drain(..) {
let mut file = std::fs::File::open(&f.path)
.with_context(|| format!("opening `{}` to compute its File ID", f.path.display()))?;
let mut head = vec![0u8; 16 * 1024];
let mut read = 0usize;
while read < head.len() {
match file.read(&mut head[read..])? {
0 => break,
n => read += n,
}
}
head.truncate(read);
let md5_16k = packet::md5(&head);
let file_id = packet::compute_file_id(&md5_16k, f.size, &f.display_name);
keyed.push((file_id, f));
}
keyed.sort_by(|(a, _), (b, _)| packet::cmp_file_id(a, b));
*files = keyed.into_iter().map(|(_, f)| f).collect();
Ok(())
}
const CHUNK_SIZE: usize = 8 * 1024 * 1024;
struct HashSink {
out: IngestHashes,
current: crate::encoder::FileHasher,
}
fn emit_logical_slice(
worker: &Par2Worker,
logical: Vec<u8>,
actual_len: usize,
is_last: bool,
window: Option<SliceWindow>,
hash: Option<&mut HashSink>,
) {
if let Some(h) = hash {
let checksum = h.current.update_and_hash_slice(&logical, actual_len);
h.out.checksums.push(checksum);
if is_last {
let finished = std::mem::take(&mut h.current);
h.out.hashes.push(finished.finish());
}
}
if let Some(w) = window {
let mut part = worker.take_buffer(w.len);
part.clear();
if w.offset < logical.len() {
let end = (w.offset + w.len).min(logical.len());
part.extend_from_slice(&logical[w.offset..end]);
}
part.resize(w.len, 0);
let win_actual = actual_len.saturating_sub(w.offset).min(w.len);
worker.send_slice(part, win_actual, is_last);
} else {
worker.send_slice(logical, actual_len, is_last);
}
}
fn feed_chunk(
chunk: &[u8],
worker: &Par2Worker,
slice_size: usize,
slice_accum: &mut Vec<u8>,
held: &mut Option<Vec<u8>>,
window: Option<SliceWindow>,
mut hash: Option<&mut HashSink>,
) {
let mut chunk_pos = 0;
while chunk_pos < chunk.len() {
let space = slice_size - slice_accum.len();
let take = space.min(chunk.len() - chunk_pos);
slice_accum.extend_from_slice(&chunk[chunk_pos..chunk_pos + take]);
chunk_pos += take;
if slice_accum.len() >= slice_size {
let next = worker.take_buffer(slice_size);
let padded = std::mem::replace(slice_accum, next);
if let Some(previous) = held.replace(padded) {
emit_logical_slice(
worker,
previous,
slice_size,
false,
window,
hash.as_deref_mut(),
);
}
}
}
}
pub async fn ingest_files(
files: &[InputFile],
worker: &Par2Worker,
slice_size: usize,
) -> Result<()> {
ingest_files_with(files, worker, slice_size, None, |_| Ok(())).await
}
pub async fn ingest_files_with<F>(
files: &[InputFile],
worker: &Par2Worker,
slice_size: usize,
cancelled: Option<&AtomicBool>,
after_file: F,
) -> Result<()>
where
F: FnMut(&InputFile) -> Result<()>,
{
ingest_files_ex(files, worker, slice_size, cancelled, after_file, None, None).await
}
pub async fn ingest_files_with_progress<F, P>(
files: &[InputFile],
worker: &Par2Worker,
slice_size: usize,
cancelled: Option<&AtomicBool>,
after_file: F,
on_read: P,
) -> Result<()>
where
F: FnMut(&InputFile) -> Result<()>,
P: FnMut(usize) -> Result<()>,
{
ingest_files_ex_impl(
files, worker, slice_size, cancelled, after_file, None, None, on_read,
)
.await
}
pub async fn ingest_files_ex<F>(
files: &[InputFile],
worker: &Par2Worker,
slice_size: usize,
cancelled: Option<&AtomicBool>,
after_file: F,
window: Option<SliceWindow>,
hash_out: Option<&mut IngestHashes>,
) -> Result<()>
where
F: FnMut(&InputFile) -> Result<()>,
{
ingest_files_ex_impl(
files,
worker,
slice_size,
cancelled,
after_file,
window,
hash_out,
|_| Ok(()),
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn ingest_files_ex_impl<F, P>(
files: &[InputFile],
worker: &Par2Worker,
slice_size: usize,
cancelled: Option<&AtomicBool>,
mut after_file: F,
window: Option<SliceWindow>,
mut hash_out: Option<&mut IngestHashes>,
mut on_read: P,
) -> Result<()>
where
F: FnMut(&InputFile) -> Result<()>,
P: FnMut(usize) -> Result<()>,
{
let mut hash = hash_out.as_mut().map(|_| HashSink {
out: IngestHashes::default(),
current: crate::encoder::FileHasher::new(),
});
for file_info in files {
if cancelled.is_some_and(|c| c.load(Ordering::Relaxed)) {
return Ok(());
}
if file_info.size == 0 {
continue;
}
let mut slice_accum = worker.take_buffer(slice_size);
slice_accum.clear();
let mut held: Option<Vec<u8>> = None;
if (file_info.size as usize) <= CHUNK_SIZE {
let path = file_info.path.clone();
tokio::task::block_in_place(|| -> Result<()> {
let buf = std::fs::read(&path)
.with_context(|| format!("reading `{}`", path.display()))?;
on_read(buf.len())?;
feed_chunk(
&buf,
worker,
slice_size,
&mut slice_accum,
&mut held,
window,
hash.as_mut(),
);
Ok(())
})?;
} else {
let (tx, mut rx) = tokio::sync::mpsc::channel::<Vec<u8>>(4);
let path = file_info.path.clone();
let reader_handle = tokio::task::spawn_blocking(move || {
use std::fs::File;
use std::io::Read;
let mut file = File::open(&path)?;
loop {
let mut buf = vec![0u8; CHUNK_SIZE];
let n = file.read(&mut buf)?;
if n == 0 {
break;
}
buf.truncate(n);
if tx.blocking_send(buf).is_err() {
break;
}
}
Ok::<_, anyhow::Error>(())
});
while let Some(chunk) = rx.recv().await {
if cancelled.is_some_and(|c| c.load(Ordering::Relaxed)) {
drop(rx);
let _ = reader_handle.await;
return Ok(());
}
on_read(chunk.len())?;
tokio::task::block_in_place(|| {
feed_chunk(
&chunk,
worker,
slice_size,
&mut slice_accum,
&mut held,
window,
hash.as_mut(),
);
});
}
reader_handle.await??;
}
if !slice_accum.is_empty() {
if let Some(previous) = held.take() {
tokio::task::block_in_place(|| {
emit_logical_slice(worker, previous, slice_size, false, window, hash.as_mut());
});
}
let actual_len = slice_accum.len();
slice_accum.resize(slice_size, 0);
tokio::task::block_in_place(|| {
emit_logical_slice(worker, slice_accum, actual_len, true, window, hash.as_mut());
});
} else if let Some(last) = held.take() {
tokio::task::block_in_place(|| {
emit_logical_slice(worker, last, slice_size, true, window, hash.as_mut());
});
}
after_file(file_info)?;
}
if let (Some(dst), Some(src)) = (hash_out.as_mut(), hash) {
**dst = src.out;
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn memory_layout_fits_in_one_pass() {
let plan = plan_memory_layout(1 << 20, 200, 1 << 30);
assert_eq!(plan.recovery_per_pass, 200);
assert_eq!(plan.slice_chunk, 1 << 20);
}
#[test]
fn memory_layout_prefers_slice_chunk_when_all_dests_fit() {
let slice = 6 * 1024 * 1024;
let plan = plan_memory_layout(slice, 1000, 1024 * 1024 * 1024);
assert_eq!(plan.recovery_per_pass, 1000);
assert!(plan.slice_chunk < slice);
assert!(plan.slice_chunk >= 4096);
assert!(plan.slice_chunk.is_multiple_of(32));
assert!(1000 * plan.slice_chunk <= 1024 * 1024 * 1024);
}
#[test]
fn memory_layout_falls_back_to_recovery_passes_when_chunk_tiny() {
let plan = plan_memory_layout(1 << 20, 10_000, 8 * 1024);
assert_eq!(plan.slice_chunk, 1 << 20);
assert!(plan.recovery_per_pass < 10_000);
assert!(plan.recovery_per_pass >= 1);
}
#[test]
fn sort_files_by_file_id_orders_by_ascending_file_id_not_input_order() {
let dir = std::env::temp_dir().join(format!(
"parmesan-ops-test-{}-{:?}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&dir).unwrap();
let names = ["zzz.bin", "aaa.bin", "mmm.bin"];
let mut files = Vec::new();
for (i, name) in names.iter().enumerate() {
let path = dir.join(name);
let data = vec![i as u8; 100 + i * 37];
std::fs::write(&path, &data).unwrap();
files.push(InputFile {
path,
display_name: (*name).to_string(),
size: data.len() as u64,
});
}
let mut expected_ids = Vec::new();
for f in &files {
let bytes = std::fs::read(&f.path).unwrap();
let md5_16k = crate::packet::md5(&bytes);
expected_ids.push(crate::packet::compute_file_id(
&md5_16k,
f.size,
&f.display_name,
));
}
expected_ids.sort_by(crate::packet::cmp_file_id);
sort_files_by_file_id(&mut files).unwrap();
let got_ids: Vec<[u8; 16]> = files
.iter()
.map(|f| {
let bytes = std::fs::read(&f.path).unwrap();
let md5_16k = crate::packet::md5(&bytes);
crate::packet::compute_file_id(&md5_16k, f.size, &f.display_name)
})
.collect();
assert_eq!(got_ids, expected_ids);
assert!(got_ids
.windows(2)
.all(|w| crate::packet::cmp_file_id(&w[0], &w[1]).is_le()));
std::fs::remove_dir_all(&dir).ok();
}
}