use std::fs::{File, OpenOptions};
use std::io::{Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use std::sync::Mutex;
use crate::error::{Error, Result};
use crate::proto::{Packet, PacketType};
const MANIFEST_MAGIC: [u8; 4] = *b"PMAN";
const MANIFEST_VERSION: u8 = 1;
const MANIFEST_HEADER_LEN: usize = 32;
#[derive(Debug, Clone, Copy)]
pub struct AssemblyStats {
pub chunks_total: u32,
pub chunks_received: u32,
pub has_name: bool,
}
#[derive(Debug, Eq, PartialEq)]
pub enum IngestOutcome {
Accepted,
Duplicate,
Complete,
Mismatched,
Ignored,
}
pub struct Reassembler {
holding: PathBuf,
finals: PathBuf,
write_lock: Mutex<()>,
}
impl Reassembler {
pub fn new(
holding: impl Into<PathBuf>,
finals: impl Into<PathBuf>,
) -> Result<Self> {
let holding = holding.into();
let finals = finals.into();
std::fs::create_dir_all(&holding)?;
std::fs::create_dir_all(&finals)?;
Ok(Self {
holding,
finals,
write_lock: Mutex::new(()),
})
}
pub fn incoming_path(&self, file_id: i64) -> PathBuf {
self.holding.join(format!("{file_id}.incoming"))
}
pub fn manifest_path(&self, file_id: i64) -> PathBuf {
self.holding.join(format!("{file_id}.parmanifest"))
}
pub fn ingest(&self, pkt: &Packet) -> Result<IngestOutcome> {
match pkt.ptype {
PacketType::Data | PacketType::Retransmit => self.ingest_chunk(pkt),
PacketType::Manifest | PacketType::NameOnly => self.ingest_manifest(pkt),
PacketType::Heartbeat => Ok(IngestOutcome::Ignored),
}
}
fn ingest_chunk(&self, pkt: &Packet) -> Result<IngestOutcome> {
let _g = self.write_lock.lock().unwrap();
let payload_size = pkt.payload.len();
let mut manifest = match Manifest::open(self.manifest_path(pkt.file_id).as_path()) {
Ok(m) => m,
Err(Error::NotFound(_)) => {
let assumed_payload = if pkt.flags.is_last() && pkt.num_chunks > 1 {
return Ok(IngestOutcome::Mismatched);
} else {
payload_size as u32
};
Manifest::create(
self.manifest_path(pkt.file_id).as_path(),
assumed_payload,
pkt.num_chunks,
"",
)?
}
Err(e) => return Err(e),
};
if manifest.num_chunks != pkt.num_chunks {
return Ok(IngestOutcome::Mismatched);
}
if manifest.chunk_payload_size == 0 {
if pkt.flags.is_last() && pkt.num_chunks > 1 {
return Ok(IngestOutcome::Mismatched);
}
manifest.chunk_payload_size = payload_size as u32;
manifest.save()?;
}
let data_path = self.incoming_path(pkt.file_id);
let mut data = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(&data_path)?;
let payload = manifest.chunk_payload_size as u64;
let needed_min = (pkt.chunk_id as u64) * payload + payload_size as u64;
if data.metadata()?.len() < needed_min {
data.set_len(needed_min)?;
}
data.seek(SeekFrom::Start(pkt.chunk_id as u64 * payload))?;
data.write_all(&pkt.payload)?;
data.sync_data()?;
if pkt.flags.is_last() {
let real_size = (pkt.chunk_id as u64) * payload + payload_size as u64;
data.set_len(real_size)?;
data.sync_all()?;
manifest.file_size_hint = real_size;
}
let already_set = manifest.is_set(pkt.chunk_id);
if already_set {
return Ok(IngestOutcome::Duplicate);
}
manifest.set(pkt.chunk_id);
manifest.save()?;
if manifest.is_complete() && !manifest.name.is_empty() {
Ok(IngestOutcome::Complete)
} else {
Ok(IngestOutcome::Accepted)
}
}
fn ingest_manifest(&self, pkt: &Packet) -> Result<IngestOutcome> {
let _g = self.write_lock.lock().unwrap();
let path_str = String::from_utf8_lossy(&pkt.payload).into_owned();
let manifest_path = self.manifest_path(pkt.file_id);
let mut manifest = match Manifest::open(&manifest_path) {
Ok(m) => m,
Err(Error::NotFound(_)) => {
Manifest::create(&manifest_path, 0, pkt.num_chunks, &path_str)?
}
Err(e) => return Err(e),
};
if manifest.num_chunks != pkt.num_chunks {
return Ok(IngestOutcome::Mismatched);
}
if manifest.name.is_empty() {
manifest.name = path_str;
manifest.save()?;
}
if manifest.is_complete() && !manifest.name.is_empty() {
Ok(IngestOutcome::Complete)
} else {
Ok(IngestOutcome::Accepted)
}
}
pub fn missing_ranges(&self, file_id: i64) -> Result<Vec<(u32, u32)>> {
let m = Manifest::open(&self.manifest_path(file_id))?;
Ok(m.missing_ranges())
}
pub fn has_name(&self, file_id: i64) -> Result<bool> {
let m = Manifest::open(&self.manifest_path(file_id))?;
Ok(!m.name.is_empty())
}
pub fn in_flight(&self) -> Result<Vec<i64>> {
let mut ids = Vec::new();
for entry in std::fs::read_dir(&self.holding)? {
let entry = entry?;
let name = entry.file_name();
let s = name.to_string_lossy();
if let Some(stem) = s.strip_suffix(".parmanifest") {
if let Ok(id) = stem.parse::<i64>() {
ids.push(id);
}
}
}
ids.sort_unstable();
Ok(ids)
}
pub fn finalize(&self, file_id: i64) -> Result<PathBuf> {
let _g = self.write_lock.lock().unwrap();
let manifest_path = self.manifest_path(file_id);
let manifest = Manifest::open(&manifest_path)?;
if !manifest.is_complete() {
return Err(Error::BadManifest("finalize before complete".into()));
}
if manifest.name.is_empty() {
return Err(Error::BadManifest("missing filename".into()));
}
let dest = crate::reconcile::ground_path(&self.finals, &manifest.name);
if let Some(parent) = dest.parent() {
std::fs::create_dir_all(parent)?;
}
let incoming = self.incoming_path(file_id);
if std::fs::rename(&incoming, &dest).is_err() {
std::fs::copy(&incoming, &dest)?;
std::fs::remove_file(&incoming)?;
}
std::fs::remove_file(&manifest_path)?;
Ok(dest)
}
pub fn stats(&self, file_id: i64) -> Result<AssemblyStats> {
let m = Manifest::open(&self.manifest_path(file_id))?;
let total = m.num_chunks;
let missing: u32 = m.missing_ranges().iter().map(|(_, c)| *c).sum();
Ok(AssemblyStats {
chunks_total: total,
chunks_received: total.saturating_sub(missing),
has_name: !m.name.is_empty(),
})
}
pub fn age_secs(&self, file_id: i64) -> Result<u64> {
let mp = self.manifest_path(file_id);
let meta = std::fs::metadata(&mp)?;
let modified = meta.modified()?;
Ok(modified
.elapsed()
.map(|d| d.as_secs())
.unwrap_or(0))
}
}
#[derive(Debug)]
struct Manifest {
path: PathBuf,
chunk_payload_size: u32,
num_chunks: u32,
file_size_hint: u64,
name: String,
bitmap: Vec<u8>,
}
impl Manifest {
fn bitmap_len(num_chunks: u32) -> usize {
((num_chunks + 7) / 8) as usize
}
fn create(path: &Path, chunk_payload_size: u32, num_chunks: u32, name: &str) -> Result<Self> {
let m = Self {
path: path.to_path_buf(),
chunk_payload_size,
num_chunks,
file_size_hint: 0,
name: name.to_owned(),
bitmap: vec![0; Self::bitmap_len(num_chunks)],
};
m.save()?;
Ok(m)
}
fn open(path: &Path) -> Result<Self> {
let mut f = match File::open(path) {
Ok(f) => f,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
return Err(Error::NotFound(path.to_path_buf()));
}
Err(e) => return Err(e.into()),
};
let mut buf = Vec::new();
f.read_to_end(&mut buf)?;
if buf.len() < MANIFEST_HEADER_LEN {
return Err(Error::BadManifest("short header".into()));
}
if buf[0..4] != MANIFEST_MAGIC {
return Err(Error::BadManifest("bad magic".into()));
}
if buf[4] != MANIFEST_VERSION {
return Err(Error::BadManifest("bad version".into()));
}
let chunk_payload_size = u32::from_le_bytes(buf[8..12].try_into().unwrap());
let num_chunks = u32::from_le_bytes(buf[12..16].try_into().unwrap());
let file_size_hint = u64::from_le_bytes(buf[16..24].try_into().unwrap());
let name_len = u32::from_le_bytes(buf[24..28].try_into().unwrap()) as usize;
let expected_crc = u32::from_le_bytes(buf[28..32].try_into().unwrap());
let mut tmp = buf.clone();
tmp[28..32].fill(0);
let actual_crc = crc32fast::hash(&tmp[..MANIFEST_HEADER_LEN]);
if actual_crc != expected_crc {
return Err(Error::BadManifest("crc mismatch".into()));
}
let name_end = MANIFEST_HEADER_LEN + name_len;
let bitmap_len = Self::bitmap_len(num_chunks);
if buf.len() < name_end + bitmap_len {
return Err(Error::BadManifest("truncated".into()));
}
let name = String::from_utf8_lossy(&buf[MANIFEST_HEADER_LEN..name_end]).into_owned();
let bitmap = buf[name_end..name_end + bitmap_len].to_vec();
Ok(Self {
path: path.to_path_buf(),
chunk_payload_size,
num_chunks,
file_size_hint,
name,
bitmap,
})
}
fn save(&self) -> Result<()> {
let bitmap_len = Self::bitmap_len(self.num_chunks);
debug_assert_eq!(self.bitmap.len(), bitmap_len);
let name_bytes = self.name.as_bytes();
let mut buf = Vec::with_capacity(MANIFEST_HEADER_LEN + name_bytes.len() + bitmap_len);
buf.extend_from_slice(&MANIFEST_MAGIC);
buf.push(MANIFEST_VERSION);
buf.push(0); buf.extend_from_slice(&[0u8, 0u8]); buf.extend_from_slice(&self.chunk_payload_size.to_le_bytes());
buf.extend_from_slice(&self.num_chunks.to_le_bytes());
buf.extend_from_slice(&self.file_size_hint.to_le_bytes());
buf.extend_from_slice(&(name_bytes.len() as u32).to_le_bytes());
buf.extend_from_slice(&[0u8; 4]); let crc = crc32fast::hash(&buf[..MANIFEST_HEADER_LEN]);
buf[28..32].copy_from_slice(&crc.to_le_bytes());
buf.extend_from_slice(name_bytes);
buf.extend_from_slice(&self.bitmap);
let tmp = self.path.with_extension("parmanifest.tmp");
{
let mut f = OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.open(&tmp)?;
f.write_all(&buf)?;
f.sync_all()?;
}
std::fs::rename(&tmp, &self.path)?;
Ok(())
}
fn is_set(&self, chunk_id: u32) -> bool {
let i = (chunk_id / 8) as usize;
let b = chunk_id % 8;
i < self.bitmap.len() && self.bitmap[i] & (1 << b) != 0
}
fn set(&mut self, chunk_id: u32) {
let i = (chunk_id / 8) as usize;
let b = chunk_id % 8;
if i < self.bitmap.len() {
self.bitmap[i] |= 1 << b;
}
}
fn is_complete(&self) -> bool {
let total_bytes = self.bitmap.len();
let full_bytes = (self.num_chunks / 8) as usize;
if self.bitmap[..full_bytes].iter().any(|&b| b != 0xff) {
return false;
}
let leftover = self.num_chunks % 8;
if leftover != 0 && full_bytes < total_bytes {
let mask = (1u8 << leftover) - 1;
if self.bitmap[full_bytes] & mask != mask {
return false;
}
}
true
}
fn missing_ranges(&self) -> Vec<(u32, u32)> {
let mut out = Vec::new();
let mut i = 0u32;
let total = self.num_chunks;
while i < total {
if !self.is_set(i) {
let start = i;
while i < total && !self.is_set(i) {
i += 1;
}
out.push((start, i - start));
} else {
i += 1;
}
}
out
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::proto::{Packet, PacketFlags, PacketType};
use tempfile::tempdir;
fn pkt(file_id: i64, chunk_id: u32, num_chunks: u32, last: bool, data: &[u8]) -> Packet {
let flags = if last {
PacketFlags::empty().last()
} else {
PacketFlags::empty()
};
Packet {
ptype: PacketType::Data,
flags,
file_id,
chunk_id,
num_chunks,
payload: data.to_vec(),
}
}
#[test]
fn three_chunks_complete_with_name() {
let dir = tempdir().unwrap();
let r = Reassembler::new(dir.path().join("hold"), dir.path().join("final")).unwrap();
let chunk_size = 4;
let p1 = pkt(1, 0, 3, false, b"abcd");
let p2 = pkt(1, 1, 3, false, b"efgh");
let p3 = pkt(1, 2, 3, true, b"ij");
assert_eq!(r.ingest(&p1).unwrap(), IngestOutcome::Accepted);
assert_eq!(r.ingest(&p2).unwrap(), IngestOutcome::Accepted);
assert_eq!(r.ingest(&p3).unwrap(), IngestOutcome::Accepted);
assert!(r.has_name(1).unwrap() == false);
let manifest_pkt = Packet {
ptype: PacketType::Manifest,
flags: PacketFlags::empty(),
file_id: 1,
chunk_id: 0,
num_chunks: 3,
payload: b"my/dir/file.bin".to_vec(),
};
assert_eq!(r.ingest(&manifest_pkt).unwrap(), IngestOutcome::Complete);
let final_path = r.finalize(1).unwrap();
let bytes = std::fs::read(&final_path).unwrap();
assert_eq!(bytes, b"abcdefghij");
assert!(final_path.ends_with("my/dir/file.bin"));
assert_eq!(chunk_size, 4); }
#[test]
fn duplicate_chunk_is_reported() {
let dir = tempdir().unwrap();
let r = Reassembler::new(dir.path().join("hold"), dir.path().join("final")).unwrap();
let p = pkt(2, 0, 2, false, b"xx");
assert_eq!(r.ingest(&p).unwrap(), IngestOutcome::Accepted);
assert_eq!(r.ingest(&p).unwrap(), IngestOutcome::Duplicate);
}
#[test]
fn missing_ranges_are_compact() {
let dir = tempdir().unwrap();
let r = Reassembler::new(dir.path().join("hold"), dir.path().join("final")).unwrap();
for i in [0u32, 1, 4, 5, 9] {
r.ingest(&pkt(7, i, 10, i == 9, b"aa")).unwrap();
}
let m = r.missing_ranges(7).unwrap();
assert_eq!(m, vec![(2, 2), (6, 3)]);
}
}