use crate::container::{ContainerBuilder, ContainerIndex, EntryMeta};
use crate::error::{Error, Result};
use std::path::{Path, PathBuf};
pub const MAX_DEPTH: usize = 64;
#[derive(Debug, Clone)]
pub struct SkippedEntry {
pub path: String,
pub reason: SkipReason,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SkipReason {
Symlink,
NotRegular,
Unreadable,
TooDeep,
}
impl SkipReason {
#[must_use]
pub const fn code(self) -> &'static str {
match self {
Self::Symlink => "symlink",
Self::NotRegular => "not_regular",
Self::Unreadable => "unreadable",
Self::TooDeep => "too_deep",
}
}
}
#[derive(Debug)]
pub struct PackedFolder {
pub payload: Vec<u8>,
pub index: ContainerIndex,
pub root_name: String,
pub file_count: usize,
pub dir_count: usize,
pub link_count: usize,
pub skipped: Vec<SkippedEntry>,
}
pub type ProgressFn<'a> = &'a mut dyn FnMut(&str, u64);
#[derive(Debug)]
pub struct WalkItem {
pub path: PathBuf,
pub comps: Vec<String>,
pub is_dir: bool,
pub meta: EntryMeta,
pub link_target: Option<String>,
}
pub fn walk_dir(
root: &Path,
skipped: &mut Vec<SkippedEntry>,
visit: &mut dyn FnMut(&WalkItem) -> Result<()>,
) -> Result<()> {
let mut stack: Vec<(PathBuf, Vec<String>, usize)> = vec![(root.to_path_buf(), Vec::new(), 0)];
while let Some((dir, rel, depth)) = stack.pop() {
if depth > MAX_DEPTH {
skipped.push(SkippedEntry {
path: rel.join("/"),
reason: SkipReason::TooDeep,
});
continue;
}
let Ok(rd) = std::fs::read_dir(&dir) else {
skipped.push(SkippedEntry {
path: rel.join("/"),
reason: SkipReason::Unreadable,
});
continue;
};
let mut items: Vec<_> = rd.flatten().collect();
items.sort_by_key(std::fs::DirEntry::file_name);
let mut subdirs = Vec::new();
for item in items {
let path = item.path();
let name = item.file_name().to_string_lossy().into_owned();
let mut comps = rel.clone();
comps.push(name);
let Ok(md) = path.symlink_metadata() else {
skipped.push(SkippedEntry {
path: comps.join("/"),
reason: SkipReason::Unreadable,
});
continue;
};
let meta = meta_from_fs(&md);
if md.file_type().is_symlink() {
match std::fs::read_link(&path) {
Ok(target) => {
visit(&WalkItem {
path,
comps,
is_dir: false,
meta,
link_target: Some(target.to_string_lossy().into_owned()),
})?;
}
Err(_) => skipped.push(SkippedEntry {
path: comps.join("/"),
reason: SkipReason::Unreadable,
}),
}
continue; }
if md.is_dir() {
visit(&WalkItem {
path: path.clone(),
comps: comps.clone(),
is_dir: true,
meta,
link_target: None,
})?;
subdirs.push((path, comps, depth.saturating_add(1)));
} else if md.is_file() {
visit(&WalkItem {
path,
comps,
is_dir: false,
meta,
link_target: None,
})?;
} else {
skipped.push(SkippedEntry {
path: comps.join("/"),
reason: SkipReason::NotRegular,
});
}
}
subdirs.reverse();
stack.extend(subdirs);
}
Ok(())
}
pub fn pack_folder(root: &Path, mut progress: Option<ProgressFn<'_>>) -> Result<PackedFolder> {
if !root.is_dir() {
return Err(Error::Io(std::io::Error::new(
std::io::ErrorKind::NotADirectory,
"pack_folder expects a directory",
)));
}
let root_name = root
.file_name()
.map(|s| s.to_string_lossy().into_owned())
.unwrap_or_else(|| String::from("folder"));
let mut builder = ContainerBuilder::new(root_name.clone());
let mut payload = Vec::new();
let mut skipped = Vec::new();
let mut file_count = 0usize;
let mut dir_count = 0usize;
let mut link_count = 0usize;
let mut unreadable: Vec<String> = Vec::new();
walk_dir(root, &mut skipped, &mut |item| {
if let Some(target) = &item.link_target {
builder.add_symlink(item.comps.clone(), target.clone(), item.meta.clone())?;
link_count = link_count.saturating_add(1);
} else if item.is_dir {
builder.add_dir(item.comps.clone(), item.meta.clone())?;
dir_count = dir_count.saturating_add(1);
} else {
let Ok(data) = std::fs::read(&item.path) else {
unreadable.push(item.comps.join("/"));
return Ok(());
};
let size = data.len() as u64;
let hash = crate::util::blake2b_256(&data);
builder.add_file(item.comps.clone(), size, Some(hash), item.meta.clone())?;
payload.extend_from_slice(&data);
file_count = file_count.saturating_add(1);
if let Some(cb) = progress.as_deref_mut() {
cb(&item.comps.join("/"), size);
}
}
Ok(())
})?;
let index = builder.finish()?;
skipped.extend(unreadable.into_iter().map(|path| SkippedEntry {
path,
reason: SkipReason::Unreadable,
}));
Ok(PackedFolder {
payload,
index,
root_name,
file_count,
dir_count,
link_count,
skipped,
})
}
#[derive(Debug)]
pub struct PackPlan {
pub root_name: String,
items: Vec<PlanItem>,
pub file_count: usize,
pub dir_count: usize,
pub link_count: usize,
pub skipped: Vec<SkippedEntry>,
pub total_size: u64,
}
#[derive(Debug)]
enum PlanItem {
Dir { comps: Vec<String>, meta: EntryMeta },
Link { comps: Vec<String>, target: String, meta: EntryMeta },
File { path: PathBuf, comps: Vec<String>, size: u64, meta: EntryMeta },
}
pub fn plan_folder(root: &Path) -> Result<PackPlan> {
let root_name = root
.file_name()
.map(|s| s.to_string_lossy().into_owned())
.unwrap_or_else(|| String::from("folder"));
let mut items = Vec::new();
let mut skipped = Vec::new();
let mut unreadable: Vec<String> = Vec::new();
let (mut files, mut dirs, mut links) = (0usize, 0usize, 0usize);
let mut total = 0u64;
walk_dir(root, &mut skipped, &mut |item| {
if let Some(target) = &item.link_target {
items.push(PlanItem::Link {
comps: item.comps.clone(),
target: target.clone(),
meta: item.meta.clone(),
});
links = links.saturating_add(1);
} else if item.is_dir {
items.push(PlanItem::Dir { comps: item.comps.clone(), meta: item.meta.clone() });
dirs = dirs.saturating_add(1);
} else {
let size = std::fs::File::open(&item.path).and_then(|f| f.metadata()).map(|m| m.len());
let Ok(size) = size else {
unreadable.push(item.comps.join("/"));
return Ok(());
};
items.push(PlanItem::File {
path: item.path.clone(),
comps: item.comps.clone(),
size,
meta: item.meta.clone(),
});
files = files.saturating_add(1);
total = total.saturating_add(size);
}
Ok(())
})?;
skipped.extend(unreadable.into_iter().map(|path| SkippedEntry {
path,
reason: SkipReason::Unreadable,
}));
Ok(PackPlan {
root_name,
items,
file_count: files,
dir_count: dirs,
link_count: links,
skipped,
total_size: total,
})
}
impl PackPlan {
fn index_with(&self, hashes: Option<&[[u8; 32]]>) -> Result<ContainerIndex> {
let mut builder = ContainerBuilder::new(self.root_name.clone());
let mut fi = 0usize;
for it in &self.items {
match it {
PlanItem::Dir { comps, meta } => builder.add_dir(comps.clone(), meta.clone())?,
PlanItem::Link { comps, target, meta } => {
builder.add_symlink(comps.clone(), target.clone(), meta.clone())?;
}
PlanItem::File { comps, size, meta, .. } => {
let hash = match hashes {
Some(h) => *h.get(fi).ok_or(Error::MalformedHeader {
reason: "pack plan hash count mismatch",
})?,
None => [0u8; 32],
};
builder.add_file(comps.clone(), *size, Some(hash), meta.clone())?;
fi = fi.saturating_add(1);
}
}
}
builder.finish()
}
pub fn placeholder_index(&self) -> Result<Vec<u8>> {
Ok(self.index_with(None)?.encode())
}
pub fn finish(&self, hashes: &[[u8; 32]]) -> Result<ContainerIndex> {
if hashes.len() != self.file_count {
return Err(Error::MalformedHeader { reason: "pack plan hash count mismatch" });
}
self.index_with(Some(hashes))
}
#[must_use]
pub fn reader(&self) -> PackReader {
let files = self
.items
.iter()
.filter_map(|it| match it {
PlanItem::File { path, size, .. } => Some((path.clone(), *size)),
_ => None,
})
.collect();
PackReader {
files,
next: 0,
cur: None,
hashes: std::rc::Rc::new(std::cell::RefCell::new(Vec::new())),
}
}
}
type CurrentFile = (std::io::Take<std::fs::File>, u64, blake2::Blake2b<blake2::digest::consts::U32>);
pub struct PackReader {
files: Vec<(PathBuf, u64)>,
next: usize,
cur: Option<CurrentFile>,
hashes: std::rc::Rc<std::cell::RefCell<Vec<[u8; 32]>>>,
}
impl std::fmt::Debug for PackReader {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("PackReader")
.field("files", &self.files.len())
.field("next", &self.next)
.finish_non_exhaustive()
}
}
impl PackReader {
#[must_use]
pub fn hashes(&self) -> std::rc::Rc<std::cell::RefCell<Vec<[u8; 32]>>> {
std::rc::Rc::clone(&self.hashes)
}
}
impl std::io::Read for PackReader {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
use blake2::Digest;
loop {
if self.cur.is_none() {
let Some((path, size)) = self.files.get(self.next) else {
return Ok(0);
};
let f = std::fs::File::open(path)?;
let size = *size;
self.cur = Some((std::io::Read::take(f, size), size, blake2::Blake2b::new()));
self.next = self.next.saturating_add(1);
}
let Some((take, remaining, hasher)) = self.cur.as_mut() else {
return Ok(0);
};
let n = take.read(buf)?;
if n > 0 {
hasher.update(buf.get(..n).unwrap_or(&[]));
*remaining = remaining.saturating_sub(n as u64);
return Ok(n);
}
if *remaining != 0 {
return Err(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"file shrank while packing",
));
}
if let Some((take, _, hasher)) = self.cur.take() {
let mut f = take.into_inner();
let mut probe = [0u8; 1];
if f.read(&mut probe)? != 0 {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"file grew while packing",
));
}
self.hashes.borrow_mut().push(hasher.finalize().into());
}
}
}
}
fn meta_from_fs(md: &std::fs::Metadata) -> EntryMeta {
let mut m = EntryMeta::default();
if let Ok(t) = md.modified() {
m.mtime_ns = system_time_to_ns(t);
}
if let Ok(t) = md.created() {
m.btime_ns = system_time_to_ns(t);
}
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
m.mode = Some(md.mode());
m.uid = Some(md.uid());
m.gid = Some(md.gid());
}
m
}
fn system_time_to_ns(t: std::time::SystemTime) -> Option<i128> {
t.duration_since(std::time::UNIX_EPOCH)
.ok()
.and_then(|d| i128::try_from(d.as_nanos()).ok())
}
#[cfg(test)]
mod tests {
use super::*;
fn make_tree(tag: &str) -> PathBuf {
let root = std::env::temp_dir().join(format!("omy-pack-{tag}"));
let _ = std::fs::remove_dir_all(&root);
std::fs::create_dir_all(root.join("sub/deeper")).unwrap();
std::fs::write(root.join("a.txt"), b"aaa").unwrap();
std::fs::write(root.join("b.txt"), b"bbbb").unwrap();
std::fs::write(root.join("sub/c.txt"), b"cc").unwrap();
std::fs::write(root.join("sub/deeper/d.txt"), b"d").unwrap();
root
}
#[test]
fn packs_all_files_and_dirs() {
let root = make_tree("basic");
let p = pack_folder(&root, None).unwrap();
assert_eq!(p.file_count, 4, "四个文件都要打进去");
assert_eq!(p.dir_count, 2, "sub 与 sub/deeper");
assert_eq!(p.payload.len(), 3 + 4 + 2 + 1, "载荷是所有文件内容拼接");
assert_eq!(p.root_name, "omy-pack-basic");
assert!(p.skipped.is_empty(), "正常目录不该有跳过项");
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn payload_ranges_match_index() {
let root = make_tree("ranges");
let p = pack_folder(&root, None).unwrap();
for e in &p.index.entries {
let Some((start, len)) = e.range() else {
continue; };
let s = usize::try_from(start).unwrap();
let end = s + usize::try_from(len).unwrap();
let slice = p.payload.get(s..end).expect("区间必须落在载荷内");
assert_eq!(slice.len() as u64, len);
let disk = std::fs::read(root.join(e.path.join("/"))).unwrap();
assert_eq!(slice, disk.as_slice(), "{} 的内容对不上", e.display_path());
}
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn ordering_is_byte_order_not_filesystem_order() {
let root = std::env::temp_dir().join("omy-pack-order");
let _ = std::fs::remove_dir_all(&root);
std::fs::create_dir_all(&root).unwrap();
std::fs::write(root.join("a.txt"), b"aaa").unwrap();
std::fs::write(root.join("Z.txt"), b"z").unwrap();
let p = pack_folder(&root, None).unwrap();
let names: Vec<String> = p
.index
.entries
.iter()
.filter(|e| e.range().is_some())
.map(|e| e.display_path())
.collect();
assert_eq!(
names,
vec!["Z.txt", "a.txt"],
"必须按字节序排列(大写 Z 在小写 a 之前),而不是沿用文件系统顺序"
);
assert_eq!(p.payload, b"zaaa", "载荷拼接顺序要与索引一致");
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn ordering_is_stable() {
let root = make_tree("stable");
let a = pack_folder(&root, None).unwrap();
let b = pack_folder(&root, None).unwrap();
assert_eq!(a.payload, b.payload, "两次打包的载荷必须一致");
assert_eq!(a.index.encode(), b.index.encode(), "索引也必须一致");
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn empty_dir_is_packed_without_files() {
let root = std::env::temp_dir().join("omy-pack-empty");
let _ = std::fs::remove_dir_all(&root);
std::fs::create_dir_all(root.join("nothing")).unwrap();
let p = pack_folder(&root, None).unwrap();
assert_eq!(p.file_count, 0);
assert_eq!(p.dir_count, 1, "空目录本身也要记进索引");
assert!(p.payload.is_empty());
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn rejects_non_directory() {
let f = std::env::temp_dir().join("omy-pack-not-a-dir.txt");
std::fs::write(&f, b"x").unwrap();
assert!(pack_folder(&f, None).is_err(), "对文件调用必须报错");
let _ = std::fs::remove_file(&f);
}
#[test]
fn progress_reports_every_file() {
let root = make_tree("progress");
let mut seen = Vec::new();
let mut cb = |name: &str, size: u64| seen.push((name.to_owned(), size));
let p = pack_folder(&root, Some(&mut cb)).unwrap();
assert_eq!(seen.len(), p.file_count, "每个文件都要回调一次");
let total: u64 = seen.iter().map(|(_, s)| s).sum();
assert_eq!(total as usize, p.payload.len(), "回调报的大小之和要等于载荷");
let _ = std::fs::remove_dir_all(&root);
}
#[cfg(unix)]
#[test]
fn symlinks_are_stored_not_followed() {
let root = std::env::temp_dir().join("omy-pack-symlink");
let _ = std::fs::remove_dir_all(&root);
std::fs::create_dir_all(&root).unwrap();
std::fs::write(root.join("real.txt"), b"real").unwrap();
std::os::unix::fs::symlink("real.txt", root.join("link.txt")).unwrap();
let p = pack_folder(&root, None).unwrap();
assert_eq!(p.file_count, 1, "链接不算文件,只有 real.txt 是");
assert_eq!(p.link_count, 1, "链接要计数");
assert!(p.skipped.is_empty(), "不再跳过,实际 {:?}", p.skipped);
assert_eq!(p.payload, b"real", "链接不该往载荷里加内容");
let link = p
.index
.entries
.iter()
.find(|e| e.path == ["link.txt"])
.expect("索引里要有链接条目");
assert_eq!(link.kind, crate::container::EntryKind::Symlink);
assert_eq!(
link.target.as_deref(),
Some("real.txt"),
"目标要原样存下来,否则还原出的链接指向错误位置"
);
assert_eq!(link.size, 0, "链接不占载荷区间");
let _ = std::fs::remove_dir_all(&root);
}
#[cfg(windows)]
#[test]
fn directory_junctions_are_stored_not_followed() {
let root = std::env::temp_dir().join("omy-pack-junction");
let _ = std::fs::remove_dir_all(&root);
std::fs::create_dir_all(root.join("real")).unwrap();
std::fs::write(root.join("real/inside.txt"), b"content").unwrap();
let link = root.join("link");
let st = std::process::Command::new("cmd")
.args(["/c", "mklink", "/J"])
.arg(&link)
.arg(root.join("real"))
.output();
let Ok(o) = st else { return };
if !o.status.success() || !link.exists() {
return;
}
let p = pack_folder(&root, None).unwrap();
assert_eq!(
p.file_count, 1,
"只有 real/inside.txt 该被打包;联接被跟随的话会变成 2"
);
assert_eq!(p.link_count, 1, "联接要作为链接存下来");
assert_eq!(
p.payload, b"content",
"载荷里只该有一份内容;被跟随会变成两份"
);
let e = p
.index
.entries
.iter()
.find(|e| e.path == ["link"])
.expect("索引里要有联接条目");
assert_eq!(e.kind, crate::container::EntryKind::Symlink);
assert!(
e.target.as_deref().is_some_and(|t| !t.is_empty()),
"联接的目标不能是空串,否则还原时无从下手"
);
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn skip_reason_codes_are_stable() {
assert_eq!(SkipReason::Symlink.code(), "symlink");
assert_eq!(SkipReason::NotRegular.code(), "not_regular");
assert_eq!(SkipReason::Unreadable.code(), "unreadable");
assert_eq!(SkipReason::TooDeep.code(), "too_deep");
}
}