use std::os::fd::AsRawFd;
use std::path::Path;
use crate::{cstring, io, Error, Support, UnsupportedReason};
const UF_COMPRESSED: u32 = 0x0000_0020;
const DECMPFS_MAGIC: u32 = 0x636d_7066; const BLOCK: usize = 0x1_0000; const XATTR_NOFOLLOW: libc::c_int = 0x0001;
const COMPRESSION_LZVN: i32 = 0x900;
const COMPRESSION_LZFSE: i32 = 0x801;
pub(crate) const STREAMING_THRESHOLD: usize = 64 * 1024 * 1024;
fn should_stream_resource_fork(raw_len: usize, threshold: usize) -> bool {
raw_len > threshold
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Codec {
Lzvn,
Lzfse,
}
impl Codec {
const fn compression_type(self) -> u32 {
match self {
Self::Lzvn => 8,
Self::Lzfse => 12,
}
}
const fn algorithm(self) -> i32 {
match self {
Self::Lzvn => COMPRESSION_LZVN,
Self::Lzfse => COMPRESSION_LZFSE,
}
}
}
#[link(name = "compression")]
extern "C" {
fn compression_decode_buffer(
dst_buffer: *mut u8,
dst_size: usize,
src_buffer: *const u8,
src_size: usize,
scratch_buffer: *mut u8,
algorithm: i32,
) -> usize;
fn compression_encode_buffer(
dst_buffer: *mut u8,
dst_size: usize,
src_buffer: *const u8,
src_size: usize,
scratch_buffer: *mut u8,
algorithm: i32,
) -> usize;
fn compression_encode_scratch_buffer_size(algorithm: i32) -> usize;
}
fn resource_fork_too_large() -> Error {
Error::Io {
context: "decmpfs resource fork exceeds u32 offsets",
source: std::io::Error::from_raw_os_error(libc::EFBIG),
}
}
fn statfs(path: &Path) -> Result<libc::statfs, Error> {
let cpath = cstring(path)?;
let mut buf: libc::statfs = unsafe { std::mem::zeroed() };
if unsafe { libc::statfs(cpath.as_ptr(), &mut buf) } != 0 {
return Err(io("statfs"));
}
Ok(buf)
}
pub(crate) fn detect(path: &Path) -> Result<Support, Error> {
let buf = statfs(path)?;
let name: Vec<u8> = buf
.f_fstypename
.iter()
.take_while(|&&c| c != 0)
.map(|&c| c as u8)
.collect();
Ok(classify_fs(
buf.f_flags & (libc::MNT_LOCAL as u32) != 0,
&name,
))
}
fn classify_fs(is_local: bool, fstype: &[u8]) -> Support {
if !is_local {
return Support::Unsupported(UnsupportedReason::NetworkOrOverlay);
}
if fstype == b"apfs" || fstype == b"hfs" {
Support::Supported
} else {
Support::Unsupported(UnsupportedReason::Filesystem)
}
}
fn st_flags(path: &Path) -> Result<u32, Error> {
let cpath = cstring(path)?;
let mut st: libc::stat = unsafe { std::mem::zeroed() };
if unsafe { libc::lstat(cpath.as_ptr(), &mut st) } != 0 {
return Err(io("lstat"));
}
Ok(st.st_flags)
}
pub(crate) fn is_already_compressed(path: &Path) -> Result<bool, Error> {
Ok(st_flags(path)? & UF_COMPRESSED != 0)
}
pub(crate) fn compressed_on_disk(path: &Path) -> Result<Option<bool>, Error> {
Ok(Some(is_already_compressed(path)?))
}
fn compress_block_with_codec(src: &[u8], scratch: &mut [u8], codec: Codec) -> Option<Vec<u8>> {
let mut dst = vec![0u8; src.len() + src.len() / 16 + 1024];
let n = unsafe {
compression_encode_buffer(
dst.as_mut_ptr(),
dst.len(),
src.as_ptr(),
src.len(),
scratch.as_mut_ptr(),
codec.algorithm(),
)
};
if n == 0 {
return None;
}
dst.truncate(n);
Some(dst)
}
#[cfg(test)]
fn compress_block(src: &[u8], scratch: &mut [u8]) -> Option<Vec<u8>> {
compress_block_with_codec(src, scratch, Codec::Lzvn)
}
#[derive(Debug, PartialEq, Eq)]
enum ResourceForkPlan {
Plain,
Compressed { table_len: usize, total_len: usize },
}
fn resource_fork_table_len(num_blocks: usize) -> Result<usize, Error> {
num_blocks
.checked_add(1)
.and_then(|entries| entries.checked_mul(std::mem::size_of::<u32>()))
.ok_or_else(resource_fork_too_large)
}
fn plan_resource_fork(
raw_len: usize,
num_blocks: usize,
encoded_len: usize,
) -> Result<ResourceForkPlan, Error> {
let table_len = resource_fork_table_len(num_blocks)?;
let total_len = table_len
.checked_add(encoded_len)
.ok_or_else(resource_fork_too_large)?;
if total_len >= raw_len {
return Ok(ResourceForkPlan::Plain);
}
if total_len > u32::MAX as usize {
return Err(resource_fork_too_large());
}
Ok(ResourceForkPlan::Compressed {
table_len,
total_len,
})
}
fn compress_blocks(raw: &[u8], codec: Codec) -> Option<Vec<Vec<u8>>> {
let num_blocks = raw.len().div_ceil(BLOCK).max(1);
let scratch_len = unsafe { compression_encode_scratch_buffer_size(codec.algorithm()) };
let workers = if std::env::var_os("DECMPFS_SERIAL").is_some() {
1
} else {
std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1)
.min(num_blocks)
};
if workers <= 1 || num_blocks < 8 {
let mut scratch = vec![0u8; scratch_len];
return raw
.chunks(BLOCK)
.map(|chunk| compress_block_with_codec(chunk, &mut scratch, codec))
.collect();
}
let bytes_per_worker = num_blocks.div_ceil(workers) * BLOCK;
let parts: Vec<Option<Vec<Vec<u8>>>> = std::thread::scope(|scope| {
let handles: Vec<_> = raw
.chunks(bytes_per_worker)
.map(|region| {
scope.spawn(move || {
let mut scratch = vec![0u8; scratch_len];
region
.chunks(BLOCK)
.map(|chunk| compress_block_with_codec(chunk, &mut scratch, codec))
.collect::<Option<Vec<Vec<u8>>>>()
})
})
.collect();
handles
.into_iter()
.map(|handle| handle.join().ok().flatten())
.collect()
});
let mut out = Vec::with_capacity(num_blocks);
for part in parts {
out.extend(part?);
}
Some(out)
}
fn build_resource_fork_with_codec(raw: &[u8], codec: Codec) -> Result<Option<Vec<u8>>, Error> {
let num_blocks = raw.len().div_ceil(BLOCK).max(1);
let Some(blocks) = compress_blocks(raw, codec) else {
return Ok(None);
};
let encoded_len = blocks
.iter()
.try_fold(0usize, |sum, block| sum.checked_add(block.len()))
.ok_or_else(resource_fork_too_large)?;
let ResourceForkPlan::Compressed {
table_len,
total_len,
} = plan_resource_fork(raw.len(), num_blocks, encoded_len)?
else {
return Ok(None);
};
let mut out = Vec::with_capacity(total_len);
let mut offset = u32::try_from(table_len).map_err(|_| resource_fork_too_large())?;
out.extend_from_slice(&offset.to_le_bytes());
for block in &blocks {
offset = offset
.checked_add(u32::try_from(block.len()).map_err(|_| resource_fork_too_large())?)
.ok_or_else(resource_fork_too_large)?;
out.extend_from_slice(&offset.to_le_bytes());
}
for block in &blocks {
out.extend_from_slice(block);
}
debug_assert_eq!(out.len(), total_len);
Ok(Some(out))
}
#[cfg(test)]
fn build_resource_fork(raw: &[u8]) -> Result<Option<Vec<u8>>, Error> {
build_resource_fork_with_codec(raw, Codec::Lzvn)
}
struct InMemoryResourceFork {
codec: Codec,
bytes: Vec<u8>,
}
fn build_in_memory_resource_fork(raw: &[u8]) -> Result<Option<InMemoryResourceFork>, Error> {
for codec in [Codec::Lzvn, Codec::Lzfse] {
if let Some(bytes) = build_resource_fork_with_codec(raw, codec)? {
return Ok(Some(InMemoryResourceFork { codec, bytes }));
}
}
Ok(None)
}
fn write_streaming_resource_fork(path: &Path, raw: &[u8], codec: Codec) -> Result<bool, Error> {
use std::io::{Seek, Write};
use std::sync::atomic::{AtomicBool, Ordering};
let num_blocks = raw.len().div_ceil(BLOCK).max(1);
let table_len = resource_fork_table_len(num_blocks)?;
if table_len >= raw.len() || table_len > u32::MAX as usize {
return Ok(false);
}
let fork_path = path.join("..namedfork").join("rsrc");
let mut file = std::fs::OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.open(fork_path)
.map_err(|source| Error::Io {
context: "open resource fork",
source,
})?;
file.set_len(table_len as u64).map_err(|source| Error::Io {
context: "reserve resource-fork table",
source,
})?;
file
.seek(std::io::SeekFrom::Start(table_len as u64))
.map_err(|source| Error::Io {
context: "seek resource-fork payload",
source,
})?;
let mut writer = std::io::BufWriter::with_capacity(1 << 20, file);
let workers = if std::env::var_os("DECMPFS_SERIAL").is_some() {
1
} else {
std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1)
.min(num_blocks)
};
let scratch_len = unsafe { compression_encode_scratch_buffer_size(codec.algorithm()) };
let cancelled = AtomicBool::new(false);
let mut offsets = Vec::with_capacity(num_blocks + 1);
offsets.push(u32::try_from(table_len).map_err(|_| resource_fork_too_large())?);
let mut offset = table_len;
let won = std::thread::scope(|scope| -> Result<bool, Error> {
let mut receivers = Vec::with_capacity(workers);
for worker in 0..workers {
let (sender, receiver) = std::sync::mpsc::sync_channel(1);
receivers.push(receiver);
let cancelled = &cancelled;
scope.spawn(move || {
let mut scratch = vec![0u8; scratch_len];
let mut block_index = worker;
while block_index < num_blocks && !cancelled.load(Ordering::Relaxed) {
let start = block_index * BLOCK;
let end = start.saturating_add(BLOCK).min(raw.len());
let encoded = compress_block_with_codec(&raw[start..end], &mut scratch, codec);
if sender.send(encoded).is_err() {
break;
}
block_index += workers;
}
});
}
let result = (|| -> Result<bool, Error> {
for block_index in 0..num_blocks {
let Some(block) = receivers[block_index % workers].recv().ok().flatten() else {
return Ok(false);
};
let Some(next_offset) = offset.checked_add(block.len()) else {
return Ok(false);
};
if next_offset >= raw.len() || next_offset > u32::MAX as usize {
return Ok(false);
}
writer.write_all(&block).map_err(|source| Error::Io {
context: "write resource-fork block",
source,
})?;
offset = next_offset;
offsets.push(u32::try_from(offset).map_err(|_| resource_fork_too_large())?);
}
Ok(true)
})();
cancelled.store(true, Ordering::Relaxed);
drop(receivers);
result
})?;
if !won {
return Ok(false);
}
debug_assert_eq!(offsets.len(), num_blocks + 1);
let mut table = Vec::with_capacity(table_len);
for offset in offsets {
table.extend_from_slice(&offset.to_le_bytes());
}
debug_assert_eq!(table.len(), table_len);
writer
.seek(std::io::SeekFrom::Start(0))
.and_then(|_| writer.write_all(&table))
.and_then(|_| writer.flush())
.map_err(|source| Error::Io {
context: "finish resource fork",
source,
})?;
writer.get_ref().sync_all().map_err(|source| Error::Io {
context: "sync resource fork",
source,
})?;
Ok(true)
}
fn build_streaming_resource_fork(path: &Path, raw: &[u8]) -> Result<Option<Codec>, Error> {
for codec in [Codec::Lzfse, Codec::Lzvn] {
if write_streaming_resource_fork(path, raw, codec)? {
return Ok(Some(codec));
}
}
Ok(None)
}
enum StreamingState {
Encoding(StreamingEncoding),
Plain(std::fs::File),
Closed,
}
struct StreamingEncoding {
file: std::fs::File,
fork: std::io::BufWriter<std::fs::File>,
scratch: Vec<u8>,
partial: Vec<u8>,
offsets: Vec<u32>,
encoded_offset: usize,
}
pub(crate) struct StreamingWriter {
path: std::path::PathBuf,
expected_len: usize,
written: usize,
state: StreamingState,
complete: bool,
}
impl StreamingEncoding {
fn write_block(&mut self, raw: &[u8], expected_len: usize) -> Result<bool, Error> {
let Some(encoded) = compress_block_with_codec(raw, &mut self.scratch, Codec::Lzfse) else {
return Ok(false);
};
let mut decoded = vec![0u8; raw.len()];
let decoded_len = unsafe {
compression_decode_buffer(
decoded.as_mut_ptr(),
decoded.len(),
encoded.as_ptr(),
encoded.len(),
std::ptr::null_mut(),
Codec::Lzfse.algorithm(),
)
};
if decoded_len != raw.len() || decoded != raw {
return Ok(false);
}
let Some(next_offset) = self.encoded_offset.checked_add(encoded.len()) else {
return Ok(false);
};
if next_offset >= expected_len || next_offset > u32::MAX as usize {
return Ok(false);
}
use std::io::Write;
self.fork.write_all(&encoded).map_err(|source| Error::Io {
context: "write streaming resource-fork block",
source,
})?;
self.encoded_offset = next_offset;
self
.offsets
.push(u32::try_from(next_offset).map_err(|_| resource_fork_too_large())?);
Ok(true)
}
}
fn streaming_fallback_path(path: &Path) -> std::path::PathBuf {
static FALLBACK_SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let seq = FALLBACK_SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let name = path.file_name().map_or_else(
|| std::borrow::Cow::Borrowed("stream"),
|n| n.to_string_lossy(),
);
path.with_file_name(format!(".{name}.plain-{}-{seq}.tmp", std::process::id()))
}
fn decode_streaming_prefix(
path: &Path,
encoding: &mut StreamingEncoding,
current: &[u8],
expected_len: usize,
) -> Result<(std::path::PathBuf, std::fs::File), Error> {
use std::io::{Read, Seek, Write};
encoding.fork.flush().map_err(|source| Error::Io {
context: "flush streaming resource fork",
source,
})?;
encoding
.fork
.get_ref()
.sync_all()
.map_err(|source| Error::Io {
context: "sync streaming resource fork",
source,
})?;
let fork_path = path.join("..namedfork").join("rsrc");
let mut fork = std::fs::File::open(fork_path).map_err(|source| Error::Io {
context: "open streaming resource fork for fallback",
source,
})?;
let fallback = streaming_fallback_path(path);
let mut plain = std::fs::OpenOptions::new()
.read(true)
.write(true)
.create_new(true)
.open(&fallback)
.map_err(|source| Error::Io {
context: "create streaming plain fallback",
source,
})?;
let decoded = (|| -> Result<(), Error> {
for (block_index, pair) in encoding.offsets.windows(2).enumerate() {
let start = pair[0] as u64;
let encoded_len = (pair[1] - pair[0]) as usize;
let mut encoded = vec![0u8; encoded_len];
fork
.seek(std::io::SeekFrom::Start(start))
.and_then(|_| fork.read_exact(&mut encoded))
.map_err(|source| Error::Io {
context: "read streaming resource fork for fallback",
source,
})?;
let raw_len = expected_len
.saturating_sub(block_index.saturating_mul(BLOCK))
.min(BLOCK);
let mut raw = vec![0u8; raw_len];
let raw_len = unsafe {
compression_decode_buffer(
raw.as_mut_ptr(),
raw.len(),
encoded.as_ptr(),
encoded.len(),
std::ptr::null_mut(),
Codec::Lzfse.algorithm(),
)
};
if raw_len != raw.len() {
return Err(Error::Io {
context: "decode streaming resource fork for fallback",
source: std::io::Error::from(std::io::ErrorKind::InvalidData),
});
}
plain.write_all(&raw).map_err(|source| Error::Io {
context: "write streaming plain fallback",
source,
})?;
}
plain.write_all(current).map_err(|source| Error::Io {
context: "write current streaming fallback block",
source,
})
})();
if let Err(err) = decoded {
drop(plain);
let _ = std::fs::remove_file(&fallback);
return Err(err);
}
Ok((fallback, plain))
}
fn streaming_kernel_matches(
path: &Path,
encoding: &StreamingEncoding,
expected_len: usize,
) -> Result<bool, Error> {
use std::io::{Read, Seek};
let mut logical = match std::fs::File::open(path) {
Ok(file) => file,
Err(_) => return Ok(false),
};
let fork_path = path.join("..namedfork").join("rsrc");
let mut fork = std::fs::File::open(fork_path).map_err(|source| Error::Io {
context: "open finished streaming resource fork",
source,
})?;
for (block_index, pair) in encoding.offsets.windows(2).enumerate() {
let encoded_len = (pair[1] - pair[0]) as usize;
let mut encoded = vec![0u8; encoded_len];
fork
.seek(std::io::SeekFrom::Start(pair[0] as u64))
.and_then(|_| fork.read_exact(&mut encoded))
.map_err(|source| Error::Io {
context: "read finished streaming resource fork",
source,
})?;
let raw_len = expected_len
.saturating_sub(block_index.saturating_mul(BLOCK))
.min(BLOCK);
let mut decoded = vec![0u8; raw_len];
let decoded_len = unsafe {
compression_decode_buffer(
decoded.as_mut_ptr(),
decoded.len(),
encoded.as_ptr(),
encoded.len(),
std::ptr::null_mut(),
Codec::Lzfse.algorithm(),
)
};
if decoded_len != raw_len {
return Ok(false);
}
let mut kernel = vec![0u8; raw_len];
if logical.read_exact(&mut kernel).is_err() || kernel != decoded {
return Ok(false);
}
}
let mut extra = [0u8; 1];
Ok(logical.read(&mut extra).is_ok_and(|len| len == 0))
}
impl StreamingWriter {
pub(crate) fn new(path: &Path, expected_len: usize) -> Result<Self, Error> {
let num_blocks = expected_len.div_ceil(BLOCK).max(1);
let table_len = resource_fork_table_len(num_blocks)?;
if expected_len == 0 || table_len >= expected_len || table_len > u32::MAX as usize {
let file = std::fs::OpenOptions::new()
.read(true)
.write(true)
.create_new(true)
.open(path)
.map_err(|source| Error::Io {
context: "create streaming plain temp",
source,
})?;
return Ok(Self {
path: path.to_path_buf(),
expected_len,
written: 0,
state: StreamingState::Plain(file),
complete: false,
});
}
let file = std::fs::OpenOptions::new()
.read(true)
.write(true)
.create_new(true)
.open(path)
.map_err(|source| Error::Io {
context: "create streaming decmpfs temp",
source,
})?;
let fork_file = (|| -> Result<std::fs::File, Error> {
let fork_path = path.join("..namedfork").join("rsrc");
let mut fork_file = std::fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open(fork_path)
.map_err(|source| Error::Io {
context: "open streaming resource fork",
source,
})?;
use std::io::Seek;
fork_file
.set_len(table_len as u64)
.map_err(|source| Error::Io {
context: "reserve streaming resource-fork table",
source,
})?;
fork_file
.seek(std::io::SeekFrom::Start(table_len as u64))
.map_err(|source| Error::Io {
context: "seek streaming resource-fork payload",
source,
})?;
Ok(fork_file)
})();
let fork_file = match fork_file {
Ok(fork_file) => fork_file,
Err(error) => {
drop(file);
let _ = std::fs::remove_file(path);
return Err(error);
}
};
let scratch_len = unsafe { compression_encode_scratch_buffer_size(Codec::Lzfse.algorithm()) };
Ok(Self {
path: path.to_path_buf(),
expected_len,
written: 0,
state: StreamingState::Encoding(StreamingEncoding {
file,
fork: std::io::BufWriter::with_capacity(1 << 20, fork_file),
scratch: vec![0u8; scratch_len],
partial: Vec::with_capacity(BLOCK),
offsets: vec![u32::try_from(table_len).map_err(|_| resource_fork_too_large())?],
encoded_offset: table_len,
}),
complete: false,
})
}
fn switch_to_plain(&mut self, current: &[u8]) -> Result<(), Error> {
let StreamingState::Encoding(mut encoding) =
std::mem::replace(&mut self.state, StreamingState::Closed)
else {
return Err(Error::Io {
context: "switch streaming writer to plain",
source: std::io::Error::from(std::io::ErrorKind::InvalidInput),
});
};
let (fallback, mut plain) =
decode_streaming_prefix(&self.path, &mut encoding, current, self.expected_len)?;
drop(encoding);
if let Err(source) = std::fs::remove_file(&self.path) {
let _ = std::fs::remove_file(&fallback);
return Err(Error::Io {
context: "remove streaming decmpfs temp",
source,
});
}
if let Err(source) = std::fs::rename(&fallback, &self.path) {
let _ = std::fs::remove_file(&fallback);
return Err(Error::Io {
context: "adopt streaming plain fallback",
source,
});
}
use std::io::Seek;
plain
.seek(std::io::SeekFrom::End(0))
.map_err(|source| Error::Io {
context: "seek streaming plain fallback",
source,
})?;
self.state = StreamingState::Plain(plain);
Ok(())
}
pub(crate) fn write_all(&mut self, mut input: &[u8]) -> Result<(), Error> {
let next_written = self
.written
.checked_add(input.len())
.filter(|&len| len <= self.expected_len)
.ok_or_else(|| Error::Io {
context: "stream exceeds expected length",
source: std::io::Error::from(std::io::ErrorKind::InvalidData),
})?;
while !input.is_empty() {
match &mut self.state {
StreamingState::Plain(file) => {
use std::io::Write;
file.write_all(input).map_err(|source| Error::Io {
context: "write streaming plain temp",
source,
})?;
input = &[];
}
StreamingState::Encoding(encoding) => {
let take = (BLOCK - encoding.partial.len()).min(input.len());
encoding.partial.extend_from_slice(&input[..take]);
input = &input[take..];
if encoding.partial.len() == BLOCK {
let block = std::mem::replace(&mut encoding.partial, Vec::with_capacity(BLOCK));
if !encoding.write_block(&block, self.expected_len)? {
self.switch_to_plain(&block)?;
}
}
}
StreamingState::Closed => {
return Err(Error::Io {
context: "write closed streaming writer",
source: std::io::Error::from(std::io::ErrorKind::BrokenPipe),
});
}
}
}
self.written = next_written;
Ok(())
}
pub(crate) fn finish(&mut self) -> Result<bool, Error> {
if self.written != self.expected_len {
return Err(Error::Io {
context: "finish incomplete streaming writer",
source: std::io::Error::from(std::io::ErrorKind::UnexpectedEof),
});
}
let partial = match &mut self.state {
StreamingState::Encoding(encoding) if !encoding.partial.is_empty() => Some(
std::mem::replace(&mut encoding.partial, Vec::with_capacity(BLOCK)),
),
_ => None,
};
if let Some(block) = partial {
let won = match &mut self.state {
StreamingState::Encoding(encoding) => encoding.write_block(&block, self.expected_len)?,
_ => false,
};
if !won {
self.switch_to_plain(&block)?;
}
}
let compressed = match std::mem::replace(&mut self.state, StreamingState::Closed) {
StreamingState::Plain(file) => {
file.sync_all().map_err(|source| Error::Io {
context: "sync streaming plain temp",
source,
})?;
false
}
StreamingState::Encoding(mut encoding) => {
use std::io::{Seek, Write};
let mut table = Vec::with_capacity(encoding.offsets.len() * std::mem::size_of::<u32>());
for offset in &encoding.offsets {
table.extend_from_slice(&offset.to_le_bytes());
}
encoding
.fork
.seek(std::io::SeekFrom::Start(0))
.and_then(|_| encoding.fork.write_all(&table))
.and_then(|_| encoding.fork.flush())
.map_err(|source| Error::Io {
context: "finish streaming resource fork",
source,
})?;
encoding
.fork
.get_ref()
.sync_all()
.map_err(|source| Error::Io {
context: "sync finished streaming resource fork",
source,
})?;
let cpath = cstring(&self.path)?;
setxattr(
&cpath,
c"com.apple.decmpfs",
&decmpfs_header(Codec::Lzfse, self.expected_len),
)?;
if unsafe { libc::fchflags(encoding.file.as_raw_fd(), UF_COMPRESSED) } != 0 {
return Err(io("fchflags streaming temp"));
}
encoding.file.sync_all().map_err(|source| Error::Io {
context: "sync streaming decmpfs temp",
source,
})?;
if streaming_kernel_matches(&self.path, &encoding, self.expected_len)? {
true
} else {
let (fallback, plain) =
decode_streaming_prefix(&self.path, &mut encoding, &[], self.expected_len)?;
drop(encoding);
std::fs::remove_file(&self.path).map_err(|source| Error::Io {
context: "remove failed streaming decmpfs oracle",
source,
})?;
if let Err(source) = std::fs::rename(&fallback, &self.path) {
let _ = std::fs::remove_file(&fallback);
return Err(Error::Io {
context: "publish streaming oracle fallback",
source,
});
}
plain.sync_all().map_err(|source| Error::Io {
context: "sync streaming oracle fallback",
source,
})?;
false
}
}
StreamingState::Closed => {
return Err(Error::Io {
context: "finish closed streaming writer",
source: std::io::Error::from(std::io::ErrorKind::BrokenPipe),
});
}
};
self.complete = true;
Ok(compressed)
}
}
impl Drop for StreamingWriter {
fn drop(&mut self) {
if !self.complete {
self.state = StreamingState::Closed;
let _ = std::fs::remove_file(&self.path);
}
}
}
fn decmpfs_header(codec: Codec, raw_len: usize) -> [u8; 16] {
let mut header = [0u8; 16];
header[..4].copy_from_slice(&DECMPFS_MAGIC.to_le_bytes());
header[4..8].copy_from_slice(&codec.compression_type().to_le_bytes());
header[8..].copy_from_slice(&(raw_len as u64).to_le_bytes());
header
}
fn setxattr(path: &std::ffi::CStr, name: &std::ffi::CStr, value: &[u8]) -> Result<(), Error> {
let rc = unsafe {
libc::setxattr(
path.as_ptr(),
name.as_ptr(),
value.as_ptr().cast(),
value.len(),
0,
XATTR_NOFOLLOW,
)
};
if rc != 0 {
return Err(io("setxattr"));
}
Ok(())
}
pub(crate) fn apply_inplace(path: &Path, snapshot: &[u8]) -> Result<(), Error> {
let cpath = cstring(path)?;
if unsafe { libc::access(cpath.as_ptr(), libc::W_OK) } != 0 {
return Err(io("access"));
}
let mode = std::fs::metadata(path).map(|m| m.permissions()).ok();
apply_bytes(path, snapshot, mode)
}
pub(crate) fn apply_bytes(
path: &Path,
content: &[u8],
mode: Option<std::fs::Permissions>,
) -> Result<(), Error> {
apply_bytes_with_streaming_threshold(path, content, mode, STREAMING_THRESHOLD)
}
fn apply_bytes_with_streaming_threshold(
path: &Path,
content: &[u8],
mode: Option<std::fs::Permissions>,
streaming_threshold: usize,
) -> Result<(), Error> {
let stream = should_stream_resource_fork(content.len(), streaming_threshold);
let in_memory_resource_fork = if stream {
None
} else {
build_in_memory_resource_fork(content)?
};
let dir = path.parent().ok_or_else(|| io("parent"))?;
let name = path
.file_name()
.ok_or_else(|| io("file_name"))?
.to_string_lossy();
static TMP_SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let seq = TMP_SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let nanos = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0);
let tmp = dir.join(format!(
".{name}.decmpfs-{}-{nanos}-{seq}.tmp",
std::process::id()
));
let build = (|| -> Result<(), Error> {
let create_temp = || {
std::fs::OpenOptions::new()
.read(true)
.write(true)
.create_new(true)
.open(&tmp)
.map_err(|source| Error::Io {
context: "create temp",
source,
})
};
let mut file = create_temp()?;
let ctmp = cstring(&tmp)?;
let codec = if stream {
build_streaming_resource_fork(&tmp, content)?
} else if let Some(resource_fork) = &in_memory_resource_fork {
setxattr(&ctmp, c"com.apple.ResourceFork", &resource_fork.bytes)?;
Some(resource_fork.codec)
} else {
None
};
if let Some(codec) = codec {
setxattr(
&ctmp,
c"com.apple.decmpfs",
&decmpfs_header(codec, content.len()),
)?;
if unsafe { libc::fchflags(file.as_raw_fd(), UF_COMPRESSED) } != 0 {
return Err(io("fchflags"));
}
} else {
if stream {
drop(file);
std::fs::remove_file(&tmp).map_err(|source| Error::Io {
context: "remove losing streamed temp",
source,
})?;
file = create_temp()?;
}
use std::io::Write;
file.write_all(content).map_err(|source| Error::Io {
context: "plain temp write",
source,
})?;
file.sync_all().map_err(|source| Error::Io {
context: "plain temp sync",
source,
})?;
}
Ok(())
})();
if let Err(e) = build {
let _ = std::fs::remove_file(&tmp);
return Err(e);
}
if let Some(perm) = mode {
let _ = std::fs::set_permissions(&tmp, perm);
}
if let Ok(meta) = std::fs::metadata(path) {
use std::os::unix::fs::MetadataExt;
let _ = std::os::unix::fs::chown(&tmp, Some(meta.uid()), Some(meta.gid()));
}
std::fs::rename(&tmp, path).map_err(|source| {
let _ = std::fs::remove_file(&tmp);
Error::Io {
context: "rename",
source,
}
})
}
pub(crate) fn clone_file(src: &Path, dest: &Path) -> Result<bool, Error> {
let csrc = cstring(src)?;
let cdest = cstring(dest)?;
Ok(unsafe { libc::clonefile(csrc.as_ptr(), cdest.as_ptr(), 0) } == 0)
}
#[cfg(test)]
#[cfg_attr(coverage_nightly, coverage(off))]
mod tests {
use super::*;
#[test]
fn kernel_roundtrips_decmpfs() {
let dir = std::env::temp_dir().join(format!("decmpfs-oracle-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("f.bin");
let mut raw = Vec::new();
let pat = b"the quick brown fox decmpfs lzvn resource-fork oracle line ";
while raw.len() < 2_000_000 {
raw.extend_from_slice(pat);
}
std::fs::write(&path, &raw).unwrap();
assert!(
matches!(detect(&path).unwrap(), Support::Supported),
"temp dir is local APFS/HFS+"
);
apply_inplace(&path, &raw).unwrap();
assert!(is_already_compressed(&path).unwrap(), "UF_COMPRESSED set");
assert_eq!(
compressed_on_disk(&path).unwrap(),
Some(true),
"reports compressed"
);
assert_eq!(
std::fs::read(&path).unwrap(),
raw,
"kernel read-back must equal the original bytes"
);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn incremental_writer_streams_lzfse_blocks_into_a_kernel_readable_file() {
let dir =
std::env::temp_dir().join(format!("decmpfs-incremental-oracle-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("model.bin");
let raw = b"incremental lzfse resource fork ".repeat((2 << 20) / 34 + 1);
let mut writer = StreamingWriter::new(&path, raw.len()).unwrap();
for chunk in raw.chunks(17_003) {
writer.write_all(chunk).unwrap();
}
assert!(writer.finish().unwrap(), "compressible stream must win");
assert!(is_already_compressed(&path).unwrap());
assert_eq!(std::fs::read(&path).unwrap(), raw);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn incremental_writer_reconstructs_plain_bytes_when_compression_loses() {
let dir = std::env::temp_dir().join(format!(
"decmpfs-incremental-fallback-{}",
std::process::id()
));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("random.bin");
let mut raw = Vec::with_capacity(2 << 20);
let mut x: u64 = 0x9e37_79b9_7f4a_7c15;
while raw.len() < (2 << 20) {
x ^= x << 13;
x ^= x >> 7;
x ^= x << 17;
raw.extend_from_slice(&x.to_le_bytes());
}
let mut writer = StreamingWriter::new(&path, raw.len()).unwrap();
for chunk in raw.chunks(17_003) {
writer.write_all(chunk).unwrap();
}
assert!(
!writer.finish().unwrap(),
"incompressible stream stays plain"
);
assert!(!is_already_compressed(&path).unwrap());
assert_eq!(std::fs::read(&path).unwrap(), raw);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
#[ignore]
fn write_time_probe() {
let dir = std::env::temp_dir().join(format!("decmpfs-time-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("addon.node");
let mut raw: Vec<u8> = Vec::with_capacity(40 << 20);
let mut x: u64 = 0x9e37_79b9_7f4a_7c15;
while raw.len() < (40 << 20) {
x ^= x << 13;
x ^= x >> 7;
x ^= x << 17;
raw.extend_from_slice(&x.to_le_bytes());
raw.extend_from_slice(b"native addon .node text segment padding ");
}
if !matches!(detect(&dir), Ok(Support::Supported)) {
std::fs::remove_dir_all(&dir).ok();
return;
}
let cores = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1);
let serial = std::env::var_os("DECMPFS_SERIAL").is_some();
let start = std::time::Instant::now();
apply_bytes(&path, &raw, None).unwrap();
let ms = start.elapsed().as_secs_f64() * 1e3;
eprintln!(
"decmpfs write {}MiB — {} ({} cores): {:.1} ms",
raw.len() >> 20,
if serial { "serial" } else { "parallel" },
cores,
ms,
);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn detect_and_flags_error_on_a_missing_path() {
let p = std::path::Path::new("/no/such/decmpfs/path/x.bin");
assert!(detect(p).is_err(), "statfs of a missing path errors");
assert!(
is_already_compressed(p).is_err(),
"lstat of a missing path errors"
);
}
#[test]
fn apply_inplace_errors_when_the_file_cannot_be_read() {
if unsafe { libc::geteuid() } == 0 {
return;
}
use std::os::unix::fs::PermissionsExt;
let dir = std::env::temp_dir().join(format!("decmpfs-noread-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("f.bin");
let content = b"\x7fELF unreadable";
std::fs::write(&path, content).unwrap();
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o000)).unwrap();
let out = apply_inplace(&path, content);
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).ok();
assert!(matches!(
out,
Err(Error::Io {
context: "access",
..
})
));
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn setxattr_errors_on_a_missing_path() {
let out = setxattr(c"/no/such/decmpfs/path", c"com.apple.decmpfs", b"x");
assert!(matches!(
out,
Err(Error::Io {
context: "setxattr",
..
})
));
}
#[test]
fn compress_block_returns_none_for_empty_input() {
let scratch_len = unsafe { compression_encode_scratch_buffer_size(COMPRESSION_LZVN) };
let mut scratch = vec![0u8; scratch_len];
assert!(compress_block(b"", &mut scratch).is_none());
}
#[test]
fn build_resource_fork_zero_length_is_no_gain() {
assert!(
build_resource_fork(&[]).unwrap().is_none(),
"a resource fork cannot make an empty file smaller"
);
}
#[test]
fn streaming_threshold_keeps_vite_native_addons_on_the_fast_path() {
assert!(!should_stream_resource_fork(37 << 20, STREAMING_THRESHOLD));
assert!(!should_stream_resource_fork(
STREAMING_THRESHOLD,
STREAMING_THRESHOLD
));
assert!(should_stream_resource_fork(
STREAMING_THRESHOLD + 1,
STREAMING_THRESHOLD
));
}
#[test]
fn kernel_roundtrips_forced_streaming_lzfse() {
let dir = std::env::temp_dir().join(format!("decmpfs-streaming-oracle-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("f.bin");
let raw = b"streamed lzfse decmpfs resource fork oracle ".repeat((2 << 20) / 46 + 1);
std::fs::write(&path, &raw).unwrap();
if matches!(detect(&path).unwrap(), Support::Supported) {
apply_bytes_with_streaming_threshold(&path, &raw, None, 0).unwrap();
assert!(is_already_compressed(&path).unwrap(), "UF_COMPRESSED set");
assert_eq!(
std::fs::read(&path).unwrap(),
raw,
"kernel read-back must decode the streamed type-12 resource fork"
);
let cpath = cstring(&path).unwrap();
let mut header = [0u8; 16];
let len = unsafe {
libc::getxattr(
cpath.as_ptr(),
c"com.apple.decmpfs".as_ptr(),
header.as_mut_ptr().cast(),
header.len(),
0,
XATTR_NOFOLLOW | 0x0020, )
};
assert_eq!(len, header.len() as isize);
assert_eq!(u32::from_le_bytes(header[4..8].try_into().unwrap()), 12);
}
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn in_memory_path_falls_back_to_lzfse_when_lzvn_has_no_gain() {
let mut raw = Vec::with_capacity(1 << 20);
let mut x: u64 = 0x9e37_79b9_7f4a_7c15;
while raw.len() < raw.capacity() {
x ^= x << 13;
x ^= x >> 7;
x ^= x << 17;
raw.push(if x.is_multiple_of(4) {
0
} else {
(x >> 32) as u8
});
}
assert!(
build_resource_fork_with_codec(&raw, Codec::Lzvn)
.unwrap()
.is_none(),
"fixture must reach the fallback"
);
let candidate = build_in_memory_resource_fork(&raw)
.unwrap()
.expect("LZFSE should exploit the skewed symbols");
assert_eq!(candidate.codec, Codec::Lzfse);
assert!(candidate.bytes.len() < raw.len());
}
#[test]
fn build_resource_fork_last_offset_equals_length() {
for size in [512usize, BLOCK, BLOCK + 1, BLOCK * 3 + 7] {
let raw = vec![0x41u8; size];
let Some(rf) = build_resource_fork(&raw).unwrap() else {
continue;
};
let num_blocks = size.div_ceil(BLOCK);
let last_idx = num_blocks * 4; let last = u32::from_le_bytes(rf[last_idx..last_idx + 4].try_into().unwrap()) as usize;
assert_eq!(last, rf.len(), "size {size}: last offset != buffer length");
}
}
#[test]
fn cstring_rejects_an_interior_nul() {
use std::os::unix::ffi::OsStrExt;
let p = std::path::Path::new(std::ffi::OsStr::from_bytes(b"a\0b"));
assert!(cstring(p).is_err());
}
#[test]
fn detect_rejects_a_non_apfs_filesystem() {
assert!(matches!(
detect(std::path::Path::new("/dev")),
Ok(Support::Unsupported(UnsupportedReason::Filesystem))
));
}
#[test]
fn classify_fs_covers_every_branch() {
assert!(matches!(
classify_fs(false, b"nfs"),
Support::Unsupported(UnsupportedReason::NetworkOrOverlay)
));
assert!(matches!(classify_fs(true, b"apfs"), Support::Supported));
assert!(matches!(classify_fs(true, b"hfs"), Support::Supported));
assert!(matches!(
classify_fs(true, b"ext4"),
Support::Unsupported(UnsupportedReason::Filesystem)
));
}
#[test]
fn resource_fork_plan_accepts_raw_files_beyond_the_old_limit() {
let raw_len = 4_100_000_000usize;
let num_blocks = raw_len.div_ceil(BLOCK);
assert!(matches!(
plan_resource_fork(raw_len, num_blocks, 3_000_000_000).unwrap(),
ResourceForkPlan::Compressed { .. }
));
}
#[test]
fn resource_fork_plan_accepts_raw_files_beyond_four_gib_when_the_fork_fits() {
let raw_len = 5_000_000_000usize;
let num_blocks = raw_len.div_ceil(BLOCK);
assert!(matches!(
plan_resource_fork(raw_len, num_blocks, 3_000_000_000).unwrap(),
ResourceForkPlan::Compressed { .. }
));
}
#[test]
fn resource_fork_plan_rejects_a_compressed_fork_past_u32() {
let raw_len = 5_000_000_000usize;
let num_blocks = raw_len.div_ceil(BLOCK);
match plan_resource_fork(raw_len, num_blocks, 4_400_000_000).unwrap_err() {
Error::Io { source, .. } => assert_eq!(source.raw_os_error(), Some(libc::EFBIG)),
other => panic!("expected EFBIG Io, got {other:?}"),
}
}
#[test]
fn gemini_nano_lzvn_resource_fork_is_no_gain() {
assert_eq!(
plan_resource_fork(4_269_932_544, 65_154, 4_364_775_458).unwrap(),
ResourceForkPlan::Plain
);
}
#[test]
fn gemini_nano_lzfse_resource_fork_fits_and_wins() {
assert_eq!(
plan_resource_fork(4_269_932_544, 65_154, 3_598_249_560).unwrap(),
ResourceForkPlan::Compressed {
table_len: 260_620,
total_len: 3_598_510_180,
}
);
}
#[test]
fn kernel_roundtrips_incompressible_blocks() {
let dir = std::env::temp_dir().join(format!("decmpfs-raw-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("f.bin");
let mut raw = Vec::new();
let mut x: u32 = 0x9e37_79b9;
while raw.len() < 200_000 {
x ^= x << 13;
x ^= x >> 17;
x ^= x << 5;
raw.extend_from_slice(&x.to_le_bytes());
}
std::fs::write(&path, &raw).unwrap();
if matches!(detect(&path).unwrap(), Support::Supported) {
assert!(matches!(
crate::compress_file(&path).unwrap(),
crate::Outcome::NoGain { .. }
));
assert_eq!(
std::fs::read(&path).unwrap(),
raw,
"plain fallback reads back identically"
);
assert!(
!is_already_compressed(&path).unwrap(),
"no-gain input must not carry UF_COMPRESSED"
);
}
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn apply_bytes_preserves_ownership_of_an_overwritten_file() {
let dir = std::env::temp_dir().join(format!("decmpfs-own-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("f");
std::fs::write(&path, vec![0u8; 4096]).unwrap();
if !matches!(detect(&path), Ok(Support::Supported)) {
std::fs::remove_dir_all(&dir).ok();
return;
}
use std::os::unix::fs::MetadataExt;
let before_uid = std::fs::metadata(&path).unwrap().uid();
let content = vec![0xABu8; 8192];
apply_bytes(&path, &content, None).unwrap();
let meta = std::fs::metadata(&path).unwrap();
assert_eq!(meta.uid(), before_uid, "owner preserved across the rewrite");
assert_eq!(std::fs::read(&path).unwrap(), content, "content intact");
std::fs::remove_dir_all(&dir).ok();
}
}