use std::fs::{self, File, OpenOptions};
use std::io::{self, Read, Write};
use std::path::Path;
use std::sync::atomic::Ordering;
extern "C" {
fn clock() -> libc::clock_t;
}
use crate::io::decompress_frame::decompress_lz4f;
use crate::io::decompress_legacy::decode_legacy_stream;
use crate::io::decompress_resources::DecompressResources;
use crate::io::file_io::{
is_skippable_magic_number, open_src_file, NUL_MARK, STDIN_MARK, STDOUT_MARK,
};
use crate::io::prefs::{
display_level, final_time_display, Prefs, DISPLAY_LEVEL, LEGACY_MAGICNUMBER, LZ4IO_MAGICNUMBER,
LZ4IO_SKIPPABLE0, MAGICNUMBER_SIZE,
};
use crate::io::sparse::{fwrite_sparse, fwrite_sparse_end, SPARSE_SEGMENT_SIZE};
use crate::timefn::get_time;
#[derive(Debug, Clone, Default)]
pub struct DecompressStats {
pub decompressed_bytes: u64,
}
const PT_BUF_SIZE: usize = 16 * 1024;
const SKIP_BUF_SIZE: usize = 16 * 1024;
struct SparseWriter {
file: File,
stored_skips: u64,
sparse_mode: bool,
}
impl SparseWriter {
fn new(file: File, sparse_mode: bool) -> Self {
SparseWriter {
file,
stored_skips: 0,
sparse_mode,
}
}
fn finish(&mut self) -> io::Result<()> {
let skips = self.stored_skips;
self.stored_skips = 0;
fwrite_sparse_end(&mut self.file, skips)
}
}
impl Write for SparseWriter {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
self.stored_skips = fwrite_sparse(
&mut self.file,
buf,
SPARSE_SEGMENT_SIZE,
self.stored_skips,
self.sparse_mode,
)?;
Ok(buf.len())
}
fn flush(&mut self) -> io::Result<()> {
self.file.flush()
}
}
fn pass_through<R: Read, W: Write>(
src: &mut R,
dst: &mut W,
magic_bytes: [u8; MAGICNUMBER_SIZE],
) -> io::Result<u64> {
dst.write_all(&magic_bytes)?;
let mut total = MAGICNUMBER_SIZE as u64;
let mut buf = [0u8; PT_BUF_SIZE];
loop {
let n = src.read(&mut buf)?;
if n == 0 {
break;
}
total += n as u64;
dst.write_all(&buf[..n])?;
}
Ok(total)
}
fn skip_stream<R: Read>(src: &mut R, mut offset: u32) -> io::Result<()> {
let mut buf = [0u8; SKIP_BUF_SIZE];
while offset > 0 {
let to_read = (offset as usize).min(SKIP_BUF_SIZE);
src.read_exact(&mut buf[..to_read]).map_err(|_| {
io::Error::new(
io::ErrorKind::UnexpectedEof,
"Stream error : cannot skip skippable area",
)
})?;
offset -= to_read as u32;
}
Ok(())
}
fn decompress_loop<R: Read, W: Write>(
src: &mut R,
dst: &mut W,
prefs: &Prefs,
resources: &mut DecompressResources,
) -> io::Result<u64> {
let mut filesize: u64 = 0;
let mut nb_frames: u64 = 0;
let mut pending_magic: Option<u32> = None;
loop {
let (magic, magic_bytes) = if let Some(m) = pending_magic.take() {
(m, m.to_le_bytes())
} else {
let mut mb = [0u8; MAGICNUMBER_SIZE];
match src.read(&mut mb[..1])? {
0 => break, _ => {}
}
if let Err(e) = src.read_exact(&mut mb[1..]) {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("Unrecognized header : Magic Number unreadable: {}", e),
));
}
(u32::from_le_bytes(mb), mb)
};
let folded = if is_skippable_magic_number(magic) {
LZ4IO_SKIPPABLE0
} else {
magic
};
match folded {
LZ4IO_MAGICNUMBER => {
let bytes = decompress_lz4f(src, dst, prefs, resources)?;
filesize += bytes;
}
LEGACY_MAGICNUMBER => {
display_level(4, "Detected : Legacy format \n");
let (bytes, next) = decode_legacy_stream(src, dst, prefs, resources)?;
filesize += bytes;
pending_magic = next;
}
LZ4IO_SKIPPABLE0 => {
display_level(4, "Skipping detected skippable area \n");
let mut sb = [0u8; 4];
src.read_exact(&mut sb).map_err(|_| {
io::Error::new(
io::ErrorKind::InvalidData,
"Stream error : skippable size unreadable",
)
})?;
let skip_size = u32::from_le_bytes(sb);
skip_stream(src, skip_size)?;
}
_ => {
if nb_frames == 0 {
if !prefs.test_mode && prefs.overwrite && prefs.pass_through {
let bytes = pass_through(src, dst, magic_bytes)?;
return Ok(bytes);
}
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"Unrecognized header : file cannot be decoded",
));
}
display_level(2, "Stream followed by undecodable data \n");
break;
}
}
nb_frames += 1;
}
Ok(filesize)
}
fn decompress_src_file<W: Write>(
src_path: &str,
dst: &mut W,
prefs: &Prefs,
resources: &mut DecompressResources,
) -> io::Result<u64> {
let mut src = open_src_file(src_path)?; let filesize = decompress_loop(&mut src, dst, prefs, resources)?;
if prefs.remove_src_file {
fs::remove_file(src_path)
.map_err(|e| io::Error::new(e.kind(), format!("Remove error : {}: {}", src_path, e)))?;
}
if DISPLAY_LEVEL.load(Ordering::Relaxed) >= 2 {
display_level(2, &format!("\r{:79}\r", ""));
display_level(
2,
&format!("{:<30.30} : decoded {} bytes \n", src_path, filesize),
);
}
Ok(filesize)
}
fn open_regular_dst(dst_path: &str, prefs: &Prefs) -> io::Result<File> {
if !prefs.overwrite && Path::new(dst_path).exists() {
let level = DISPLAY_LEVEL.load(Ordering::Relaxed);
if level <= 1 {
eprintln!("{} already exists; not overwritten ", dst_path);
return Err(io::Error::new(
io::ErrorKind::AlreadyExists,
format!("{}: already exists; not overwritten", dst_path),
));
}
eprint!(
"{} already exists; do you want to overwrite (y/N) ? ",
dst_path
);
let _ = io::stderr().flush();
let mut line = String::new();
io::stdin().read_line(&mut line)?;
let first = line.trim_start().chars().next().unwrap_or('\0');
if first != 'y' && first != 'Y' {
eprintln!(" not overwritten ");
return Err(io::Error::new(
io::ErrorKind::AlreadyExists,
format!("{}: not overwritten", dst_path),
));
}
}
OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.open(dst_path)
.map_err(|e| {
if DISPLAY_LEVEL.load(Ordering::Relaxed) >= 1 {
eprintln!("{}: {}", dst_path, e);
}
e
})
}
fn decompress_dst_file(
src_path: &str,
dst_path: &str,
prefs: &Prefs,
resources: &mut DecompressResources,
) -> io::Result<u64> {
let src_stat = if src_path != STDIN_MARK {
fs::metadata(src_path).ok()
} else {
None
};
let filesize = if dst_path == STDOUT_MARK {
let mut dst = io::stdout();
decompress_src_file(src_path, &mut dst, prefs, resources)?
} else if dst_path == NUL_MARK {
let mut dst = io::sink();
decompress_src_file(src_path, &mut dst, prefs, resources)?
} else {
let file = open_regular_dst(dst_path, prefs)?;
let sparse_mode = prefs.sparse_file_support > 0;
let mut sparse_writer = SparseWriter::new(file, sparse_mode);
let result = decompress_src_file(src_path, &mut sparse_writer, prefs, resources);
let finish_result = sparse_writer.finish();
let sz = result?;
finish_result?;
sz
};
let is_special_dst = dst_path == STDOUT_MARK || dst_path == NUL_MARK;
if !is_special_dst {
if let Some(meta) = &src_stat {
if let Ok(mtime) = meta.modified() {
let ft = filetime::FileTime::from_system_time(mtime);
let _ = filetime::set_file_mtime(dst_path, ft);
}
let _ = fs::set_permissions(dst_path, meta.permissions());
}
}
Ok(filesize)
}
pub fn decompress_filename(src: &str, dst: &str, prefs: &Prefs) -> io::Result<DecompressStats> {
let mut resources = DecompressResources::from_prefs(prefs)?;
let time_start = get_time();
let cpu_start = unsafe { clock() };
let result = decompress_dst_file(src, dst, prefs, &mut resources);
match result {
Ok(bytes) => Ok(DecompressStats {
decompressed_bytes: bytes,
}),
Err(e) => {
final_time_display(time_start, cpu_start, 0);
Err(e)
}
}
}
pub fn decompress_multiple_filenames(srcs: &[&str], suffix: &str, prefs: &Prefs) -> io::Result<()> {
let mut resources = DecompressResources::from_prefs(prefs)?;
let time_start = get_time();
let cpu_start = unsafe { clock() };
if !prefs.block_checksum && !prefs.stream_checksum {
display_level(4, "disabling checksum validation during decoding \n");
}
let mut total_processed: u64 = 0;
let mut missing_files: i32 = 0;
let mut skipped_files: i32 = 0;
let dst_is_special = suffix == STDOUT_MARK || suffix == NUL_MARK;
for &src_path in srcs {
if dst_is_special {
let result = if suffix == NUL_MARK {
let mut sink = io::sink();
decompress_src_file(src_path, &mut sink, prefs, &mut resources)
} else {
let mut stdout = io::stdout();
decompress_src_file(src_path, &mut stdout, prefs, &mut resources)
};
match result {
Ok(n) => total_processed += n,
Err(_) => missing_files += 1,
}
} else {
if src_path.len() <= suffix.len() || !src_path.ends_with(suffix) {
display_level(
1,
&format!(
"File extension doesn't match expected LZ4_EXTENSION ({:4}); \
will not process file: {}\n",
suffix, src_path
),
);
skipped_files += 1;
continue;
}
let out_path = &src_path[..src_path.len() - suffix.len()];
match decompress_dst_file(src_path, out_path, prefs, &mut resources) {
Ok(n) => total_processed += n,
Err(_) => missing_files += 1,
}
}
}
final_time_display(time_start, cpu_start, total_processed);
let total_failures = missing_files + skipped_files;
if total_failures > 0 {
Err(io::Error::other(format!(
"{} file(s) could not be decompressed; {} file(s) skipped",
missing_files, skipped_files
)))
} else {
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::io::prefs::Prefs;
use std::io::{Cursor, Write};
fn make_frame_stream(data: &[u8]) -> Vec<u8> {
crate::frame::compress_frame_to_vec(data)
}
fn make_legacy_stream(data: &[u8]) -> Vec<u8> {
use crate::io::prefs::LEGACY_BLOCKSIZE;
let mut stream = Vec::new();
stream.extend_from_slice(&LEGACY_MAGICNUMBER.to_le_bytes());
for chunk in data.chunks(LEGACY_BLOCKSIZE) {
let compressed = crate::block::compress_block_to_vec(chunk);
stream.extend_from_slice(&(compressed.len() as u32).to_le_bytes());
stream.extend_from_slice(&compressed);
}
stream
}
#[test]
fn pass_through_copies_magic_and_data() {
let magic = [0x01, 0x02, 0x03, 0x04u8];
let payload = b"hello world payload";
let mut src = Cursor::new(payload.as_ref());
let mut dst = Vec::new();
let total = pass_through(&mut src, &mut dst, magic).expect("pass_through should succeed");
let mut expected = magic.to_vec();
expected.extend_from_slice(payload);
assert_eq!(dst, expected);
assert_eq!(total, (MAGICNUMBER_SIZE + payload.len()) as u64);
}
#[test]
fn pass_through_empty_payload() {
let magic = [0xAA, 0xBB, 0xCC, 0xDDu8];
let mut src = Cursor::new(b"" as &[u8]);
let mut dst = Vec::new();
let total = pass_through(&mut src, &mut dst, magic).expect("pass_through should succeed");
assert_eq!(dst, magic.as_ref());
assert_eq!(total, MAGICNUMBER_SIZE as u64);
}
#[test]
fn skip_stream_discards_bytes() {
let data = b"ABCDEFGHIJ";
let mut src = Cursor::new(data.as_ref());
skip_stream(&mut src, 5).expect("skip should succeed");
let mut remaining = Vec::new();
src.read_to_end(&mut remaining).unwrap();
assert_eq!(remaining, b"FGHIJ");
}
#[test]
fn skip_stream_zero_is_noop() {
let data = b"XYZ";
let mut src = Cursor::new(data.as_ref());
skip_stream(&mut src, 0).expect("skip 0 should succeed");
let mut remaining = Vec::new();
src.read_to_end(&mut remaining).unwrap();
assert_eq!(remaining, b"XYZ");
}
#[test]
fn skip_stream_exact_length() {
let data = b"HELLO";
let mut src = Cursor::new(data.as_ref());
skip_stream(&mut src, 5).expect("skip exact should succeed");
let mut remaining = Vec::new();
src.read_to_end(&mut remaining).unwrap();
assert!(remaining.is_empty());
}
#[test]
fn decompress_loop_frame_format() {
let original: Vec<u8> = (0u8..=255).cycle().take(4096).collect();
let frame_stream = make_frame_stream(&original);
let prefs = Prefs::default();
let mut resources = DecompressResources::new(&prefs).unwrap();
let mut src = Cursor::new(frame_stream);
let mut dst = Vec::new();
let bytes = decompress_loop(&mut src, &mut dst, &prefs, &mut resources)
.expect("frame decompress should succeed");
assert_eq!(bytes as usize, original.len());
assert_eq!(dst, original);
}
#[test]
fn decompress_loop_legacy_format() {
let original = b"Hello, legacy world!";
let legacy_stream = make_legacy_stream(original);
let prefs = Prefs::default();
let mut resources = DecompressResources::new(&prefs).unwrap();
let mut src = Cursor::new(legacy_stream);
let mut dst = Vec::new();
let bytes = decompress_loop(&mut src, &mut dst, &prefs, &mut resources)
.expect("legacy decompress should succeed");
assert_eq!(bytes as usize, original.len());
assert_eq!(dst.as_slice(), original.as_ref());
}
#[test]
fn decompress_loop_skippable_frame() {
let original = b"After skippable frame";
let mut stream = Vec::new();
stream.extend_from_slice(&LZ4IO_SKIPPABLE0.to_le_bytes()); let skip_payload = b"XXXXX"; stream.extend_from_slice(&(skip_payload.len() as u32).to_le_bytes()); stream.extend_from_slice(skip_payload);
stream.extend_from_slice(&make_frame_stream(original));
let prefs = Prefs::default();
let mut resources = DecompressResources::new(&prefs).unwrap();
let mut src = Cursor::new(stream);
let mut dst = Vec::new();
let bytes = decompress_loop(&mut src, &mut dst, &prefs, &mut resources)
.expect("skippable + frame should succeed");
assert_eq!(bytes as usize, original.len());
assert_eq!(dst.as_slice(), original.as_ref());
}
#[test]
fn decompress_loop_chained_frame_then_legacy() {
let part1 = b"Part one in frame format.";
let part2 = b"Part two in legacy format.";
let mut stream = Vec::new();
stream.extend_from_slice(&make_frame_stream(part1));
stream.extend_from_slice(&make_legacy_stream(part2));
let prefs = Prefs::default();
let mut resources = DecompressResources::new(&prefs).unwrap();
let mut src = Cursor::new(stream);
let mut dst = Vec::new();
let bytes = decompress_loop(&mut src, &mut dst, &prefs, &mut resources)
.expect("chained frames should succeed");
let mut expected = part1.to_vec();
expected.extend_from_slice(part2);
assert_eq!(bytes as usize, expected.len());
assert_eq!(dst, expected);
}
#[test]
fn decompress_loop_empty_input_returns_zero() {
let prefs = Prefs::default();
let mut resources = DecompressResources::new(&prefs).unwrap();
let mut src = Cursor::new(b"" as &[u8]);
let mut dst = Vec::new();
let bytes = decompress_loop(&mut src, &mut dst, &prefs, &mut resources)
.expect("empty input should succeed");
assert_eq!(bytes, 0);
assert!(dst.is_empty());
}
#[test]
fn decompress_loop_unrecognized_magic_first_frame_returns_error() {
let mut stream = Vec::new();
stream.extend_from_slice(&0xDEADBEEFu32.to_le_bytes());
let prefs = Prefs::default(); let mut resources = DecompressResources::new(&prefs).unwrap();
let mut src = Cursor::new(stream);
let mut dst = Vec::new();
let result = decompress_loop(&mut src, &mut dst, &prefs, &mut resources);
assert!(result.is_err(), "unrecognized magic must return error");
}
#[test]
fn decompress_loop_pass_through_first_frame() {
let magic: u32 = 0xDEADBEEF;
let payload = b"raw data payload";
let mut stream = Vec::new();
stream.extend_from_slice(&magic.to_le_bytes());
stream.extend_from_slice(payload);
let mut prefs = Prefs::default();
prefs.pass_through = true;
prefs.overwrite = true;
prefs.test_mode = false;
let mut resources = DecompressResources::new(&prefs).unwrap();
let mut src = Cursor::new(stream.clone());
let mut dst = Vec::new();
let bytes = decompress_loop(&mut src, &mut dst, &prefs, &mut resources)
.expect("pass-through should succeed");
assert_eq!(bytes as usize, stream.len());
assert_eq!(dst, stream);
}
#[test]
fn decompress_loop_corrupt_frame_returns_error() {
let mut stream = Vec::new();
stream.extend_from_slice(&LZ4IO_MAGICNUMBER.to_le_bytes());
stream.extend_from_slice(b"\xFF\xFF\xFF\xFF\xFF");
let prefs = Prefs::default();
let mut resources = DecompressResources::new(&prefs).unwrap();
let mut src = Cursor::new(stream);
let mut dst = Vec::new();
let result = decompress_loop(&mut src, &mut dst, &prefs, &mut resources);
assert!(result.is_err(), "corrupt frame must return error");
}
#[test]
fn sparse_writer_write_and_finish() {
let tmp = tempfile::NamedTempFile::new().unwrap();
let file = File::create(tmp.path()).unwrap();
let mut sw = SparseWriter::new(file, false);
let data = b"hello sparse";
write!(&mut sw, "{}", std::str::from_utf8(data).unwrap()).unwrap();
sw.finish().unwrap();
let written = fs::read(tmp.path()).unwrap();
assert_eq!(written, data);
}
#[test]
fn decompress_filename_frame_format_round_trip() {
let original: Vec<u8> = (0u8..=255).cycle().take(8192).collect();
let compressed = make_frame_stream(&original);
let src_dir = tempfile::tempdir().unwrap();
let dst_dir = tempfile::tempdir().unwrap();
let src_path = src_dir.path().join("input.lz4");
let dst_path = dst_dir.path().join("output.raw");
fs::write(&src_path, &compressed).unwrap();
let prefs = Prefs::default();
let stats = decompress_filename(
src_path.to_str().unwrap(),
dst_path.to_str().unwrap(),
&prefs,
)
.expect("decompress_filename should succeed");
let decompressed = fs::read(&dst_path).unwrap();
assert_eq!(decompressed, original);
assert_eq!(stats.decompressed_bytes as usize, original.len());
}
#[test]
fn decompress_filename_legacy_format_round_trip() {
let original = b"Legacy format round-trip test data";
let compressed = make_legacy_stream(original);
let src_dir = tempfile::tempdir().unwrap();
let dst_dir = tempfile::tempdir().unwrap();
let src_path = src_dir.path().join("input.lz4");
let dst_path = dst_dir.path().join("output.raw");
fs::write(&src_path, &compressed).unwrap();
let prefs = Prefs::default();
let stats = decompress_filename(
src_path.to_str().unwrap(),
dst_path.to_str().unwrap(),
&prefs,
)
.expect("decompress_filename should succeed");
let decompressed = fs::read(&dst_path).unwrap();
assert_eq!(decompressed.as_slice(), original.as_ref());
assert_eq!(stats.decompressed_bytes as usize, original.len());
}
#[test]
fn decompress_multiple_filenames_strips_suffix() {
let suffix = ".lz4";
let original = b"multiple filenames test";
let compressed = make_frame_stream(original);
let dir = tempfile::tempdir().unwrap();
let src = dir.path().join("file.raw.lz4");
let expected_dst = dir.path().join("file.raw");
fs::write(&src, &compressed).unwrap();
let prefs = Prefs::default();
let src_str = src.to_str().unwrap();
decompress_multiple_filenames(&[src_str], suffix, &prefs).expect("should succeed");
let decompressed = fs::read(&expected_dst).unwrap();
assert_eq!(decompressed.as_slice(), original.as_ref());
}
#[test]
fn decompress_multiple_filenames_skips_wrong_extension() {
let dir = tempfile::tempdir().unwrap();
let src = dir.path().join("file.gz"); fs::write(&src, b"not an lz4 file").unwrap();
let prefs = Prefs::default();
let src_str = src.to_str().unwrap();
let result = decompress_multiple_filenames(&[src_str], ".lz4", &prefs);
assert!(
result.is_err(),
"wrong-extension file should cause a skip error"
);
}
}