use std::fs;
use std::io::{self, Read, Seek, SeekFrom};
use std::sync::atomic::Ordering;
use crate::frame::types::{
BlockChecksum, BlockMode, BlockSizeId, ContentChecksum, FrameInfo as NativeFrameInfo,
FrameType as NativeFrameType,
};
use crate::frame::{lz4f_create_decompression_context, lz4f_get_frame_info, lz4f_header_size};
use crate::io::file_io::STDIN_MARK;
use crate::io::prefs::{
DISPLAY_LEVEL, LEGACY_MAGICNUMBER, LZ4IO_MAGICNUMBER, LZ4IO_SKIPPABLE0, LZ4IO_SKIPPABLEMASK,
MAGICNUMBER_SIZE, MB,
};
const LZ4F_HEADER_SIZE_MIN: usize = 7;
const LZ4F_HEADER_SIZE_MAX: usize = 19;
const LZ4F_BLOCK_HEADER_SIZE: usize = 4;
const LZ4F_BLOCK_CHECKSUM_SIZE: usize = 4;
const LZ4F_CONTENT_CHECKSUM_SIZE: usize = 4;
const LEGACY_BLOCK_HEADER_SIZE: usize = 4;
const LEGACY_BLOCK_SIZE_MAX: usize = 8 * MB;
const LZ4F_VERSION: u32 = 100;
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum FrameType {
Lz4Frame = 0,
LegacyFrame,
SkippableFrame,
}
impl FrameType {
fn name(self) -> &'static str {
match self {
FrameType::Lz4Frame => "LZ4Frame",
FrameType::LegacyFrame => "LegacyFrame",
FrameType::SkippableFrame => "SkippableFrame",
}
}
}
struct FrameInfo {
lz4_frame_info: NativeFrameInfo,
frame_type: FrameType,
}
impl FrameInfo {
fn new() -> Self {
FrameInfo {
lz4_frame_info: NativeFrameInfo {
block_size_id: BlockSizeId::Max64Kb,
block_mode: BlockMode::Linked,
content_checksum_flag: ContentChecksum::Disabled,
frame_type: NativeFrameType::Frame,
content_size: 0,
dict_id: 0,
block_checksum_flag: BlockChecksum::Disabled,
},
frame_type: FrameType::Lz4Frame,
}
}
}
pub struct CompressedFileInfo {
pub file_name: String,
pub file_size: u64,
pub frame_count: u64,
frame_summary: FrameInfo,
pub eq_frame_types: bool,
pub eq_block_types: bool,
pub all_content_size: bool,
}
impl CompressedFileInfo {
fn new() -> Self {
CompressedFileInfo {
file_name: String::new(),
file_size: 0,
frame_count: 0,
frame_summary: FrameInfo::new(),
eq_frame_types: true,
eq_block_types: true,
all_content_size: true,
}
}
}
#[derive(PartialEq, Eq, Debug)]
enum InfoResult {
Ok,
FormatNotKnown,
NotAFile,
}
#[inline]
fn read_le32(buf: &[u8]) -> u32 {
u32::from_le_bytes([buf[0], buf[1], buf[2], buf[3]])
}
#[inline]
fn is_skippable_magic_number(magic: u32) -> bool {
(magic & LZ4IO_SKIPPABLEMASK) == LZ4IO_SKIPPABLE0
}
fn skip_blocks_data(file: &mut fs::File, block_checksum: bool, content_checksum: bool) -> u64 {
let mut buf = [0u8; LZ4F_BLOCK_HEADER_SIZE];
let mut total: u64 = 0;
loop {
match file.read_exact(&mut buf) {
Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return total,
Err(_) => return 0,
Ok(_) => {}
}
total += LZ4F_BLOCK_HEADER_SIZE as u64;
let next_cblock_size = (read_le32(&buf) & 0x7FFF_FFFF) as u64;
let next_block = next_cblock_size
+ if block_checksum {
LZ4F_BLOCK_CHECKSUM_SIZE as u64
} else {
0
};
if next_cblock_size == 0 {
if content_checksum {
if file
.seek(SeekFrom::Current(LZ4F_CONTENT_CHECKSUM_SIZE as i64))
.is_err()
{
return 0;
}
total += LZ4F_CONTENT_CHECKSUM_SIZE as u64;
}
break;
}
total += next_block;
if file.seek(SeekFrom::Current(next_block as i64)).is_err() {
return 0;
}
}
total
}
const LEGACY_FRAME_UNDECODABLE: u64 = u64::MAX;
fn skip_legacy_blocks_data(file: &mut fs::File) -> u64 {
let mut buf = [0u8; LEGACY_BLOCK_HEADER_SIZE];
let mut total: u64 = 0;
loop {
let first = file.read(&mut buf[..1]);
match first {
Ok(0) => return total, Ok(1) => {}
Ok(_) => unreachable!(),
Err(_) => return LEGACY_FRAME_UNDECODABLE,
}
match file.read_exact(&mut buf[1..4]) {
Ok(()) => {}
Err(_) => return LEGACY_FRAME_UNDECODABLE,
}
let next_cblock_size = read_le32(&buf);
if next_cblock_size == LEGACY_MAGICNUMBER
|| next_cblock_size == LZ4IO_MAGICNUMBER
|| is_skippable_magic_number(next_cblock_size)
{
if file
.seek(SeekFrom::Current(-(LEGACY_BLOCK_HEADER_SIZE as i64)))
.is_err()
{
eprintln!("Error 37 : impossible to skip backward");
std::process::exit(37);
}
break;
}
if next_cblock_size as usize > LEGACY_BLOCK_SIZE_MAX {
if DISPLAY_LEVEL.load(Ordering::Relaxed) >= 4 {
eprintln!("Error : block in legacy frame is too large");
}
return LEGACY_FRAME_UNDECODABLE;
}
total += (LEGACY_BLOCK_HEADER_SIZE as u64) + (next_cblock_size as u64);
if file
.seek(SeekFrom::Current(next_cblock_size as i64))
.is_err()
{
return LEGACY_FRAME_UNDECODABLE;
}
}
total
}
pub fn block_type_id(size_id: &BlockSizeId, block_mode: &BlockMode) -> String {
let id_digit = match size_id {
BlockSizeId::Max64Kb | BlockSizeId::Default => b'4',
BlockSizeId::Max256Kb => b'5',
BlockSizeId::Max1Mb => b'6',
BlockSizeId::Max4Mb => b'7',
};
let mode_char = match block_mode {
BlockMode::Independent => b'I',
BlockMode::Linked => b'D',
};
String::from_utf8(vec![b'B', id_digit, mode_char]).unwrap()
}
fn to_human(mut size: f64) -> String {
const UNITS: &[char] = &['\0', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'];
let mut i = 0usize;
while size >= 1024.0 && i + 1 < UNITS.len() {
size /= 1024.0;
i += 1;
}
if UNITS[i] == '\0' {
format!("{:.2}", size)
} else {
format!("{:.2}{}", size, UNITS[i])
}
}
fn base_name(path: &str) -> &str {
path.rfind('/')
.or_else(|| path.rfind('\\'))
.map(|pos| &path[pos + 1..])
.unwrap_or(path)
}
#[cfg(unix)]
fn is_stdin_regular_file() -> bool {
use nix::sys::stat::fstat;
use std::os::unix::io::BorrowedFd;
match fstat(unsafe { BorrowedFd::borrow_raw(0) }) {
Ok(stat) => (stat.st_mode & 0o0170000) == 0o0100000,
Err(_) => false,
}
}
#[cfg(not(unix))]
fn is_stdin_regular_file() -> bool {
false
}
fn get_compressed_file_info(
cfinfo: &mut CompressedFileInfo,
path: &str,
display_now: bool,
) -> InfoResult {
let mut file = match fs::File::open(path) {
Ok(f) => f,
Err(e) => {
if DISPLAY_LEVEL.load(Ordering::Relaxed) >= 1 {
eprintln!("{}: {}", path, e);
}
return InfoResult::NotAFile;
}
};
cfinfo.file_size = file.metadata().map(|m| m.len()).unwrap_or(0);
let mut result = InfoResult::FormatNotKnown;
let mut buf = [0u8; LZ4F_HEADER_SIZE_MAX];
'frame_loop: loop {
let n = match file.read(&mut buf[..MAGICNUMBER_SIZE]) {
Ok(0) => break,
Ok(n) => n,
Err(_) => break,
};
result = InfoResult::FormatNotKnown;
if n != MAGICNUMBER_SIZE {
eprintln!("Error 40 : Unrecognized header : Magic Number unreadable");
std::process::exit(40);
}
let mut magic = read_le32(&buf[..4]);
if is_skippable_magic_number(magic) {
magic = LZ4IO_SKIPPABLE0; }
let mut frame_info = FrameInfo::new();
match magic {
LZ4IO_MAGICNUMBER => {
if cfinfo.frame_summary.frame_type != FrameType::Lz4Frame {
cfinfo.eq_frame_types = false;
}
{
let n2 = match file.read(&mut buf[MAGICNUMBER_SIZE..LZ4F_HEADER_SIZE_MIN]) {
Ok(n) => n,
Err(_) => {
eprintln!("Error 71 : Error reading {}", path);
std::process::exit(71);
}
};
if n2 == 0 {
eprintln!("Error 71 : Error reading {}", path);
std::process::exit(71);
}
}
let h_size_raw = match lz4f_header_size(&buf[..LZ4F_HEADER_SIZE_MIN]) {
Ok(n) => n,
Err(_) => break 'frame_loop,
};
let mut h_size = h_size_raw;
if h_size > LZ4F_HEADER_SIZE_MIN + MAGICNUMBER_SIZE {
let extra = h_size - LZ4F_HEADER_SIZE_MIN;
let end = LZ4F_HEADER_SIZE_MIN + extra;
let n3 = match file.read(&mut buf[LZ4F_HEADER_SIZE_MIN..end]) {
Ok(n) => n,
Err(_) => {
eprintln!("Error 72 : Error reading {}", path);
std::process::exit(72);
}
};
if n3 == 0 {
eprintln!("Error 72 : Error reading {}", path);
std::process::exit(72);
}
}
let mut dctx = match lz4f_create_decompression_context(LZ4F_VERSION) {
Ok(ctx) => ctx,
Err(_) => break 'frame_loop,
};
let (native_fi, consumed, _hint) =
match lz4f_get_frame_info(&mut dctx, &buf[..h_size]) {
Ok(t) => t,
Err(_) => break 'frame_loop,
};
frame_info.lz4_frame_info = native_fi;
h_size = consumed;
if cfinfo.frame_count != 0 {
let prev = &cfinfo.frame_summary.lz4_frame_info;
let curr = &frame_info.lz4_frame_info;
let size_changed = prev.block_size_id != curr.block_size_id;
let mode_changed = prev.block_mode != curr.block_mode;
if size_changed || mode_changed {
cfinfo.eq_block_types = false;
}
}
let block_checksum = matches!(
frame_info.lz4_frame_info.block_checksum_flag,
BlockChecksum::Enabled
);
let content_checksum = matches!(
frame_info.lz4_frame_info.content_checksum_flag,
ContentChecksum::Enabled
);
let total_blocks_size =
skip_blocks_data(&mut file, block_checksum, content_checksum);
if total_blocks_size != 0 {
let b_type = block_type_id(
&frame_info.lz4_frame_info.block_size_id,
&frame_info.lz4_frame_info.block_mode,
);
let checksum_str = if content_checksum { "XXH32" } else { "-" };
if display_now {
print!(
" {:>6} {:>14} {:>5} {:>8}",
cfinfo.frame_count + 1,
frame_info.frame_type.name(),
b_type,
checksum_str
);
}
if frame_info.lz4_frame_info.content_size != 0 {
let compressed = total_blocks_size + h_size as u64;
let uncompressed = frame_info.lz4_frame_info.content_size;
let ratio = compressed as f64 / uncompressed as f64 * 100.0;
if display_now {
println!(" {:>20} {:>20} {:>9.2}%", compressed, uncompressed, ratio);
}
frame_info.lz4_frame_info.content_size +=
cfinfo.frame_summary.lz4_frame_info.content_size;
} else {
if display_now {
println!(
" {:>20} {:>20} {:>9} ",
total_blocks_size + h_size as u64,
"-",
"-"
);
}
cfinfo.all_content_size = false;
}
result = InfoResult::Ok;
}
}
LEGACY_MAGICNUMBER => {
frame_info.frame_type = FrameType::LegacyFrame;
if cfinfo.frame_summary.frame_type != FrameType::LegacyFrame
&& cfinfo.frame_count != 0
{
cfinfo.eq_frame_types = false;
}
cfinfo.eq_block_types = false;
cfinfo.all_content_size = false;
let total_blocks_size = skip_legacy_blocks_data(&mut file);
if total_blocks_size == LEGACY_FRAME_UNDECODABLE {
if DISPLAY_LEVEL.load(Ordering::Relaxed) >= 1 {
eprintln!("Corrupted legacy frame");
}
result = InfoResult::FormatNotKnown;
break 'frame_loop;
}
if total_blocks_size != 0 {
if display_now {
println!(
" {:>6} {:>14} {:>5} {:>8} {:>20} {:>20} {:>9}",
cfinfo.frame_count + 1,
frame_info.frame_type.name(),
"-",
"-",
total_blocks_size + 4, "-",
"-"
);
}
result = InfoResult::Ok;
}
}
LZ4IO_SKIPPABLE0 => {
frame_info.frame_type = FrameType::SkippableFrame;
if cfinfo.frame_summary.frame_type != FrameType::SkippableFrame
&& cfinfo.frame_count != 0
{
cfinfo.eq_frame_types = false;
}
cfinfo.eq_block_types = false;
cfinfo.all_content_size = false;
let n = match file.read(&mut buf[..4]) {
Ok(n) => n,
Err(_) => {
eprintln!("Error 42 : Stream error : skippable size unreadable");
std::process::exit(42);
}
};
if n != 4 {
eprintln!("Error 42 : Stream error : skippable size unreadable");
std::process::exit(42);
}
let size = read_le32(&buf[..4]);
if file.seek(SeekFrom::Current(size as i64)).is_err() {
eprintln!("Error 43 : Stream error : cannot skip skippable area");
std::process::exit(43);
}
if display_now {
println!(
" {:>6} {:>14} {:>5} {:>8} {:>20} {:>20} {:>9}",
cfinfo.frame_count + 1,
"SkippableFrame",
"-",
"-",
size + 8, "-",
"-"
);
}
result = InfoResult::Ok;
}
_ => {
if DISPLAY_LEVEL.load(Ordering::Relaxed) >= 3 {
eprint!("Stream followed by undecodable data ");
if let Ok(pos) = file.stream_position() {
eprint!("at position {} ", pos);
}
eprintln!();
}
result = InfoResult::FormatNotKnown;
break 'frame_loop;
}
}
if result != InfoResult::Ok {
break 'frame_loop;
}
cfinfo.frame_summary = frame_info;
cfinfo.frame_count += 1;
}
result
}
pub fn display_compressed_files_info(paths: &[&str]) -> io::Result<()> {
let display_level = DISPLAY_LEVEL.load(Ordering::Relaxed);
if display_level < 3 {
println!(
"{:>10} {:>14} {:>5} {:>11} {:>13} {:>8} Filename",
"Frames", "Type", "Block", "Compressed", "Uncompressed", "Ratio"
);
}
for (idx, &path) in paths.iter().enumerate() {
let mut cfinfo = CompressedFileInfo::new();
cfinfo.file_name = base_name(path).to_owned();
let is_regular = if path == STDIN_MARK {
is_stdin_regular_file()
} else {
fs::metadata(path)
.map(|m| m.file_type().is_file())
.unwrap_or(false)
};
if !is_regular {
if DISPLAY_LEVEL.load(Ordering::Relaxed) >= 1 {
eprintln!("lz4: {} is not a regular file", path);
}
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!("{} is not a regular file", path),
));
}
if display_level >= 3 {
println!("{}({}/{})", cfinfo.file_name, idx + 1, paths.len());
println!(
" {:>6} {:>14} {:>5} {:>8} {:>20} {:>20} {:>9}",
"Frame", "Type", "Block", "Checksum", "Compressed", "Uncompressed", "Ratio"
);
}
let op_result = get_compressed_file_info(&mut cfinfo, path, display_level >= 3);
if op_result != InfoResult::Ok {
if DISPLAY_LEVEL.load(Ordering::Relaxed) >= 1 {
eprintln!("lz4: {}: File format not recognized", path);
}
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("{}: File format not recognized", path),
));
}
if display_level >= 3 {
println!();
}
if display_level < 3 {
let frame_type_str = if cfinfo.eq_frame_types {
cfinfo.frame_summary.frame_type.name()
} else {
"-"
};
let block_type_str = if cfinfo.eq_block_types {
block_type_id(
&cfinfo.frame_summary.lz4_frame_info.block_size_id,
&cfinfo.frame_summary.lz4_frame_info.block_mode,
)
} else {
"-".to_owned()
};
let compressed_str = to_human(cfinfo.file_size as f64);
let uncompressed_str = if cfinfo.all_content_size {
to_human(cfinfo.frame_summary.lz4_frame_info.content_size as f64)
} else {
"-".to_owned()
};
print!(
"{:>10} {:>14} {:>5} {:>11} {:>13} ",
cfinfo.frame_count,
frame_type_str,
block_type_str,
compressed_str,
uncompressed_str,
);
if cfinfo.all_content_size && cfinfo.frame_summary.lz4_frame_info.content_size != 0 {
let ratio = cfinfo.file_size as f64
/ cfinfo.frame_summary.lz4_frame_info.content_size as f64
* 100.0;
println!("{:>8.2}% {} ", ratio, cfinfo.file_name);
} else {
println!("{:>8} {}", "-", cfinfo.file_name);
}
}
}
Ok(())
}