use std::str::from_utf8;
use phf::phf_set;
use super::{
Codec, Result,
block::{MAX_BLOCK, SLACK},
};
const PROBE_HEAD: u32 = 2;
const PROBE_EVERY: u32 = 64;
const PROBE_SAMPLE: usize = 16 * 1024;
const EXT_MAX: usize = 12;
static SKIP_EXT: phf::Set<&'static str> = phf_set! {
"zip", "gz", "tgz", "zst", "xz", "7z", "rar", "bz2", "lz4", "br",
"jpg", "jpeg", "png", "gif", "webp", "avif", "heic",
"mp4", "mkv", "mov", "avi", "webm", "flv",
"mp3", "aac", "flac", "ogg", "opus",
"woff", "woff2", "ttf", "otf",
"docx", "xlsx", "pptx", "pdf",
"onnx", "safetensors", "gguf", "parquet",
};
static MAGICS: &[&[u8]] = &[
b"\x89PNG\r\n\x1a\n",
b"\xff\xd8\xff",
b"PK\x03\x04",
b"\x1f\x8b",
b"\x28\xb5\x2f\xfd",
b"\xfd7zXZ\x00",
b"7z\xbc\xaf\x27\x1c",
b"Rar!\x1a\x07",
b"\x00\x00\x00\x18ftyp",
b"\x00\x00\x00\x20ftyp",
b"OggS",
b"fLaC",
];
pub fn skip_name(path: &str) -> bool {
let Some((_, ext)) = path.rsplit_once('.') else {
return false;
};
let bytes = ext.as_bytes();
if bytes.is_empty() || bytes.len() > EXT_MAX {
return false;
}
let mut buf = [0u8; EXT_MAX];
for (dst, &src) in buf.iter_mut().zip(bytes) {
*dst = src.to_ascii_lowercase();
}
from_utf8(&buf[..bytes.len()]).is_ok_and(|s| SKIP_EXT.contains(s))
}
pub fn skip_magic(head: &[u8]) -> bool {
MAGICS.iter().any(|m| head.starts_with(m))
}
pub struct Detector {
skip_name: bool,
magic_checked: bool,
skip: bool,
bad: u32,
blocks: u32,
scratch: Box<[u8]>,
}
impl Detector {
pub fn new(path: &str) -> Self {
Self::with_flags(skip_name(path), true)
}
pub fn with_flags(skip_name: bool, check_magic: bool) -> Self {
Self {
skip_name,
magic_checked: !check_magic,
skip: false,
bad: PROBE_HEAD - 1,
blocks: 0,
scratch: if skip_name {
Box::new([])
} else {
vec![0u8; MAX_BLOCK + SLACK].into_boxed_slice()
},
}
}
pub const fn skip(&self) -> bool {
self.skip
}
pub const fn reset_file(&mut self) {
self.skip = false;
self.bad = 0;
}
pub fn want(&mut self, raw: &[u8], codec: &mut Codec) -> Result<bool> {
Ok(self.compress(raw, codec)?.is_some())
}
pub fn compress(&mut self, raw: &[u8], codec: &mut Codec) -> Result<Option<&[u8]>> {
if self.skip_name {
return Ok(None);
}
self.blocks += 1;
if !self.magic_checked {
self.magic_checked = true;
if skip_magic(raw) {
self.skip = true;
return Ok(None);
}
}
if self.skip && self.blocks > PROBE_HEAD && !self.blocks.is_multiple_of(PROBE_EVERY) {
return Ok(None);
}
if self.bad > 0
&& raw.len() > PROBE_SAMPLE
&& codec
.try_compress_block(&raw[..PROBE_SAMPLE], &mut self.scratch)?
.is_none()
{
return Ok(self.on_bad());
}
if let Some(len) = codec.try_compress_block(raw, &mut self.scratch)? {
self.bad = 0;
self.skip = false;
Ok(Some(&self.scratch[..len]))
} else {
Ok(self.on_bad())
}
}
const fn on_bad(&mut self) -> Option<&[u8]> {
self.bad += 1;
if self.bad >= PROBE_HEAD {
self.skip = true;
}
None
}
}