use std::fs::File;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Container {
pub format: &'static str,
pub decompress: &'static str,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Compression {
Container(Container),
Filesystem {
allocated: u64,
len: u64,
},
}
impl std::fmt::Display for Compression {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Container(c) => write!(
f,
"the file is a {} stream, not raw GGUF — mapping it maps the compressed \
bytes, so no tensor can be read in place. Decompress it first ({})",
c.format, c.decompress
),
Self::Filesystem { allocated, len } => write!(
f,
"the file occupies {} MiB on disk for {} MiB of content, so the filesystem \
is storing it compressed (or it is sparse). The mapping still works, but \
every page fault has to decompress a block instead of handing back a \
shared page-cache page, which makes loading and inference far slower. \
Store the model uncompressed (e.g. `chattr -c`, btrfs `compress=no`, or \
`cp` it onto an uncompressed filesystem)",
allocated / (1024 * 1024),
len / (1024 * 1024)
),
}
}
}
pub const GGUF_MAGIC: &[u8] = b"GGUF";
const HEAD_BYTES: usize = 8;
const CONTAINERS: &[(&[u8], Container)] = &[
(
b"\x1f\x8b",
Container {
format: "gzip",
decompress: "gunzip",
},
),
(
b"\x28\xb5\x2f\xfd",
Container {
format: "zstd",
decompress: "unzstd",
},
),
(
b"\xfd7zXZ\x00",
Container {
format: "xz",
decompress: "unxz",
},
),
(
b"BZh",
Container {
format: "bzip2",
decompress: "bunzip2",
},
),
(
b"\x04\x22\x4d\x18",
Container {
format: "lz4",
decompress: "unlz4",
},
),
(
b"\x5d\x00\x00",
Container {
format: "lzma",
decompress: "unlzma",
},
),
(
b"\x1f\x9d",
Container {
format: "compress(1) .Z",
decompress: "uncompress",
},
),
(
b"PK\x03\x04",
Container {
format: "zip archive",
decompress: "unzip",
},
),
(
b"PK\x05\x06",
Container {
format: "zip archive",
decompress: "unzip",
},
),
(
b"7z\xbc\xaf\x27\x1c",
Container {
format: "7-Zip archive",
decompress: "7z x",
},
),
];
fn is_zlib(head: &[u8]) -> bool {
match head {
[cmf, flg, ..] => cmf & 0x0f == 8 && (u16::from(*cmf) * 256 + u16::from(*flg)) % 31 == 0,
_ => false,
}
}
const MIN_FILESYSTEM_CHECK_BYTES: u64 = 8 * 1024 * 1024;
const FILESYSTEM_RATIO_NUM: u64 = 9;
const FILESYSTEM_RATIO_DEN: u64 = 10;
pub fn detect(file: &File, expected_magic: Option<&[u8]>) -> Option<Compression> {
if let Some(container) = detect_container(file, expected_magic) {
return Some(Compression::Container(container));
}
detect_filesystem(file)
}
pub fn detect_gguf(file: &File) -> Option<Compression> {
detect(file, Some(GGUF_MAGIC))
}
fn read_head(file: &File) -> Option<(usize, [u8; HEAD_BYTES])> {
let mut head = [0u8; HEAD_BYTES];
let mut filled = 0;
while filled < head.len() {
#[cfg(unix)]
let n = {
use std::os::unix::fs::FileExt;
file.read_at(&mut head[filled..], filled as u64).ok()?
};
#[cfg(windows)]
let n = {
use std::os::windows::fs::FileExt;
file.seek_read(&mut head[filled..], filled as u64).ok()?
};
#[cfg(not(any(unix, windows)))]
let n = {
use std::io::{Seek, SeekFrom};
let mut f = file;
f.seek(SeekFrom::Start(filled as u64)).ok()?;
f.read(&mut head[filled..]).ok()?
};
if n == 0 {
break;
}
filled += n;
}
Some((filled, head))
}
fn detect_container(file: &File, expected_magic: Option<&[u8]>) -> Option<Container> {
let (n, head) = read_head(file)?;
let head = &head[..n];
if expected_magic.is_some_and(|m| head.starts_with(m)) {
return None;
}
for (magic, container) in CONTAINERS {
if head.starts_with(magic) {
return Some(*container);
}
}
if expected_magic.is_some() && is_zlib(head) {
return Some(Container {
format: "zlib",
decompress: "e.g. `pigz -dz`",
});
}
None
}
#[cfg(unix)]
fn detect_filesystem(file: &File) -> Option<Compression> {
use std::os::unix::fs::MetadataExt;
let md = file.metadata().ok()?;
let len = md.len();
if len < MIN_FILESYSTEM_CHECK_BYTES {
return None;
}
let allocated = md.blocks().checked_mul(512)?;
if allocated == 0 {
return None;
}
if allocated.saturating_mul(FILESYSTEM_RATIO_DEN) < len.saturating_mul(FILESYSTEM_RATIO_NUM) {
return Some(Compression::Filesystem { allocated, len });
}
None
}
#[cfg(not(unix))]
fn detect_filesystem(_file: &File) -> Option<Compression> {
None
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
fn temp_file(name: &str, bytes: &[u8]) -> File {
let path =
std::env::temp_dir().join(format!("joshua-compression-{}-{name}", std::process::id()));
let mut f = File::create(&path).expect("create temp file");
f.write_all(bytes).expect("write temp file");
drop(f);
let opened = File::open(&path).expect("open temp file");
let _ = std::fs::remove_file(&path); opened
}
#[test]
fn plain_gguf_is_not_flagged() {
let f = temp_file("gguf", b"GGUF\x03\x00\x00\x00rest of the header");
assert_eq!(detect_gguf(&f), None);
}
#[test]
fn gzip_is_detected() {
let f = temp_file("gz", b"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03payload");
match detect_gguf(&f) {
Some(Compression::Container(c)) => assert_eq!(c.format, "gzip"),
other => panic!("expected gzip, got {other:?}"),
}
}
#[test]
fn common_containers_are_detected() {
let cases: &[(&str, &[u8], &str)] = &[
("zstd", b"\x28\xb5\x2f\xfd\x00\x00\x00\x00", "zstd"),
("xz", b"\xfd7zXZ\x00\x00\x00", "xz"),
("bz2", b"BZh9payload", "bzip2"),
("lz4", b"\x04\x22\x4d\x18payload", "lz4"),
("zip", b"PK\x03\x04payload", "zip archive"),
("7z", b"7z\xbc\xaf\x27\x1c\x00\x00", "7-Zip archive"),
("Z", b"\x1f\x9dpayload!", "compress(1) .Z"),
("zlib", b"\x78\x9cpayload!", "zlib"),
];
for (name, bytes, expected) in cases {
let f = temp_file(name, bytes);
match detect_gguf(&f) {
Some(Compression::Container(c)) => assert_eq!(&c.format, expected, "for {name}"),
other => panic!("expected {expected} for {name}, got {other:?}"),
}
}
}
#[test]
fn unrecognised_and_tiny_files_are_left_alone() {
assert_eq!(detect_gguf(&temp_file("junk", b"not a model at all")), None);
assert_eq!(detect_gguf(&temp_file("tiny", b"G")), None);
assert_eq!(detect_gguf(&temp_file("empty", b"")), None);
}
#[test]
fn zlib_heuristic_does_not_fire_on_gguf_or_noise() {
assert!(!is_zlib(b"GGUF"));
assert!(!is_zlib(b"\x79\x9b"));
assert!(is_zlib(b"\x78\x01"));
assert!(is_zlib(b"\x78\xda"));
}
#[test]
fn zlib_heuristic_needs_a_known_magic_to_contradict() {
let f = temp_file("st", b"\x78\x9c\x00\x00\x00\x00\x00\x00{\"__met");
assert_eq!(detect(&f, None), None);
let f = temp_file("st2", b"\x78\x9c\x00\x00\x00\x00\x00\x00{\"__met");
assert!(matches!(
detect(&f, Some(GGUF_MAGIC)),
Some(Compression::Container(_))
));
}
#[cfg(unix)]
#[test]
fn sparse_file_trips_the_allocation_ratio() {
let path =
std::env::temp_dir().join(format!("joshua-compression-{}-sparse", std::process::id()));
let mut f = File::create(&path).expect("create sparse file");
f.write_all(&[0x5Au8; 64 * 1024]).expect("write prefix");
f.set_len(64 * 1024 * 1024).expect("set_len");
drop(f);
let opened = File::open(&path).expect("open sparse file");
let found = detect_gguf(&opened);
let _ = std::fs::remove_file(&path);
match found {
Some(Compression::Filesystem { allocated, len }) => {
assert_eq!(len, 64 * 1024 * 1024);
assert!(allocated < len, "allocated {allocated} len {len}");
}
None => {}
other => panic!("unexpected {other:?}"),
}
}
#[cfg(unix)]
#[test]
fn fully_allocated_file_is_not_flagged() {
let mut state = 0x2545_F491_4F6C_DD1Du64;
let bytes: Vec<u8> = (0..12 * 1024 * 1024)
.map(|_| {
state ^= state << 13;
state ^= state >> 7;
state ^= state << 17;
state as u8
})
.collect();
let f = temp_file("dense", &bytes);
assert_eq!(detect_gguf(&f), None);
}
#[test]
fn messages_name_the_problem() {
let container = Compression::Container(Container {
format: "gzip",
decompress: "gunzip",
});
let msg = container.to_string();
assert!(msg.contains("gzip") && msg.contains("gunzip"), "{msg}");
let fs = Compression::Filesystem {
allocated: 1024 * 1024,
len: 64 * 1024 * 1024,
};
let msg = fs.to_string();
assert!(msg.contains("1 MiB") && msg.contains("64 MiB"), "{msg}");
}
}