limnifs_write/file_categorizer/
executable.rs

1//! Executable categorizer — routes ELF / PE / Mach-O binaries to
2//! BCJ-x86 or BCJ-ARM64 composite codecs.
3//!
4//! ## Algorithm
5//!
6//! 1. Detect executable format by magic bytes at offset 0:
7//!    - ELF: `\x7FELF`
8//!    - PE:  `MZ` then PE header at offset stored at 0x3C
9//!    - Mach-O: `0xFEEDFACE`, `0xFEEDFACF`, `0xCEFAEDFE`, `0xCFFAEDFE`
10//! 2. Read architecture from the format-specific header field:
11//!    - ELF: `e_machine` at offset 18 (u16 LE)
12//!      - `0x3E` = x86_64 → BCJ-x86
13//!      - `0xB7` = aarch64 → BCJ-ARM64
14//!    - PE: `Machine` at offset 0 in PE header (u16 LE)
15//!      - `0x8664` = x86_64 → BCJ-x86
16//!      - `0xAA64` = aarch64 → BCJ-ARM64
17//!    - Mach-O: `cputype` at offset 4 (u32 LE for native-endian magic)
18//!      - `0x01000007` = x86_64 → BCJ-x86
19//!      - `0x0100000C` = arm64 → BCJ-ARM64
20//! 3. Pick the LZ4 variant for write-heavy profiles (faster encode)
21//!    and the ZSTD variant otherwise. We pick LZ4 by default for
22//!    speed; the tournament in `process_whole_file_drop` will
23//!    re-evaluate against ZSTD and pick the smaller result.
24//!
25//! ## Coverage
26//!
27//! All four major executable formats × the two architectures that
28//! have a BCJ filter implementation in omnizip-filters today. 32-bit
29//! ARM (no BCJ filter published), PowerPC, SPARC, IA-64 routes to
30//! plain LZ4 (no benefit from a filter we don't have).
31
32use std::path::Path;
33
34use super::{Categorization, FileCategorizer};
35use limnifs_core::codec::{CODEC_BCJ_ARM64_LZ4, CODEC_BCJ_X86_LZ4};
36
37/// Minimum size worth running through BCJ + categorizer overhead.
38const MIN_EXEC_SIZE: usize = 1024;
39
40/// ELF magic.
41const ELF_MAGIC: [u8; 4] = [0x7F, b'E', b'L', b'F'];
42/// DOS/PE magic (MZ).
43const DOS_MAGIC: [u8; 2] = [b'M', b'Z'];
44/// Mach-O magics (4 bytes each, big- and little-endian).
45const MACHO_MAGICS: &[[u8; 4]] = &[
46    [0xFE, 0xED, 0xFA, 0xCE], // 32-bit native-endian
47    [0xFE, 0xED, 0xFA, 0xCF], // 64-bit native-endian
48    [0xCE, 0xFA, 0xED, 0xFE], // 32-bit swapped-endian
49    [0xCF, 0xFA, 0xED, 0xFE], // 64-bit swapped-endian
50];
51
52/// ELF e_machine values we route.
53const EM_X86_64: u16 = 0x3E;
54const EM_AARCH64: u16 = 0xB7;
55
56/// PE Machine values we route.
57const PE_MACHINE_AMD64: u16 = 0x8664;
58const PE_MACHINE_ARM64: u16 = 0xAA64;
59
60/// Mach-O cputype values we route (CPU_TYPE_x86_64, CPU_TYPE_ARM64).
61const CPU_TYPE_X86_64: u32 = 0x0100_0007;
62const CPU_TYPE_ARM64: u32 = 0x0100_000C;
63
64/// Pick a codec for the given executable bytes, or `None` if not a
65/// recognized executable format.
66fn pick_codec(data: &[u8]) -> Option<u8> {
67    if data.len() < MIN_EXEC_SIZE {
68        return None;
69    }
70
71    if data.starts_with(&ELF_MAGIC) {
72        return parse_elf(data);
73    }
74    if data.starts_with(&DOS_MAGIC) {
75        return parse_pe(data);
76    }
77    if MACHO_MAGICS.iter().any(|m| data.starts_with(m)) {
78        return parse_macho(data);
79    }
80    None
81}
82
83fn parse_elf(data: &[u8]) -> Option<u8> {
84    // ELF64 header layout: e_ident[16], e_type(2), e_machine(2), ...
85    // e_machine is at offset 18, u16 LE on little-endian ELF
86    // (EI_DATA byte at offset 5; we assume LE for now, which covers
87    // every modern x86_64 / aarch64 binary).
88    if data.len() < 20 {
89        return None;
90    }
91    let machine = u16::from_le_bytes([data[18], data[19]]);
92    route_by_arch(machine == EM_X86_64, machine == EM_AARCH64)
93}
94
95fn parse_pe(data: &[u8]) -> Option<u8> {
96    // PE: e_lfanew at offset 0x3C (u32 LE) points to PE\0\0 header.
97    // PE header: signature(4) + machine(2).
98    if data.len() < 0x40 {
99        return None;
100    }
101    let lfanew = u32::from_le_bytes([data[0x3C], data[0x3D], data[0x3E], data[0x3F]]) as usize;
102    let pe_off = lfanew.checked_add(4)?;
103    if data.len() < pe_off + 2 {
104        return None;
105    }
106    // Confirm PE signature.
107    if &data[lfanew..lfanew + 4] != b"PE\0\0" {
108        return None;
109    }
110    let machine = u16::from_le_bytes([data[pe_off], data[pe_off + 1]]);
111    route_by_arch(machine == PE_MACHINE_AMD64, machine == PE_MACHINE_ARM64)
112}
113
114fn parse_macho(data: &[u8]) -> Option<u8> {
115    // Mach-O: magic(4), cputype(4), cpusubtype(4), filetype(4), ...
116    // cputype interpretation depends on magic endianness, but the
117    // constant value is the same in the file's native byte order.
118    if data.len() < 12 {
119        return None;
120    }
121    let magic = [data[0], data[1], data[2], data[3]];
122    let is_le = matches!(magic, [0xFE, 0xED, 0xFA, 0xCE] | [0xFE, 0xED, 0xFA, 0xCF]);
123    let cputype = if is_le {
124        u32::from_le_bytes([data[4], data[5], data[6], data[7]])
125    } else {
126        u32::from_be_bytes([data[4], data[5], data[6], data[7]])
127    };
128    route_by_arch(cputype == CPU_TYPE_X86_64, cputype == CPU_TYPE_ARM64)
129}
130
131fn route_by_arch(x86_64: bool, arm64: bool) -> Option<u8> {
132    if x86_64 {
133        Some(CODEC_BCJ_X86_LZ4)
134    } else if arm64 {
135        Some(CODEC_BCJ_ARM64_LZ4)
136    } else {
137        None
138    }
139}
140
141/// Categorizer for executable binaries. Detects ELF / PE / Mach-O
142/// magic and routes x86_64 / aarch64 architectures to BCJ-x86 /
143/// BCJ-ARM64 composite codecs.
144pub struct ExecutableCategorizer;
145
146impl FileCategorizer for ExecutableCategorizer {
147    fn name(&self) -> &'static str {
148        "executable"
149    }
150
151    fn categories(&self) -> &'static [&'static str] {
152        &["binary/executable"]
153    }
154
155    fn first_byte_hint(&self) -> Option<&'static [u8]> {
156        Some(&[0x7F, b'M', 0xFE, 0xCE, 0xCF])
157    }
158
159    fn categorize(&self, _path: &Path, data: &[u8]) -> Option<Categorization> {
160        let codec_id = pick_codec(data)?;
161        Some(Categorization {
162            codec_id,
163            codec_params: Vec::new(),
164            category: "binary/executable",
165        })
166    }
167}
168
169#[cfg(test)]
170mod tests {
171    use super::*;
172
173    fn elf_x86_64() -> Vec<u8> {
174        // Minimal ELF64 header prefix with e_machine = EM_X86_64,
175        // padded to MIN_EXEC_SIZE so the categorizer accepts it.
176        let mut v = vec![0u8; MIN_EXEC_SIZE];
177        v[0..4].copy_from_slice(&ELF_MAGIC);
178        v[4] = 2; // EI_CLASS = ELFCLASS64
179        v[5] = 1; // EI_DATA = ELFDATA2LSB
180        v[16..18].copy_from_slice(&2u16.to_le_bytes()); // e_type = ET_EXEC
181        v[18..20].copy_from_slice(&EM_X86_64.to_le_bytes());
182        v
183    }
184
185    fn elf_aarch64() -> Vec<u8> {
186        let mut v = elf_x86_64();
187        v[18..20].copy_from_slice(&EM_AARCH64.to_le_bytes());
188        v
189    }
190
191    fn elf_unknown_arch() -> Vec<u8> {
192        let mut v = elf_x86_64();
193        v[18..20].copy_from_slice(&0x1234u16.to_le_bytes());
194        v
195    }
196
197    #[test]
198    fn detects_elf_x86_64() {
199        let data = elf_x86_64();
200        assert_eq!(
201            pick_codec(&data),
202            Some(CODEC_BCJ_X86_LZ4),
203            "ELF x86_64 should route to BCJ-x86+LZ4"
204        );
205    }
206
207    #[test]
208    fn detects_elf_aarch64() {
209        let data = elf_aarch64();
210        assert_eq!(
211            pick_codec(&data),
212            Some(CODEC_BCJ_ARM64_LZ4),
213            "ELF aarch64 should route to BCJ-ARM64+LZ4"
214        );
215    }
216
217    #[test]
218    fn unknown_arch_returns_none() {
219        // Unknown architecture should NOT route — caller falls back
220        // to plain FastCDC. BCJ on a different arch would corrupt
221        // the binary.
222        let data = elf_unknown_arch();
223        assert_eq!(pick_codec(&data), None);
224    }
225
226    #[test]
227    fn small_input_returns_none() {
228        let mut v = elf_x86_64();
229        v.truncate(100); // below MIN_EXEC_SIZE
230        assert_eq!(pick_codec(&v), None);
231    }
232
233    #[test]
234    fn pe_x86_64_routes_correctly() {
235        // Construct a minimal DOS + PE header for x86_64.
236        let mut v = vec![0u8; MIN_EXEC_SIZE];
237        v[0..2].copy_from_slice(&DOS_MAGIC);
238        let pe_offset: u32 = 0x40;
239        v[0x3C..0x40].copy_from_slice(&pe_offset.to_le_bytes());
240        v[0x40..0x44].copy_from_slice(b"PE\0\0");
241        v[0x44..0x46].copy_from_slice(&PE_MACHINE_AMD64.to_le_bytes());
242        assert_eq!(pick_codec(&v), Some(CODEC_BCJ_X86_LZ4));
243    }
244
245    #[test]
246    fn macho_x86_64_routes_correctly() {
247        let mut v = vec![0u8; MIN_EXEC_SIZE];
248        v[0..4].copy_from_slice(&[0xFE, 0xED, 0xFA, 0xCF]); // MH_MAGIC_64
249        v[4..8].copy_from_slice(&CPU_TYPE_X86_64.to_le_bytes());
250        assert_eq!(pick_codec(&v), Some(CODEC_BCJ_X86_LZ4));
251    }
252
253    #[test]
254    fn non_executable_returns_none() {
255        assert_eq!(pick_codec(b"hello world text not exec"), None);
256        assert_eq!(pick_codec(&vec![0u8; 4096]), None);
257    }
258}
limnifs_write/file_categorizer/executable.rs

limnifs_write/file_categorizer/
executable.rs
