libmagic_rs/parser/loader.rs
1// Copyright (c) 2025-2026 the libmagic-rs contributors
2// SPDX-License-Identifier: Apache-2.0
3
4//! File and directory loading for magic files.
5//!
6//! Provides functions for loading magic rules from individual files and
7//! directories, with automatic format detection and error handling.
8
9use log::warn;
10
11use crate::error::ParseError;
12use crate::parser::ParsedMagic;
13use crate::parser::name_table::NameTable;
14use std::path::{Path, PathBuf};
15
16use super::format::{MagicFileFormat, detect_format};
17
18/// Maximum magic file size (1 GB).
19///
20/// Applied before loading a magic file (or any file within a magic directory)
21/// into memory to prevent memory-exhaustion `DoS` from maliciously oversized
22/// inputs.
23///
24/// This value is kept in sync with `crate::io::FileBuffer::MAX_FILE_SIZE`.
25/// The constant is duplicated (rather than imported) because this module is
26/// also pulled in by `build.rs` via `#[path]` and the build script cannot
27/// reference lib-only modules such as `crate::io`. A unit test below asserts
28/// the two constants remain equal.
29pub const MAX_MAGIC_FILE_SIZE: u64 = 1024 * 1024 * 1024;
30
31/// Reads a magic file into a `String` after verifying its size does not
32/// exceed [`MAX_MAGIC_FILE_SIZE`].
33///
34/// Returns a `ParseError` if metadata cannot be read, the file exceeds the
35/// size limit, or the file contents cannot be read.
36///
37/// # Encoding
38///
39/// Magic files are parsed as byte streams (matching GNU `file`/libmagic
40/// behavior). Real-world magic files frequently contain non-UTF-8 bytes in
41/// comments and attribution lines (e.g., Latin-1 author names). Rather than
42/// rejecting such files, invalid UTF-8 sequences are replaced with U+FFFD
43/// via [`String::from_utf8_lossy`] and a warning is logged. ASCII rule
44/// syntax is preserved byte-for-byte; replacements only affect non-ASCII
45/// text which, in practice, appears almost exclusively inside comments
46/// that are stripped before tokenization.
47fn read_magic_file_bounded(path: &Path) -> Result<String, ParseError> {
48 let metadata = std::fs::metadata(path).map_err(|e| {
49 ParseError::IoError(std::io::Error::new(
50 e.kind(),
51 format!("Failed to read metadata for '{}': {}", path.display(), e),
52 ))
53 })?;
54
55 if metadata.len() > MAX_MAGIC_FILE_SIZE {
56 return Err(ParseError::invalid_syntax(
57 0,
58 format!(
59 "Magic file '{}' is too large: {} bytes (maximum allowed: {} bytes)",
60 path.display(),
61 metadata.len(),
62 MAX_MAGIC_FILE_SIZE
63 ),
64 ));
65 }
66
67 let bytes = std::fs::read(path).map_err(ParseError::from)?;
68
69 match String::from_utf8(bytes) {
70 Ok(s) => Ok(s),
71 Err(e) => {
72 warn!(
73 "Magic file '{}' contains non-UTF-8 bytes; they were replaced with U+FFFD. \
74 Rule parsing proceeds, but replacements inside rule bodies may alter matching.",
75 path.display()
76 );
77 Ok(String::from_utf8_lossy(&e.into_bytes()).into_owned())
78 }
79 }
80}
81
82/// Whether `contents` contains at least one actual rule line -- a non-blank
83/// line that is neither a comment (`#`) nor a `!:` metadata directive
84/// (`!:mime`, `!:strength`, ...). Used by [`load_magic_directory`] to tell a
85/// file whose rules were all skipped as unparseable (unusable) apart from a
86/// genuinely empty, comment-only, or directive-only file (valid, contributes
87/// no rules). `!:` directives are stripped during preprocessing and are not
88/// rules, so counting them here would misclassify a directive-only file as
89/// "had rules but all were skipped".
90fn has_rule_lines(contents: &str) -> bool {
91 contents.lines().any(|line| {
92 let trimmed = line.trim();
93 !trimmed.is_empty() && !trimmed.starts_with('#') && !trimmed.starts_with("!:")
94 })
95}
96
97/// Loads and parses all magic files from a directory, merging them into a single rule set.
98///
99/// This function reads all regular files in the specified directory, parses each as a magic file,
100/// and combines the resulting rules into a single `Vec<MagicRule>`. Files are processed in
101/// alphabetical order by filename to ensure deterministic results.
102///
103/// # Error Handling Strategy
104///
105/// This function distinguishes between critical and non-critical errors:
106///
107/// - **Critical errors** (I/O failures, directory access issues, encoding problems):
108/// These cause immediate failure and return a `ParseError`. The function stops processing
109/// and propagates the error to the caller.
110///
111/// - **Non-critical errors** (individual file parse failures):
112/// These are logged at warn level and the file is skipped. Processing
113/// continues with remaining files.
114///
115/// # Behavior
116///
117/// - Subdirectories are skipped (not recursively processed)
118/// - Symbolic links are skipped
119/// - Empty directories return an empty rules vector
120/// - Files are processed in alphabetical order by filename
121/// - All successfully parsed rules are merged in order
122///
123/// # Examples
124///
125/// Loading a directory of magic files:
126///
127/// ```rust,no_run
128/// use libmagic_rs::parser::load_magic_directory;
129/// use std::path::Path;
130///
131/// let parsed = load_magic_directory(Path::new("/usr/share/file/magic.d"))?;
132/// println!("Loaded {} rules from directory", parsed.rules.len());
133/// # Ok::<(), libmagic_rs::ParseError>(())
134/// ```
135///
136/// Creating a Magdir-style directory structure:
137///
138/// ```rust,no_run
139/// use libmagic_rs::parser::load_magic_directory;
140/// use std::path::Path;
141///
142/// // Directory structure:
143/// // magic.d/
144/// // ├── 01-elf
145/// // ├── 02-archive
146/// // └── 03-text
147///
148/// let parsed = load_magic_directory(Path::new("./magic.d"))?;
149/// // Rules from all three files are merged in alphabetical order
150/// # Ok::<(), libmagic_rs::ParseError>(())
151/// ```
152///
153/// # Errors
154///
155/// Returns `ParseError` if:
156/// - The directory does not exist or cannot be accessed
157/// - Directory entries cannot be read
158/// - A file cannot be read due to I/O errors
159/// - A file contains invalid UTF-8 encoding
160///
161/// # Panics
162///
163/// This function does not panic under normal operation.
164// Directory iteration + per-file parse + error aggregation runs slightly over
165// the 100-line lint; splitting this module is tracked in #391.
166#[allow(clippy::too_many_lines)]
167pub fn load_magic_directory(dir_path: &Path) -> Result<ParsedMagic, ParseError> {
168 use std::fs;
169
170 // Read directory entries
171 let entries = fs::read_dir(dir_path).map_err(|e| {
172 ParseError::invalid_syntax(
173 0,
174 format!("Failed to read directory '{}': {}", dir_path.display(), e),
175 )
176 })?;
177
178 // Collect and sort entries by filename for deterministic ordering
179 let mut file_paths: Vec<std::path::PathBuf> = Vec::new();
180 for entry in entries {
181 let entry = entry.map_err(|e| {
182 ParseError::invalid_syntax(
183 0,
184 format!(
185 "Failed to read directory entry in '{}': {}",
186 dir_path.display(),
187 e
188 ),
189 )
190 })?;
191
192 let path = entry.path();
193 let file_type = entry.file_type().map_err(|e| {
194 ParseError::invalid_syntax(
195 0,
196 format!("Failed to read file type for '{}': {}", path.display(), e),
197 )
198 })?;
199
200 // Only process regular files, skip directories and symlinks.
201 // `is_file()` (not `!is_dir()`) is deliberate: sockets, FIFOs, and
202 // device nodes must also be excluded from magic-file discovery.
203 #[allow(clippy::filetype_is_file)]
204 if file_type.is_file() && !file_type.is_symlink() {
205 file_paths.push(path);
206 }
207 }
208
209 // Sort by filename for deterministic ordering
210 file_paths.sort_by_key(|path| path.file_name().map(std::ffi::OsStr::to_os_string));
211
212 // Accumulate rules and name tables from all files
213 let mut all_rules = Vec::new();
214 let mut merged_table = NameTable::empty();
215 let mut parse_failures: Vec<(PathBuf, ParseError)> = Vec::new();
216 // Files that parsed (line-tolerantly) but contributed no usable rule or
217 // name despite having non-empty content -- i.e. every rule was skipped as
218 // unparseable. Tracked so an all-unusable directory is still reported as a
219 // failure even though tolerant parsing returns Ok for each such file.
220 let mut empty_files: Vec<PathBuf> = Vec::new();
221 let mut any_success = false;
222 let file_count = file_paths.len();
223
224 for path in file_paths {
225 // Read file contents (size-bounded to prevent memory exhaustion)
226 let contents = match read_magic_file_bounded(&path) {
227 Ok(contents) => contents,
228 Err(e) => {
229 // I/O errors (including oversized files) are critical
230 return Err(ParseError::invalid_syntax(
231 0,
232 format!("Failed to read file '{}': {}", path.display(), e),
233 ));
234 }
235 };
236
237 // Parse the file (line-tolerant: unparseable rules are skipped with a
238 // warning rather than dropping the whole file).
239 match super::parse_text_magic_file_tolerant(&contents) {
240 Ok(parsed) => {
241 if parsed.rules.is_empty() && parsed.name_table.is_empty() {
242 // Contributed nothing usable. If the file had actual rule
243 // lines (not just comments/blank lines), they were all
244 // skipped as unparseable -- record it so an entirely-
245 // unusable directory is still reported as a failure. A
246 // genuinely empty or comment-only file is valid and is NOT
247 // recorded (it simply contributes no rules).
248 if has_rule_lines(&contents) {
249 empty_files.push(path);
250 }
251 } else {
252 any_success = true;
253 all_rules.extend(parsed.rules);
254 merged_table.merge(parsed.name_table);
255 }
256 }
257 Err(e) => {
258 // Hard (preprocess-level) failure -- track for reporting.
259 parse_failures.push((path, e));
260 }
261 }
262 }
263
264 // A directory has loaded only if some file contributed a usable rule or
265 // name-table entry. `any_success` (not `all_rules.is_empty()`) so that a
266 // directory of pure `name`-subroutine files is not mistaken for failure.
267 // With line-tolerant parsing, a file whose rules were all skipped returns
268 // Ok with nothing; a directory where every content-bearing file is like
269 // that (or fails preprocessing) has loaded nothing and is reported as a
270 // failure -- preserving the pre-tolerance contract for a wholly-unusable
271 // directory.
272 if !any_success && (!parse_failures.is_empty() || !empty_files.is_empty()) {
273 use std::fmt::Write;
274
275 let mut problems: Vec<String> = parse_failures
276 .iter()
277 .map(|(path, e)| format!(" - {}: {}", path.display(), e))
278 .collect();
279 problems.extend(
280 empty_files
281 .iter()
282 .map(|path| format!(" - {}: no usable rules (all skipped)", path.display())),
283 );
284
285 let mut message = format!(
286 "All {file_count} magic file(s) in directory failed to parse or produced no usable rules"
287 );
288 let shown = problems.iter().take(3).cloned().collect::<Vec<_>>();
289 if !shown.is_empty() {
290 message.push_str(":\n");
291 message.push_str(&shown.join("\n"));
292 if problems.len() > 3 {
293 // fmt::Write to a String is infallible; discard the Result
294 // rather than unwrap so the no-panic policy holds regardless.
295 #[allow(clippy::let_underscore_must_use)]
296 let _ = write!(
297 message,
298 "\n ... and {} more",
299 problems.len().saturating_sub(3)
300 );
301 }
302 }
303
304 return Err(ParseError::invalid_syntax(0, message));
305 }
306
307 // Log warnings for partial failures (some files parsed, some failed)
308 for (path, e) in &parse_failures {
309 warn!("Failed to parse '{}': {}", path.display(), e);
310 }
311
312 Ok(ParsedMagic {
313 rules: all_rules,
314 name_table: merged_table,
315 })
316}
317
318/// Loads magic rules from a file or directory, automatically detecting the format.
319///
320/// This is the unified entry point for loading magic rules from the filesystem. It
321/// automatically detects whether the path points to a text magic file, a directory
322/// containing magic files, or a binary compiled magic file, and dispatches to the
323/// appropriate handler.
324///
325/// # Format Detection and Handling
326///
327/// The function uses [`detect_format()`] to determine the file type and handles each
328/// format as follows:
329///
330/// - **Text format**: Reads the file contents and parses using [`super::parse_text_magic_file()`]
331/// - **Directory format**: Loads all magic files from the directory using [`load_magic_directory()`]
332/// - **Binary format**: Returns an error with guidance to use the `--use-builtin` option
333///
334/// # Arguments
335///
336/// * `path` - Path to a magic file or directory. Can be absolute or relative.
337///
338/// # Returns
339///
340/// Returns `Ok(Vec<MagicRule>)` containing all successfully parsed magic rules. For
341/// directories, rules from all files are merged in alphabetical order by filename.
342///
343/// # Errors
344///
345/// This function returns a [`ParseError`] in the following cases:
346///
347/// - **File not found**: The specified path does not exist
348/// - **Unsupported format**: The file is a binary compiled magic file (`.mgc`)
349/// - **Parse errors**: The magic file contains syntax errors or invalid rules
350/// - **I/O errors**: File system errors during reading (permissions, disk errors, etc.)
351///
352/// # Examples
353///
354/// ## Loading a text magic file
355///
356/// ```no_run
357/// use libmagic_rs::parser::load_magic_file;
358/// use std::path::Path;
359///
360/// let parsed = load_magic_file(Path::new("/usr/share/misc/magic"))?;
361/// println!("Loaded {} magic rules", parsed.rules.len());
362/// # Ok::<(), libmagic_rs::ParseError>(())
363/// ```
364///
365/// ## Loading a directory of magic files
366///
367/// ```no_run
368/// use libmagic_rs::parser::load_magic_file;
369/// use std::path::Path;
370///
371/// let parsed = load_magic_file(Path::new("/usr/share/misc/magic.d"))?;
372/// println!("Loaded {} rules from directory", parsed.rules.len());
373/// # Ok::<(), libmagic_rs::ParseError>(())
374/// ```
375///
376/// ## Handling binary format errors
377///
378/// ```no_run
379/// use libmagic_rs::parser::load_magic_file;
380/// use std::path::Path;
381///
382/// match load_magic_file(Path::new("/usr/share/misc/magic.mgc")) {
383/// Ok(parsed) => println!("Loaded {} rules", parsed.rules.len()),
384/// Err(e) => {
385/// eprintln!("Error loading magic file: {}", e);
386/// eprintln!("Hint: Use --use-builtin for binary files");
387/// }
388/// }
389/// # Ok::<(), libmagic_rs::ParseError>(())
390/// ```
391///
392/// # Security
393///
394/// This function delegates to [`super::parse_text_magic_file()`] or [`load_magic_directory()`]
395/// based on format detection. Security considerations are handled by those functions:
396///
397/// - Rule hierarchy depth is bounded during parsing
398/// - Invalid syntax is rejected with descriptive errors
399/// - Binary `.mgc` files are rejected (not parsed)
400///
401/// A 1 GB size limit (`MAX_MAGIC_FILE_SIZE`, module-internal) is enforced on each file loaded
402/// (both standalone files and files within a directory) to prevent memory
403/// exhaustion from maliciously oversized inputs. Files exceeding the limit are
404/// rejected with a `ParseError` before their contents are read.
405///
406/// # See Also
407///
408/// - [`detect_format()`] - Format detection logic
409/// - [`super::parse_text_magic_file()`] - Text file parser
410/// - [`load_magic_directory()`] - Directory loader
411pub fn load_magic_file(path: &Path) -> Result<ParsedMagic, ParseError> {
412 // Detect the magic file format
413 let format = detect_format(path)?;
414
415 // Dispatch to appropriate handler based on format
416 match format {
417 MagicFileFormat::Text => {
418 // Read file contents (size-bounded) and parse as text magic file
419 let content = read_magic_file_bounded(path)?;
420 super::parse_text_magic_file_tolerant(&content)
421 }
422 MagicFileFormat::Directory => {
423 // Load all magic files from directory
424 load_magic_directory(path)
425 }
426 MagicFileFormat::Binary => {
427 // Binary compiled magic files are not supported
428 Err(ParseError::unsupported_format(
429 0,
430 "binary .mgc file",
431 "Binary compiled magic files (.mgc) are not supported for parsing.\n\
432 Use the --use-builtin option to use the built-in magic rules instead,\n\
433 or provide a text-based magic file or directory.",
434 ))
435 }
436 }
437}
438
439#[cfg(test)]
440mod tests {
441 // Restriction lints without an allow-*-in-tests config option;
442 // tests create exactly one level under a fresh TempDir.
443 #![allow(clippy::create_dir)]
444
445 use super::*;
446
447 // ============================================================
448 // Tests for load_magic_directory (6+ test cases)
449 // ============================================================
450
451 #[test]
452 fn test_load_directory_critical_error_io() {
453 use std::path::Path;
454
455 let non_existent = Path::new("/this/should/not/exist/anywhere/at/all");
456 let result = load_magic_directory(non_existent);
457
458 assert!(
459 result.is_err(),
460 "Should return error for non-existent directory"
461 );
462 let err = result.unwrap_err();
463 assert!(err.to_string().contains("Failed to read directory"));
464 }
465
466 #[test]
467 fn test_load_directory_non_critical_error_parse() {
468 use std::fs;
469 use tempfile::TempDir;
470
471 let temp_dir = TempDir::new().expect("Failed to create temp dir");
472
473 // Create a valid file
474 let valid_path = temp_dir.path().join("valid.magic");
475 fs::write(&valid_path, "0 string \\x01\\x02 valid\n").expect("Failed to write valid file");
476
477 // Create an invalid file
478 let invalid_path = temp_dir.path().join("invalid.magic");
479 fs::write(&invalid_path, "this is invalid syntax\n").expect("Failed to write invalid file");
480
481 // Should succeed, loading only the valid file
482 let parsed = load_magic_directory(temp_dir.path()).expect("Should load valid files");
483
484 assert_eq!(parsed.rules.len(), 1, "Should load only valid file");
485 assert_eq!(parsed.rules[0].message, "valid");
486 }
487
488 #[test]
489 fn test_load_directory_empty_files() {
490 use std::fs;
491 use tempfile::TempDir;
492
493 let temp_dir = TempDir::new().expect("Failed to create temp dir");
494
495 // Create an empty file
496 let empty_path = temp_dir.path().join("empty.magic");
497 fs::write(&empty_path, "").expect("Failed to write empty file");
498
499 // Create a file with only comments
500 let comments_path = temp_dir.path().join("comments.magic");
501 fs::write(&comments_path, "# Just comments\n# Nothing else\n")
502 .expect("Failed to write comments file");
503
504 // Should succeed with no rules
505 let parsed = load_magic_directory(temp_dir.path()).expect("Should handle empty files");
506
507 assert_eq!(
508 parsed.rules.len(),
509 0,
510 "Empty files should contribute no rules"
511 );
512 }
513
514 #[test]
515 fn test_load_directory_all_content_bearing_but_all_rules_skipped_errors() {
516 use std::fs;
517 use tempfile::TempDir;
518
519 let temp_dir = TempDir::new().expect("Failed to create temp dir");
520
521 // Two files that BOTH have content (non-comment rule lines) but whose
522 // every rule is unparseable. Under line-tolerant parsing (GOTCHAS S3.11)
523 // each file returns Ok with zero rules, but the directory contributed
524 // nothing usable, so `load_magic_directory` must still Err -- preserving
525 // the pre-tolerance "all failed" contract via the `empty_files` /
526 // `has_rule_lines` tracking. This is distinct from a directory of
527 // genuinely-empty/comment-only files (test_load_directory_empty_files),
528 // which is a valid no-op success, and from the mixed valid+invalid case
529 // (test_load_directory_non_critical_error_parse), which succeeds because
530 // one file contributed a rule.
531 fs::write(
532 temp_dir.path().join("bad1.magic"),
533 "notanoffset badtype whatever\nalso not a valid rule line\n",
534 )
535 .expect("Failed to write bad1");
536 fs::write(
537 temp_dir.path().join("bad2.magic"),
538 "still invalid syntax here\n",
539 )
540 .expect("Failed to write bad2");
541
542 let err = load_magic_directory(temp_dir.path()).expect_err(
543 "a directory whose content-bearing files all parse to zero rules must fail",
544 );
545 let msg = err.to_string();
546 assert!(
547 msg.contains("failed to parse"),
548 "error must report the all-failed contract: {msg}"
549 );
550 assert!(
551 msg.contains("no usable rules (all skipped)"),
552 "error must attribute the content-bearing-but-all-skipped files: {msg}"
553 );
554 }
555
556 #[test]
557 fn test_load_directory_binary_files() {
558 use std::fs;
559 use tempfile::TempDir;
560
561 let temp_dir = TempDir::new().expect("Failed to create temp dir");
562
563 // Create a binary file (invalid UTF-8). Lossy conversion turns this
564 // into U+FFFD characters that the grammar parser cannot interpret as
565 // a rule; the directory loader treats that as a non-critical parse
566 // failure and skips the file.
567 let binary_path = temp_dir.path().join("binary.dat");
568 fs::write(&binary_path, [0xFF, 0xFE, 0xFF, 0xFE]).expect("Failed to write binary file");
569
570 // Create a valid text file
571 let valid_path = temp_dir.path().join("valid.magic");
572 fs::write(&valid_path, "0 string \\x01\\x02 valid\n").expect("Failed to write valid file");
573
574 let parsed = load_magic_directory(temp_dir.path())
575 .expect("Directory with a binary file alongside a valid file should still load");
576
577 assert_eq!(
578 parsed.rules.len(),
579 1,
580 "Only the valid magic file should contribute rules"
581 );
582 assert_eq!(parsed.rules[0].message, "valid");
583 }
584
585 #[test]
586 fn test_load_directory_mixed_extensions() {
587 use std::fs;
588 use tempfile::TempDir;
589
590 let temp_dir = TempDir::new().expect("Failed to create temp dir");
591
592 // Create files with different extensions
593 fs::write(
594 temp_dir.path().join("file.magic"),
595 "0 string \\x01\\x02 magic\n",
596 )
597 .expect("Failed to write .magic file");
598 fs::write(
599 temp_dir.path().join("file.txt"),
600 "0 string \\x03\\x04 txt\n",
601 )
602 .expect("Failed to write .txt file");
603 fs::write(temp_dir.path().join("noext"), "0 string \\x05\\x06 noext\n")
604 .expect("Failed to write no-ext file");
605
606 let parsed = load_magic_directory(temp_dir.path())
607 .expect("Should load all files regardless of extension");
608
609 assert_eq!(
610 parsed.rules.len(),
611 3,
612 "Should process all files regardless of extension"
613 );
614
615 let messages: Vec<&str> = parsed.rules.iter().map(|r| r.message.as_str()).collect();
616 assert!(messages.contains(&"magic"));
617 assert!(messages.contains(&"txt"));
618 assert!(messages.contains(&"noext"));
619 }
620
621 #[test]
622 fn test_load_directory_alphabetical_ordering() {
623 use std::fs;
624 use tempfile::TempDir;
625
626 let temp_dir = TempDir::new().expect("Failed to create temp dir");
627
628 // Create files in non-alphabetical order - using valid magic syntax with hex escapes
629 fs::write(
630 temp_dir.path().join("03-third"),
631 "0 string \\x07\\x08\\x09 third\n",
632 )
633 .expect("Failed to write third file");
634 fs::write(
635 temp_dir.path().join("01-first"),
636 "0 string \\x01\\x02\\x03 first\n",
637 )
638 .expect("Failed to write first file");
639 fs::write(
640 temp_dir.path().join("02-second"),
641 "0 string \\x04\\x05\\x06 second\n",
642 )
643 .expect("Failed to write second file");
644
645 let parsed = load_magic_directory(temp_dir.path()).expect("Should load directory in order");
646
647 assert_eq!(parsed.rules.len(), 3);
648 // Should be sorted alphabetically by filename
649 assert_eq!(parsed.rules[0].message, "first");
650 assert_eq!(parsed.rules[1].message, "second");
651 assert_eq!(parsed.rules[2].message, "third");
652 }
653
654 // ============================================================
655 // Tests for load_magic_file (5+ test cases)
656 // ============================================================
657
658 #[test]
659 fn test_load_magic_file_text_format() {
660 use std::fs;
661 use tempfile::TempDir;
662
663 let temp_dir = TempDir::new().expect("Failed to create temp dir");
664 let magic_file = temp_dir.path().join("magic.txt");
665
666 // Create text magic file with valid content
667 fs::write(&magic_file, "0 string \\x7fELF ELF executable\n")
668 .expect("Failed to write magic file");
669
670 // Load using load_magic_file
671 let parsed = load_magic_file(&magic_file).expect("Failed to load text magic file");
672
673 assert_eq!(parsed.rules.len(), 1);
674 assert_eq!(parsed.rules[0].message, "ELF executable");
675 }
676
677 #[test]
678 fn test_load_magic_file_directory_format() {
679 use std::fs;
680 use tempfile::TempDir;
681
682 let temp_dir = TempDir::new().expect("Failed to create temp dir");
683 let magic_dir = temp_dir.path().join("magic.d");
684 fs::create_dir(&magic_dir).expect("Failed to create magic directory");
685
686 // Create multiple files in directory
687 fs::write(
688 magic_dir.join("00_elf"),
689 "0 string \\x7fELF ELF executable\n",
690 )
691 .expect("Failed to write elf file");
692 fs::write(
693 magic_dir.join("01_zip"),
694 "0 string \\x50\\x4b\\x03\\x04 ZIP archive\n",
695 )
696 .expect("Failed to write zip file");
697
698 // Load using load_magic_file
699 let parsed = load_magic_file(&magic_dir).expect("Failed to load directory");
700
701 assert_eq!(parsed.rules.len(), 2);
702 assert_eq!(parsed.rules[0].message, "ELF executable");
703 assert_eq!(parsed.rules[1].message, "ZIP archive");
704 }
705
706 #[test]
707 fn test_load_magic_file_binary_format_error() {
708 use std::fs::File;
709 use std::io::Write;
710 use tempfile::TempDir;
711
712 let temp_dir = TempDir::new().expect("Failed to create temp dir");
713 let binary_file = temp_dir.path().join("magic.mgc");
714
715 // Create binary file with .mgc magic number
716 let mut file = File::create(&binary_file).expect("Failed to create binary file");
717 let magic_number: [u8; 4] = [0x1C, 0x04, 0x1E, 0xF1]; // Little-endian 0xF11E041C
718 file.write_all(&magic_number)
719 .expect("Failed to write magic number");
720
721 // Attempt to load binary file
722 let result = load_magic_file(&binary_file);
723
724 assert!(result.is_err(), "Should fail to load binary .mgc file");
725
726 let error = result.unwrap_err();
727 let error_msg = error.to_string();
728
729 // Verify error mentions unsupported format and --use-builtin
730 assert!(
731 error_msg.contains("Binary") || error_msg.contains("binary"),
732 "Error should mention binary format: {error_msg}",
733 );
734 assert!(
735 error_msg.contains("--use-builtin") || error_msg.contains("built-in"),
736 "Error should mention --use-builtin option: {error_msg}",
737 );
738 }
739
740 #[test]
741 fn test_load_magic_file_io_error() {
742 use std::path::Path;
743
744 // Try to load non-existent file
745 let non_existent = Path::new("/this/path/should/not/exist/magic.txt");
746 let result = load_magic_file(non_existent);
747
748 assert!(result.is_err(), "Should fail for non-existent file");
749 }
750
751 #[test]
752 fn test_load_magic_file_tolerates_unparseable_rule_and_keeps_valid_ones() {
753 use std::fs;
754 use tempfile::TempDir;
755
756 let temp_dir = TempDir::new().expect("Failed to create temp dir");
757 let mixed_file = temp_dir.path().join("mixed.magic");
758
759 // A valid rule, an unparseable rule (missing offset), then another valid
760 // rule. Runtime loading is line-tolerant (GNU `file` semantics, GOTCHAS
761 // S3.11): the bad rule is skipped with a warning and the valid rules on
762 // either side survive, instead of the whole file being dropped. (This
763 // is what lets real system magic files keep their common-format
764 // detection despite a stray construct this parser cannot yet handle.)
765 fs::write(
766 &mixed_file,
767 "0 string GOOD1 first good rule\nstring test invalid\n0 string GOOD2 second good rule\n",
768 )
769 .expect("Failed to write file");
770
771 let parsed = load_magic_file(&mixed_file)
772 .expect("runtime load must tolerate an unparseable rule, not abort the whole file");
773 let msgs: Vec<&str> = parsed.rules.iter().map(|r| r.message.as_str()).collect();
774 assert!(
775 msgs.contains(&"first good rule"),
776 "a valid rule before the bad one must survive: {msgs:?}"
777 );
778 assert!(
779 msgs.contains(&"second good rule"),
780 "a valid rule after the bad one must survive: {msgs:?}"
781 );
782 assert_eq!(
783 parsed.rules.len(),
784 2,
785 "the unparseable rule must be dropped, keeping exactly the two valid ones: {msgs:?}"
786 );
787 }
788
789 #[test]
790 fn test_load_magic_file_drops_subtree_of_unparseable_rule_without_reattaching() {
791 use std::fs;
792 use tempfile::TempDir;
793
794 let temp_dir = TempDir::new().expect("Failed to create temp dir");
795 let file = temp_dir.path().join("subtree.magic");
796
797 // A valid parent, then an UNPARSEABLE rule (no valid offset) that owns a
798 // `>`-indented child, then a valid sibling back at the original level.
799 // GOTCHAS S3.11: when the bad rule is skipped, its deeper-indented
800 // subtree must be dropped WITH it (`skip_subtree_deeper_than` in
801 // `hierarchy.rs`). The orphaned child must NOT silently re-attach to the
802 // previous level-0 rule (`GOOD1`) nor survive as a stray top-level rule,
803 // and the trailing `GOOD2` proves the skip threshold resets so a later
804 // sibling at the original indent still parses.
805 fs::write(
806 &file,
807 "0 string GOOD1 parent rule\n\
808 notanoffset badtype orphan parent\n\
809 >0 byte x orphaned child that must be dropped\n\
810 0 string GOOD2 sibling after the dropped subtree\n",
811 )
812 .expect("Failed to write file");
813
814 let parsed = load_magic_file(&file)
815 .expect("runtime load must tolerate the unparseable rule and its subtree");
816
817 let top_msgs: Vec<&str> = parsed.rules.iter().map(|r| r.message.as_str()).collect();
818 assert_eq!(
819 parsed.rules.len(),
820 2,
821 "exactly the two valid top-level rules survive: {top_msgs:?}"
822 );
823 assert!(
824 top_msgs.contains(&"parent rule"),
825 "the valid rule before the bad one must survive: {top_msgs:?}"
826 );
827 assert!(
828 top_msgs.contains(&"sibling after the dropped subtree"),
829 "the sibling after the dropped subtree must parse (threshold reset): {top_msgs:?}"
830 );
831
832 // The orphaned child must appear NOWHERE: not re-attached to the
833 // preceding level-0 rule, and not as any surviving rule's child.
834 let good1 = parsed
835 .rules
836 .iter()
837 .find(|r| r.message == "parent rule")
838 .expect("GOOD1 must be present");
839 assert!(
840 good1.children.is_empty(),
841 "the dropped child must not re-attach to the previous level-0 rule: {:?}",
842 good1
843 .children
844 .iter()
845 .map(|c| c.message.as_str())
846 .collect::<Vec<_>>()
847 );
848 let orphan_reattached = parsed
849 .rules
850 .iter()
851 .flat_map(|r| r.children.iter())
852 .any(|c| c.message.contains("orphaned child"));
853 assert!(
854 !orphan_reattached,
855 "the orphaned child of the unparseable rule must be dropped entirely"
856 );
857 }
858
859 #[test]
860 fn test_max_magic_file_size_matches_file_buffer_limit() {
861 // Ensure the duplicated limit stays in sync with FileBuffer::MAX_FILE_SIZE.
862 // loader.rs cannot `use crate::io::FileBuffer` at module scope because
863 // build.rs pulls this file in via `#[path]`, but tests compile as part
864 // of the library and can reach it fine.
865 assert_eq!(
866 MAX_MAGIC_FILE_SIZE,
867 crate::io::FileBuffer::MAX_FILE_SIZE,
868 "MAX_MAGIC_FILE_SIZE must match FileBuffer::MAX_FILE_SIZE"
869 );
870 }
871
872 #[test]
873 fn test_load_magic_file_rejects_oversized_file() {
874 use std::fs::File;
875 use tempfile::TempDir;
876
877 let temp_dir = TempDir::new().expect("Failed to create temp dir");
878 let oversized = temp_dir.path().join("huge.magic");
879
880 // Create a sparse file whose reported size exceeds MAX_MAGIC_FILE_SIZE
881 // without actually consuming that much disk space.
882 let file = File::create(&oversized).expect("Failed to create oversized file");
883 file.set_len(MAX_MAGIC_FILE_SIZE + 1)
884 .expect("Failed to set sparse file length");
885 drop(file);
886
887 let result = load_magic_file(&oversized);
888
889 assert!(
890 result.is_err(),
891 "Loading a file larger than MAX_MAGIC_FILE_SIZE must fail"
892 );
893
894 let err_msg = result.unwrap_err().to_string();
895 assert!(
896 err_msg.contains("too large"),
897 "Error should indicate size limit violation, got: {err_msg}"
898 );
899 assert!(
900 err_msg.contains(&MAX_MAGIC_FILE_SIZE.to_string()),
901 "Error should mention the maximum allowed size, got: {err_msg}"
902 );
903 }
904
905 #[test]
906 fn test_load_magic_file_tolerates_non_utf8_in_comment() {
907 // Regression: /usr/share/file/magic/filesystems on macOS contains a
908 // Latin-1 `ß` (0xdf) in a contributor attribution comment. Previously
909 // this was rejected by `fs::read_to_string` with an opaque "stream
910 // did not contain valid UTF-8" error. The loader must now tolerate
911 // non-UTF-8 bytes in comments (and anywhere else they appear) by
912 // lossily replacing them.
913 use std::fs;
914 use tempfile::TempDir;
915
916 let temp_dir = TempDir::new().expect("Failed to create temp dir");
917 let magic_path = temp_dir.path().join("with-latin1-comment.magic");
918
919 let mut bytes: Vec<u8> = Vec::new();
920 bytes.extend_from_slice(b"# From: Thomas Wei");
921 bytes.push(0xdf); // invalid UTF-8 (Latin-1 encoding of `ß`)
922 bytes.extend_from_slice(b"schuh <thomas@example.invalid>\n");
923 bytes.extend_from_slice(b"0 string \\x7fELF ELF executable\n");
924 fs::write(&magic_path, &bytes).expect("Failed to write magic file with non-UTF-8 byte");
925
926 let parsed = load_magic_file(&magic_path)
927 .expect("Magic file with non-UTF-8 bytes in a comment must still load");
928
929 assert_eq!(
930 parsed.rules.len(),
931 1,
932 "The ELF rule should be parsed; the comment is stripped"
933 );
934 assert_eq!(parsed.rules[0].message, "ELF executable");
935 }
936
937 #[test]
938 fn test_load_directory_merges_name_tables() {
939 use std::fs;
940 use tempfile::TempDir;
941
942 let temp_dir = TempDir::new().expect("Failed to create temp dir");
943
944 // Each file defines a different named subroutine.
945 fs::write(
946 temp_dir.path().join("00_first"),
947 "0 name sub_a\n>0 byte 1 a-body\n",
948 )
949 .expect("Failed to write sub_a file");
950 fs::write(
951 temp_dir.path().join("01_second"),
952 "0 name sub_b\n>0 byte 2 b-body\n",
953 )
954 .expect("Failed to write sub_b file");
955
956 let parsed =
957 load_magic_directory(temp_dir.path()).expect("Should load both name subroutines");
958
959 // Both `name` rules are hoisted out, so top-level rules list is empty.
960 assert_eq!(parsed.rules.len(), 0);
961 assert!(parsed.name_table.get("sub_a").is_some());
962 assert!(parsed.name_table.get("sub_b").is_some());
963 }
964}