provenant/utils/file/
mod.rs

1// SPDX-FileCopyrightText: Provenant contributors
2// SPDX-License-Identifier: Apache-2.0
3
4//! File-level utilities split by concern:
5//!
6//! - [`path`]: filesystem metadata, glob exclusion, extension/name predicates
7//! - [`encoding`]: byte-to-text decoding and "looks like text" heuristics
8//! - [`format_sniff`]: magic-byte and container-format detection
9//! - [`classify`]: mime type and file-info classification surface
10//! - [`extract`]: text extraction for license/copyright detection
11//! - [`image_metadata`]: EXIF/XMP image metadata text extraction
12//! - [`pdf`]: bounded PDF text extraction
13//!
14//! The public API is re-exported here so existing `crate::utils::file::*`
15//! paths continue to resolve unchanged.
16
17mod classify;
18mod encoding;
19mod extract;
20mod format_sniff;
21mod image_metadata;
22mod path;
23mod pdf;
24
25pub use classify::{FileInfoClassification, classify_file_info, detect_mime_type};
26pub use encoding::decode_bytes_to_string;
27pub use extract::{ExtractedTextKind, extract_printable_strings, extract_text_for_detection};
28pub use path::{get_creation_date, is_path_excluded};
29
30pub(crate) use extract::{
31    augment_license_detection_text, extract_text_for_detection_with_diagnostics,
32};
33
34#[cfg(test)]
35mod tests {
36    use image::ImageFormat;
37    use std::path::Path;
38
39    use crate::copyright::detect_copyrights;
40
41    use super::classify::classify_file_info;
42    use super::encoding::CORRUPTED_UTF16_BOM_PREFIX;
43    use super::extract::{
44        ExtractedTextKind, LARGE_OPAQUE_BINARY_SKIP_BYTES, extract_printable_strings,
45        extract_text_for_detection, extract_text_for_detection_with_diagnostics,
46        windows_metadata_or_empty_result,
47    };
48    use super::image_metadata::{MAX_XMP_PACKET_BYTES, extract_raw_xmp_packet};
49    use super::pdf::{MAX_PDF_TEXT_EXTRACTION_BYTES, normalize_pdf_heading_comparison_text};
50
51    fn png_chunk(chunk_type: &[u8; 4], data: &[u8]) -> Vec<u8> {
52        let mut out = Vec::new();
53        out.extend_from_slice(&(data.len() as u32).to_be_bytes());
54        out.extend_from_slice(chunk_type);
55        out.extend_from_slice(data);
56        out.extend_from_slice(&0u32.to_be_bytes());
57        out
58    }
59
60    fn build_png_with_xmp(xmp: &str) -> Vec<u8> {
61        let mut bytes = Vec::new();
62        bytes.extend_from_slice(b"\x89PNG\r\n\x1a\n");
63
64        let ihdr = [
65            0, 0, 0, 1, // width
66            0, 0, 0, 1, // height
67            8, // bit depth
68            2, // color type
69            0, // compression
70            0, // filter
71            0, // interlace
72        ];
73        bytes.extend_from_slice(&png_chunk(b"IHDR", &ihdr));
74
75        let mut itxt = Vec::new();
76        itxt.extend_from_slice(b"XML:com.adobe.xmp");
77        itxt.push(0); // keyword terminator
78        itxt.push(0); // compression flag
79        itxt.push(0); // compression method
80        itxt.push(0); // language tag terminator
81        itxt.push(0); // translated keyword terminator
82        itxt.extend_from_slice(xmp.as_bytes());
83        bytes.extend_from_slice(&png_chunk(b"iTXt", &itxt));
84
85        bytes.extend_from_slice(&png_chunk(b"IEND", &[]));
86        bytes
87    }
88
89    #[test]
90    fn test_extract_text_for_detection_skips_jar_archives() {
91        let path = Path::new(
92            "testdata/license-golden/datadriven/lic1/do-not_detect-licenses-in-archive.jar",
93        );
94        let bytes = std::fs::read(path).expect("failed to read jar fixture");
95
96        let (text, kind) = extract_text_for_detection(path, &bytes);
97
98        assert!(text.is_empty());
99        assert_eq!(kind, ExtractedTextKind::None);
100    }
101
102    #[test]
103    fn test_extract_text_for_detection_reads_pdf_fixture_text() {
104        let path = Path::new("testdata/license-golden/datadriven/lic2/bsd-new_156.pdf");
105        let bytes = std::fs::read(path).expect("failed to read pdf fixture");
106
107        let (text, kind) = extract_text_for_detection(path, &bytes);
108
109        assert_eq!(kind, ExtractedTextKind::Pdf);
110        assert!(text.contains("Redistribution and use in source and binary forms"));
111    }
112
113    #[test]
114    fn test_extract_text_for_detection_prefers_first_pdf_page_before_full_document() {
115        let path =
116            Path::new("testdata/license-golden/datadriven/lic4/should_detect_something_5.pdf");
117        let bytes = std::fs::read(path).expect("failed to read pdf fixture");
118
119        let (text, kind) = extract_text_for_detection(path, &bytes);
120
121        assert_eq!(kind, ExtractedTextKind::Pdf);
122        assert!(text.contains("SUN INDUSTRY STANDARDS SOURCE LICENSE"));
123        assert!(!text.contains("DISCLAIMER OF WARRANTY"));
124    }
125
126    #[test]
127    fn test_extract_text_for_detection_does_not_duplicate_pdf_heading_prefix() {
128        let path =
129            Path::new("testdata/license-golden/datadriven/lic4/should_detect_something_5.pdf");
130        let bytes = std::fs::read(path).expect("failed to read pdf fixture");
131
132        let (text, kind) = extract_text_for_detection(path, &bytes);
133
134        assert_eq!(kind, ExtractedTextKind::Pdf);
135
136        let normalized = normalize_pdf_heading_comparison_text(&text);
137        let heading =
138            normalize_pdf_heading_comparison_text("SUN INDUSTRY STANDARDS SOURCE LICENSE");
139        assert_eq!(normalized.matches(&heading).count(), 1);
140    }
141
142    #[test]
143    fn test_extract_text_for_detection_reads_pdf_fixture_without_pdf_extension() {
144        let path = Path::new("testdata/license-golden/datadriven/lic2/bsd-new_156.pdf");
145        let bytes = std::fs::read(path).expect("failed to read pdf fixture");
146
147        let (text, kind) = extract_text_for_detection(Path::new("renamed.bin"), &bytes);
148
149        assert_eq!(kind, ExtractedTextKind::Pdf);
150        assert!(text.contains("Redistribution and use in source and binary forms"));
151    }
152
153    #[test]
154    fn test_extract_text_for_detection_skips_oversized_pdf_payload() {
155        let mut bytes = b"%PDF-1.7\n".to_vec();
156        bytes.resize(MAX_PDF_TEXT_EXTRACTION_BYTES + 1, b'0');
157
158        let (text, kind, scan_error) =
159            extract_text_for_detection_with_diagnostics(Path::new("oversized.pdf"), &bytes);
160
161        assert!(text.is_empty());
162        assert_eq!(kind, ExtractedTextKind::None);
163        assert!(
164            scan_error
165                .as_deref()
166                .is_some_and(|message| message.contains("PDF text extraction skipped"))
167        );
168    }
169
170    #[test]
171    fn test_extract_text_for_detection_reports_terminal_pdf_failure() {
172        let malformed = b"%PDF-1.7\nthis is not a valid pdf object graph\n";
173
174        let (text, kind, scan_error) =
175            extract_text_for_detection_with_diagnostics(Path::new("broken.pdf"), malformed);
176
177        assert!(text.is_empty());
178        assert_eq!(kind, ExtractedTextKind::None);
179        let scan_error = scan_error.expect("terminal pdf failure should be surfaced");
180        assert!(scan_error.contains("PDF text extraction failed after"));
181    }
182
183    #[test]
184    fn test_extract_text_for_detection_skips_large_opaque_binary_blobs() {
185        let bytes = vec![0_u8; LARGE_OPAQUE_BINARY_SKIP_BYTES + 8];
186
187        let (text, kind) = extract_text_for_detection(Path::new("model.bin"), &bytes);
188
189        assert!(text.is_empty());
190        assert_eq!(kind, ExtractedTextKind::None);
191    }
192
193    #[test]
194    fn test_extract_text_for_detection_keeps_large_binaries_with_promising_strings() {
195        let mut bytes = vec![0_u8; LARGE_OPAQUE_BINARY_SKIP_BYTES + 8];
196        let text = b"Copyright 2026 Example Project!!!";
197        bytes[..text.len()].copy_from_slice(text);
198        let second_offset = LARGE_OPAQUE_BINARY_SKIP_BYTES / 2;
199        bytes[second_offset..second_offset + text.len()].copy_from_slice(text);
200
201        let (text, kind) = extract_text_for_detection(Path::new("weights.bin"), &bytes);
202
203        assert_ne!(kind, ExtractedTextKind::None);
204        assert!(text.contains("Copyright 2026 Example Project"));
205    }
206
207    #[test]
208    fn test_extract_text_for_detection_skips_large_binary_with_unstructured_runs() {
209        let mut bytes = vec![0_u8; LARGE_OPAQUE_BINARY_SKIP_BYTES + 8];
210        let noise = b"(c) $1234567890ABCDEF[]{}--==++";
211        bytes[..noise.len()].copy_from_slice(noise);
212        let second_offset = LARGE_OPAQUE_BINARY_SKIP_BYTES / 2;
213        bytes[second_offset..second_offset + noise.len()].copy_from_slice(noise);
214
215        let (text, kind) = extract_text_for_detection(Path::new("tensor.bin"), &bytes);
216
217        assert!(text.is_empty());
218        assert_eq!(kind, ExtractedTextKind::None);
219    }
220
221    #[test]
222    fn test_extract_text_for_detection_uses_windows_executable_metadata() {
223        let path = Path::new("testdata/compiled-binary-golden/win_pe/libiconv2.dll");
224        let bytes = std::fs::read(path).expect("read PE fixture");
225
226        let (text, kind) = extract_text_for_detection(path, &bytes);
227
228        assert_eq!(kind, ExtractedTextKind::BinaryStrings);
229        assert!(text.contains("License: This program is free software"));
230        assert!(text.contains("LegalCopyright:"));
231    }
232
233    #[test]
234    fn test_extract_text_for_detection_keeps_windows_metadata_for_large_pe_without_sampled_signal()
235    {
236        let path = Path::new("testdata/compiled-binary-golden/win_pe/libiconv2.dll");
237        let mut bytes = std::fs::read(path).expect("read PE fixture");
238        bytes.resize(LARGE_OPAQUE_BINARY_SKIP_BYTES + 8, 0);
239
240        let (text, kind) = extract_text_for_detection(path, &bytes);
241
242        assert_ne!(kind, ExtractedTextKind::None);
243        assert!(!text.trim().is_empty());
244    }
245
246    #[test]
247    fn test_windows_metadata_or_empty_result_preserves_metadata() {
248        let (text, kind, scan_error) =
249            windows_metadata_or_empty_result(Some("LegalCopyright: Example Corp".to_string()));
250
251        assert_eq!(kind, ExtractedTextKind::WindowsExecutableMetadata);
252        assert_eq!(text, "LegalCopyright: Example Corp");
253        assert!(scan_error.is_none());
254    }
255
256    #[test]
257    fn test_extract_text_for_detection_keeps_image_author_separate_from_title_and_description() {
258        let xmp = r#"<x:xmpmeta xmlns:x="adobe:ns:meta/"><rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"><rdf:Description xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:creator>Chinmay Garde</dc:creator><dc:title>Bay Bridge At Night</dc:title><dc:description>Embarcadero in the evening on Delta 3200</dc:description></rdf:Description></rdf:RDF></x:xmpmeta>"#;
259        let bytes = build_png_with_xmp(xmp);
260
261        let (text, kind) = extract_text_for_detection(Path::new("fixture.png"), &bytes);
262
263        assert_eq!(kind, ExtractedTextKind::ImageMetadata);
264        assert!(text.contains("Author: Chinmay Garde"), "text: {text:?}");
265        assert!(
266            text.contains("Title: Bay Bridge At Night"),
267            "text: {text:?}"
268        );
269        assert!(
270            text.contains("Description: Embarcadero in the evening on Delta 3200"),
271            "text: {text:?}"
272        );
273
274        let (_copyrights, _holders, authors) = detect_copyrights(&text, None);
275        assert_eq!(
276            authors
277                .iter()
278                .map(|a| a.author.as_str())
279                .collect::<Vec<_>>(),
280            vec!["Chinmay Garde"],
281            "authors: {authors:?}; text: {text:?}"
282        );
283    }
284
285    #[test]
286    fn test_extract_text_for_detection_skips_large_binary_with_single_isolated_string_run() {
287        let mut bytes = vec![0_u8; LARGE_OPAQUE_BINARY_SKIP_BYTES + 8];
288        let text = b"Copyright 2026 Example Project!!!";
289        bytes[..text.len()].copy_from_slice(text);
290
291        let (text, kind) = extract_text_for_detection(Path::new("opaque.bin"), &bytes);
292
293        assert!(text.is_empty());
294        assert_eq!(kind, ExtractedTextKind::None);
295    }
296
297    #[test]
298    fn test_extract_text_for_detection_keeps_large_binary_with_single_contact_rich_window() {
299        let mut bytes = vec![0_u8; LARGE_OPAQUE_BINARY_SKIP_BYTES + 8];
300        let text = b"Andreas Schneider <asn@redhat.com> Rob Crittenden (rcritten@redhat.com) Mr. Sam <sam@email-scan.com> https://publicsuffix.org/ http://tukaani.org/xz/";
301        bytes[..text.len()].copy_from_slice(text);
302
303        let (text, kind) = extract_text_for_detection(Path::new("rootfs.bin"), &bytes);
304
305        assert_ne!(kind, ExtractedTextKind::None);
306        assert!(text.contains("asn@redhat.com"));
307        assert!(text.contains("https://publicsuffix.org/"));
308    }
309
310    #[test]
311    fn test_extract_text_for_detection_keeps_large_macho_with_off_window_legal_markers() {
312        let mut bytes = vec![0_u8; LARGE_OPAQUE_BINARY_SKIP_BYTES * 2];
313        bytes[..4].copy_from_slice(&[0xCF, 0xFA, 0xED, 0xFE]);
314        let apache_notice = b"// Licensed under the Apache License, Version 2.0 (the \"License\");\n// http://www.apache.org/licenses/LICENSE-2.0\n// SPDX-License-Identifier: Apache-2.0\n";
315        let insert_offset = 200 * 1024;
316        bytes[insert_offset..insert_offset + apache_notice.len()].copy_from_slice(apache_notice);
317
318        let (text, kind) = extract_text_for_detection(Path::new("node"), &bytes);
319
320        assert_eq!(kind, ExtractedTextKind::BinaryStrings);
321        assert!(text.contains("Apache License, Version 2.0"), "{text}");
322        assert!(
323            text.contains("SPDX-License-Identifier: Apache-2.0"),
324            "{text}"
325        );
326    }
327
328    #[test]
329    fn test_extract_text_for_detection_keeps_large_macho_with_unicode_notice_markers() {
330        let mut bytes = vec![0_u8; LARGE_OPAQUE_BINARY_SKIP_BYTES * 2];
331        bytes[..4].copy_from_slice(&[0xCF, 0xFA, 0xED, 0xFE]);
332        let unicode_notice = b"Copyright (c) 1991-2024 Unicode, Inc.\nFor terms of use, see http://www.unicode.org/copyright.html\n";
333        let insert_offset = 700 * 1024;
334        bytes[insert_offset..insert_offset + unicode_notice.len()].copy_from_slice(unicode_notice);
335
336        let (text, kind) = extract_text_for_detection(Path::new("node"), &bytes);
337
338        assert_eq!(kind, ExtractedTextKind::BinaryStrings);
339        assert!(text.contains("Unicode, Inc."), "{text}");
340        assert!(text.contains("unicode.org/copyright.html"), "{text}");
341    }
342
343    #[test]
344    fn test_extract_text_for_detection_does_not_reopen_single_window_legal_noise_for_non_macho() {
345        let mut bytes = vec![0_u8; LARGE_OPAQUE_BINARY_SKIP_BYTES * 2];
346        let apache_notice = b"// Licensed under the Apache License, Version 2.0 (the \"License\");\n// http://www.apache.org/licenses/LICENSE-2.0\n// SPDX-License-Identifier: Apache-2.0\n";
347        let insert_offset = 200 * 1024;
348        bytes[insert_offset..insert_offset + apache_notice.len()].copy_from_slice(apache_notice);
349
350        let (text, kind) = extract_text_for_detection(Path::new("model.bin"), &bytes);
351
352        assert!(text.is_empty());
353        assert_eq!(kind, ExtractedTextKind::None);
354    }
355
356    #[test]
357    fn test_extract_text_for_detection_avoids_latin1_decode_for_binary_blob_noise() {
358        let bytes = vec![
359            0x28, 0x63, 0x29, 0x20, 0x4b, 0x30, 0x0e, 0x71, 0x86, 0x20, 0x62, 0x24, 0x4c,
360        ];
361
362        let (text, kind) = extract_text_for_detection(Path::new("fixture.blb"), &bytes);
363
364        assert_eq!(kind, ExtractedTextKind::BinaryStrings);
365        assert_eq!(text, "(c) K0\n b$L");
366    }
367
368    #[test]
369    fn test_extract_raw_xmp_packet_rejects_oversized_png_itxt_payload() {
370        let xmp = "A".repeat(MAX_XMP_PACKET_BYTES + 1);
371        let bytes = build_png_with_xmp(&xmp);
372
373        assert!(extract_raw_xmp_packet(&bytes, ImageFormat::Png).is_none());
374    }
375
376    #[test]
377    fn test_extract_text_for_detection_skips_zip_like_archives() {
378        let zip_bytes = b"PK\x03\x04\x14\x00\x00\x00\x08\x00artifact";
379
380        let (whl_text, whl_kind) = extract_text_for_detection(Path::new("demo.whl"), zip_bytes);
381        let (crate_text, crate_kind) =
382            extract_text_for_detection(Path::new("demo.crate"), zip_bytes);
383
384        assert!(whl_text.is_empty());
385        assert_eq!(whl_kind, ExtractedTextKind::None);
386        assert!(crate_text.is_empty());
387        assert_eq!(crate_kind, ExtractedTextKind::None);
388    }
389
390    #[test]
391    fn test_extract_text_for_detection_keeps_binary_strings_for_lib_fixtures() {
392        let path =
393            Path::new("testdata/copyright-golden/copyrights/copyright_php_lib-php_embed_lib.lib");
394        let bytes = std::fs::read(path).expect("failed to read lib fixture");
395
396        let (text, kind) = extract_text_for_detection(path, &bytes);
397
398        assert_ne!(kind, ExtractedTextKind::None);
399        assert!(text.contains("Copyright nexB and others (c) 2012"));
400    }
401
402    #[test]
403    fn test_extract_text_for_detection_reads_font_metadata() {
404        let path = Path::new("testdata/font-fixtures/Lato-Bold.ttf");
405        let bytes = std::fs::read(path).expect("failed to read font fixture");
406
407        let (text, kind) = extract_text_for_detection(path, &bytes);
408
409        assert_eq!(kind, ExtractedTextKind::FontMetadata);
410        assert!(text.contains("License Description:"), "{text}");
411        assert!(
412            text.contains("Open Font License") || text.contains("OFL"),
413            "{text}"
414        );
415        assert!(text.contains("Lato"), "{text}");
416    }
417
418    #[test]
419    fn test_extract_printable_strings_scales_cap_for_medium_binary_files() {
420        let bytes = b"abcd\0".repeat(525_000);
421
422        let text = extract_printable_strings(&bytes);
423
424        assert!(
425            text.len() > 2_000_000,
426            "unexpected truncation at {}",
427            text.len()
428        );
429        assert!(text.ends_with("abcd"));
430    }
431
432    #[test]
433    fn test_extract_text_for_detection_decodes_svg_fixture_text() {
434        let path = Path::new(
435            "testdata/license-golden/datadriven/external/fossology-tests/Public-domain/biohazard.svg",
436        );
437        let bytes = std::fs::read(path).expect("failed to read svg fixture");
438
439        let (text, kind) = extract_text_for_detection(path, &bytes);
440
441        assert_eq!(kind, ExtractedTextKind::Decoded);
442        assert!(text.contains("creativecommons.org/licenses/publicdomain"));
443    }
444
445    #[test]
446    fn test_extract_text_for_detection_preserves_blank_comment_lines_in_utf8_source() {
447        let path = Path::new("testdata/plugin_email_url/files/IMarkerActionFilter.java");
448        let bytes = std::fs::read(path).expect("failed to read java fixture");
449
450        let (text, kind) = extract_text_for_detection(path, &bytes);
451
452        assert_eq!(kind, ExtractedTextKind::Decoded);
453        let lines: Vec<_> = text.lines().collect();
454        assert_eq!(lines.get(2).copied(), Some(" *"));
455        assert_eq!(
456            lines.get(3).copied(),
457            Some(" *https://github.com/rpm-software-management")
458        );
459        assert_eq!(lines.get(5).copied(), Some("https://gitlab.com/Conan_Kudo"));
460    }
461
462    #[test]
463    fn test_extract_text_for_detection_decodes_rtf_fixture_text() {
464        let path = Path::new(
465            "testdata/license-golden/datadriven/external/fossology-tests/LGPL/License.rtf",
466        );
467        let bytes = std::fs::read(path).expect("failed to read rtf fixture");
468
469        let (text, kind) = extract_text_for_detection(path, &bytes);
470
471        assert_eq!(kind, ExtractedTextKind::Decoded);
472        assert!(text.contains("GNU Lesser General Public"));
473        assert!(text.contains("version"));
474        assert!(text.contains("2.1 of the License"));
475    }
476
477    #[test]
478    fn test_classify_file_info_marks_empty_files_as_text_not_source() {
479        let classification = classify_file_info(Path::new("test.txt"), b"");
480
481        assert_eq!(classification.mime_type, "inode/x-empty");
482        assert_eq!(classification.file_type, "empty");
483        assert!(!classification.is_binary);
484        assert!(classification.is_text);
485        assert!(!classification.is_source);
486        assert_eq!(classification.programming_language, None);
487    }
488
489    #[test]
490    fn test_classify_file_info_keeps_json_out_of_programming_language() {
491        let classification = classify_file_info(Path::new("package.json"), br#"{"name":"demo"}"#);
492
493        assert_eq!(classification.mime_type, "application/json");
494        assert_eq!(classification.file_type, "JSON text data");
495        assert!(classification.is_text);
496        assert!(!classification.is_source);
497        assert_eq!(classification.programming_language, None);
498    }
499
500    #[test]
501    fn test_classify_file_info_does_not_label_invalid_json_text_as_json() {
502        let classification =
503            classify_file_info(Path::new("broken.json"), b"{ definitely not json\n");
504
505        assert_eq!(classification.mime_type, "text/plain");
506        assert_eq!(classification.file_type, "UTF-8 Unicode text");
507        assert!(classification.is_text);
508        assert!(!classification.is_binary);
509    }
510
511    #[test]
512    fn test_classify_file_info_does_not_label_binary_json_garbage_as_json() {
513        let classification =
514            classify_file_info(Path::new("broken.json"), &[0xff, 0x00, 0x01, 0x02]);
515
516        assert_eq!(classification.mime_type, "application/octet-stream");
517        assert_eq!(classification.file_type, "data");
518        assert!(classification.is_binary);
519        assert!(!classification.is_text);
520    }
521
522    #[test]
523    fn test_classify_file_info_treats_valid_utf16_json_with_bom_as_text() {
524        let classification = classify_file_info(
525            Path::new("utf16.json"),
526            &[
527                0xFF, 0xFE, 0x5B, 0x00, 0x22, 0x00, 0xE9, 0x00, 0x22, 0x00, 0x5D, 0x00,
528            ],
529        );
530
531        assert!(!classification.is_binary);
532        assert!(classification.is_text);
533        assert_eq!(classification.mime_type, "application/json");
534        assert_eq!(classification.file_type, "JSON text data");
535    }
536
537    #[test]
538    fn test_classify_file_info_treats_valid_utf16be_json_without_bom_as_text() {
539        let classification = classify_file_info(
540            Path::new("utf16be.json"),
541            &[0x00, 0x5B, 0x00, 0x22, 0x00, 0xE9, 0x00, 0x22, 0x00, 0x5D],
542        );
543
544        assert!(!classification.is_binary);
545        assert!(classification.is_text);
546        assert_eq!(classification.mime_type, "application/json");
547        assert_eq!(classification.file_type, "JSON text data");
548    }
549
550    #[test]
551    fn test_classify_file_info_treats_small_valid_utf16be_json_literal_as_text() {
552        let classification =
553            classify_file_info(Path::new("utf16be-literal.json"), &[0x00, 0x5B, 0x00, 0x5D]);
554
555        assert!(!classification.is_binary);
556        assert!(classification.is_text);
557        assert_eq!(classification.mime_type, "application/json");
558        assert_eq!(classification.file_type, "JSON text data");
559    }
560
561    #[test]
562    fn test_extract_text_for_detection_decodes_utf16be_text_with_corrupted_bom_prefix() {
563        let mut bytes = CORRUPTED_UTF16_BOM_PREFIX.to_vec();
564        for code_unit in
565            "Licensed to the Apache Software Foundation\nApache License, Version 2.0".encode_utf16()
566        {
567            bytes.extend_from_slice(&code_unit.to_be_bytes());
568        }
569
570        let (text, kind) = extract_text_for_detection(Path::new("notice.ftl"), &bytes);
571
572        assert_eq!(kind, ExtractedTextKind::Decoded);
573        assert!(text.contains("Apache Software Foundation"), "{text}");
574        assert!(text.contains("Apache License, Version 2.0"), "{text}");
575    }
576
577    #[test]
578    fn test_classify_file_info_treats_small_valid_json_literals_as_text() {
579        let classification = classify_file_info(Path::new("true.json"), b"true");
580
581        assert!(!classification.is_binary);
582        assert!(classification.is_text);
583        assert_eq!(classification.mime_type, "application/json");
584        assert_eq!(classification.file_type, "JSON text data");
585    }
586
587    #[test]
588    fn test_classify_file_info_treats_json_wrapped_invalid_utf8_sequences_as_text() {
589        let classification = classify_file_info(
590            Path::new("wrapped.json"),
591            &[0x5B, 0x22, 0xE6, 0x97, 0xA5, 0xD1, 0x88, 0xFA, 0x22, 0x5D],
592        );
593
594        assert!(!classification.is_binary);
595        assert!(classification.is_text);
596        assert_eq!(classification.mime_type, "text/plain");
597        assert_eq!(classification.file_type, "text, with no line terminators");
598    }
599
600    #[test]
601    fn test_classify_file_info_keeps_lone_ff_json_byte_binary() {
602        let classification =
603            classify_file_info(Path::new("lone-ff.json"), &[0x5B, 0x22, 0xFF, 0x22, 0x5D]);
604
605        assert!(classification.is_binary);
606        assert!(!classification.is_text);
607        assert_eq!(classification.mime_type, "application/octet-stream");
608        assert_eq!(classification.file_type, "data");
609    }
610
611    #[test]
612    fn test_classify_file_info_keeps_nul_heavy_crash_json_binary() {
613        let classification = classify_file_info(
614            Path::new("crash.json"),
615            &[
616                0xFE, 0x90, 0x00, 0x00, 0x00, 0x93, 0x5B, 0x5B, 0x32, 0x38, 0x36,
617            ],
618        );
619
620        assert!(classification.is_binary);
621        assert!(!classification.is_text);
622        assert_eq!(classification.mime_type, "application/octet-stream");
623    }
624
625    #[test]
626    fn test_classify_file_info_treats_dockerfile_as_source() {
627        let classification = classify_file_info(Path::new("Dockerfile"), b"FROM scratch\n");
628
629        assert_eq!(
630            classification.programming_language.as_deref(),
631            Some("Dockerfile")
632        );
633        assert!(classification.is_source);
634        assert!(!classification.is_script);
635        assert_eq!(
636            classification.file_type,
637            "Dockerfile source, UTF-8 Unicode text"
638        );
639    }
640
641    #[test]
642    fn test_classify_file_info_treats_makefile_as_text_not_source() {
643        let classification = classify_file_info(Path::new("Makefile"), b"all:\n\techo hi\n");
644
645        assert_eq!(classification.programming_language, None);
646        assert!(classification.is_text);
647        assert!(!classification.is_source);
648        assert!(!classification.is_script);
649        assert_eq!(classification.file_type, "UTF-8 Unicode text");
650    }
651
652    #[test]
653    fn test_classify_file_info_marks_supported_package_archives() {
654        let zip_bytes = b"PK\x03\x04\x14\x00\x00\x00";
655
656        let egg = classify_file_info(Path::new("demo.egg"), zip_bytes);
657        let nupkg = classify_file_info(Path::new("demo.nupkg"), zip_bytes);
658
659        assert!(egg.is_archive);
660        assert_eq!(egg.mime_type, "application/zip");
661        assert_eq!(egg.file_type, "Zip archive data");
662        assert!(nupkg.is_archive);
663        assert_eq!(nupkg.mime_type, "application/zip");
664        assert_eq!(nupkg.file_type, "Zip archive data");
665    }
666
667    #[test]
668    fn test_classify_file_info_marks_png_as_binary_media() {
669        let png_bytes = b"\x89PNG\r\n\x1a\n\x00\x00\x00\x0dIHDR";
670
671        let classification = classify_file_info(Path::new("logo.png"), png_bytes);
672
673        assert_eq!(classification.mime_type, "image/png");
674        assert_eq!(classification.file_type, "PNG image data");
675        assert!(classification.is_binary);
676        assert!(!classification.is_text);
677        assert!(classification.is_media);
678        assert!(!classification.is_archive);
679        assert!(!classification.is_source);
680    }
681
682    #[test]
683    fn test_classify_file_info_marks_pdf_as_binary_document() {
684        let pdf_bytes = b"%PDF-1.7\n1 0 obj\n<< /Type /Catalog >>\n";
685
686        let classification = classify_file_info(Path::new("report.pdf"), pdf_bytes);
687
688        assert_eq!(classification.mime_type, "application/pdf");
689        assert_eq!(classification.file_type, "PDF document");
690        assert!(classification.is_binary);
691        assert!(!classification.is_text);
692        assert!(!classification.is_archive);
693        assert!(!classification.is_media);
694    }
695
696    #[test]
697    fn test_classify_file_info_marks_binary_blobs_as_binary() {
698        let classification =
699            classify_file_info(Path::new("blob.bin"), &[0, 159, 146, 150, 0, 1, 2, 3, 4, 5]);
700
701        assert!(classification.is_binary);
702        assert!(!classification.is_text);
703        assert!(!classification.is_source);
704        assert_eq!(classification.programming_language, None);
705    }
706
707    #[test]
708    fn test_classify_file_info_treats_yaml_as_text_not_source() {
709        let classification = classify_file_info(Path::new("config.yaml"), b"key: value\n");
710
711        assert_eq!(classification.programming_language, None);
712        assert!(classification.is_text);
713        assert!(!classification.is_source);
714        assert_eq!(classification.file_type, "YAML text data");
715    }
716
717    #[test]
718    fn test_classify_file_info_classifies_common_build_manifests() {
719        let gradle = classify_file_info(Path::new("build.gradle"), b"plugins { id 'java' }\n");
720        let flake = classify_file_info(Path::new("flake.nix"), b"{ inputs, ... }: {}\n");
721        let cmake = classify_file_info(
722            Path::new("toolchain.cmake"),
723            b"set(CMAKE_CXX_STANDARD 20)\n",
724        );
725        let gitmodules = classify_file_info(
726            Path::new(".gitmodules"),
727            b"[submodule \"demo\"]\n\tpath = vendor/demo\n",
728        );
729
730        assert_eq!(gradle.programming_language.as_deref(), Some("Groovy"));
731        assert!(gradle.is_source);
732        assert_eq!(gradle.mime_type, "text/plain");
733        assert_eq!(gradle.file_type, "Groovy source, UTF-8 Unicode text");
734
735        assert_eq!(flake.programming_language.as_deref(), Some("Nix"));
736        assert!(flake.is_source);
737        assert_eq!(flake.mime_type, "text/plain");
738        assert_eq!(flake.file_type, "Nix source, UTF-8 Unicode text");
739
740        assert_eq!(cmake.programming_language.as_deref(), Some("CMake"));
741        assert!(cmake.is_source);
742        assert_eq!(cmake.file_type, "CMake source, UTF-8 Unicode text");
743
744        assert_eq!(gitmodules.programming_language, None);
745        assert!(gitmodules.is_text);
746        assert!(!gitmodules.is_source);
747        assert_eq!(gitmodules.file_type, "Git configuration text");
748    }
749
750    #[test]
751    fn test_classify_file_info_labels_cpp_headers_and_ipp_separately() {
752        let header = classify_file_info(
753            Path::new("include/demo.hpp"),
754            b"#pragma once\nclass Demo {};\n",
755        );
756        let ipp = classify_file_info(
757            Path::new("include/detail/demo.ipp"),
758            b"template <class T> void parse() {}\n",
759        );
760
761        assert_eq!(header.programming_language.as_deref(), Some("C++"));
762        assert!(header.is_source);
763        assert!(!header.is_script);
764        assert_eq!(header.file_type, "C++ source, UTF-8 Unicode text");
765
766        assert_eq!(ipp.programming_language, None);
767        assert!(!ipp.is_source);
768        assert!(!ipp.is_script);
769        assert_eq!(ipp.file_type, "UTF-8 Unicode text");
770    }
771
772    #[test]
773    fn test_classify_file_info_preserves_specific_shell_family_labels() {
774        let bash = classify_file_info(Path::new("bin/run"), b"#!/usr/bin/env bash\necho hi\n");
775
776        assert_eq!(bash.programming_language.as_deref(), Some("Bash"));
777        assert!(bash.is_script);
778        assert_eq!(bash.file_type, "bash script, UTF-8 Unicode text executable");
779    }
780
781    #[test]
782    fn test_classify_file_info_marks_jamfile_as_source() {
783        let jamfile = classify_file_info(Path::new("Jamfile"), b"lib boost_json ;\n");
784
785        assert_eq!(jamfile.programming_language.as_deref(), Some("Jamfile"));
786        assert!(jamfile.is_source);
787        assert!(!jamfile.is_script);
788        assert_eq!(jamfile.file_type, "Jamfile source, UTF-8 Unicode text");
789    }
790
791    #[test]
792    fn test_classify_file_info_labels_javascript_shebang_scripts() {
793        let classification = classify_file_info(
794            Path::new("bin/run"),
795            b"#!/usr/bin/env node\nconsole.log('hello');\n",
796        );
797
798        assert_eq!(
799            classification.programming_language.as_deref(),
800            Some("JavaScript")
801        );
802        assert!(classification.is_script);
803        assert_eq!(
804            classification.file_type,
805            "javascript script, UTF-8 Unicode text executable"
806        );
807    }
808
809    #[test]
810    fn test_classify_file_info_uses_non_utf8_text_labels_for_latin1_scripts() {
811        let classification = classify_file_info(
812            Path::new("script.py"),
813            b"# coding: latin-1\nprint(\"caf\xe9\")\n",
814        );
815
816        assert_eq!(
817            classification.programming_language.as_deref(),
818            Some("Python")
819        );
820        assert!(classification.is_script);
821        assert_eq!(classification.file_type, "python script, text executable");
822    }
823
824    #[test]
825    fn test_classify_file_info_treats_textual_tga_as_media() {
826        let classification = classify_file_info(Path::new("texture.tga"), b"not really a tga\n");
827
828        assert!(classification.is_media);
829        assert!(classification.is_text);
830        assert!(!classification.is_binary);
831    }
832
833    #[test]
834    fn test_classify_file_info_keeps_binaryish_source_extension_out_of_text_path() {
835        let classification =
836            classify_file_info(Path::new("main.ts"), &[0x80, 0x81, 0x82, 0x83, 0x84, 0x85]);
837
838        assert!(classification.is_binary);
839        assert!(!classification.is_text);
840        assert!(!classification.is_source);
841        assert_eq!(classification.programming_language, None);
842    }
843
844    #[test]
845    fn test_extract_text_for_detection_skips_unsupported_image_formats() {
846        let gif_bytes = b"GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;";
847
848        let (text, kind) = extract_text_for_detection(Path::new("tiny.gif"), gif_bytes);
849
850        assert!(text.is_empty());
851        assert_eq!(kind, ExtractedTextKind::None);
852    }
853
854    #[test]
855    fn test_classify_file_info_preserves_language_detection_precedence_matrix() {
856        let cases = [
857            (
858                Path::new("bin/run"),
859                b"#!/usr/bin/env node\nconsole.log('hello');\n".as_slice(),
860                Some("JavaScript"),
861                true,
862                true,
863            ),
864            (
865                Path::new("Dockerfile"),
866                b"FROM scratch\n".as_slice(),
867                Some("Dockerfile"),
868                true,
869                false,
870            ),
871            (
872                Path::new("package.json"),
873                br#"{"name":"demo"}"#.as_slice(),
874                None,
875                false,
876                false,
877            ),
878            (
879                Path::new("config.yaml"),
880                b"key: value\n".as_slice(),
881                None,
882                false,
883                false,
884            ),
885            (
886                Path::new("Makefile"),
887                b"all:\n\techo hi\n".as_slice(),
888                None,
889                false,
890                false,
891            ),
892        ];
893
894        for (path, bytes, expected_language, expected_is_source, expected_is_script) in cases {
895            let classification = classify_file_info(path, bytes);
896
897            assert_eq!(
898                classification.programming_language.as_deref(),
899                expected_language,
900                "unexpected language for {}",
901                path.display()
902            );
903            assert_eq!(
904                classification.is_source,
905                expected_is_source,
906                "unexpected is_source for {}",
907                path.display()
908            );
909            assert_eq!(
910                classification.is_script,
911                expected_is_script,
912                "unexpected is_script for {}",
913                path.display()
914            );
915        }
916    }
917}
provenant/utils/file/mod.rs

provenant/utils/file/
mod.rs
