provenant/utils/file/
mod.rs

1// SPDX-FileCopyrightText: Provenant contributors
2// SPDX-License-Identifier: Apache-2.0
3
4//! File-level utilities split by concern:
5//!
6//! - [`path`]: filesystem metadata, glob exclusion, extension/name predicates
7//! - [`encoding`]: byte-to-text decoding and "looks like text" heuristics
8//! - [`format_sniff`]: magic-byte and container-format detection
9//! - [`classify`]: mime type and file-info classification surface
10//! - [`extract`]: text extraction for license/copyright detection
11//! - [`image_metadata`]: EXIF/XMP image metadata text extraction
12//! - [`pdf`]: bounded PDF text extraction
13//!
14//! The public API is re-exported here so existing `crate::utils::file::*`
15//! paths continue to resolve unchanged.
16
17mod classify;
18mod encoding;
19mod extract;
20mod format_sniff;
21mod image_metadata;
22mod path;
23mod pdf;
24
25pub use classify::{FileInfoClassification, classify_file_info, detect_mime_type};
26pub use encoding::decode_bytes_to_string;
27pub use extract::{ExtractedTextKind, extract_printable_strings, extract_text_for_detection};
28pub use path::{get_creation_date, is_path_excluded};
29
30pub(crate) use extract::{
31    augment_license_detection_text, extract_text_for_detection_with_diagnostics,
32};
33
34#[cfg(test)]
35mod tests {
36    use image::ImageFormat;
37    use std::path::Path;
38
39    use crate::copyright::detect_copyrights;
40    use crate::models::DiagnosticSeverity;
41
42    use super::classify::classify_file_info;
43    use super::encoding::{
44        CORRUPTED_UTF16_BOM_PREFIX, NEAR_BINARY_SKIP_DIAGNOSTIC, decode_bytes_to_string,
45        decode_bytes_to_string_with_diagnostic,
46    };
47    use super::extract::{
48        ExtractedTextKind, LARGE_OPAQUE_BINARY_SKIP_BYTES, extract_printable_strings,
49        extract_text_for_detection, extract_text_for_detection_with_diagnostics,
50        windows_metadata_or_empty_result,
51    };
52    use super::image_metadata::{MAX_XMP_PACKET_BYTES, extract_raw_xmp_packet};
53    use super::pdf::{MAX_PDF_TEXT_EXTRACTION_BYTES, normalize_pdf_heading_comparison_text};
54
55    fn png_chunk(chunk_type: &[u8; 4], data: &[u8]) -> Vec<u8> {
56        let mut out = Vec::new();
57        out.extend_from_slice(&(data.len() as u32).to_be_bytes());
58        out.extend_from_slice(chunk_type);
59        out.extend_from_slice(data);
60        out.extend_from_slice(&0u32.to_be_bytes());
61        out
62    }
63
64    fn build_png_with_xmp(xmp: &str) -> Vec<u8> {
65        let mut bytes = Vec::new();
66        bytes.extend_from_slice(b"\x89PNG\r\n\x1a\n");
67
68        let ihdr = [
69            0, 0, 0, 1, // width
70            0, 0, 0, 1, // height
71            8, // bit depth
72            2, // color type
73            0, // compression
74            0, // filter
75            0, // interlace
76        ];
77        bytes.extend_from_slice(&png_chunk(b"IHDR", &ihdr));
78
79        let mut itxt = Vec::new();
80        itxt.extend_from_slice(b"XML:com.adobe.xmp");
81        itxt.push(0); // keyword terminator
82        itxt.push(0); // compression flag
83        itxt.push(0); // compression method
84        itxt.push(0); // language tag terminator
85        itxt.push(0); // translated keyword terminator
86        itxt.extend_from_slice(xmp.as_bytes());
87        bytes.extend_from_slice(&png_chunk(b"iTXt", &itxt));
88
89        bytes.extend_from_slice(&png_chunk(b"IEND", &[]));
90        bytes
91    }
92
93    #[test]
94    fn test_extract_text_for_detection_skips_jar_archives() {
95        let path = Path::new(
96            "testdata/license-golden/datadriven/lic1/do-not_detect-licenses-in-archive.jar",
97        );
98        let bytes = std::fs::read(path).expect("failed to read jar fixture");
99
100        let (text, kind) = extract_text_for_detection(path, &bytes);
101
102        assert!(text.is_empty());
103        assert_eq!(kind, ExtractedTextKind::None);
104    }
105
106    #[test]
107    fn test_extract_text_for_detection_reads_pdf_fixture_text() {
108        let path = Path::new("testdata/license-golden/datadriven/lic2/bsd-new_156.pdf");
109        let bytes = std::fs::read(path).expect("failed to read pdf fixture");
110
111        let (text, kind) = extract_text_for_detection(path, &bytes);
112
113        assert_eq!(kind, ExtractedTextKind::Pdf);
114        assert!(text.contains("Redistribution and use in source and binary forms"));
115    }
116
117    #[test]
118    fn test_extract_text_for_detection_prefers_first_pdf_page_before_full_document() {
119        let path =
120            Path::new("testdata/license-golden/datadriven/lic4/should_detect_something_5.pdf");
121        let bytes = std::fs::read(path).expect("failed to read pdf fixture");
122
123        let (text, kind) = extract_text_for_detection(path, &bytes);
124
125        assert_eq!(kind, ExtractedTextKind::Pdf);
126        assert!(text.contains("SUN INDUSTRY STANDARDS SOURCE LICENSE"));
127        assert!(!text.contains("DISCLAIMER OF WARRANTY"));
128    }
129
130    #[test]
131    fn test_extract_text_for_detection_does_not_duplicate_pdf_heading_prefix() {
132        let path =
133            Path::new("testdata/license-golden/datadriven/lic4/should_detect_something_5.pdf");
134        let bytes = std::fs::read(path).expect("failed to read pdf fixture");
135
136        let (text, kind) = extract_text_for_detection(path, &bytes);
137
138        assert_eq!(kind, ExtractedTextKind::Pdf);
139
140        let normalized = normalize_pdf_heading_comparison_text(&text);
141        let heading =
142            normalize_pdf_heading_comparison_text("SUN INDUSTRY STANDARDS SOURCE LICENSE");
143        assert_eq!(normalized.matches(&heading).count(), 1);
144    }
145
146    #[test]
147    fn test_extract_text_for_detection_reads_pdf_fixture_without_pdf_extension() {
148        let path = Path::new("testdata/license-golden/datadriven/lic2/bsd-new_156.pdf");
149        let bytes = std::fs::read(path).expect("failed to read pdf fixture");
150
151        let (text, kind) = extract_text_for_detection(Path::new("renamed.bin"), &bytes);
152
153        assert_eq!(kind, ExtractedTextKind::Pdf);
154        assert!(text.contains("Redistribution and use in source and binary forms"));
155    }
156
157    #[test]
158    fn test_extract_text_for_detection_skips_oversized_pdf_payload() {
159        let mut bytes = b"%PDF-1.7\n".to_vec();
160        bytes.resize(MAX_PDF_TEXT_EXTRACTION_BYTES + 1, b'0');
161
162        let (text, kind, scan_error) =
163            extract_text_for_detection_with_diagnostics(Path::new("oversized.pdf"), &bytes);
164
165        assert!(text.is_empty());
166        assert_eq!(kind, ExtractedTextKind::None);
167        let scan_error = scan_error.expect("oversized pdf should be surfaced");
168        assert!(scan_error.message.contains("PDF text extraction skipped"));
169        assert_eq!(
170            scan_error.severity,
171            DiagnosticSeverity::Error,
172            "a real PDF extraction failure must remain an error"
173        );
174    }
175
176    #[test]
177    fn test_extract_text_for_detection_reports_terminal_pdf_failure() {
178        let malformed = b"%PDF-1.7\nthis is not a valid pdf object graph\n";
179
180        let (text, kind, scan_error) =
181            extract_text_for_detection_with_diagnostics(Path::new("broken.pdf"), malformed);
182
183        assert!(text.is_empty());
184        assert_eq!(kind, ExtractedTextKind::None);
185        let scan_error = scan_error.expect("terminal pdf failure should be surfaced");
186        assert!(
187            scan_error
188                .message
189                .contains("PDF text extraction failed after")
190        );
191        assert_eq!(
192            scan_error.severity,
193            DiagnosticSeverity::Error,
194            "a terminal PDF failure must remain an error"
195        );
196    }
197
198    #[test]
199    fn test_extract_text_for_detection_skips_large_opaque_binary_blobs() {
200        let bytes = vec![0_u8; LARGE_OPAQUE_BINARY_SKIP_BYTES + 8];
201
202        let (text, kind) = extract_text_for_detection(Path::new("model.bin"), &bytes);
203
204        assert!(text.is_empty());
205        assert_eq!(kind, ExtractedTextKind::None);
206    }
207
208    #[test]
209    fn test_extract_text_for_detection_keeps_large_binaries_with_promising_strings() {
210        let mut bytes = vec![0_u8; LARGE_OPAQUE_BINARY_SKIP_BYTES + 8];
211        let text = b"Copyright 2026 Example Project!!!";
212        bytes[..text.len()].copy_from_slice(text);
213        let second_offset = LARGE_OPAQUE_BINARY_SKIP_BYTES / 2;
214        bytes[second_offset..second_offset + text.len()].copy_from_slice(text);
215
216        let (text, kind) = extract_text_for_detection(Path::new("weights.bin"), &bytes);
217
218        assert_ne!(kind, ExtractedTextKind::None);
219        assert!(text.contains("Copyright 2026 Example Project"));
220    }
221
222    #[test]
223    fn test_extract_text_for_detection_skips_large_binary_with_unstructured_runs() {
224        let mut bytes = vec![0_u8; LARGE_OPAQUE_BINARY_SKIP_BYTES + 8];
225        let noise = b"(c) $1234567890ABCDEF[]{}--==++";
226        bytes[..noise.len()].copy_from_slice(noise);
227        let second_offset = LARGE_OPAQUE_BINARY_SKIP_BYTES / 2;
228        bytes[second_offset..second_offset + noise.len()].copy_from_slice(noise);
229
230        let (text, kind) = extract_text_for_detection(Path::new("tensor.bin"), &bytes);
231
232        assert!(text.is_empty());
233        assert_eq!(kind, ExtractedTextKind::None);
234    }
235
236    #[test]
237    fn test_extract_text_for_detection_uses_windows_executable_metadata() {
238        let path = Path::new("testdata/compiled-binary-golden/win_pe/libiconv2.dll");
239        let bytes = std::fs::read(path).expect("read PE fixture");
240
241        let (text, kind) = extract_text_for_detection(path, &bytes);
242
243        assert_eq!(kind, ExtractedTextKind::BinaryStrings);
244        assert!(text.contains("License: This program is free software"));
245        assert!(text.contains("LegalCopyright:"));
246    }
247
248    #[test]
249    fn test_extract_text_for_detection_keeps_windows_metadata_for_large_pe_without_sampled_signal()
250    {
251        let path = Path::new("testdata/compiled-binary-golden/win_pe/libiconv2.dll");
252        let mut bytes = std::fs::read(path).expect("read PE fixture");
253        bytes.resize(LARGE_OPAQUE_BINARY_SKIP_BYTES + 8, 0);
254
255        let (text, kind) = extract_text_for_detection(path, &bytes);
256
257        assert_ne!(kind, ExtractedTextKind::None);
258        assert!(!text.trim().is_empty());
259    }
260
261    #[test]
262    fn test_windows_metadata_or_empty_result_preserves_metadata() {
263        let (text, kind, scan_error) =
264            windows_metadata_or_empty_result(Some("LegalCopyright: Example Corp".to_string()));
265
266        assert_eq!(kind, ExtractedTextKind::WindowsExecutableMetadata);
267        assert_eq!(text, "LegalCopyright: Example Corp");
268        assert!(scan_error.is_none());
269    }
270
271    #[test]
272    fn test_extract_text_for_detection_keeps_image_author_separate_from_title_and_description() {
273        let xmp = r#"<x:xmpmeta xmlns:x="adobe:ns:meta/"><rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"><rdf:Description xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:creator>Chinmay Garde</dc:creator><dc:title>Bay Bridge At Night</dc:title><dc:description>Embarcadero in the evening on Delta 3200</dc:description></rdf:Description></rdf:RDF></x:xmpmeta>"#;
274        let bytes = build_png_with_xmp(xmp);
275
276        let (text, kind) = extract_text_for_detection(Path::new("fixture.png"), &bytes);
277
278        assert_eq!(kind, ExtractedTextKind::ImageMetadata);
279        assert!(text.contains("Author: Chinmay Garde"), "text: {text:?}");
280        assert!(
281            text.contains("Title: Bay Bridge At Night"),
282            "text: {text:?}"
283        );
284        assert!(
285            text.contains("Description: Embarcadero in the evening on Delta 3200"),
286            "text: {text:?}"
287        );
288
289        let (_copyrights, _holders, authors) = detect_copyrights(&text, None);
290        assert_eq!(
291            authors
292                .iter()
293                .map(|a| a.author.as_str())
294                .collect::<Vec<_>>(),
295            vec!["Chinmay Garde"],
296            "authors: {authors:?}; text: {text:?}"
297        );
298    }
299
300    #[test]
301    fn test_extract_text_for_detection_skips_large_binary_with_single_isolated_string_run() {
302        let mut bytes = vec![0_u8; LARGE_OPAQUE_BINARY_SKIP_BYTES + 8];
303        let text = b"Copyright 2026 Example Project!!!";
304        bytes[..text.len()].copy_from_slice(text);
305
306        let (text, kind) = extract_text_for_detection(Path::new("opaque.bin"), &bytes);
307
308        assert!(text.is_empty());
309        assert_eq!(kind, ExtractedTextKind::None);
310    }
311
312    #[test]
313    fn test_extract_text_for_detection_keeps_large_binary_with_single_contact_rich_window() {
314        let mut bytes = vec![0_u8; LARGE_OPAQUE_BINARY_SKIP_BYTES + 8];
315        let text = b"Andreas Schneider <asn@redhat.com> Rob Crittenden (rcritten@redhat.com) Mr. Sam <sam@email-scan.com> https://publicsuffix.org/ http://tukaani.org/xz/";
316        bytes[..text.len()].copy_from_slice(text);
317
318        let (text, kind) = extract_text_for_detection(Path::new("rootfs.bin"), &bytes);
319
320        assert_ne!(kind, ExtractedTextKind::None);
321        assert!(text.contains("asn@redhat.com"));
322        assert!(text.contains("https://publicsuffix.org/"));
323    }
324
325    #[test]
326    fn test_extract_text_for_detection_keeps_large_macho_with_off_window_legal_markers() {
327        let mut bytes = vec![0_u8; LARGE_OPAQUE_BINARY_SKIP_BYTES * 2];
328        bytes[..4].copy_from_slice(&[0xCF, 0xFA, 0xED, 0xFE]);
329        let apache_notice = b"// Licensed under the Apache License, Version 2.0 (the \"License\");\n// http://www.apache.org/licenses/LICENSE-2.0\n// SPDX-License-Identifier: Apache-2.0\n";
330        let insert_offset = 200 * 1024;
331        bytes[insert_offset..insert_offset + apache_notice.len()].copy_from_slice(apache_notice);
332
333        let (text, kind) = extract_text_for_detection(Path::new("node"), &bytes);
334
335        assert_eq!(kind, ExtractedTextKind::BinaryStrings);
336        assert!(text.contains("Apache License, Version 2.0"), "{text}");
337        assert!(
338            text.contains("SPDX-License-Identifier: Apache-2.0"),
339            "{text}"
340        );
341    }
342
343    #[test]
344    fn test_extract_text_for_detection_keeps_large_macho_with_unicode_notice_markers() {
345        let mut bytes = vec![0_u8; LARGE_OPAQUE_BINARY_SKIP_BYTES * 2];
346        bytes[..4].copy_from_slice(&[0xCF, 0xFA, 0xED, 0xFE]);
347        let unicode_notice = b"Copyright (c) 1991-2024 Unicode, Inc.\nFor terms of use, see http://www.unicode.org/copyright.html\n";
348        let insert_offset = 700 * 1024;
349        bytes[insert_offset..insert_offset + unicode_notice.len()].copy_from_slice(unicode_notice);
350
351        let (text, kind) = extract_text_for_detection(Path::new("node"), &bytes);
352
353        assert_eq!(kind, ExtractedTextKind::BinaryStrings);
354        assert!(text.contains("Unicode, Inc."), "{text}");
355        assert!(text.contains("unicode.org/copyright.html"), "{text}");
356    }
357
358    #[test]
359    fn test_extract_text_for_detection_does_not_reopen_single_window_legal_noise_for_non_macho() {
360        let mut bytes = vec![0_u8; LARGE_OPAQUE_BINARY_SKIP_BYTES * 2];
361        let apache_notice = b"// Licensed under the Apache License, Version 2.0 (the \"License\");\n// http://www.apache.org/licenses/LICENSE-2.0\n// SPDX-License-Identifier: Apache-2.0\n";
362        let insert_offset = 200 * 1024;
363        bytes[insert_offset..insert_offset + apache_notice.len()].copy_from_slice(apache_notice);
364
365        let (text, kind) = extract_text_for_detection(Path::new("model.bin"), &bytes);
366
367        assert!(text.is_empty());
368        assert_eq!(kind, ExtractedTextKind::None);
369    }
370
371    #[test]
372    fn test_extract_text_for_detection_avoids_latin1_decode_for_binary_blob_noise() {
373        let bytes = vec![
374            0x28, 0x63, 0x29, 0x20, 0x4b, 0x30, 0x0e, 0x71, 0x86, 0x20, 0x62, 0x24, 0x4c,
375        ];
376
377        let (text, kind) = extract_text_for_detection(Path::new("fixture.blb"), &bytes);
378
379        assert_eq!(kind, ExtractedTextKind::BinaryStrings);
380        assert_eq!(text, "(c) K0\n b$L");
381    }
382
383    #[test]
384    fn test_extract_raw_xmp_packet_rejects_oversized_png_itxt_payload() {
385        let xmp = "A".repeat(MAX_XMP_PACKET_BYTES + 1);
386        let bytes = build_png_with_xmp(&xmp);
387
388        assert!(extract_raw_xmp_packet(&bytes, ImageFormat::Png).is_none());
389    }
390
391    #[test]
392    fn test_extract_text_for_detection_skips_zip_like_archives() {
393        let zip_bytes = b"PK\x03\x04\x14\x00\x00\x00\x08\x00artifact";
394
395        let (whl_text, whl_kind) = extract_text_for_detection(Path::new("demo.whl"), zip_bytes);
396        let (crate_text, crate_kind) =
397            extract_text_for_detection(Path::new("demo.crate"), zip_bytes);
398
399        assert!(whl_text.is_empty());
400        assert_eq!(whl_kind, ExtractedTextKind::None);
401        assert!(crate_text.is_empty());
402        assert_eq!(crate_kind, ExtractedTextKind::None);
403    }
404
405    #[test]
406    fn test_extract_text_for_detection_keeps_binary_strings_for_lib_fixtures() {
407        let path =
408            Path::new("testdata/copyright-golden/copyrights/copyright_php_lib-php_embed_lib.lib");
409        let bytes = std::fs::read(path).expect("failed to read lib fixture");
410
411        let (text, kind) = extract_text_for_detection(path, &bytes);
412
413        assert_ne!(kind, ExtractedTextKind::None);
414        assert!(text.contains("Copyright nexB and others (c) 2012"));
415    }
416
417    #[test]
418    fn test_extract_text_for_detection_reads_font_metadata() {
419        let path = Path::new("testdata/font-fixtures/Lato-Bold.ttf");
420        let bytes = std::fs::read(path).expect("failed to read font fixture");
421
422        let (text, kind) = extract_text_for_detection(path, &bytes);
423
424        assert_eq!(kind, ExtractedTextKind::FontMetadata);
425        assert!(text.contains("License Description:"), "{text}");
426        assert!(
427            text.contains("Open Font License") || text.contains("OFL"),
428            "{text}"
429        );
430        assert!(text.contains("Lato"), "{text}");
431    }
432
433    #[test]
434    fn test_extract_printable_strings_scales_cap_for_medium_binary_files() {
435        let bytes = b"abcd\0".repeat(525_000);
436
437        let text = extract_printable_strings(&bytes);
438
439        assert!(
440            text.len() > 2_000_000,
441            "unexpected truncation at {}",
442            text.len()
443        );
444        assert!(text.ends_with("abcd"));
445    }
446
447    #[test]
448    fn test_extract_text_for_detection_decodes_svg_fixture_text() {
449        let path = Path::new(
450            "testdata/license-golden/datadriven/external/fossology-tests/Public-domain/biohazard.svg",
451        );
452        let bytes = std::fs::read(path).expect("failed to read svg fixture");
453
454        let (text, kind) = extract_text_for_detection(path, &bytes);
455
456        assert_eq!(kind, ExtractedTextKind::Decoded);
457        assert!(text.contains("creativecommons.org/licenses/publicdomain"));
458    }
459
460    #[test]
461    fn test_extract_text_for_detection_preserves_blank_comment_lines_in_utf8_source() {
462        let path = Path::new("testdata/plugin_email_url/files/IMarkerActionFilter.java");
463        let bytes = std::fs::read(path).expect("failed to read java fixture");
464
465        let (text, kind) = extract_text_for_detection(path, &bytes);
466
467        assert_eq!(kind, ExtractedTextKind::Decoded);
468        let lines: Vec<_> = text.lines().collect();
469        assert_eq!(lines.get(2).copied(), Some(" *"));
470        assert_eq!(
471            lines.get(3).copied(),
472            Some(" *https://github.com/rpm-software-management")
473        );
474        assert_eq!(lines.get(5).copied(), Some("https://gitlab.com/Conan_Kudo"));
475    }
476
477    #[test]
478    fn test_extract_text_for_detection_decodes_rtf_fixture_text() {
479        let path = Path::new(
480            "testdata/license-golden/datadriven/external/fossology-tests/LGPL/License.rtf",
481        );
482        let bytes = std::fs::read(path).expect("failed to read rtf fixture");
483
484        let (text, kind) = extract_text_for_detection(path, &bytes);
485
486        assert_eq!(kind, ExtractedTextKind::Decoded);
487        assert!(text.contains("GNU Lesser General Public"));
488        assert!(text.contains("version"));
489        assert!(text.contains("2.1 of the License"));
490    }
491
492    #[test]
493    fn test_classify_file_info_marks_empty_files_as_text_not_source() {
494        let classification = classify_file_info(Path::new("test.txt"), b"");
495
496        assert_eq!(classification.mime_type, "inode/x-empty");
497        assert_eq!(classification.file_type, "empty");
498        assert!(!classification.is_binary);
499        assert!(classification.is_text);
500        assert!(!classification.is_source);
501        assert_eq!(classification.programming_language, None);
502    }
503
504    #[test]
505    fn test_classify_file_info_keeps_json_out_of_programming_language() {
506        let classification = classify_file_info(Path::new("package.json"), br#"{"name":"demo"}"#);
507
508        assert_eq!(classification.mime_type, "application/json");
509        assert_eq!(classification.file_type, "JSON text data");
510        assert!(classification.is_text);
511        assert!(!classification.is_source);
512        assert_eq!(classification.programming_language, None);
513    }
514
515    #[test]
516    fn test_classify_file_info_does_not_label_invalid_json_text_as_json() {
517        let classification =
518            classify_file_info(Path::new("broken.json"), b"{ definitely not json\n");
519
520        assert_eq!(classification.mime_type, "text/plain");
521        assert_eq!(classification.file_type, "UTF-8 Unicode text");
522        assert!(classification.is_text);
523        assert!(!classification.is_binary);
524    }
525
526    #[test]
527    fn test_classify_file_info_does_not_label_binary_json_garbage_as_json() {
528        let classification =
529            classify_file_info(Path::new("broken.json"), &[0xff, 0x00, 0x01, 0x02]);
530
531        assert_eq!(classification.mime_type, "application/octet-stream");
532        assert_eq!(classification.file_type, "data");
533        assert!(classification.is_binary);
534        assert!(!classification.is_text);
535    }
536
537    #[test]
538    fn test_classify_file_info_treats_valid_utf16_json_with_bom_as_text() {
539        let classification = classify_file_info(
540            Path::new("utf16.json"),
541            &[
542                0xFF, 0xFE, 0x5B, 0x00, 0x22, 0x00, 0xE9, 0x00, 0x22, 0x00, 0x5D, 0x00,
543            ],
544        );
545
546        assert!(!classification.is_binary);
547        assert!(classification.is_text);
548        assert_eq!(classification.mime_type, "application/json");
549        assert_eq!(classification.file_type, "JSON text data");
550    }
551
552    #[test]
553    fn test_classify_file_info_treats_valid_utf16be_json_without_bom_as_text() {
554        let classification = classify_file_info(
555            Path::new("utf16be.json"),
556            &[0x00, 0x5B, 0x00, 0x22, 0x00, 0xE9, 0x00, 0x22, 0x00, 0x5D],
557        );
558
559        assert!(!classification.is_binary);
560        assert!(classification.is_text);
561        assert_eq!(classification.mime_type, "application/json");
562        assert_eq!(classification.file_type, "JSON text data");
563    }
564
565    #[test]
566    fn test_classify_file_info_treats_small_valid_utf16be_json_literal_as_text() {
567        let classification =
568            classify_file_info(Path::new("utf16be-literal.json"), &[0x00, 0x5B, 0x00, 0x5D]);
569
570        assert!(!classification.is_binary);
571        assert!(classification.is_text);
572        assert_eq!(classification.mime_type, "application/json");
573        assert_eq!(classification.file_type, "JSON text data");
574    }
575
576    #[test]
577    fn test_extract_text_for_detection_decodes_utf16be_text_with_corrupted_bom_prefix() {
578        let mut bytes = CORRUPTED_UTF16_BOM_PREFIX.to_vec();
579        for code_unit in
580            "Licensed to the Apache Software Foundation\nApache License, Version 2.0".encode_utf16()
581        {
582            bytes.extend_from_slice(&code_unit.to_be_bytes());
583        }
584
585        let (text, kind) = extract_text_for_detection(Path::new("notice.ftl"), &bytes);
586
587        assert_eq!(kind, ExtractedTextKind::Decoded);
588        assert!(text.contains("Apache Software Foundation"), "{text}");
589        assert!(text.contains("Apache License, Version 2.0"), "{text}");
590    }
591
592    #[test]
593    fn test_classify_file_info_treats_small_valid_json_literals_as_text() {
594        let classification = classify_file_info(Path::new("true.json"), b"true");
595
596        assert!(!classification.is_binary);
597        assert!(classification.is_text);
598        assert_eq!(classification.mime_type, "application/json");
599        assert_eq!(classification.file_type, "JSON text data");
600    }
601
602    #[test]
603    fn test_classify_file_info_treats_json_wrapped_invalid_utf8_sequences_as_text() {
604        let classification = classify_file_info(
605            Path::new("wrapped.json"),
606            &[0x5B, 0x22, 0xE6, 0x97, 0xA5, 0xD1, 0x88, 0xFA, 0x22, 0x5D],
607        );
608
609        assert!(!classification.is_binary);
610        assert!(classification.is_text);
611        assert_eq!(classification.mime_type, "text/plain");
612        assert_eq!(classification.file_type, "text, with no line terminators");
613    }
614
615    #[test]
616    fn test_classify_file_info_keeps_lone_ff_json_byte_binary() {
617        let classification =
618            classify_file_info(Path::new("lone-ff.json"), &[0x5B, 0x22, 0xFF, 0x22, 0x5D]);
619
620        assert!(classification.is_binary);
621        assert!(!classification.is_text);
622        assert_eq!(classification.mime_type, "application/octet-stream");
623        assert_eq!(classification.file_type, "data");
624    }
625
626    #[test]
627    fn test_classify_file_info_keeps_nul_heavy_crash_json_binary() {
628        let classification = classify_file_info(
629            Path::new("crash.json"),
630            &[
631                0xFE, 0x90, 0x00, 0x00, 0x00, 0x93, 0x5B, 0x5B, 0x32, 0x38, 0x36,
632            ],
633        );
634
635        assert!(classification.is_binary);
636        assert!(!classification.is_text);
637        assert_eq!(classification.mime_type, "application/octet-stream");
638    }
639
640    #[test]
641    fn test_classify_file_info_treats_dockerfile_as_source() {
642        let classification = classify_file_info(Path::new("Dockerfile"), b"FROM scratch\n");
643
644        assert_eq!(
645            classification.programming_language.as_deref(),
646            Some("Dockerfile")
647        );
648        assert!(classification.is_source);
649        assert!(!classification.is_script);
650        assert_eq!(
651            classification.file_type,
652            "Dockerfile source, UTF-8 Unicode text"
653        );
654    }
655
656    #[test]
657    fn test_classify_file_info_treats_makefile_as_text_not_source() {
658        let classification = classify_file_info(Path::new("Makefile"), b"all:\n\techo hi\n");
659
660        assert_eq!(classification.programming_language, None);
661        assert!(classification.is_text);
662        assert!(!classification.is_source);
663        assert!(!classification.is_script);
664        assert_eq!(classification.file_type, "UTF-8 Unicode text");
665    }
666
667    #[test]
668    fn test_classify_file_info_marks_supported_package_archives() {
669        let zip_bytes = b"PK\x03\x04\x14\x00\x00\x00";
670
671        let egg = classify_file_info(Path::new("demo.egg"), zip_bytes);
672        let nupkg = classify_file_info(Path::new("demo.nupkg"), zip_bytes);
673
674        assert!(egg.is_archive);
675        assert_eq!(egg.mime_type, "application/zip");
676        assert_eq!(egg.file_type, "Zip archive data");
677        assert!(nupkg.is_archive);
678        assert_eq!(nupkg.mime_type, "application/zip");
679        assert_eq!(nupkg.file_type, "Zip archive data");
680    }
681
682    #[test]
683    fn test_classify_file_info_marks_png_as_binary_media() {
684        let png_bytes = b"\x89PNG\r\n\x1a\n\x00\x00\x00\x0dIHDR";
685
686        let classification = classify_file_info(Path::new("logo.png"), png_bytes);
687
688        assert_eq!(classification.mime_type, "image/png");
689        assert_eq!(classification.file_type, "PNG image data");
690        assert!(classification.is_binary);
691        assert!(!classification.is_text);
692        assert!(classification.is_media);
693        assert!(!classification.is_archive);
694        assert!(!classification.is_source);
695    }
696
697    #[test]
698    fn test_classify_file_info_marks_pdf_as_binary_document() {
699        let pdf_bytes = b"%PDF-1.7\n1 0 obj\n<< /Type /Catalog >>\n";
700
701        let classification = classify_file_info(Path::new("report.pdf"), pdf_bytes);
702
703        assert_eq!(classification.mime_type, "application/pdf");
704        assert_eq!(classification.file_type, "PDF document");
705        assert!(classification.is_binary);
706        assert!(!classification.is_text);
707        assert!(!classification.is_archive);
708        assert!(!classification.is_media);
709    }
710
711    #[test]
712    fn test_classify_file_info_marks_binary_blobs_as_binary() {
713        let classification =
714            classify_file_info(Path::new("blob.bin"), &[0, 159, 146, 150, 0, 1, 2, 3, 4, 5]);
715
716        assert!(classification.is_binary);
717        assert!(!classification.is_text);
718        assert!(!classification.is_source);
719        assert_eq!(classification.programming_language, None);
720    }
721
722    #[test]
723    fn test_classify_file_info_treats_yaml_as_text_not_source() {
724        let classification = classify_file_info(Path::new("config.yaml"), b"key: value\n");
725
726        assert_eq!(classification.programming_language, None);
727        assert!(classification.is_text);
728        assert!(!classification.is_source);
729        assert_eq!(classification.file_type, "YAML text data");
730    }
731
732    #[test]
733    fn test_classify_file_info_classifies_common_build_manifests() {
734        let gradle = classify_file_info(Path::new("build.gradle"), b"plugins { id 'java' }\n");
735        let flake = classify_file_info(Path::new("flake.nix"), b"{ inputs, ... }: {}\n");
736        let cmake = classify_file_info(
737            Path::new("toolchain.cmake"),
738            b"set(CMAKE_CXX_STANDARD 20)\n",
739        );
740        let gitmodules = classify_file_info(
741            Path::new(".gitmodules"),
742            b"[submodule \"demo\"]\n\tpath = vendor/demo\n",
743        );
744
745        assert_eq!(gradle.programming_language.as_deref(), Some("Groovy"));
746        assert!(gradle.is_source);
747        assert_eq!(gradle.mime_type, "text/plain");
748        assert_eq!(gradle.file_type, "Groovy source, UTF-8 Unicode text");
749
750        assert_eq!(flake.programming_language.as_deref(), Some("Nix"));
751        assert!(flake.is_source);
752        assert_eq!(flake.mime_type, "text/plain");
753        assert_eq!(flake.file_type, "Nix source, UTF-8 Unicode text");
754
755        assert_eq!(cmake.programming_language.as_deref(), Some("CMake"));
756        assert!(cmake.is_source);
757        assert_eq!(cmake.file_type, "CMake source, UTF-8 Unicode text");
758
759        assert_eq!(gitmodules.programming_language, None);
760        assert!(gitmodules.is_text);
761        assert!(!gitmodules.is_source);
762        assert_eq!(gitmodules.file_type, "Git configuration text");
763    }
764
765    #[test]
766    fn test_classify_file_info_labels_cpp_headers_and_ipp_separately() {
767        let header = classify_file_info(
768            Path::new("include/demo.hpp"),
769            b"#pragma once\nclass Demo {};\n",
770        );
771        let ipp = classify_file_info(
772            Path::new("include/detail/demo.ipp"),
773            b"template <class T> void parse() {}\n",
774        );
775
776        assert_eq!(header.programming_language.as_deref(), Some("C++"));
777        assert!(header.is_source);
778        assert!(!header.is_script);
779        assert_eq!(header.file_type, "C++ source, UTF-8 Unicode text");
780
781        assert_eq!(ipp.programming_language, None);
782        assert!(!ipp.is_source);
783        assert!(!ipp.is_script);
784        assert_eq!(ipp.file_type, "UTF-8 Unicode text");
785    }
786
787    #[test]
788    fn test_classify_file_info_preserves_specific_shell_family_labels() {
789        let bash = classify_file_info(Path::new("bin/run"), b"#!/usr/bin/env bash\necho hi\n");
790
791        assert_eq!(bash.programming_language.as_deref(), Some("Bash"));
792        assert!(bash.is_script);
793        assert_eq!(bash.file_type, "bash script, UTF-8 Unicode text executable");
794    }
795
796    #[test]
797    fn test_classify_file_info_marks_jamfile_as_source() {
798        let jamfile = classify_file_info(Path::new("Jamfile"), b"lib boost_json ;\n");
799
800        assert_eq!(jamfile.programming_language.as_deref(), Some("Jamfile"));
801        assert!(jamfile.is_source);
802        assert!(!jamfile.is_script);
803        assert_eq!(jamfile.file_type, "Jamfile source, UTF-8 Unicode text");
804    }
805
806    #[test]
807    fn test_classify_file_info_labels_javascript_shebang_scripts() {
808        let classification = classify_file_info(
809            Path::new("bin/run"),
810            b"#!/usr/bin/env node\nconsole.log('hello');\n",
811        );
812
813        assert_eq!(
814            classification.programming_language.as_deref(),
815            Some("JavaScript")
816        );
817        assert!(classification.is_script);
818        assert_eq!(
819            classification.file_type,
820            "javascript script, UTF-8 Unicode text executable"
821        );
822    }
823
824    #[test]
825    fn test_classify_file_info_uses_non_utf8_text_labels_for_latin1_scripts() {
826        let classification = classify_file_info(
827            Path::new("script.py"),
828            b"# coding: latin-1\nprint(\"caf\xe9\")\n",
829        );
830
831        assert_eq!(
832            classification.programming_language.as_deref(),
833            Some("Python")
834        );
835        assert!(classification.is_script);
836        assert_eq!(classification.file_type, "python script, text executable");
837    }
838
839    #[test]
840    fn test_classify_file_info_treats_textual_tga_as_media() {
841        let classification = classify_file_info(Path::new("texture.tga"), b"not really a tga\n");
842
843        assert!(classification.is_media);
844        assert!(classification.is_text);
845        assert!(!classification.is_binary);
846    }
847
848    #[test]
849    fn test_classify_file_info_keeps_binaryish_source_extension_out_of_text_path() {
850        let classification =
851            classify_file_info(Path::new("main.ts"), &[0x80, 0x81, 0x82, 0x83, 0x84, 0x85]);
852
853        assert!(classification.is_binary);
854        assert!(!classification.is_text);
855        assert!(!classification.is_source);
856        assert_eq!(classification.programming_language, None);
857    }
858
859    #[test]
860    fn test_extract_text_for_detection_skips_unsupported_image_formats() {
861        let gif_bytes = b"GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;";
862
863        let (text, kind) = extract_text_for_detection(Path::new("tiny.gif"), gif_bytes);
864
865        assert!(text.is_empty());
866        assert_eq!(kind, ExtractedTextKind::None);
867    }
868
869    #[test]
870    fn test_classify_file_info_preserves_language_detection_precedence_matrix() {
871        let cases = [
872            (
873                Path::new("bin/run"),
874                b"#!/usr/bin/env node\nconsole.log('hello');\n".as_slice(),
875                Some("JavaScript"),
876                true,
877                true,
878            ),
879            (
880                Path::new("Dockerfile"),
881                b"FROM scratch\n".as_slice(),
882                Some("Dockerfile"),
883                true,
884                false,
885            ),
886            (
887                Path::new("package.json"),
888                br#"{"name":"demo"}"#.as_slice(),
889                None,
890                false,
891                false,
892            ),
893            (
894                Path::new("config.yaml"),
895                b"key: value\n".as_slice(),
896                None,
897                false,
898                false,
899            ),
900            (
901                Path::new("Makefile"),
902                b"all:\n\techo hi\n".as_slice(),
903                None,
904                false,
905                false,
906            ),
907        ];
908
909        for (path, bytes, expected_language, expected_is_source, expected_is_script) in cases {
910            let classification = classify_file_info(path, bytes);
911
912            assert_eq!(
913                classification.programming_language.as_deref(),
914                expected_language,
915                "unexpected language for {}",
916                path.display()
917            );
918            assert_eq!(
919                classification.is_source,
920                expected_is_source,
921                "unexpected is_source for {}",
922                path.display()
923            );
924            assert_eq!(
925                classification.is_script,
926                expected_is_script,
927                "unexpected is_script for {}",
928                path.display()
929            );
930        }
931    }
932
933    // ----------------------------------------------------------------------
934    // Encoding and line-ending edge cases (issue #1030).
935    //
936    // These tests PIN the current decoding/line-ending contract. Several of
937    // them document behavior that is intentionally accepted ScanCode /
938    // UnicodeDammit parity (mojibake from Latin-1 fallback) or known sharp
939    // edges (lone-CR line endings, the 10% control-byte cliff). They are not
940    // assertions that the current behavior is ideal; they exist so any future
941    // change to it is deliberate and reviewed.
942    //
943    // Fixtures are built from explicit byte literals (e.g. `b"...\r..."`) so
944    // editor/source normalization can never silently change the line endings
945    // under test.
946    // ----------------------------------------------------------------------
947
948    /// Builds invalid-UTF-8 bytes with a controllable ratio of control bytes
949    /// to printable bytes, so threshold behavior can be pinned precisely.
950    ///
951    /// Each entry contributes one printable byte (`A`, valid ASCII) and a 0x80
952    /// high byte makes the buffer invalid UTF-8 so it takes the Latin-1 path.
953    /// Control bytes use 0x01 (counted by `has_binary_control_chars`).
954    fn invalid_utf8_with_control_ratio(printable: usize, control: usize) -> Vec<u8> {
955        let mut bytes = vec![0x80]; // forces invalid UTF-8
956        bytes.extend(std::iter::repeat_n(b'A', printable));
957        bytes.extend(std::iter::repeat_n(0x01, control));
958        bytes
959    }
960
961    #[test]
962    fn decode_bytes_control_char_threshold_just_under_decodes_as_latin1() {
963        // Total 100 bytes, exactly 10 control bytes => 10%. The threshold is
964        // `control > len / 10`, so 10 is NOT over the cliff: decoded as Latin-1.
965        let bytes = invalid_utf8_with_control_ratio(89, 10);
966        assert_eq!(bytes.len(), 100);
967
968        let (decoded, diagnostic) = decode_bytes_to_string_with_diagnostic(&bytes);
969
970        assert!(
971            !decoded.is_empty(),
972            "input at the 10% boundary must still decode (Latin-1 fallback)"
973        );
974        assert!(decoded.contains('A'));
975        assert!(diagnostic.is_none());
976    }
977
978    #[test]
979    fn decode_bytes_control_char_threshold_just_over_returns_empty_with_diagnostic() {
980        // Total 100 bytes, 11 control bytes => 11% > 10%, over the cliff:
981        // empty string plus a structured diagnostic.
982        let bytes = invalid_utf8_with_control_ratio(88, 11);
983        assert_eq!(bytes.len(), 100);
984
985        let (decoded, diagnostic) = decode_bytes_to_string_with_diagnostic(&bytes);
986
987        assert!(
988            decoded.is_empty(),
989            "input over the 10% control-byte threshold must decode to empty"
990        );
991        let diagnostic = diagnostic.expect("near-binary skip must emit a diagnostic");
992        assert_eq!(diagnostic.message, NEAR_BINARY_SKIP_DIAGNOSTIC);
993        assert_eq!(
994            diagnostic.severity,
995            DiagnosticSeverity::Info,
996            "benign binary-content skip must be informational, not an error"
997        );
998        // The thin public wrapper preserves the empty-string result contract.
999        assert!(decode_bytes_to_string(&bytes).is_empty());
1000    }
1001
1002    #[test]
1003    fn decode_bytes_latin1_high_bytes_produce_mojibake_parity() {
1004        // Windows-1252 "café" (é = 0xE9) and Shift-JIS "日" (0x93 0xFA) are not
1005        // valid UTF-8; the Latin-1 fallback maps each byte to U+0000..=U+00FF,
1006        // producing mojibake. This is accepted ScanCode/UnicodeDammit parity.
1007        let windows_1252 = b"caf\xE9";
1008        let decoded = decode_bytes_to_string(windows_1252);
1009        // 0xE9 -> U+00E9 (é). Latin-1 happens to agree with Windows-1252 here.
1010        assert_eq!(decoded, "caf\u{00E9}");
1011
1012        let shift_jis = b"\x93\xFA"; // Shift-JIS for "日"
1013        let decoded = decode_bytes_to_string(shift_jis);
1014        // Mojibake: bytes mapped directly, NOT decoded to "日".
1015        assert_eq!(decoded, "\u{0093}\u{00FA}");
1016        assert!(!decoded.contains('日'));
1017    }
1018
1019    #[test]
1020    fn decode_bytes_utf8_bom_is_preserved_as_leading_codepoint() {
1021        // UTF-8 BOM (EF BB BF) is valid UTF-8: it decodes to U+FEFF and is NOT
1022        // stripped by the decoder. Pin that contract here.
1023        let bytes = b"\xEF\xBB\xBFhello";
1024        let decoded = decode_bytes_to_string(bytes);
1025        assert_eq!(decoded, "\u{FEFF}hello");
1026    }
1027
1028    /// Encodes an ASCII string as UTF-16 with the given endianness, prefixed by
1029    /// the matching BOM. Only ASCII codepoints are used in the fixtures.
1030    fn utf16_with_bom(text: &str, little_endian: bool) -> Vec<u8> {
1031        let mut bytes = if little_endian {
1032            vec![0xFF, 0xFE]
1033        } else {
1034            vec![0xFE, 0xFF]
1035        };
1036        for ch in text.chars() {
1037            let unit = ch as u16;
1038            if little_endian {
1039                bytes.extend_from_slice(&unit.to_le_bytes());
1040            } else {
1041                bytes.extend_from_slice(&unit.to_be_bytes());
1042            }
1043        }
1044        bytes
1045    }
1046
1047    #[test]
1048    fn decode_bytes_utf16_le_bom_is_decoded_and_bom_stripped() {
1049        // UTF-16 LE BOM (FF FE) followed by text: the UTF-16 path consumes the
1050        // BOM and returns the text without a leading U+FEFF. The body must be
1051        // long enough to pass the text-shape gate (>= 3 visible chars).
1052        let bytes = utf16_with_bom("hello world", true);
1053        let decoded = decode_bytes_to_string(&bytes);
1054        assert_eq!(decoded, "hello world");
1055        assert!(!decoded.starts_with('\u{FEFF}'));
1056    }
1057
1058    #[test]
1059    fn decode_bytes_utf16_be_bom_is_decoded_and_bom_stripped() {
1060        let bytes = utf16_with_bom("hello world", false);
1061        let decoded = decode_bytes_to_string(&bytes);
1062        assert_eq!(decoded, "hello world");
1063        assert!(!decoded.starts_with('\u{FEFF}'));
1064    }
1065
1066    #[test]
1067    fn decode_bytes_short_utf16_bom_text_fails_text_shape_gate() {
1068        // Very short UTF-16 BOM bodies (< 3 visible chars) do NOT pass the
1069        // text-shape heuristic and decode to empty. Pinned so any change to the
1070        // minimum-visible threshold is deliberate.
1071        let bytes = utf16_with_bom("hi", true);
1072        assert_eq!(decode_bytes_to_string(&bytes), "");
1073    }
1074
1075    #[test]
1076    fn decode_bytes_corrupted_utf16_bom_prefix_is_stripped() {
1077        // The hardcoded "corrupted UTF-16 BOM" prefix (EF BF BD repeated, i.e.
1078        // two U+FFFD replacement chars) is stripped before UTF-16 detection.
1079        // Body is UTF-16 LE for "hello world test text long enough".
1080        let mut bytes = CORRUPTED_UTF16_BOM_PREFIX.to_vec();
1081        for ch in "hello world test text long enough".chars() {
1082            bytes.push(ch as u8);
1083            bytes.push(0x00);
1084        }
1085        let decoded = decode_bytes_to_string(&bytes);
1086        assert_eq!(decoded, "hello world test text long enough");
1087        // The corrupted-BOM marker must not survive in the decoded output.
1088        assert!(!decoded.contains('\u{FFFD}'));
1089    }
1090
1091    #[test]
1092    fn decoded_text_lone_cr_line_endings_are_not_split_by_lines() {
1093        // Classic-Mac (CR-only) line endings: `str::lines()` splits on `\n` and
1094        // `\r\n`, but NOT a lone `\r`. So three logical lines collapse into one
1095        // `lines()` element. This is a known sharp edge that can misattribute
1096        // email/url/copyright line numbers; pinned here rather than changed,
1097        // since fixing line-splitting would shift detection line numbers
1098        // broadly. Flagged as a follow-up in the issue.
1099        let bytes = b"first line\rsecond line\rthird line";
1100        let decoded = decode_bytes_to_string(bytes);
1101        assert_eq!(decoded, "first line\rsecond line\rthird line");
1102
1103        let line_count = decoded.lines().count();
1104        assert_eq!(
1105            line_count, 1,
1106            "lone-CR line endings are currently NOT split by str::lines()"
1107        );
1108
1109        // For contrast, LF and CRLF DO split into separate lines.
1110        let lf = decode_bytes_to_string(b"a\nb\nc");
1111        assert_eq!(lf.lines().count(), 3);
1112        let crlf = decode_bytes_to_string(b"a\r\nb\r\nc");
1113        assert_eq!(crlf.lines().count(), 3);
1114    }
1115
1116    #[test]
1117    fn extract_text_near_binary_invalid_utf8_surfaces_skip_diagnostic() {
1118        // End-to-end through the extraction entry point. An `.svg` path forces
1119        // the decoded-text branch (`should_try_decoded_text`) even for
1120        // near-binary bytes, so the diagnostic threading is exercised. The
1121        // bytes are invalid UTF-8 with a high control-byte ratio and no
1122        // recoverable printable strings, so the file drops from detection AND
1123        // surfaces the structured near-binary skip diagnostic.
1124        let path = Path::new("garbled.svg");
1125        let bytes = invalid_utf8_with_control_ratio(1, 60);
1126
1127        let (text, kind, diagnostic) = extract_text_for_detection_with_diagnostics(path, &bytes);
1128
1129        assert!(text.is_empty());
1130        assert_eq!(kind, ExtractedTextKind::None);
1131        let diagnostic = diagnostic.expect("near-binary skip must emit a diagnostic");
1132        assert_eq!(diagnostic.message, NEAR_BINARY_SKIP_DIAGNOSTIC);
1133        assert_eq!(diagnostic.severity, DiagnosticSeverity::Info);
1134    }
1135}
provenant/utils/file/mod.rs

provenant/utils/file/
mod.rs
