provenant/utils/file/
mod.rs

1// SPDX-FileCopyrightText: Provenant contributors
2// SPDX-License-Identifier: Apache-2.0
3
4//! File-level utilities split by concern:
5//!
6//! - [`path`]: filesystem metadata, glob exclusion, extension/name predicates
7//! - [`encoding`]: byte-to-text decoding and "looks like text" heuristics
8//! - [`format_sniff`]: magic-byte and container-format detection
9//! - [`classify`]: mime type and file-info classification surface
10//! - [`extract`]: text extraction for license/copyright detection
11//! - [`image_metadata`]: EXIF/XMP image metadata text extraction
12//! - [`pdf`]: bounded PDF text extraction
13//!
14//! The public API is re-exported here so existing `crate::utils::file::*`
15//! paths continue to resolve unchanged.
16
17mod classify;
18mod encoding;
19mod extract;
20mod format_sniff;
21mod image_metadata;
22mod path;
23mod pdf;
24
25pub use classify::{FileInfoClassification, classify_file_info, detect_mime_type};
26pub use encoding::decode_bytes_to_string;
27pub use extract::{ExtractedTextKind, extract_printable_strings, extract_text_for_detection};
28pub use path::{get_creation_date, is_path_excluded};
29
30pub(crate) use extract::{
31    augment_license_detection_text, extract_text_for_detection_with_diagnostics,
32};
33
34#[cfg(test)]
35mod tests {
36    use image::ImageFormat;
37    use std::path::Path;
38
39    use crate::copyright::detect_copyrights;
40
41    use super::classify::classify_file_info;
42    use super::encoding::{
43        CORRUPTED_UTF16_BOM_PREFIX, NEAR_BINARY_SKIP_DIAGNOSTIC, decode_bytes_to_string,
44        decode_bytes_to_string_with_diagnostic,
45    };
46    use super::extract::{
47        ExtractedTextKind, LARGE_OPAQUE_BINARY_SKIP_BYTES, extract_printable_strings,
48        extract_text_for_detection, extract_text_for_detection_with_diagnostics,
49        windows_metadata_or_empty_result,
50    };
51    use super::image_metadata::{MAX_XMP_PACKET_BYTES, extract_raw_xmp_packet};
52    use super::pdf::{MAX_PDF_TEXT_EXTRACTION_BYTES, normalize_pdf_heading_comparison_text};
53
54    fn png_chunk(chunk_type: &[u8; 4], data: &[u8]) -> Vec<u8> {
55        let mut out = Vec::new();
56        out.extend_from_slice(&(data.len() as u32).to_be_bytes());
57        out.extend_from_slice(chunk_type);
58        out.extend_from_slice(data);
59        out.extend_from_slice(&0u32.to_be_bytes());
60        out
61    }
62
63    fn build_png_with_xmp(xmp: &str) -> Vec<u8> {
64        let mut bytes = Vec::new();
65        bytes.extend_from_slice(b"\x89PNG\r\n\x1a\n");
66
67        let ihdr = [
68            0, 0, 0, 1, // width
69            0, 0, 0, 1, // height
70            8, // bit depth
71            2, // color type
72            0, // compression
73            0, // filter
74            0, // interlace
75        ];
76        bytes.extend_from_slice(&png_chunk(b"IHDR", &ihdr));
77
78        let mut itxt = Vec::new();
79        itxt.extend_from_slice(b"XML:com.adobe.xmp");
80        itxt.push(0); // keyword terminator
81        itxt.push(0); // compression flag
82        itxt.push(0); // compression method
83        itxt.push(0); // language tag terminator
84        itxt.push(0); // translated keyword terminator
85        itxt.extend_from_slice(xmp.as_bytes());
86        bytes.extend_from_slice(&png_chunk(b"iTXt", &itxt));
87
88        bytes.extend_from_slice(&png_chunk(b"IEND", &[]));
89        bytes
90    }
91
92    #[test]
93    fn test_extract_text_for_detection_skips_jar_archives() {
94        let path = Path::new(
95            "testdata/license-golden/datadriven/lic1/do-not_detect-licenses-in-archive.jar",
96        );
97        let bytes = std::fs::read(path).expect("failed to read jar fixture");
98
99        let (text, kind) = extract_text_for_detection(path, &bytes);
100
101        assert!(text.is_empty());
102        assert_eq!(kind, ExtractedTextKind::None);
103    }
104
105    #[test]
106    fn test_extract_text_for_detection_reads_pdf_fixture_text() {
107        let path = Path::new("testdata/license-golden/datadriven/lic2/bsd-new_156.pdf");
108        let bytes = std::fs::read(path).expect("failed to read pdf fixture");
109
110        let (text, kind) = extract_text_for_detection(path, &bytes);
111
112        assert_eq!(kind, ExtractedTextKind::Pdf);
113        assert!(text.contains("Redistribution and use in source and binary forms"));
114    }
115
116    #[test]
117    fn test_extract_text_for_detection_prefers_first_pdf_page_before_full_document() {
118        let path =
119            Path::new("testdata/license-golden/datadriven/lic4/should_detect_something_5.pdf");
120        let bytes = std::fs::read(path).expect("failed to read pdf fixture");
121
122        let (text, kind) = extract_text_for_detection(path, &bytes);
123
124        assert_eq!(kind, ExtractedTextKind::Pdf);
125        assert!(text.contains("SUN INDUSTRY STANDARDS SOURCE LICENSE"));
126        assert!(!text.contains("DISCLAIMER OF WARRANTY"));
127    }
128
129    #[test]
130    fn test_extract_text_for_detection_does_not_duplicate_pdf_heading_prefix() {
131        let path =
132            Path::new("testdata/license-golden/datadriven/lic4/should_detect_something_5.pdf");
133        let bytes = std::fs::read(path).expect("failed to read pdf fixture");
134
135        let (text, kind) = extract_text_for_detection(path, &bytes);
136
137        assert_eq!(kind, ExtractedTextKind::Pdf);
138
139        let normalized = normalize_pdf_heading_comparison_text(&text);
140        let heading =
141            normalize_pdf_heading_comparison_text("SUN INDUSTRY STANDARDS SOURCE LICENSE");
142        assert_eq!(normalized.matches(&heading).count(), 1);
143    }
144
145    #[test]
146    fn test_extract_text_for_detection_reads_pdf_fixture_without_pdf_extension() {
147        let path = Path::new("testdata/license-golden/datadriven/lic2/bsd-new_156.pdf");
148        let bytes = std::fs::read(path).expect("failed to read pdf fixture");
149
150        let (text, kind) = extract_text_for_detection(Path::new("renamed.bin"), &bytes);
151
152        assert_eq!(kind, ExtractedTextKind::Pdf);
153        assert!(text.contains("Redistribution and use in source and binary forms"));
154    }
155
156    #[test]
157    fn test_extract_text_for_detection_skips_oversized_pdf_payload() {
158        let mut bytes = b"%PDF-1.7\n".to_vec();
159        bytes.resize(MAX_PDF_TEXT_EXTRACTION_BYTES + 1, b'0');
160
161        let (text, kind, scan_error) =
162            extract_text_for_detection_with_diagnostics(Path::new("oversized.pdf"), &bytes);
163
164        assert!(text.is_empty());
165        assert_eq!(kind, ExtractedTextKind::None);
166        assert!(
167            scan_error
168                .as_deref()
169                .is_some_and(|message| message.contains("PDF text extraction skipped"))
170        );
171    }
172
173    #[test]
174    fn test_extract_text_for_detection_reports_terminal_pdf_failure() {
175        let malformed = b"%PDF-1.7\nthis is not a valid pdf object graph\n";
176
177        let (text, kind, scan_error) =
178            extract_text_for_detection_with_diagnostics(Path::new("broken.pdf"), malformed);
179
180        assert!(text.is_empty());
181        assert_eq!(kind, ExtractedTextKind::None);
182        let scan_error = scan_error.expect("terminal pdf failure should be surfaced");
183        assert!(scan_error.contains("PDF text extraction failed after"));
184    }
185
186    #[test]
187    fn test_extract_text_for_detection_skips_large_opaque_binary_blobs() {
188        let bytes = vec![0_u8; LARGE_OPAQUE_BINARY_SKIP_BYTES + 8];
189
190        let (text, kind) = extract_text_for_detection(Path::new("model.bin"), &bytes);
191
192        assert!(text.is_empty());
193        assert_eq!(kind, ExtractedTextKind::None);
194    }
195
196    #[test]
197    fn test_extract_text_for_detection_keeps_large_binaries_with_promising_strings() {
198        let mut bytes = vec![0_u8; LARGE_OPAQUE_BINARY_SKIP_BYTES + 8];
199        let text = b"Copyright 2026 Example Project!!!";
200        bytes[..text.len()].copy_from_slice(text);
201        let second_offset = LARGE_OPAQUE_BINARY_SKIP_BYTES / 2;
202        bytes[second_offset..second_offset + text.len()].copy_from_slice(text);
203
204        let (text, kind) = extract_text_for_detection(Path::new("weights.bin"), &bytes);
205
206        assert_ne!(kind, ExtractedTextKind::None);
207        assert!(text.contains("Copyright 2026 Example Project"));
208    }
209
210    #[test]
211    fn test_extract_text_for_detection_skips_large_binary_with_unstructured_runs() {
212        let mut bytes = vec![0_u8; LARGE_OPAQUE_BINARY_SKIP_BYTES + 8];
213        let noise = b"(c) $1234567890ABCDEF[]{}--==++";
214        bytes[..noise.len()].copy_from_slice(noise);
215        let second_offset = LARGE_OPAQUE_BINARY_SKIP_BYTES / 2;
216        bytes[second_offset..second_offset + noise.len()].copy_from_slice(noise);
217
218        let (text, kind) = extract_text_for_detection(Path::new("tensor.bin"), &bytes);
219
220        assert!(text.is_empty());
221        assert_eq!(kind, ExtractedTextKind::None);
222    }
223
224    #[test]
225    fn test_extract_text_for_detection_uses_windows_executable_metadata() {
226        let path = Path::new("testdata/compiled-binary-golden/win_pe/libiconv2.dll");
227        let bytes = std::fs::read(path).expect("read PE fixture");
228
229        let (text, kind) = extract_text_for_detection(path, &bytes);
230
231        assert_eq!(kind, ExtractedTextKind::BinaryStrings);
232        assert!(text.contains("License: This program is free software"));
233        assert!(text.contains("LegalCopyright:"));
234    }
235
236    #[test]
237    fn test_extract_text_for_detection_keeps_windows_metadata_for_large_pe_without_sampled_signal()
238    {
239        let path = Path::new("testdata/compiled-binary-golden/win_pe/libiconv2.dll");
240        let mut bytes = std::fs::read(path).expect("read PE fixture");
241        bytes.resize(LARGE_OPAQUE_BINARY_SKIP_BYTES + 8, 0);
242
243        let (text, kind) = extract_text_for_detection(path, &bytes);
244
245        assert_ne!(kind, ExtractedTextKind::None);
246        assert!(!text.trim().is_empty());
247    }
248
249    #[test]
250    fn test_windows_metadata_or_empty_result_preserves_metadata() {
251        let (text, kind, scan_error) =
252            windows_metadata_or_empty_result(Some("LegalCopyright: Example Corp".to_string()));
253
254        assert_eq!(kind, ExtractedTextKind::WindowsExecutableMetadata);
255        assert_eq!(text, "LegalCopyright: Example Corp");
256        assert!(scan_error.is_none());
257    }
258
259    #[test]
260    fn test_extract_text_for_detection_keeps_image_author_separate_from_title_and_description() {
261        let xmp = r#"<x:xmpmeta xmlns:x="adobe:ns:meta/"><rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"><rdf:Description xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:creator>Chinmay Garde</dc:creator><dc:title>Bay Bridge At Night</dc:title><dc:description>Embarcadero in the evening on Delta 3200</dc:description></rdf:Description></rdf:RDF></x:xmpmeta>"#;
262        let bytes = build_png_with_xmp(xmp);
263
264        let (text, kind) = extract_text_for_detection(Path::new("fixture.png"), &bytes);
265
266        assert_eq!(kind, ExtractedTextKind::ImageMetadata);
267        assert!(text.contains("Author: Chinmay Garde"), "text: {text:?}");
268        assert!(
269            text.contains("Title: Bay Bridge At Night"),
270            "text: {text:?}"
271        );
272        assert!(
273            text.contains("Description: Embarcadero in the evening on Delta 3200"),
274            "text: {text:?}"
275        );
276
277        let (_copyrights, _holders, authors) = detect_copyrights(&text, None);
278        assert_eq!(
279            authors
280                .iter()
281                .map(|a| a.author.as_str())
282                .collect::<Vec<_>>(),
283            vec!["Chinmay Garde"],
284            "authors: {authors:?}; text: {text:?}"
285        );
286    }
287
288    #[test]
289    fn test_extract_text_for_detection_skips_large_binary_with_single_isolated_string_run() {
290        let mut bytes = vec![0_u8; LARGE_OPAQUE_BINARY_SKIP_BYTES + 8];
291        let text = b"Copyright 2026 Example Project!!!";
292        bytes[..text.len()].copy_from_slice(text);
293
294        let (text, kind) = extract_text_for_detection(Path::new("opaque.bin"), &bytes);
295
296        assert!(text.is_empty());
297        assert_eq!(kind, ExtractedTextKind::None);
298    }
299
300    #[test]
301    fn test_extract_text_for_detection_keeps_large_binary_with_single_contact_rich_window() {
302        let mut bytes = vec![0_u8; LARGE_OPAQUE_BINARY_SKIP_BYTES + 8];
303        let text = b"Andreas Schneider <asn@redhat.com> Rob Crittenden (rcritten@redhat.com) Mr. Sam <sam@email-scan.com> https://publicsuffix.org/ http://tukaani.org/xz/";
304        bytes[..text.len()].copy_from_slice(text);
305
306        let (text, kind) = extract_text_for_detection(Path::new("rootfs.bin"), &bytes);
307
308        assert_ne!(kind, ExtractedTextKind::None);
309        assert!(text.contains("asn@redhat.com"));
310        assert!(text.contains("https://publicsuffix.org/"));
311    }
312
313    #[test]
314    fn test_extract_text_for_detection_keeps_large_macho_with_off_window_legal_markers() {
315        let mut bytes = vec![0_u8; LARGE_OPAQUE_BINARY_SKIP_BYTES * 2];
316        bytes[..4].copy_from_slice(&[0xCF, 0xFA, 0xED, 0xFE]);
317        let apache_notice = b"// Licensed under the Apache License, Version 2.0 (the \"License\");\n// http://www.apache.org/licenses/LICENSE-2.0\n// SPDX-License-Identifier: Apache-2.0\n";
318        let insert_offset = 200 * 1024;
319        bytes[insert_offset..insert_offset + apache_notice.len()].copy_from_slice(apache_notice);
320
321        let (text, kind) = extract_text_for_detection(Path::new("node"), &bytes);
322
323        assert_eq!(kind, ExtractedTextKind::BinaryStrings);
324        assert!(text.contains("Apache License, Version 2.0"), "{text}");
325        assert!(
326            text.contains("SPDX-License-Identifier: Apache-2.0"),
327            "{text}"
328        );
329    }
330
331    #[test]
332    fn test_extract_text_for_detection_keeps_large_macho_with_unicode_notice_markers() {
333        let mut bytes = vec![0_u8; LARGE_OPAQUE_BINARY_SKIP_BYTES * 2];
334        bytes[..4].copy_from_slice(&[0xCF, 0xFA, 0xED, 0xFE]);
335        let unicode_notice = b"Copyright (c) 1991-2024 Unicode, Inc.\nFor terms of use, see http://www.unicode.org/copyright.html\n";
336        let insert_offset = 700 * 1024;
337        bytes[insert_offset..insert_offset + unicode_notice.len()].copy_from_slice(unicode_notice);
338
339        let (text, kind) = extract_text_for_detection(Path::new("node"), &bytes);
340
341        assert_eq!(kind, ExtractedTextKind::BinaryStrings);
342        assert!(text.contains("Unicode, Inc."), "{text}");
343        assert!(text.contains("unicode.org/copyright.html"), "{text}");
344    }
345
346    #[test]
347    fn test_extract_text_for_detection_does_not_reopen_single_window_legal_noise_for_non_macho() {
348        let mut bytes = vec![0_u8; LARGE_OPAQUE_BINARY_SKIP_BYTES * 2];
349        let apache_notice = b"// Licensed under the Apache License, Version 2.0 (the \"License\");\n// http://www.apache.org/licenses/LICENSE-2.0\n// SPDX-License-Identifier: Apache-2.0\n";
350        let insert_offset = 200 * 1024;
351        bytes[insert_offset..insert_offset + apache_notice.len()].copy_from_slice(apache_notice);
352
353        let (text, kind) = extract_text_for_detection(Path::new("model.bin"), &bytes);
354
355        assert!(text.is_empty());
356        assert_eq!(kind, ExtractedTextKind::None);
357    }
358
359    #[test]
360    fn test_extract_text_for_detection_avoids_latin1_decode_for_binary_blob_noise() {
361        let bytes = vec![
362            0x28, 0x63, 0x29, 0x20, 0x4b, 0x30, 0x0e, 0x71, 0x86, 0x20, 0x62, 0x24, 0x4c,
363        ];
364
365        let (text, kind) = extract_text_for_detection(Path::new("fixture.blb"), &bytes);
366
367        assert_eq!(kind, ExtractedTextKind::BinaryStrings);
368        assert_eq!(text, "(c) K0\n b$L");
369    }
370
371    #[test]
372    fn test_extract_raw_xmp_packet_rejects_oversized_png_itxt_payload() {
373        let xmp = "A".repeat(MAX_XMP_PACKET_BYTES + 1);
374        let bytes = build_png_with_xmp(&xmp);
375
376        assert!(extract_raw_xmp_packet(&bytes, ImageFormat::Png).is_none());
377    }
378
379    #[test]
380    fn test_extract_text_for_detection_skips_zip_like_archives() {
381        let zip_bytes = b"PK\x03\x04\x14\x00\x00\x00\x08\x00artifact";
382
383        let (whl_text, whl_kind) = extract_text_for_detection(Path::new("demo.whl"), zip_bytes);
384        let (crate_text, crate_kind) =
385            extract_text_for_detection(Path::new("demo.crate"), zip_bytes);
386
387        assert!(whl_text.is_empty());
388        assert_eq!(whl_kind, ExtractedTextKind::None);
389        assert!(crate_text.is_empty());
390        assert_eq!(crate_kind, ExtractedTextKind::None);
391    }
392
393    #[test]
394    fn test_extract_text_for_detection_keeps_binary_strings_for_lib_fixtures() {
395        let path =
396            Path::new("testdata/copyright-golden/copyrights/copyright_php_lib-php_embed_lib.lib");
397        let bytes = std::fs::read(path).expect("failed to read lib fixture");
398
399        let (text, kind) = extract_text_for_detection(path, &bytes);
400
401        assert_ne!(kind, ExtractedTextKind::None);
402        assert!(text.contains("Copyright nexB and others (c) 2012"));
403    }
404
405    #[test]
406    fn test_extract_text_for_detection_reads_font_metadata() {
407        let path = Path::new("testdata/font-fixtures/Lato-Bold.ttf");
408        let bytes = std::fs::read(path).expect("failed to read font fixture");
409
410        let (text, kind) = extract_text_for_detection(path, &bytes);
411
412        assert_eq!(kind, ExtractedTextKind::FontMetadata);
413        assert!(text.contains("License Description:"), "{text}");
414        assert!(
415            text.contains("Open Font License") || text.contains("OFL"),
416            "{text}"
417        );
418        assert!(text.contains("Lato"), "{text}");
419    }
420
421    #[test]
422    fn test_extract_printable_strings_scales_cap_for_medium_binary_files() {
423        let bytes = b"abcd\0".repeat(525_000);
424
425        let text = extract_printable_strings(&bytes);
426
427        assert!(
428            text.len() > 2_000_000,
429            "unexpected truncation at {}",
430            text.len()
431        );
432        assert!(text.ends_with("abcd"));
433    }
434
435    #[test]
436    fn test_extract_text_for_detection_decodes_svg_fixture_text() {
437        let path = Path::new(
438            "testdata/license-golden/datadriven/external/fossology-tests/Public-domain/biohazard.svg",
439        );
440        let bytes = std::fs::read(path).expect("failed to read svg fixture");
441
442        let (text, kind) = extract_text_for_detection(path, &bytes);
443
444        assert_eq!(kind, ExtractedTextKind::Decoded);
445        assert!(text.contains("creativecommons.org/licenses/publicdomain"));
446    }
447
448    #[test]
449    fn test_extract_text_for_detection_preserves_blank_comment_lines_in_utf8_source() {
450        let path = Path::new("testdata/plugin_email_url/files/IMarkerActionFilter.java");
451        let bytes = std::fs::read(path).expect("failed to read java fixture");
452
453        let (text, kind) = extract_text_for_detection(path, &bytes);
454
455        assert_eq!(kind, ExtractedTextKind::Decoded);
456        let lines: Vec<_> = text.lines().collect();
457        assert_eq!(lines.get(2).copied(), Some(" *"));
458        assert_eq!(
459            lines.get(3).copied(),
460            Some(" *https://github.com/rpm-software-management")
461        );
462        assert_eq!(lines.get(5).copied(), Some("https://gitlab.com/Conan_Kudo"));
463    }
464
465    #[test]
466    fn test_extract_text_for_detection_decodes_rtf_fixture_text() {
467        let path = Path::new(
468            "testdata/license-golden/datadriven/external/fossology-tests/LGPL/License.rtf",
469        );
470        let bytes = std::fs::read(path).expect("failed to read rtf fixture");
471
472        let (text, kind) = extract_text_for_detection(path, &bytes);
473
474        assert_eq!(kind, ExtractedTextKind::Decoded);
475        assert!(text.contains("GNU Lesser General Public"));
476        assert!(text.contains("version"));
477        assert!(text.contains("2.1 of the License"));
478    }
479
480    #[test]
481    fn test_classify_file_info_marks_empty_files_as_text_not_source() {
482        let classification = classify_file_info(Path::new("test.txt"), b"");
483
484        assert_eq!(classification.mime_type, "inode/x-empty");
485        assert_eq!(classification.file_type, "empty");
486        assert!(!classification.is_binary);
487        assert!(classification.is_text);
488        assert!(!classification.is_source);
489        assert_eq!(classification.programming_language, None);
490    }
491
492    #[test]
493    fn test_classify_file_info_keeps_json_out_of_programming_language() {
494        let classification = classify_file_info(Path::new("package.json"), br#"{"name":"demo"}"#);
495
496        assert_eq!(classification.mime_type, "application/json");
497        assert_eq!(classification.file_type, "JSON text data");
498        assert!(classification.is_text);
499        assert!(!classification.is_source);
500        assert_eq!(classification.programming_language, None);
501    }
502
503    #[test]
504    fn test_classify_file_info_does_not_label_invalid_json_text_as_json() {
505        let classification =
506            classify_file_info(Path::new("broken.json"), b"{ definitely not json\n");
507
508        assert_eq!(classification.mime_type, "text/plain");
509        assert_eq!(classification.file_type, "UTF-8 Unicode text");
510        assert!(classification.is_text);
511        assert!(!classification.is_binary);
512    }
513
514    #[test]
515    fn test_classify_file_info_does_not_label_binary_json_garbage_as_json() {
516        let classification =
517            classify_file_info(Path::new("broken.json"), &[0xff, 0x00, 0x01, 0x02]);
518
519        assert_eq!(classification.mime_type, "application/octet-stream");
520        assert_eq!(classification.file_type, "data");
521        assert!(classification.is_binary);
522        assert!(!classification.is_text);
523    }
524
525    #[test]
526    fn test_classify_file_info_treats_valid_utf16_json_with_bom_as_text() {
527        let classification = classify_file_info(
528            Path::new("utf16.json"),
529            &[
530                0xFF, 0xFE, 0x5B, 0x00, 0x22, 0x00, 0xE9, 0x00, 0x22, 0x00, 0x5D, 0x00,
531            ],
532        );
533
534        assert!(!classification.is_binary);
535        assert!(classification.is_text);
536        assert_eq!(classification.mime_type, "application/json");
537        assert_eq!(classification.file_type, "JSON text data");
538    }
539
540    #[test]
541    fn test_classify_file_info_treats_valid_utf16be_json_without_bom_as_text() {
542        let classification = classify_file_info(
543            Path::new("utf16be.json"),
544            &[0x00, 0x5B, 0x00, 0x22, 0x00, 0xE9, 0x00, 0x22, 0x00, 0x5D],
545        );
546
547        assert!(!classification.is_binary);
548        assert!(classification.is_text);
549        assert_eq!(classification.mime_type, "application/json");
550        assert_eq!(classification.file_type, "JSON text data");
551    }
552
553    #[test]
554    fn test_classify_file_info_treats_small_valid_utf16be_json_literal_as_text() {
555        let classification =
556            classify_file_info(Path::new("utf16be-literal.json"), &[0x00, 0x5B, 0x00, 0x5D]);
557
558        assert!(!classification.is_binary);
559        assert!(classification.is_text);
560        assert_eq!(classification.mime_type, "application/json");
561        assert_eq!(classification.file_type, "JSON text data");
562    }
563
564    #[test]
565    fn test_extract_text_for_detection_decodes_utf16be_text_with_corrupted_bom_prefix() {
566        let mut bytes = CORRUPTED_UTF16_BOM_PREFIX.to_vec();
567        for code_unit in
568            "Licensed to the Apache Software Foundation\nApache License, Version 2.0".encode_utf16()
569        {
570            bytes.extend_from_slice(&code_unit.to_be_bytes());
571        }
572
573        let (text, kind) = extract_text_for_detection(Path::new("notice.ftl"), &bytes);
574
575        assert_eq!(kind, ExtractedTextKind::Decoded);
576        assert!(text.contains("Apache Software Foundation"), "{text}");
577        assert!(text.contains("Apache License, Version 2.0"), "{text}");
578    }
579
580    #[test]
581    fn test_classify_file_info_treats_small_valid_json_literals_as_text() {
582        let classification = classify_file_info(Path::new("true.json"), b"true");
583
584        assert!(!classification.is_binary);
585        assert!(classification.is_text);
586        assert_eq!(classification.mime_type, "application/json");
587        assert_eq!(classification.file_type, "JSON text data");
588    }
589
590    #[test]
591    fn test_classify_file_info_treats_json_wrapped_invalid_utf8_sequences_as_text() {
592        let classification = classify_file_info(
593            Path::new("wrapped.json"),
594            &[0x5B, 0x22, 0xE6, 0x97, 0xA5, 0xD1, 0x88, 0xFA, 0x22, 0x5D],
595        );
596
597        assert!(!classification.is_binary);
598        assert!(classification.is_text);
599        assert_eq!(classification.mime_type, "text/plain");
600        assert_eq!(classification.file_type, "text, with no line terminators");
601    }
602
603    #[test]
604    fn test_classify_file_info_keeps_lone_ff_json_byte_binary() {
605        let classification =
606            classify_file_info(Path::new("lone-ff.json"), &[0x5B, 0x22, 0xFF, 0x22, 0x5D]);
607
608        assert!(classification.is_binary);
609        assert!(!classification.is_text);
610        assert_eq!(classification.mime_type, "application/octet-stream");
611        assert_eq!(classification.file_type, "data");
612    }
613
614    #[test]
615    fn test_classify_file_info_keeps_nul_heavy_crash_json_binary() {
616        let classification = classify_file_info(
617            Path::new("crash.json"),
618            &[
619                0xFE, 0x90, 0x00, 0x00, 0x00, 0x93, 0x5B, 0x5B, 0x32, 0x38, 0x36,
620            ],
621        );
622
623        assert!(classification.is_binary);
624        assert!(!classification.is_text);
625        assert_eq!(classification.mime_type, "application/octet-stream");
626    }
627
628    #[test]
629    fn test_classify_file_info_treats_dockerfile_as_source() {
630        let classification = classify_file_info(Path::new("Dockerfile"), b"FROM scratch\n");
631
632        assert_eq!(
633            classification.programming_language.as_deref(),
634            Some("Dockerfile")
635        );
636        assert!(classification.is_source);
637        assert!(!classification.is_script);
638        assert_eq!(
639            classification.file_type,
640            "Dockerfile source, UTF-8 Unicode text"
641        );
642    }
643
644    #[test]
645    fn test_classify_file_info_treats_makefile_as_text_not_source() {
646        let classification = classify_file_info(Path::new("Makefile"), b"all:\n\techo hi\n");
647
648        assert_eq!(classification.programming_language, None);
649        assert!(classification.is_text);
650        assert!(!classification.is_source);
651        assert!(!classification.is_script);
652        assert_eq!(classification.file_type, "UTF-8 Unicode text");
653    }
654
655    #[test]
656    fn test_classify_file_info_marks_supported_package_archives() {
657        let zip_bytes = b"PK\x03\x04\x14\x00\x00\x00";
658
659        let egg = classify_file_info(Path::new("demo.egg"), zip_bytes);
660        let nupkg = classify_file_info(Path::new("demo.nupkg"), zip_bytes);
661
662        assert!(egg.is_archive);
663        assert_eq!(egg.mime_type, "application/zip");
664        assert_eq!(egg.file_type, "Zip archive data");
665        assert!(nupkg.is_archive);
666        assert_eq!(nupkg.mime_type, "application/zip");
667        assert_eq!(nupkg.file_type, "Zip archive data");
668    }
669
670    #[test]
671    fn test_classify_file_info_marks_png_as_binary_media() {
672        let png_bytes = b"\x89PNG\r\n\x1a\n\x00\x00\x00\x0dIHDR";
673
674        let classification = classify_file_info(Path::new("logo.png"), png_bytes);
675
676        assert_eq!(classification.mime_type, "image/png");
677        assert_eq!(classification.file_type, "PNG image data");
678        assert!(classification.is_binary);
679        assert!(!classification.is_text);
680        assert!(classification.is_media);
681        assert!(!classification.is_archive);
682        assert!(!classification.is_source);
683    }
684
685    #[test]
686    fn test_classify_file_info_marks_pdf_as_binary_document() {
687        let pdf_bytes = b"%PDF-1.7\n1 0 obj\n<< /Type /Catalog >>\n";
688
689        let classification = classify_file_info(Path::new("report.pdf"), pdf_bytes);
690
691        assert_eq!(classification.mime_type, "application/pdf");
692        assert_eq!(classification.file_type, "PDF document");
693        assert!(classification.is_binary);
694        assert!(!classification.is_text);
695        assert!(!classification.is_archive);
696        assert!(!classification.is_media);
697    }
698
699    #[test]
700    fn test_classify_file_info_marks_binary_blobs_as_binary() {
701        let classification =
702            classify_file_info(Path::new("blob.bin"), &[0, 159, 146, 150, 0, 1, 2, 3, 4, 5]);
703
704        assert!(classification.is_binary);
705        assert!(!classification.is_text);
706        assert!(!classification.is_source);
707        assert_eq!(classification.programming_language, None);
708    }
709
710    #[test]
711    fn test_classify_file_info_treats_yaml_as_text_not_source() {
712        let classification = classify_file_info(Path::new("config.yaml"), b"key: value\n");
713
714        assert_eq!(classification.programming_language, None);
715        assert!(classification.is_text);
716        assert!(!classification.is_source);
717        assert_eq!(classification.file_type, "YAML text data");
718    }
719
720    #[test]
721    fn test_classify_file_info_classifies_common_build_manifests() {
722        let gradle = classify_file_info(Path::new("build.gradle"), b"plugins { id 'java' }\n");
723        let flake = classify_file_info(Path::new("flake.nix"), b"{ inputs, ... }: {}\n");
724        let cmake = classify_file_info(
725            Path::new("toolchain.cmake"),
726            b"set(CMAKE_CXX_STANDARD 20)\n",
727        );
728        let gitmodules = classify_file_info(
729            Path::new(".gitmodules"),
730            b"[submodule \"demo\"]\n\tpath = vendor/demo\n",
731        );
732
733        assert_eq!(gradle.programming_language.as_deref(), Some("Groovy"));
734        assert!(gradle.is_source);
735        assert_eq!(gradle.mime_type, "text/plain");
736        assert_eq!(gradle.file_type, "Groovy source, UTF-8 Unicode text");
737
738        assert_eq!(flake.programming_language.as_deref(), Some("Nix"));
739        assert!(flake.is_source);
740        assert_eq!(flake.mime_type, "text/plain");
741        assert_eq!(flake.file_type, "Nix source, UTF-8 Unicode text");
742
743        assert_eq!(cmake.programming_language.as_deref(), Some("CMake"));
744        assert!(cmake.is_source);
745        assert_eq!(cmake.file_type, "CMake source, UTF-8 Unicode text");
746
747        assert_eq!(gitmodules.programming_language, None);
748        assert!(gitmodules.is_text);
749        assert!(!gitmodules.is_source);
750        assert_eq!(gitmodules.file_type, "Git configuration text");
751    }
752
753    #[test]
754    fn test_classify_file_info_labels_cpp_headers_and_ipp_separately() {
755        let header = classify_file_info(
756            Path::new("include/demo.hpp"),
757            b"#pragma once\nclass Demo {};\n",
758        );
759        let ipp = classify_file_info(
760            Path::new("include/detail/demo.ipp"),
761            b"template <class T> void parse() {}\n",
762        );
763
764        assert_eq!(header.programming_language.as_deref(), Some("C++"));
765        assert!(header.is_source);
766        assert!(!header.is_script);
767        assert_eq!(header.file_type, "C++ source, UTF-8 Unicode text");
768
769        assert_eq!(ipp.programming_language, None);
770        assert!(!ipp.is_source);
771        assert!(!ipp.is_script);
772        assert_eq!(ipp.file_type, "UTF-8 Unicode text");
773    }
774
775    #[test]
776    fn test_classify_file_info_preserves_specific_shell_family_labels() {
777        let bash = classify_file_info(Path::new("bin/run"), b"#!/usr/bin/env bash\necho hi\n");
778
779        assert_eq!(bash.programming_language.as_deref(), Some("Bash"));
780        assert!(bash.is_script);
781        assert_eq!(bash.file_type, "bash script, UTF-8 Unicode text executable");
782    }
783
784    #[test]
785    fn test_classify_file_info_marks_jamfile_as_source() {
786        let jamfile = classify_file_info(Path::new("Jamfile"), b"lib boost_json ;\n");
787
788        assert_eq!(jamfile.programming_language.as_deref(), Some("Jamfile"));
789        assert!(jamfile.is_source);
790        assert!(!jamfile.is_script);
791        assert_eq!(jamfile.file_type, "Jamfile source, UTF-8 Unicode text");
792    }
793
794    #[test]
795    fn test_classify_file_info_labels_javascript_shebang_scripts() {
796        let classification = classify_file_info(
797            Path::new("bin/run"),
798            b"#!/usr/bin/env node\nconsole.log('hello');\n",
799        );
800
801        assert_eq!(
802            classification.programming_language.as_deref(),
803            Some("JavaScript")
804        );
805        assert!(classification.is_script);
806        assert_eq!(
807            classification.file_type,
808            "javascript script, UTF-8 Unicode text executable"
809        );
810    }
811
812    #[test]
813    fn test_classify_file_info_uses_non_utf8_text_labels_for_latin1_scripts() {
814        let classification = classify_file_info(
815            Path::new("script.py"),
816            b"# coding: latin-1\nprint(\"caf\xe9\")\n",
817        );
818
819        assert_eq!(
820            classification.programming_language.as_deref(),
821            Some("Python")
822        );
823        assert!(classification.is_script);
824        assert_eq!(classification.file_type, "python script, text executable");
825    }
826
827    #[test]
828    fn test_classify_file_info_treats_textual_tga_as_media() {
829        let classification = classify_file_info(Path::new("texture.tga"), b"not really a tga\n");
830
831        assert!(classification.is_media);
832        assert!(classification.is_text);
833        assert!(!classification.is_binary);
834    }
835
836    #[test]
837    fn test_classify_file_info_keeps_binaryish_source_extension_out_of_text_path() {
838        let classification =
839            classify_file_info(Path::new("main.ts"), &[0x80, 0x81, 0x82, 0x83, 0x84, 0x85]);
840
841        assert!(classification.is_binary);
842        assert!(!classification.is_text);
843        assert!(!classification.is_source);
844        assert_eq!(classification.programming_language, None);
845    }
846
847    #[test]
848    fn test_extract_text_for_detection_skips_unsupported_image_formats() {
849        let gif_bytes = b"GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;";
850
851        let (text, kind) = extract_text_for_detection(Path::new("tiny.gif"), gif_bytes);
852
853        assert!(text.is_empty());
854        assert_eq!(kind, ExtractedTextKind::None);
855    }
856
857    #[test]
858    fn test_classify_file_info_preserves_language_detection_precedence_matrix() {
859        let cases = [
860            (
861                Path::new("bin/run"),
862                b"#!/usr/bin/env node\nconsole.log('hello');\n".as_slice(),
863                Some("JavaScript"),
864                true,
865                true,
866            ),
867            (
868                Path::new("Dockerfile"),
869                b"FROM scratch\n".as_slice(),
870                Some("Dockerfile"),
871                true,
872                false,
873            ),
874            (
875                Path::new("package.json"),
876                br#"{"name":"demo"}"#.as_slice(),
877                None,
878                false,
879                false,
880            ),
881            (
882                Path::new("config.yaml"),
883                b"key: value\n".as_slice(),
884                None,
885                false,
886                false,
887            ),
888            (
889                Path::new("Makefile"),
890                b"all:\n\techo hi\n".as_slice(),
891                None,
892                false,
893                false,
894            ),
895        ];
896
897        for (path, bytes, expected_language, expected_is_source, expected_is_script) in cases {
898            let classification = classify_file_info(path, bytes);
899
900            assert_eq!(
901                classification.programming_language.as_deref(),
902                expected_language,
903                "unexpected language for {}",
904                path.display()
905            );
906            assert_eq!(
907                classification.is_source,
908                expected_is_source,
909                "unexpected is_source for {}",
910                path.display()
911            );
912            assert_eq!(
913                classification.is_script,
914                expected_is_script,
915                "unexpected is_script for {}",
916                path.display()
917            );
918        }
919    }
920
921    // ----------------------------------------------------------------------
922    // Encoding and line-ending edge cases (issue #1030).
923    //
924    // These tests PIN the current decoding/line-ending contract. Several of
925    // them document behavior that is intentionally accepted ScanCode /
926    // UnicodeDammit parity (mojibake from Latin-1 fallback) or known sharp
927    // edges (lone-CR line endings, the 10% control-byte cliff). They are not
928    // assertions that the current behavior is ideal; they exist so any future
929    // change to it is deliberate and reviewed.
930    //
931    // Fixtures are built from explicit byte literals (e.g. `b"...\r..."`) so
932    // editor/source normalization can never silently change the line endings
933    // under test.
934    // ----------------------------------------------------------------------
935
936    /// Builds invalid-UTF-8 bytes with a controllable ratio of control bytes
937    /// to printable bytes, so threshold behavior can be pinned precisely.
938    ///
939    /// Each entry contributes one printable byte (`A`, valid ASCII) and a 0x80
940    /// high byte makes the buffer invalid UTF-8 so it takes the Latin-1 path.
941    /// Control bytes use 0x01 (counted by `has_binary_control_chars`).
942    fn invalid_utf8_with_control_ratio(printable: usize, control: usize) -> Vec<u8> {
943        let mut bytes = vec![0x80]; // forces invalid UTF-8
944        bytes.extend(std::iter::repeat_n(b'A', printable));
945        bytes.extend(std::iter::repeat_n(0x01, control));
946        bytes
947    }
948
949    #[test]
950    fn decode_bytes_control_char_threshold_just_under_decodes_as_latin1() {
951        // Total 100 bytes, exactly 10 control bytes => 10%. The threshold is
952        // `control > len / 10`, so 10 is NOT over the cliff: decoded as Latin-1.
953        let bytes = invalid_utf8_with_control_ratio(89, 10);
954        assert_eq!(bytes.len(), 100);
955
956        let (decoded, diagnostic) = decode_bytes_to_string_with_diagnostic(&bytes);
957
958        assert!(
959            !decoded.is_empty(),
960            "input at the 10% boundary must still decode (Latin-1 fallback)"
961        );
962        assert!(decoded.contains('A'));
963        assert!(diagnostic.is_none());
964    }
965
966    #[test]
967    fn decode_bytes_control_char_threshold_just_over_returns_empty_with_diagnostic() {
968        // Total 100 bytes, 11 control bytes => 11% > 10%, over the cliff:
969        // empty string plus a structured diagnostic.
970        let bytes = invalid_utf8_with_control_ratio(88, 11);
971        assert_eq!(bytes.len(), 100);
972
973        let (decoded, diagnostic) = decode_bytes_to_string_with_diagnostic(&bytes);
974
975        assert!(
976            decoded.is_empty(),
977            "input over the 10% control-byte threshold must decode to empty"
978        );
979        assert_eq!(diagnostic.as_deref(), Some(NEAR_BINARY_SKIP_DIAGNOSTIC));
980        // The thin public wrapper preserves the empty-string result contract.
981        assert!(decode_bytes_to_string(&bytes).is_empty());
982    }
983
984    #[test]
985    fn decode_bytes_latin1_high_bytes_produce_mojibake_parity() {
986        // Windows-1252 "café" (é = 0xE9) and Shift-JIS "日" (0x93 0xFA) are not
987        // valid UTF-8; the Latin-1 fallback maps each byte to U+0000..=U+00FF,
988        // producing mojibake. This is accepted ScanCode/UnicodeDammit parity.
989        let windows_1252 = b"caf\xE9";
990        let decoded = decode_bytes_to_string(windows_1252);
991        // 0xE9 -> U+00E9 (é). Latin-1 happens to agree with Windows-1252 here.
992        assert_eq!(decoded, "caf\u{00E9}");
993
994        let shift_jis = b"\x93\xFA"; // Shift-JIS for "日"
995        let decoded = decode_bytes_to_string(shift_jis);
996        // Mojibake: bytes mapped directly, NOT decoded to "日".
997        assert_eq!(decoded, "\u{0093}\u{00FA}");
998        assert!(!decoded.contains('日'));
999    }
1000
1001    #[test]
1002    fn decode_bytes_utf8_bom_is_preserved_as_leading_codepoint() {
1003        // UTF-8 BOM (EF BB BF) is valid UTF-8: it decodes to U+FEFF and is NOT
1004        // stripped by the decoder. Pin that contract here.
1005        let bytes = b"\xEF\xBB\xBFhello";
1006        let decoded = decode_bytes_to_string(bytes);
1007        assert_eq!(decoded, "\u{FEFF}hello");
1008    }
1009
1010    /// Encodes an ASCII string as UTF-16 with the given endianness, prefixed by
1011    /// the matching BOM. Only ASCII codepoints are used in the fixtures.
1012    fn utf16_with_bom(text: &str, little_endian: bool) -> Vec<u8> {
1013        let mut bytes = if little_endian {
1014            vec![0xFF, 0xFE]
1015        } else {
1016            vec![0xFE, 0xFF]
1017        };
1018        for ch in text.chars() {
1019            let unit = ch as u16;
1020            if little_endian {
1021                bytes.extend_from_slice(&unit.to_le_bytes());
1022            } else {
1023                bytes.extend_from_slice(&unit.to_be_bytes());
1024            }
1025        }
1026        bytes
1027    }
1028
1029    #[test]
1030    fn decode_bytes_utf16_le_bom_is_decoded_and_bom_stripped() {
1031        // UTF-16 LE BOM (FF FE) followed by text: the UTF-16 path consumes the
1032        // BOM and returns the text without a leading U+FEFF. The body must be
1033        // long enough to pass the text-shape gate (>= 3 visible chars).
1034        let bytes = utf16_with_bom("hello world", true);
1035        let decoded = decode_bytes_to_string(&bytes);
1036        assert_eq!(decoded, "hello world");
1037        assert!(!decoded.starts_with('\u{FEFF}'));
1038    }
1039
1040    #[test]
1041    fn decode_bytes_utf16_be_bom_is_decoded_and_bom_stripped() {
1042        let bytes = utf16_with_bom("hello world", false);
1043        let decoded = decode_bytes_to_string(&bytes);
1044        assert_eq!(decoded, "hello world");
1045        assert!(!decoded.starts_with('\u{FEFF}'));
1046    }
1047
1048    #[test]
1049    fn decode_bytes_short_utf16_bom_text_fails_text_shape_gate() {
1050        // Very short UTF-16 BOM bodies (< 3 visible chars) do NOT pass the
1051        // text-shape heuristic and decode to empty. Pinned so any change to the
1052        // minimum-visible threshold is deliberate.
1053        let bytes = utf16_with_bom("hi", true);
1054        assert_eq!(decode_bytes_to_string(&bytes), "");
1055    }
1056
1057    #[test]
1058    fn decode_bytes_corrupted_utf16_bom_prefix_is_stripped() {
1059        // The hardcoded "corrupted UTF-16 BOM" prefix (EF BF BD repeated, i.e.
1060        // two U+FFFD replacement chars) is stripped before UTF-16 detection.
1061        // Body is UTF-16 LE for "hello world test text long enough".
1062        let mut bytes = CORRUPTED_UTF16_BOM_PREFIX.to_vec();
1063        for ch in "hello world test text long enough".chars() {
1064            bytes.push(ch as u8);
1065            bytes.push(0x00);
1066        }
1067        let decoded = decode_bytes_to_string(&bytes);
1068        assert_eq!(decoded, "hello world test text long enough");
1069        // The corrupted-BOM marker must not survive in the decoded output.
1070        assert!(!decoded.contains('\u{FFFD}'));
1071    }
1072
1073    #[test]
1074    fn decoded_text_lone_cr_line_endings_are_not_split_by_lines() {
1075        // Classic-Mac (CR-only) line endings: `str::lines()` splits on `\n` and
1076        // `\r\n`, but NOT a lone `\r`. So three logical lines collapse into one
1077        // `lines()` element. This is a known sharp edge that can misattribute
1078        // email/url/copyright line numbers; pinned here rather than changed,
1079        // since fixing line-splitting would shift detection line numbers
1080        // broadly. Flagged as a follow-up in the issue.
1081        let bytes = b"first line\rsecond line\rthird line";
1082        let decoded = decode_bytes_to_string(bytes);
1083        assert_eq!(decoded, "first line\rsecond line\rthird line");
1084
1085        let line_count = decoded.lines().count();
1086        assert_eq!(
1087            line_count, 1,
1088            "lone-CR line endings are currently NOT split by str::lines()"
1089        );
1090
1091        // For contrast, LF and CRLF DO split into separate lines.
1092        let lf = decode_bytes_to_string(b"a\nb\nc");
1093        assert_eq!(lf.lines().count(), 3);
1094        let crlf = decode_bytes_to_string(b"a\r\nb\r\nc");
1095        assert_eq!(crlf.lines().count(), 3);
1096    }
1097
1098    #[test]
1099    fn extract_text_near_binary_invalid_utf8_surfaces_skip_diagnostic() {
1100        // End-to-end through the extraction entry point. An `.svg` path forces
1101        // the decoded-text branch (`should_try_decoded_text`) even for
1102        // near-binary bytes, so the diagnostic threading is exercised. The
1103        // bytes are invalid UTF-8 with a high control-byte ratio and no
1104        // recoverable printable strings, so the file drops from detection AND
1105        // surfaces the structured near-binary skip diagnostic.
1106        let path = Path::new("garbled.svg");
1107        let bytes = invalid_utf8_with_control_ratio(1, 60);
1108
1109        let (text, kind, diagnostic) = extract_text_for_detection_with_diagnostics(path, &bytes);
1110
1111        assert!(text.is_empty());
1112        assert_eq!(kind, ExtractedTextKind::None);
1113        assert_eq!(diagnostic.as_deref(), Some(NEAR_BINARY_SKIP_DIAGNOSTIC));
1114    }
1115}
provenant/utils/file/mod.rs

provenant/utils/file/
mod.rs
