Skip to main content

provenant/scanner/
mod.rs

1// SPDX-FileCopyrightText: Provenant contributors
2// SPDX-License-Identifier: Apache-2.0
3
4mod collect;
5pub(crate) mod process;
6
7use crate::license_detection::LicenseDetectionEngine;
8use crate::models::FileInfo;
9
10pub struct ProcessResult {
11    pub files: Vec<FileInfo>,
12    pub excluded_count: usize,
13}
14
15#[derive(Debug, Clone, Copy)]
16pub struct LicenseScanOptions {
17    pub include_text: bool,
18    pub include_text_diagnostics: bool,
19    pub include_diagnostics: bool,
20    pub unknown_licenses: bool,
21    pub enable_sequence_matching: bool,
22    pub min_score: u8,
23}
24
25impl Default for LicenseScanOptions {
26    fn default() -> Self {
27        Self {
28            include_text: false,
29            include_text_diagnostics: false,
30            include_diagnostics: false,
31            unknown_licenses: false,
32            enable_sequence_matching: true,
33            min_score: 0,
34        }
35    }
36}
37
38#[derive(Debug, Clone)]
39pub struct TextDetectionOptions {
40    pub collect_info: bool,
41    pub detect_packages: bool,
42    pub detect_application_packages: bool,
43    pub detect_system_packages: bool,
44    pub detect_packages_in_compiled: bool,
45    pub detect_copyrights: bool,
46    pub detect_generated: bool,
47    pub detect_emails: bool,
48    pub detect_urls: bool,
49    pub max_emails: usize,
50    pub max_urls: usize,
51    pub timeout_seconds: f64,
52}
53
54impl Default for TextDetectionOptions {
55    fn default() -> Self {
56        Self {
57            collect_info: false,
58            detect_packages: false,
59            detect_application_packages: false,
60            detect_system_packages: false,
61            detect_packages_in_compiled: false,
62            detect_copyrights: true,
63            detect_generated: false,
64            detect_emails: false,
65            detect_urls: false,
66            max_emails: 50,
67            max_urls: 50,
68            timeout_seconds: 120.0,
69        }
70    }
71}
72
73pub fn scan_options_fingerprint(
74    text_options: &TextDetectionOptions,
75    license_options: LicenseScanOptions,
76    license_engine: Option<&LicenseDetectionEngine>,
77) -> String {
78    let (license_enabled, rules_count, first_rule_id, last_rule_id) = match license_engine {
79        Some(engine) => {
80            let rules = &engine.index().rules_by_rid;
81            (
82                true,
83                rules.len(),
84                rules
85                    .first()
86                    .map(|rule| rule.identifier.as_str())
87                    .unwrap_or(""),
88                rules
89                    .last()
90                    .map(|rule| rule.identifier.as_str())
91                    .unwrap_or(""),
92            )
93        }
94        None => (false, 0, "", ""),
95    };
96
97    format!(
98        "tool_version={};info={};packages={};app_packages={};system_packages={};compiled_packages={};copyrights={};generated={};emails={};urls={};max_emails={};max_urls={};timeout={:.6};license_enabled={};rules_count={};first_rule_id={};last_rule_id={};license_text={};license_text_diagnostics={};license_diagnostics={};unknown_licenses={};sequence_matching={};license_score={}",
99        crate::version::BUILD_VERSION,
100        text_options.collect_info,
101        text_options.detect_packages,
102        text_options.detect_application_packages,
103        text_options.detect_system_packages,
104        text_options.detect_packages_in_compiled,
105        text_options.detect_copyrights,
106        text_options.detect_generated,
107        text_options.detect_emails,
108        text_options.detect_urls,
109        text_options.max_emails,
110        text_options.max_urls,
111        text_options.timeout_seconds,
112        license_enabled,
113        rules_count,
114        first_rule_id,
115        last_rule_id,
116        license_options.include_text,
117        license_options.include_text_diagnostics,
118        license_options.include_diagnostics,
119        license_options.unknown_licenses,
120        license_options.enable_sequence_matching,
121        license_options.min_score,
122    )
123}
124
125pub use self::collect::{
126    CollectedPaths, CollectionFrontier, collect_paths, collect_selected_paths,
127};
128#[allow(unused_imports)]
129pub use self::process::{
130    MemoryMode, process_collected, process_collected_sequential,
131    process_collected_with_memory_limit, process_collected_with_memory_limit_sequential,
132};
133
134#[cfg(test)]
135mod tests {
136    use std::fs;
137    use std::path::PathBuf;
138    use std::sync::Arc;
139
140    use object::pe;
141    use tempfile::TempDir;
142
143    use crate::cache::build_collection_exclude_patterns;
144    use crate::license_detection::{LicenseDetectionEngine, MatcherKind};
145    use crate::models::{DatasourceId, FileType, PackageType as FilePackageType};
146    use crate::progress::{ProgressMode, ScanProgress};
147
148    use super::{
149        CollectionFrontier, LicenseScanOptions, MemoryMode, TextDetectionOptions, collect_paths,
150        collect_selected_paths, process_collected, process_collected_with_memory_limit,
151        scan_options_fingerprint,
152    };
153
154    fn build_sparse_oversized_rpm_with_filename(
155        temp_dir: &TempDir,
156        package_name: &str,
157        filename: &str,
158    ) -> PathBuf {
159        let file_path = temp_dir.path().join(filename);
160        rpm::PackageBuilder::new(package_name, "1.0", "MIT", "x86_64", "Demo RPM package")
161            .release("1")
162            .build()
163            .expect("build rpm fixture")
164            .write_file(&file_path)
165            .expect("write rpm fixture");
166        fs::OpenOptions::new()
167            .write(true)
168            .open(&file_path)
169            .expect("open rpm fixture for sparse extension")
170            .set_len(100 * 1024 * 1024 + 1_048_576)
171            .expect("extend rpm fixture");
172        file_path
173    }
174
175    fn build_sparse_oversized_rpm(temp_dir: &TempDir, name: &str) -> PathBuf {
176        build_sparse_oversized_rpm_with_filename(
177            temp_dir,
178            name,
179            &format!("{name}-1.0-1.x86_64.rpm"),
180        )
181    }
182
183    fn build_sparse_oversized_pack_rpm(temp_dir: &TempDir, name: &str) -> PathBuf {
184        build_sparse_oversized_rpm_with_filename(
185            temp_dir,
186            name,
187            &format!("{name}-1.0-1.x86_64.pack"),
188        )
189    }
190
191    #[test]
192    fn default_options_keep_copyright_detection_enabled() {
193        let options = TextDetectionOptions::default();
194        assert!(!options.detect_packages);
195        assert!(options.detect_copyrights);
196    }
197
198    #[test]
199    fn test_scan_options_fingerprint_changes_with_license_score() {
200        let text_options = TextDetectionOptions::default();
201        let default_fingerprint = scan_options_fingerprint(
202            &text_options,
203            LicenseScanOptions {
204                min_score: 0,
205                ..LicenseScanOptions::default()
206            },
207            None,
208        );
209        let filtered_fingerprint = scan_options_fingerprint(
210            &text_options,
211            LicenseScanOptions {
212                min_score: 70,
213                ..LicenseScanOptions::default()
214            },
215            None,
216        );
217
218        assert_ne!(default_fingerprint, filtered_fingerprint);
219    }
220
221    fn scan_single_file(
222        file_name: &str,
223        content: &str,
224        options: &TextDetectionOptions,
225    ) -> crate::models::FileInfo {
226        let temp_dir = TempDir::new().expect("create temp dir");
227        let file_path = temp_dir.path().join(file_name);
228        fs::write(&file_path, content).expect("write test file");
229
230        let progress = Arc::new(ScanProgress::new(ProgressMode::Quiet));
231        let collected = collect_paths(temp_dir.path(), 0, &[]);
232        let result = process_collected(
233            &collected,
234            progress,
235            None,
236            LicenseScanOptions::default(),
237            options,
238        );
239
240        result
241            .files
242            .into_iter()
243            .find(|entry| {
244                entry.file_type == FileType::File && entry.path == file_path.to_string_lossy()
245            })
246            .expect("scanned file entry")
247    }
248
249    fn scan_file_at_relative_path(
250        relative_path: &str,
251        content: &[u8],
252        options: &TextDetectionOptions,
253    ) -> crate::models::FileInfo {
254        let temp_dir = TempDir::new().expect("create temp dir");
255        let file_path = temp_dir.path().join(relative_path);
256        if let Some(parent) = file_path.parent() {
257            fs::create_dir_all(parent).expect("create parent dirs");
258        }
259        fs::write(&file_path, content).expect("write test file");
260
261        let progress = Arc::new(ScanProgress::new(ProgressMode::Quiet));
262        let collected = collect_paths(temp_dir.path(), 0, &[]);
263        let result = process_collected(
264            &collected,
265            progress,
266            None,
267            LicenseScanOptions::default(),
268            options,
269        );
270
271        result
272            .files
273            .into_iter()
274            .find(|entry| {
275                entry.file_type == FileType::File && entry.path == file_path.to_string_lossy()
276            })
277            .expect("scanned file entry")
278    }
279
280    fn scan_single_file_with_license_engine(
281        file_name: &str,
282        content: &str,
283        options: &TextDetectionOptions,
284    ) -> crate::models::FileInfo {
285        let temp_dir = TempDir::new().expect("create temp dir");
286        let file_path = temp_dir.path().join(file_name);
287        fs::write(&file_path, content).expect("write test file");
288
289        let progress = Arc::new(ScanProgress::new(ProgressMode::Quiet));
290        let collected = collect_paths(temp_dir.path(), 0, &[]);
291        let engine =
292            Arc::new(LicenseDetectionEngine::from_embedded().expect("initialize license engine"));
293        let result = process_collected(
294            &collected,
295            progress,
296            Some(engine),
297            LicenseScanOptions::default(),
298            options,
299        );
300
301        result
302            .files
303            .into_iter()
304            .find(|entry| {
305                entry.file_type == FileType::File && entry.path == file_path.to_string_lossy()
306            })
307            .expect("scanned file entry")
308    }
309
310    fn scan_single_file_with_license_engine_and_options(
311        file_name: &str,
312        content: &str,
313        options: &TextDetectionOptions,
314        license_options: LicenseScanOptions,
315    ) -> crate::models::FileInfo {
316        let temp_dir = TempDir::new().expect("create temp dir");
317        let file_path = temp_dir.path().join(file_name);
318        fs::write(&file_path, content).expect("write test file");
319
320        let progress = Arc::new(ScanProgress::new(ProgressMode::Quiet));
321        let collected = collect_paths(temp_dir.path(), 0, &[]);
322        let engine =
323            Arc::new(LicenseDetectionEngine::from_embedded().expect("initialize license engine"));
324        let result =
325            process_collected(&collected, progress, Some(engine), license_options, options);
326
327        result
328            .files
329            .into_iter()
330            .find(|entry| {
331                entry.file_type == FileType::File && entry.path == file_path.to_string_lossy()
332            })
333            .expect("scanned file entry")
334    }
335
336    #[test]
337    fn scanner_can_disable_sequence_matching_for_partial_license_hits() {
338        let partial_mit = r#"Permission is hereby granted, free of charge, to any person obtaining a copy
339of this software and associated documentation files (the "Software"), to deal
340in the Software without restriction, including without limitation the rights
341to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
342copies of the Software."#;
343
344        let with_sequence = scan_single_file_with_license_engine_and_options(
345            "partial-license.txt",
346            partial_mit,
347            &TextDetectionOptions::default(),
348            LicenseScanOptions::default(),
349        );
350        let without_sequence = scan_single_file_with_license_engine_and_options(
351            "partial-license.txt",
352            partial_mit,
353            &TextDetectionOptions::default(),
354            LicenseScanOptions {
355                enable_sequence_matching: false,
356                ..LicenseScanOptions::default()
357            },
358        );
359
360        assert!(
361            !with_sequence.license_detections.is_empty(),
362            "partial MIT snippet should be detected with sequence matching enabled"
363        );
364        assert!(
365            without_sequence
366                .license_detections
367                .iter()
368                .all(|detection| detection
369                    .matches
370                    .iter()
371                    .all(|m| m.matcher != MatcherKind::Seq)),
372            "sequence matching disabled should remove all seq detections; detections={:?} clues={:?}",
373            without_sequence.license_detections,
374            without_sequence.license_clues,
375        );
376    }
377
378    #[test]
379    fn scanner_does_not_emit_cc_pdm_false_positive_clue_for_issue4859_snippet() {
380        let text =
381            std::fs::read_to_string("testdata/license-detection-regressions/issue4859_exact.html")
382                .expect("issue4859 fixture should be readable");
383
384        let scanned = scan_single_file_with_license_engine_and_options(
385            "issue4859_exact.html",
386            &text,
387            &TextDetectionOptions::default(),
388            LicenseScanOptions::default(),
389        );
390
391        assert!(
392            scanned.license_detections.is_empty(),
393            "issue4859 snippet should not produce a concrete detection: {:?}",
394            scanned.license_detections
395        );
396        assert!(
397            scanned.license_clues.is_empty(),
398            "issue4859 snippet should not surface a low-quality cc-pdm clue: {:?}",
399            scanned.license_clues
400        );
401    }
402
403    #[test]
404    fn scanner_reports_repeated_email_occurrences() {
405        let options = TextDetectionOptions {
406            collect_info: false,
407            detect_packages: false,
408            detect_application_packages: false,
409            detect_system_packages: false,
410            detect_packages_in_compiled: false,
411            detect_copyrights: false,
412            detect_generated: false,
413            detect_emails: true,
414            detect_urls: false,
415            max_emails: 50,
416            max_urls: 50,
417            timeout_seconds: 120.0,
418        };
419        let scanned = scan_single_file(
420            "contacts.txt",
421            "linux@3ware.com\nlinux@3ware.com\nandre@suse.com\nlinux@3ware.com\n",
422            &options,
423        );
424
425        let emails: Vec<(&str, usize)> = scanned
426            .emails
427            .iter()
428            .map(|email| (email.email.as_str(), email.start_line.get()))
429            .collect();
430
431        assert_eq!(emails.len(), 4, "emails: {emails:#?}");
432        assert_eq!(
433            emails,
434            vec![
435                ("linux@3ware.com", 1),
436                ("linux@3ware.com", 2),
437                ("andre@suse.com", 3),
438                ("linux@3ware.com", 4),
439            ]
440        );
441    }
442
443    #[test]
444    fn scanner_skips_pem_certificate_text_detection() {
445        let options = TextDetectionOptions {
446            collect_info: false,
447            detect_packages: false,
448            detect_application_packages: false,
449            detect_system_packages: false,
450            detect_packages_in_compiled: false,
451            detect_copyrights: true,
452            detect_generated: false,
453            detect_emails: true,
454            detect_urls: true,
455            max_emails: 50,
456            max_urls: 50,
457            timeout_seconds: 120.0,
458        };
459        let pem_fixture = concat!(
460            "-----BEGIN CERTIFICATE-----\n",
461            "MIID8TCCAtmgAwIBAgIQQT1yx/RrH4FDffHSKFTfmjANBgkqhkiG9w0BAQUFADCB\n",
462            "ijELMAkGA1UEBhMCQ0gxEDAOBgNVBAoTB1dJU2VLZXkxGzAZBgNVBAsTEkNvcHly\n",
463            "-----END CERTIFICATE-----\n",
464            "Certificate:\n",
465            "    Data:\n",
466            "        Signature Algorithm: sha1WithRSAEncryption\n",
467            "        Issuer: C=CH, O=WISeKey, OU=Copyright (c) 2005, OU=OISTE Foundation Endorsed\n",
468            "        Subject: C=CH, O=WISeKey, OU=Copyright (c) 2005, OU=OISTE Foundation Endorsed\n",
469            "        Contact: cert-owner@example.com\n",
470        );
471        let scanned = scan_single_file("cert.pem", pem_fixture, &options);
472
473        assert!(
474            scanned.copyrights.is_empty(),
475            "copyrights: {:#?}",
476            scanned.copyrights
477        );
478        assert!(
479            scanned.holders.is_empty(),
480            "holders: {:#?}",
481            scanned.holders
482        );
483        assert!(
484            scanned.authors.is_empty(),
485            "authors: {:#?}",
486            scanned.authors
487        );
488        assert!(scanned.emails.is_empty(), "emails: {:#?}", scanned.emails);
489        assert!(scanned.urls.is_empty(), "urls: {:#?}", scanned.urls);
490        assert!(
491            scanned.license_detections.is_empty(),
492            "licenses: {:#?}",
493            scanned.license_detections
494        );
495        assert!(
496            scanned.license_clues.is_empty(),
497            "license clues: {:#?}",
498            scanned.license_clues
499        );
500    }
501
502    #[test]
503    fn scanner_keeps_source_headers_when_pem_blocks_are_embedded() {
504        let options = TextDetectionOptions {
505            collect_info: false,
506            detect_packages: false,
507            detect_application_packages: false,
508            detect_system_packages: false,
509            detect_packages_in_compiled: false,
510            detect_copyrights: true,
511            detect_generated: false,
512            detect_emails: false,
513            detect_urls: true,
514            max_emails: 50,
515            max_urls: 50,
516            timeout_seconds: 120.0,
517        };
518        let fixture = concat!(
519            "/*\n",
520            "Copyright 2022 The Kubernetes Authors.\n\n",
521            "Licensed under the Apache License, Version 2.0 (the \"License\");\n",
522            "you may not use this file except in compliance with the License.\n",
523            "You may obtain a copy of the License at\n\n",
524            "    http://www.apache.org/licenses/LICENSE-2.0\n",
525            "*/\n\n",
526            "package storage\n\n",
527            "const validCert = `\n",
528            "-----BEGIN CERTIFICATE-----\n",
529            "MIIDmTCCAoGgAwIBAgIUWQ==\n",
530            "-----END CERTIFICATE-----\n",
531            "`\n",
532        );
533        let temp_dir = TempDir::new().expect("create temp dir");
534        let file_path = temp_dir.path().join("storage_test.go");
535        fs::write(&file_path, fixture).expect("write fixture");
536
537        let progress = Arc::new(ScanProgress::new(ProgressMode::Quiet));
538        let collected = collect_paths(temp_dir.path(), 0, &[]);
539        let engine =
540            Arc::new(LicenseDetectionEngine::from_embedded().expect("initialize license engine"));
541        let result = process_collected(
542            &collected,
543            progress,
544            Some(engine),
545            LicenseScanOptions::default(),
546            &options,
547        );
548        let scanned = result
549            .files
550            .into_iter()
551            .find(|entry| {
552                entry.file_type == FileType::File && entry.path == file_path.to_string_lossy()
553            })
554            .expect("scanned file entry");
555
556        assert!(
557            scanned
558                .copyrights
559                .iter()
560                .any(|c| c.copyright == "Copyright 2022 The Kubernetes Authors."),
561            "copyrights: {:#?}",
562            scanned.copyrights
563        );
564        assert!(
565            scanned
566                .holders
567                .iter()
568                .any(|h| h.holder == "The Kubernetes Authors"),
569            "holders: {:#?}",
570            scanned.holders
571        );
572        assert!(
573            scanned
574                .urls
575                .iter()
576                .any(|u| u.url == "http://www.apache.org/licenses/LICENSE-2.0"),
577            "urls: {:#?}",
578            scanned.urls
579        );
580        assert_eq!(
581            scanned.detected_license_expression.as_deref(),
582            Some("Apache-2.0")
583        );
584    }
585
586    #[test]
587    fn scanner_detects_structured_credits_authors() {
588        let options = TextDetectionOptions {
589            collect_info: false,
590            detect_packages: false,
591            detect_application_packages: false,
592            detect_system_packages: false,
593            detect_packages_in_compiled: false,
594            detect_copyrights: true,
595            detect_generated: false,
596            detect_emails: false,
597            detect_urls: false,
598            max_emails: 50,
599            max_urls: 50,
600            timeout_seconds: 120.0,
601        };
602        let credits_fixture = concat!(
603            "N: Jack Lloyd\n",
604            "E: lloyd@randombit.net\n",
605            "W: http://www.randombit.net/\n",
606        );
607        let scanned = scan_single_file("CREDITS", credits_fixture, &options);
608
609        let authors: Vec<(&str, usize, usize)> = scanned
610            .authors
611            .iter()
612            .map(|author| {
613                (
614                    author.author.as_str(),
615                    author.start_line.get(),
616                    author.end_line.get(),
617                )
618            })
619            .collect();
620
621        assert_eq!(
622            authors,
623            vec![(
624                "Jack Lloyd lloyd@randombit.net http://www.randombit.net/",
625                1,
626                3,
627            )]
628        );
629        assert!(scanned.copyrights.is_empty());
630        assert!(scanned.holders.is_empty());
631    }
632
633    #[test]
634    fn scanner_uses_or_for_alternative_license_header() {
635        let fixture =
636            include_str!("../../testdata/license-golden/datadriven/external/boost-json-d2s.ipp");
637        let temp_dir = TempDir::new().expect("create temp dir");
638        let file_path = temp_dir.path().join("d2s.ipp");
639        fs::write(&file_path, fixture).expect("write fixture");
640
641        let progress = Arc::new(ScanProgress::new(ProgressMode::Quiet));
642        let collected = collect_paths(temp_dir.path(), 0, &[]);
643        let engine =
644            Arc::new(LicenseDetectionEngine::from_embedded().expect("initialize license engine"));
645        let result = process_collected(
646            &collected,
647            progress,
648            Some(engine),
649            LicenseScanOptions::default(),
650            &TextDetectionOptions::default(),
651        );
652        let scanned = result
653            .files
654            .into_iter()
655            .find(|entry| {
656                entry.file_type == FileType::File && entry.path == file_path.to_string_lossy()
657            })
658            .expect("scanned file entry");
659
660        assert_eq!(
661            scanned.detected_license_expression.as_deref(),
662            Some("Apache-2.0 OR BSL-1.0")
663        );
664        assert!(
665            scanned.license_clues.is_empty(),
666            "license clues: {:#?}",
667            scanned.license_clues
668        );
669        assert_eq!(
670            scanned.license_detections.len(),
671            1,
672            "detections: {:#?}",
673            scanned.license_detections
674        );
675
676        let detection = &scanned.license_detections[0];
677        assert_eq!(detection.license_expression_spdx, "Apache-2.0 OR BSL-1.0");
678
679        let match_expressions: Vec<_> = detection
680            .matches
681            .iter()
682            .map(|m| m.license_expression_spdx.as_str())
683            .collect();
684        assert_eq!(match_expressions, vec!["Apache-2.0", "BSL-1.0"]);
685    }
686
687    #[test]
688    fn scanner_sets_generated_flag_when_enabled() {
689        let options = TextDetectionOptions {
690            collect_info: false,
691            detect_packages: false,
692            detect_application_packages: false,
693            detect_system_packages: false,
694            detect_packages_in_compiled: false,
695            detect_copyrights: false,
696            detect_generated: true,
697            detect_emails: false,
698            detect_urls: false,
699            max_emails: 50,
700            max_urls: 50,
701            timeout_seconds: 120.0,
702        };
703        let scanned = scan_single_file(
704            "generated.c",
705            "/* DO NOT EDIT THIS FILE - it is machine generated */\n",
706            &options,
707        );
708
709        assert_eq!(scanned.is_generated, Some(true));
710    }
711
712    #[test]
713    fn scanner_leaves_generated_flag_unset_when_disabled() {
714        let options = TextDetectionOptions {
715            collect_info: false,
716            detect_packages: false,
717            detect_application_packages: false,
718            detect_system_packages: false,
719            detect_packages_in_compiled: false,
720            detect_copyrights: false,
721            detect_generated: false,
722            detect_emails: false,
723            detect_urls: false,
724            max_emails: 50,
725            max_urls: 50,
726            timeout_seconds: 120.0,
727        };
728        let scanned = scan_single_file(
729            "generated.c",
730            "/* DO NOT EDIT THIS FILE - it is machine generated */\n",
731            &options,
732        );
733
734        assert_eq!(scanned.is_generated, None);
735    }
736
737    #[test]
738    fn scanner_populates_info_surface_when_enabled() {
739        let options = TextDetectionOptions {
740            collect_info: true,
741            detect_packages: false,
742            detect_application_packages: false,
743            detect_system_packages: false,
744            detect_packages_in_compiled: false,
745            detect_copyrights: false,
746            detect_generated: false,
747            detect_emails: false,
748            detect_urls: false,
749            max_emails: 50,
750            max_urls: 50,
751            timeout_seconds: 120.0,
752        };
753        let scanned = scan_single_file(
754            "script.py",
755            "#!/usr/bin/env python3\nprint(\"hello\")\n",
756            &options,
757        );
758
759        assert!(scanned.sha1.is_some());
760        assert!(scanned.md5.is_some());
761        assert!(scanned.sha256.is_some());
762        assert!(scanned.sha1_git.is_some());
763        assert!(scanned.mime_type.is_some());
764        assert!(scanned.date.is_some());
765        assert_eq!(scanned.programming_language.as_deref(), Some("Python"));
766        assert_eq!(scanned.is_text, Some(true));
767        assert_eq!(scanned.is_script, Some(true));
768        assert_eq!(scanned.is_source, Some(true));
769    }
770
771    #[test]
772    fn scanner_treats_latin1_python_sources_as_textual_scripts() {
773        let options = TextDetectionOptions {
774            collect_info: true,
775            detect_packages: false,
776            detect_application_packages: false,
777            detect_system_packages: false,
778            detect_packages_in_compiled: false,
779            detect_copyrights: false,
780            detect_generated: false,
781            detect_emails: false,
782            detect_urls: false,
783            max_emails: 50,
784            max_urls: 50,
785            timeout_seconds: 120.0,
786        };
787        let latin1_python = b"# coding: latin-1\nprint(\"caf\xe9\")\n# comment padding\n";
788        let scanned = scan_file_at_relative_path("script.py", latin1_python, &options);
789
790        assert_eq!(scanned.programming_language.as_deref(), Some("Python"));
791        assert_eq!(
792            scanned.file_type_label.as_deref(),
793            Some("python script, text executable")
794        );
795        assert_eq!(scanned.is_binary, Some(false));
796        assert_eq!(scanned.is_text, Some(true));
797        assert_eq!(scanned.is_script, Some(true));
798        assert_eq!(scanned.is_source, Some(true));
799    }
800
801    #[test]
802    fn scanner_skips_findings_for_zip_like_archives() {
803        let options = TextDetectionOptions {
804            collect_info: true,
805            detect_packages: false,
806            detect_application_packages: false,
807            detect_system_packages: false,
808            detect_packages_in_compiled: false,
809            detect_copyrights: true,
810            detect_generated: false,
811            detect_emails: true,
812            detect_urls: true,
813            max_emails: 50,
814            max_urls: 50,
815            timeout_seconds: 120.0,
816        };
817        let archive_like = b"PK\x03\x04\x14\x00\x00\x00\x08\x00MIT License\ncontact@example.com\nhttps://example.com\n";
818        let scanned = scan_file_at_relative_path("demo.whl", archive_like, &options);
819
820        assert_eq!(scanned.mime_type.as_deref(), Some("application/zip"));
821        assert_eq!(scanned.is_archive, Some(true));
822        assert!(scanned.license_detections.is_empty());
823        assert!(scanned.copyrights.is_empty());
824        assert!(scanned.emails.is_empty());
825        assert!(scanned.urls.is_empty());
826    }
827
828    #[test]
829    fn scanner_treats_typescript_sources_as_text_not_video_media() {
830        let options = TextDetectionOptions {
831            collect_info: true,
832            detect_packages: false,
833            detect_application_packages: false,
834            detect_system_packages: false,
835            detect_packages_in_compiled: false,
836            detect_copyrights: false,
837            detect_generated: false,
838            detect_emails: false,
839            detect_urls: false,
840            max_emails: 50,
841            max_urls: 50,
842            timeout_seconds: 120.0,
843        };
844        let scanned = scan_single_file("main.ts", "export const answer: number = 42;\n", &options);
845
846        assert_eq!(scanned.programming_language.as_deref(), Some("TypeScript"));
847        assert_eq!(scanned.mime_type.as_deref(), Some("text/plain"));
848        assert_eq!(
849            scanned.file_type_label.as_deref(),
850            Some("TypeScript source, UTF-8 Unicode text")
851        );
852        assert_eq!(scanned.is_text, Some(true));
853        assert_eq!(scanned.is_media, Some(false));
854        assert_eq!(scanned.is_script, Some(false));
855        assert_eq!(scanned.is_source, Some(true));
856    }
857
858    #[test]
859    fn scanner_normalizes_sparse_ts_files_away_from_video_mime() {
860        let options = TextDetectionOptions {
861            collect_info: true,
862            detect_packages: false,
863            detect_application_packages: false,
864            detect_system_packages: false,
865            detect_packages_in_compiled: false,
866            detect_copyrights: false,
867            detect_generated: false,
868            detect_emails: false,
869            detect_urls: false,
870            max_emails: 50,
871            max_urls: 50,
872            timeout_seconds: 120.0,
873        };
874        let scanned = scan_single_file("main.ts", "// comment-only TypeScript fixture\n", &options);
875
876        assert_eq!(scanned.mime_type.as_deref(), Some("text/plain"));
877        assert_eq!(
878            scanned.file_type_label.as_deref(),
879            Some("TypeScript source, UTF-8 Unicode text")
880        );
881        assert_eq!(scanned.is_text, Some(true));
882        assert_eq!(scanned.is_media, Some(false));
883        assert_eq!(scanned.is_script, Some(false));
884        assert_eq!(scanned.is_source, Some(true));
885    }
886
887    #[test]
888    fn scanner_treats_empty_files_like_scancode_info_surface() {
889        let options = TextDetectionOptions {
890            collect_info: true,
891            detect_packages: false,
892            detect_application_packages: false,
893            detect_system_packages: false,
894            detect_packages_in_compiled: false,
895            detect_copyrights: false,
896            detect_generated: false,
897            detect_emails: false,
898            detect_urls: false,
899            max_emails: 50,
900            max_urls: 50,
901            timeout_seconds: 120.0,
902        };
903        let scanned = scan_single_file("test.txt", "", &options);
904
905        assert_eq!(scanned.mime_type.as_deref(), Some("inode/x-empty"));
906        assert_eq!(scanned.file_type_label.as_deref(), Some("empty"));
907        assert_eq!(scanned.programming_language, None);
908        assert_eq!(scanned.is_binary, Some(false));
909        assert_eq!(scanned.is_text, Some(true));
910        assert_eq!(scanned.is_archive, Some(false));
911        assert_eq!(scanned.is_media, Some(false));
912        assert_eq!(scanned.is_source, Some(false));
913        assert_eq!(scanned.is_script, Some(false));
914    }
915
916    #[test]
917    fn scanner_treats_package_json_as_text_not_source() {
918        let options = TextDetectionOptions {
919            collect_info: true,
920            detect_packages: false,
921            detect_application_packages: false,
922            detect_system_packages: false,
923            detect_packages_in_compiled: false,
924            detect_copyrights: false,
925            detect_generated: false,
926            detect_emails: false,
927            detect_urls: false,
928            max_emails: 50,
929            max_urls: 50,
930            timeout_seconds: 120.0,
931        };
932        let scanned = scan_single_file("package.json", r#"{"name":"demo"}"#, &options);
933
934        assert_eq!(scanned.mime_type.as_deref(), Some("application/json"));
935        assert_eq!(scanned.file_type_label.as_deref(), Some("JSON text data"));
936        assert_eq!(scanned.programming_language, None);
937        assert_eq!(scanned.is_text, Some(true));
938        assert_eq!(scanned.is_source, Some(false));
939        assert_eq!(scanned.is_script, Some(false));
940    }
941
942    #[test]
943    fn scanner_classifies_gradle_and_nix_manifests_as_source() {
944        let options = TextDetectionOptions {
945            collect_info: true,
946            detect_packages: false,
947            detect_application_packages: false,
948            detect_system_packages: false,
949            detect_packages_in_compiled: false,
950            detect_copyrights: false,
951            detect_generated: false,
952            detect_emails: false,
953            detect_urls: false,
954            max_emails: 50,
955            max_urls: 50,
956            timeout_seconds: 120.0,
957        };
958
959        let gradle = scan_single_file("build.gradle", "plugins { id 'java' }\n", &options);
960        let nix = scan_single_file("flake.nix", "{ inputs, ... }: {}\n", &options);
961
962        assert_eq!(gradle.programming_language.as_deref(), Some("Groovy"));
963        assert_eq!(gradle.mime_type.as_deref(), Some("text/plain"));
964        assert_eq!(gradle.is_source, Some(true));
965        assert_eq!(gradle.is_script, Some(false));
966
967        assert_eq!(nix.programming_language.as_deref(), Some("Nix"));
968        assert_eq!(nix.mime_type.as_deref(), Some("text/plain"));
969        assert_eq!(nix.is_source, Some(true));
970        assert_eq!(nix.is_script, Some(false));
971    }
972
973    #[test]
974    fn scanner_treats_gitmodules_as_text_not_source() {
975        let options = TextDetectionOptions {
976            collect_info: true,
977            detect_packages: false,
978            detect_application_packages: false,
979            detect_system_packages: false,
980            detect_packages_in_compiled: false,
981            detect_copyrights: false,
982            detect_generated: false,
983            detect_emails: false,
984            detect_urls: false,
985            max_emails: 50,
986            max_urls: 50,
987            timeout_seconds: 120.0,
988        };
989        let scanned = scan_file_at_relative_path(
990            ".gitmodules",
991            b"[submodule \"demo\"]\n\tpath = vendor/demo\n",
992            &options,
993        );
994
995        assert_eq!(scanned.programming_language, None);
996        assert_eq!(
997            scanned.file_type_label.as_deref(),
998            Some("Git configuration text")
999        );
1000        assert_eq!(scanned.is_text, Some(true));
1001        assert_eq!(scanned.is_source, Some(false));
1002        assert_eq!(scanned.is_script, Some(false));
1003    }
1004
1005    #[test]
1006    fn scanner_treats_javascript_shebang_files_as_scripts() {
1007        let options = TextDetectionOptions {
1008            collect_info: true,
1009            detect_packages: false,
1010            detect_application_packages: false,
1011            detect_system_packages: false,
1012            detect_packages_in_compiled: false,
1013            detect_copyrights: false,
1014            detect_generated: false,
1015            detect_emails: false,
1016            detect_urls: false,
1017            max_emails: 50,
1018            max_urls: 50,
1019            timeout_seconds: 120.0,
1020        };
1021        let scanned = scan_file_at_relative_path(
1022            "bin/run",
1023            b"#!/usr/bin/env node\nconsole.log('hello');\n",
1024            &options,
1025        );
1026
1027        assert_eq!(scanned.programming_language.as_deref(), Some("JavaScript"));
1028        assert_eq!(
1029            scanned.file_type_label.as_deref(),
1030            Some("javascript script, UTF-8 Unicode text executable")
1031        );
1032        assert_eq!(scanned.is_script, Some(true));
1033        assert_eq!(scanned.is_source, Some(true));
1034    }
1035
1036    #[test]
1037    fn scanner_treats_dockerfile_as_source() {
1038        let options = TextDetectionOptions {
1039            collect_info: true,
1040            detect_packages: false,
1041            detect_application_packages: false,
1042            detect_system_packages: false,
1043            detect_packages_in_compiled: false,
1044            detect_copyrights: false,
1045            detect_generated: false,
1046            detect_emails: false,
1047            detect_urls: false,
1048            max_emails: 50,
1049            max_urls: 50,
1050            timeout_seconds: 120.0,
1051        };
1052        let scanned = scan_single_file("Dockerfile", "FROM scratch\n", &options);
1053
1054        assert_eq!(scanned.programming_language.as_deref(), Some("Dockerfile"));
1055        assert_eq!(
1056            scanned.file_type_label.as_deref(),
1057            Some("Dockerfile source, UTF-8 Unicode text")
1058        );
1059        assert_eq!(scanned.is_source, Some(true));
1060        assert_eq!(scanned.is_script, Some(false));
1061    }
1062
1063    #[test]
1064    fn scanner_treats_makefile_as_text_not_source() {
1065        let options = TextDetectionOptions {
1066            collect_info: true,
1067            detect_packages: false,
1068            detect_application_packages: false,
1069            detect_system_packages: false,
1070            detect_packages_in_compiled: false,
1071            detect_copyrights: false,
1072            detect_generated: false,
1073            detect_emails: false,
1074            detect_urls: false,
1075            max_emails: 50,
1076            max_urls: 50,
1077            timeout_seconds: 120.0,
1078        };
1079        let scanned = scan_single_file("Makefile", "all:\n\techo hi\n", &options);
1080
1081        assert_eq!(scanned.programming_language, None);
1082        assert_eq!(
1083            scanned.file_type_label.as_deref(),
1084            Some("UTF-8 Unicode text")
1085        );
1086        assert_eq!(scanned.is_text, Some(true));
1087        assert_eq!(scanned.is_source, Some(false));
1088        assert_eq!(scanned.is_script, Some(false));
1089    }
1090
1091    #[test]
1092    fn scanner_omits_info_surface_when_disabled() {
1093        let options = TextDetectionOptions {
1094            collect_info: false,
1095            detect_packages: false,
1096            detect_application_packages: false,
1097            detect_system_packages: false,
1098            detect_packages_in_compiled: false,
1099            detect_copyrights: false,
1100            detect_generated: false,
1101            detect_emails: false,
1102            detect_urls: false,
1103            max_emails: 50,
1104            max_urls: 50,
1105            timeout_seconds: 120.0,
1106        };
1107        let scanned = scan_single_file(
1108            "script.py",
1109            "#!/usr/bin/env python3\nprint(\"hello\")\n",
1110            &options,
1111        );
1112
1113        assert!(scanned.sha1.is_none());
1114        assert!(scanned.md5.is_none());
1115        assert!(scanned.sha256.is_none());
1116        assert!(scanned.sha1_git.is_none());
1117        assert!(scanned.mime_type.is_none());
1118        assert!(scanned.date.is_none());
1119        assert!(scanned.programming_language.is_none());
1120        assert!(scanned.is_binary.is_none());
1121        assert!(scanned.is_text.is_none());
1122        assert!(scanned.is_archive.is_none());
1123        assert!(scanned.is_media.is_none());
1124        assert!(scanned.is_script.is_none());
1125        assert!(scanned.is_source.is_none());
1126    }
1127
1128    #[test]
1129    fn scanner_skips_package_parsing_when_disabled() {
1130        let options = TextDetectionOptions {
1131            collect_info: false,
1132            detect_packages: false,
1133            detect_application_packages: false,
1134            detect_system_packages: false,
1135            detect_packages_in_compiled: false,
1136            detect_copyrights: false,
1137            detect_generated: false,
1138            detect_emails: false,
1139            detect_urls: false,
1140            max_emails: 50,
1141            max_urls: 50,
1142            timeout_seconds: 120.0,
1143        };
1144        let scanned = scan_single_file(
1145            "package.json",
1146            r#"{"name":"demo","version":"1.0.0"}"#,
1147            &options,
1148        );
1149
1150        assert!(
1151            scanned.package_data.is_empty(),
1152            "package_data: {:#?}",
1153            scanned.package_data
1154        );
1155    }
1156
1157    #[test]
1158    fn scanner_parses_package_manifests_when_enabled() {
1159        let options = TextDetectionOptions {
1160            collect_info: false,
1161            detect_packages: true,
1162            detect_application_packages: true,
1163            detect_system_packages: false,
1164            detect_packages_in_compiled: false,
1165            detect_copyrights: false,
1166            detect_generated: false,
1167            detect_emails: false,
1168            detect_urls: false,
1169            max_emails: 50,
1170            max_urls: 50,
1171            timeout_seconds: 120.0,
1172        };
1173        let scanned = scan_single_file(
1174            "package.json",
1175            r#"{"name":"demo","version":"1.0.0"}"#,
1176            &options,
1177        );
1178
1179        assert_eq!(
1180            scanned.package_data.len(),
1181            1,
1182            "package_data: {:#?}",
1183            scanned.package_data
1184        );
1185    }
1186
1187    #[test]
1188    fn scanner_parses_oversized_rpm_in_package_only_mode_without_size_warning() {
1189        let temp_dir = TempDir::new().expect("create temp dir");
1190        let file_path = build_sparse_oversized_rpm(&temp_dir, "oversized-demo");
1191
1192        let progress = Arc::new(ScanProgress::new(ProgressMode::Quiet));
1193        let collected = collect_paths(temp_dir.path(), 0, &[]);
1194        let result = process_collected(
1195            &collected,
1196            progress,
1197            None,
1198            LicenseScanOptions::default(),
1199            &TextDetectionOptions {
1200                collect_info: false,
1201                detect_packages: true,
1202                detect_application_packages: true,
1203                detect_system_packages: false,
1204                detect_packages_in_compiled: false,
1205                detect_copyrights: false,
1206                detect_generated: false,
1207                detect_emails: false,
1208                detect_urls: false,
1209                max_emails: 50,
1210                max_urls: 50,
1211                timeout_seconds: 120.0,
1212            },
1213        );
1214
1215        let scanned = result
1216            .files
1217            .into_iter()
1218            .find(|entry| {
1219                entry.file_type == FileType::File && entry.path == file_path.to_string_lossy()
1220            })
1221            .expect("scanned file entry");
1222
1223        assert!(
1224            scanned.scan_diagnostics.is_empty(),
1225            "scan_diagnostics: {:#?}",
1226            scanned.scan_diagnostics
1227        );
1228        assert_eq!(
1229            scanned.package_data.len(),
1230            1,
1231            "package_data: {:#?}",
1232            scanned.package_data
1233        );
1234        assert_eq!(
1235            scanned.package_data[0].datasource_id,
1236            Some(DatasourceId::RpmArchive)
1237        );
1238        assert_eq!(
1239            scanned.package_data[0].name.as_deref(),
1240            Some("oversized-demo")
1241        );
1242        assert_eq!(scanned.package_data[0].version.as_deref(), Some("1.0-1"));
1243    }
1244
1245    #[test]
1246    fn scanner_parses_oversized_rpm_with_info_without_timeout_or_size_warning() {
1247        let temp_dir = TempDir::new().expect("create temp dir");
1248        let file_path = build_sparse_oversized_rpm(&temp_dir, "oversized-info-demo");
1249
1250        let progress = Arc::new(ScanProgress::new(ProgressMode::Quiet));
1251        let collected = collect_paths(temp_dir.path(), 0, &[]);
1252        let result = process_collected(
1253            &collected,
1254            progress,
1255            None,
1256            LicenseScanOptions::default(),
1257            &TextDetectionOptions {
1258                collect_info: true,
1259                detect_packages: true,
1260                detect_application_packages: true,
1261                detect_system_packages: false,
1262                detect_packages_in_compiled: false,
1263                detect_copyrights: false,
1264                detect_generated: false,
1265                detect_emails: false,
1266                detect_urls: false,
1267                max_emails: 50,
1268                max_urls: 50,
1269                timeout_seconds: 120.0,
1270            },
1271        );
1272
1273        let scanned = result
1274            .files
1275            .into_iter()
1276            .find(|entry| {
1277                entry.file_type == FileType::File && entry.path == file_path.to_string_lossy()
1278            })
1279            .expect("scanned file entry");
1280
1281        assert!(
1282            scanned.scan_diagnostics.is_empty(),
1283            "scan_diagnostics: {:#?}",
1284            scanned.scan_diagnostics
1285        );
1286        assert_eq!(
1287            scanned.package_data.len(),
1288            1,
1289            "package_data: {:#?}",
1290            scanned.package_data
1291        );
1292        assert_eq!(
1293            scanned.package_data[0].datasource_id,
1294            Some(DatasourceId::RpmArchive)
1295        );
1296        assert_eq!(
1297            scanned.package_data[0].name.as_deref(),
1298            Some("oversized-info-demo")
1299        );
1300        assert!(scanned.sha1.is_some());
1301        assert!(scanned.md5.is_some());
1302        assert!(scanned.sha256.is_some());
1303        assert!(scanned.sha1_git.is_some());
1304        assert_eq!(scanned.mime_type.as_deref(), Some("application/x-rpm"));
1305        assert_eq!(scanned.file_type_label.as_deref(), Some("RPM package"));
1306        assert_eq!(scanned.is_binary, Some(true));
1307        assert_eq!(scanned.is_text, Some(false));
1308        assert_eq!(scanned.is_archive, Some(true));
1309    }
1310
1311    #[test]
1312    fn scanner_parses_oversized_pack_rpm_in_package_only_mode_without_size_warning() {
1313        let temp_dir = TempDir::new().expect("create temp dir");
1314        let file_path = build_sparse_oversized_pack_rpm(&temp_dir, "oversized-pack-demo");
1315
1316        let progress = Arc::new(ScanProgress::new(ProgressMode::Quiet));
1317        let collected = collect_paths(temp_dir.path(), 0, &[]);
1318        let result = process_collected(
1319            &collected,
1320            progress,
1321            None,
1322            LicenseScanOptions::default(),
1323            &TextDetectionOptions {
1324                collect_info: false,
1325                detect_packages: true,
1326                detect_application_packages: true,
1327                detect_system_packages: false,
1328                detect_packages_in_compiled: false,
1329                detect_copyrights: false,
1330                detect_generated: false,
1331                detect_emails: false,
1332                detect_urls: false,
1333                max_emails: 50,
1334                max_urls: 50,
1335                timeout_seconds: 120.0,
1336            },
1337        );
1338
1339        let scanned = result
1340            .files
1341            .into_iter()
1342            .find(|entry| {
1343                entry.file_type == FileType::File && entry.path == file_path.to_string_lossy()
1344            })
1345            .expect("scanned file entry");
1346
1347        assert!(
1348            scanned.scan_diagnostics.is_empty(),
1349            "scan_diagnostics: {:#?}",
1350            scanned.scan_diagnostics
1351        );
1352        assert_eq!(
1353            scanned.package_data.len(),
1354            1,
1355            "package_data: {:#?}",
1356            scanned.package_data
1357        );
1358        assert_eq!(
1359            scanned.package_data[0].datasource_id,
1360            Some(DatasourceId::RpmArchive)
1361        );
1362        assert_eq!(
1363            scanned.package_data[0].name.as_deref(),
1364            Some("oversized-pack-demo")
1365        );
1366    }
1367
1368    #[test]
1369    fn scanner_parses_oversized_pack_rpm_with_info_without_timeout_or_size_warning() {
1370        let temp_dir = TempDir::new().expect("create temp dir");
1371        let file_path = build_sparse_oversized_pack_rpm(&temp_dir, "oversized-pack-info-demo");
1372
1373        let progress = Arc::new(ScanProgress::new(ProgressMode::Quiet));
1374        let collected = collect_paths(temp_dir.path(), 0, &[]);
1375        let result = process_collected(
1376            &collected,
1377            progress,
1378            None,
1379            LicenseScanOptions::default(),
1380            &TextDetectionOptions {
1381                collect_info: true,
1382                detect_packages: true,
1383                detect_application_packages: true,
1384                detect_system_packages: false,
1385                detect_packages_in_compiled: false,
1386                detect_copyrights: false,
1387                detect_generated: false,
1388                detect_emails: false,
1389                detect_urls: false,
1390                max_emails: 50,
1391                max_urls: 50,
1392                timeout_seconds: 120.0,
1393            },
1394        );
1395
1396        let scanned = result
1397            .files
1398            .into_iter()
1399            .find(|entry| {
1400                entry.file_type == FileType::File && entry.path == file_path.to_string_lossy()
1401            })
1402            .expect("scanned file entry");
1403
1404        assert!(
1405            scanned.scan_diagnostics.is_empty(),
1406            "scan_diagnostics: {:#?}",
1407            scanned.scan_diagnostics
1408        );
1409        assert_eq!(
1410            scanned.package_data.len(),
1411            1,
1412            "package_data: {:#?}",
1413            scanned.package_data
1414        );
1415        assert_eq!(
1416            scanned.package_data[0].datasource_id,
1417            Some(DatasourceId::RpmArchive)
1418        );
1419        assert_eq!(
1420            scanned.package_data[0].name.as_deref(),
1421            Some("oversized-pack-info-demo")
1422        );
1423        assert!(scanned.sha1.is_some());
1424        assert!(scanned.md5.is_some());
1425        assert!(scanned.sha256.is_some());
1426        assert!(scanned.sha1_git.is_some());
1427        assert_eq!(scanned.mime_type.as_deref(), Some("application/x-rpm"));
1428        assert_eq!(scanned.file_type_label.as_deref(), Some("RPM package"));
1429        assert_eq!(scanned.is_binary, Some(true));
1430        assert_eq!(scanned.is_text, Some(false));
1431        assert_eq!(scanned.is_archive, Some(true));
1432    }
1433
1434    #[test]
1435    fn scanner_skips_application_packages_when_only_system_packages_enabled() {
1436        let options = TextDetectionOptions {
1437            collect_info: false,
1438            detect_packages: true,
1439            detect_application_packages: false,
1440            detect_system_packages: true,
1441            detect_packages_in_compiled: false,
1442            detect_copyrights: false,
1443            detect_generated: false,
1444            detect_emails: false,
1445            detect_urls: false,
1446            max_emails: 50,
1447            max_urls: 50,
1448            timeout_seconds: 120.0,
1449        };
1450        let scanned = scan_single_file(
1451            "package.json",
1452            r#"{"name":"demo","version":"1.0.0"}"#,
1453            &options,
1454        );
1455
1456        assert!(
1457            scanned.package_data.is_empty(),
1458            "package_data: {:#?}",
1459            scanned.package_data
1460        );
1461    }
1462
1463    #[test]
1464    fn scanner_parses_system_package_files_when_enabled() {
1465        let options = TextDetectionOptions {
1466            collect_info: false,
1467            detect_packages: true,
1468            detect_application_packages: false,
1469            detect_system_packages: true,
1470            detect_packages_in_compiled: false,
1471            detect_copyrights: false,
1472            detect_generated: false,
1473            detect_emails: false,
1474            detect_urls: false,
1475            max_emails: 50,
1476            max_urls: 50,
1477            timeout_seconds: 120.0,
1478        };
1479        let scanned = scan_file_at_relative_path(
1480            "var/lib/dpkg/status",
1481            b"Package: demo\nVersion: 1.0\nArchitecture: all\nDescription: demo package\n\n",
1482            &options,
1483        );
1484
1485        assert!(
1486            !scanned.package_data.is_empty(),
1487            "package_data: {:#?}",
1488            scanned.package_data
1489        );
1490    }
1491
1492    #[test]
1493    fn scanner_only_parses_compiled_packages_when_package_in_compiled_is_enabled() {
1494        if std::process::Command::new("go")
1495            .arg("version")
1496            .status()
1497            .is_err()
1498        {
1499            return;
1500        }
1501
1502        let temp_dir = TempDir::new().expect("create temp dir");
1503        fs::write(
1504            temp_dir.path().join("go.mod"),
1505            "module example.com/demo\n\ngo 1.23.0\n",
1506        )
1507        .expect("write go.mod");
1508        fs::write(
1509            temp_dir.path().join("main.go"),
1510            "package main\nfunc main() {}\n",
1511        )
1512        .expect("write main.go");
1513        let file_path = temp_dir.path().join("demo");
1514        let status = std::process::Command::new("go")
1515            .current_dir(temp_dir.path())
1516            .args(["build", "-o"])
1517            .arg(&file_path)
1518            .status()
1519            .expect("run go build");
1520        assert!(status.success());
1521
1522        let progress = Arc::new(ScanProgress::new(ProgressMode::Quiet));
1523        let collected = collect_paths(temp_dir.path(), 0, &[]);
1524
1525        let without_compiled = process_collected(
1526            &collected,
1527            Arc::clone(&progress),
1528            None,
1529            LicenseScanOptions::default(),
1530            &TextDetectionOptions {
1531                collect_info: false,
1532                detect_packages: true,
1533                detect_application_packages: true,
1534                detect_system_packages: false,
1535                detect_packages_in_compiled: false,
1536                detect_copyrights: false,
1537                detect_generated: false,
1538                detect_emails: false,
1539                detect_urls: false,
1540                max_emails: 50,
1541                max_urls: 50,
1542                timeout_seconds: 120.0,
1543            },
1544        );
1545        let with_compiled = process_collected(
1546            &collected,
1547            progress,
1548            None,
1549            LicenseScanOptions::default(),
1550            &TextDetectionOptions {
1551                collect_info: false,
1552                detect_packages: true,
1553                detect_application_packages: true,
1554                detect_system_packages: false,
1555                detect_packages_in_compiled: true,
1556                detect_copyrights: false,
1557                detect_generated: false,
1558                detect_emails: false,
1559                detect_urls: false,
1560                max_emails: 50,
1561                max_urls: 50,
1562                timeout_seconds: 120.0,
1563            },
1564        );
1565
1566        let without_compiled = without_compiled
1567            .files
1568            .into_iter()
1569            .find(|entry| entry.file_type == FileType::File && entry.path.ends_with("/demo"))
1570            .expect("compiled artifact present");
1571        let with_compiled = with_compiled
1572            .files
1573            .into_iter()
1574            .find(|entry| entry.file_type == FileType::File && entry.path.ends_with("/demo"))
1575            .expect("compiled artifact present");
1576
1577        assert!(
1578            without_compiled.package_data.is_empty(),
1579            "package_data: {:#?}",
1580            without_compiled.package_data
1581        );
1582        assert!(!with_compiled.package_data.is_empty());
1583    }
1584
1585    #[test]
1586    fn scanner_parses_windows_executable_packages_under_normal_package_scan() {
1587        let temp_dir = TempDir::new().expect("create temp dir");
1588        let file_path = temp_dir.path().join("libiconv2.dll");
1589        let fixture = fs::read("testdata/compiled-binary-golden/win_pe/libiconv2.dll")
1590            .expect("read PE fixture");
1591        fs::write(&file_path, fixture).expect("write PE fixture");
1592
1593        let progress = Arc::new(ScanProgress::new(ProgressMode::Quiet));
1594        let collected = collect_paths(temp_dir.path(), 0, &[]);
1595
1596        let without_package = process_collected(
1597            &collected,
1598            Arc::clone(&progress),
1599            None,
1600            LicenseScanOptions::default(),
1601            &TextDetectionOptions {
1602                collect_info: false,
1603                detect_packages: false,
1604                detect_application_packages: false,
1605                detect_system_packages: false,
1606                detect_packages_in_compiled: false,
1607                detect_copyrights: false,
1608                detect_generated: false,
1609                detect_emails: false,
1610                detect_urls: false,
1611                max_emails: 50,
1612                max_urls: 50,
1613                timeout_seconds: 120.0,
1614            },
1615        );
1616        let with_package = process_collected(
1617            &collected,
1618            progress,
1619            None,
1620            LicenseScanOptions::default(),
1621            &TextDetectionOptions {
1622                collect_info: false,
1623                detect_packages: true,
1624                detect_application_packages: true,
1625                detect_system_packages: false,
1626                detect_packages_in_compiled: false,
1627                detect_copyrights: false,
1628                detect_generated: false,
1629                detect_emails: false,
1630                detect_urls: false,
1631                max_emails: 50,
1632                max_urls: 50,
1633                timeout_seconds: 120.0,
1634            },
1635        );
1636
1637        let without_package = without_package
1638            .files
1639            .into_iter()
1640            .find(|entry| {
1641                entry.file_type == FileType::File && entry.path.ends_with("/libiconv2.dll")
1642            })
1643            .expect("compiled artifact present");
1644        let with_package = with_package
1645            .files
1646            .into_iter()
1647            .find(|entry| {
1648                entry.file_type == FileType::File && entry.path.ends_with("/libiconv2.dll")
1649            })
1650            .expect("compiled artifact present");
1651
1652        assert!(without_package.package_data.is_empty());
1653        assert_eq!(with_package.package_data.len(), 1);
1654        assert_eq!(
1655            with_package.package_data[0].package_type,
1656            Some(FilePackageType::Winexe)
1657        );
1658        assert_eq!(
1659            with_package.package_data[0].datasource_id,
1660            Some(DatasourceId::WindowsExecutable)
1661        );
1662    }
1663
1664    #[test]
1665    fn scanner_keeps_nsis_and_windows_executable_package_data_together() {
1666        let temp_dir = TempDir::new().expect("create temp dir");
1667        let file_path = temp_dir.path().join("nsis-with-version.exe");
1668        let mut fixture = fs::read("testdata/compiled-binary-golden/win_pe/libiconv2.dll")
1669            .expect("read PE fixture");
1670        if fixture.len() < 70_000 {
1671            fixture.resize(70_000, 0);
1672        }
1673        fixture.extend_from_slice(b"Nullsoft.NSIS.exehead");
1674        fs::write(&file_path, fixture).expect("write synthetic NSIS PE fixture");
1675
1676        let progress = Arc::new(ScanProgress::new(ProgressMode::Quiet));
1677        let collected = collect_paths(temp_dir.path(), 0, &[]);
1678        let result = process_collected(
1679            &collected,
1680            progress,
1681            None,
1682            LicenseScanOptions::default(),
1683            &TextDetectionOptions {
1684                collect_info: false,
1685                detect_packages: true,
1686                detect_application_packages: true,
1687                detect_system_packages: false,
1688                detect_packages_in_compiled: false,
1689                detect_copyrights: false,
1690                detect_generated: false,
1691                detect_emails: false,
1692                detect_urls: false,
1693                max_emails: 50,
1694                max_urls: 50,
1695                timeout_seconds: 120.0,
1696            },
1697        );
1698
1699        let scanned = result
1700            .files
1701            .into_iter()
1702            .find(|entry| {
1703                entry.file_type == FileType::File && entry.path.ends_with("/nsis-with-version.exe")
1704            })
1705            .expect("compiled artifact present");
1706
1707        assert_eq!(
1708            scanned.package_data.len(),
1709            2,
1710            "package_data: {:#?}",
1711            scanned.package_data
1712        );
1713        assert!(
1714            scanned
1715                .package_data
1716                .iter()
1717                .any(|pkg| pkg.datasource_id == Some(DatasourceId::NsisInstaller))
1718        );
1719        assert!(
1720            scanned
1721                .package_data
1722                .iter()
1723                .any(|pkg| pkg.datasource_id == Some(DatasourceId::WindowsExecutable))
1724        );
1725    }
1726
1727    #[test]
1728    fn scanner_detects_license_from_font_metadata() {
1729        let temp_dir = TempDir::new().expect("create temp dir");
1730        let file_path = temp_dir.path().join("Lato-Bold.ttf");
1731        let fixture = fs::read("testdata/font-fixtures/Lato-Bold.ttf").expect("read font fixture");
1732        fs::write(&file_path, fixture).expect("write font fixture");
1733
1734        let progress = Arc::new(ScanProgress::new(ProgressMode::Quiet));
1735        let collected = collect_paths(temp_dir.path(), 0, &[]);
1736        let engine =
1737            Arc::new(LicenseDetectionEngine::from_embedded().expect("initialize license engine"));
1738        let result = process_collected(
1739            &collected,
1740            progress,
1741            Some(engine),
1742            LicenseScanOptions::default(),
1743            &TextDetectionOptions::default(),
1744        );
1745        let scanned = result
1746            .files
1747            .into_iter()
1748            .find(|entry| {
1749                entry.file_type == FileType::File && entry.path == file_path.to_string_lossy()
1750            })
1751            .expect("scanned file entry");
1752
1753        assert!(
1754            scanned.detected_license_expression.is_some(),
1755            "license detections: {:#?}",
1756            scanned.license_detections
1757        );
1758        assert!(
1759            scanned
1760                .detected_license_expression
1761                .as_deref()
1762                .is_some_and(
1763                    |expression| expression.contains("OFL-1.1") || expression.contains("ofl-1.1")
1764                ),
1765            "license expression: {:?}",
1766            scanned.detected_license_expression
1767        );
1768    }
1769
1770    #[test]
1771    fn scanner_detects_license_from_windows_executable_metadata() {
1772        let temp_dir = TempDir::new().expect("create temp dir");
1773        let file_path = temp_dir.path().join("libiconv2.dll");
1774        let fixture = fs::read("testdata/compiled-binary-golden/win_pe/libiconv2.dll")
1775            .expect("read PE fixture");
1776        fs::write(&file_path, fixture).expect("write PE fixture");
1777
1778        let progress = Arc::new(ScanProgress::new(ProgressMode::Quiet));
1779        let collected = collect_paths(temp_dir.path(), 0, &[]);
1780        let engine =
1781            Arc::new(LicenseDetectionEngine::from_embedded().expect("initialize license engine"));
1782        let result = process_collected(
1783            &collected,
1784            progress,
1785            Some(engine),
1786            LicenseScanOptions::default(),
1787            &TextDetectionOptions::default(),
1788        );
1789        let scanned = result
1790            .files
1791            .into_iter()
1792            .find(|entry| {
1793                entry.file_type == FileType::File && entry.path == file_path.to_string_lossy()
1794            })
1795            .expect("scanned file entry");
1796
1797        assert!(
1798            scanned.detected_license_expression.is_some(),
1799            "license detections: {:#?}",
1800            scanned.license_detections
1801        );
1802        assert!(
1803            scanned
1804                .detected_license_expression
1805                .as_deref()
1806                .is_some_and(|expression| {
1807                    expression.contains("lgpl") || expression.contains("LGPL")
1808                }),
1809            "license expression: {:?}",
1810            scanned.detected_license_expression
1811        );
1812    }
1813
1814    #[test]
1815    fn scanner_detects_license_from_windows_executable_security_notice() {
1816        fn synthetic_pe_with_security_notice(notice: &str) -> Vec<u8> {
1817            let cert_payload = notice
1818                .encode_utf16()
1819                .flat_map(|unit| unit.to_le_bytes())
1820                .collect::<Vec<_>>();
1821            let cert_len = (8 + cert_payload.len()) as u32;
1822            let mut cert = Vec::new();
1823            cert.extend_from_slice(&cert_len.to_le_bytes());
1824            cert.extend_from_slice(&0x0200u16.to_le_bytes());
1825            cert.extend_from_slice(&0x0002u16.to_le_bytes());
1826            cert.extend_from_slice(&cert_payload);
1827            while !cert.len().is_multiple_of(8) {
1828                cert.push(0);
1829            }
1830
1831            let offset = 0x200usize;
1832            let size = cert.len();
1833            let optional_header_size = 224usize;
1834            let pe_header_offset = 0x80usize;
1835            let nt_headers_offset = pe_header_offset + 4;
1836            let optional_header_offset = nt_headers_offset + 20;
1837            let data_directory_offset = optional_header_offset + 96;
1838            let security_directory_offset =
1839                data_directory_offset + pe::IMAGE_DIRECTORY_ENTRY_SECURITY * 8;
1840            let total_len = offset + size;
1841            let mut bytes = vec![0u8; total_len];
1842
1843            bytes[0..2].copy_from_slice(b"MZ");
1844            bytes[0x3c..0x40].copy_from_slice(&(pe_header_offset as u32).to_le_bytes());
1845            bytes[pe_header_offset..pe_header_offset + 4].copy_from_slice(b"PE\0\0");
1846
1847            bytes[nt_headers_offset..nt_headers_offset + 2]
1848                .copy_from_slice(&0x014cu16.to_le_bytes());
1849            bytes[nt_headers_offset + 16..nt_headers_offset + 18]
1850                .copy_from_slice(&(optional_header_size as u16).to_le_bytes());
1851
1852            bytes[optional_header_offset..optional_header_offset + 2]
1853                .copy_from_slice(&0x010bu16.to_le_bytes());
1854            bytes[optional_header_offset + 92..optional_header_offset + 96]
1855                .copy_from_slice(&16u32.to_le_bytes());
1856            bytes[security_directory_offset..security_directory_offset + 4]
1857                .copy_from_slice(&(offset as u32).to_le_bytes());
1858            bytes[security_directory_offset + 4..security_directory_offset + 8]
1859                .copy_from_slice(&(size as u32).to_le_bytes());
1860            bytes[offset..offset + size].copy_from_slice(&cert);
1861
1862            bytes
1863        }
1864
1865        let temp_dir = TempDir::new().expect("create temp dir");
1866        let file_path = temp_dir.path().join("signed.dll");
1867        let fixture = synthetic_pe_with_security_notice(
1868            "use of this Certificate constitutes acceptance of the DigiCert CP/CPS and the Relying Party Agreement which limit liability and are incorporated herein by reference.",
1869        );
1870        fs::write(&file_path, fixture).expect("write PE fixture");
1871
1872        let progress = Arc::new(ScanProgress::new(ProgressMode::Quiet));
1873        let collected = collect_paths(temp_dir.path(), 0, &[]);
1874        let engine =
1875            Arc::new(LicenseDetectionEngine::from_embedded().expect("initialize license engine"));
1876        let result = process_collected(
1877            &collected,
1878            progress,
1879            Some(engine),
1880            LicenseScanOptions::default(),
1881            &TextDetectionOptions::default(),
1882        );
1883        let scanned = result
1884            .files
1885            .into_iter()
1886            .find(|entry| {
1887                entry.file_type == FileType::File && entry.path == file_path.to_string_lossy()
1888            })
1889            .expect("scanned file entry");
1890
1891        assert!(
1892            scanned
1893                .detected_license_expression
1894                .as_deref()
1895                .is_some_and(|expression| expression.contains("proprietary-license")),
1896            "license expression: {:?}, detections: {:#?}",
1897            scanned.detected_license_expression,
1898            scanned.license_detections
1899        );
1900    }
1901
1902    #[test]
1903    fn scanner_detects_cc_by_license_from_markdown_comment_banner() {
1904        let scanned = scan_single_file_with_license_engine(
1905            "navbar.md",
1906            "<!-- Documentation licensed under CC BY 4.0 -->\n<!-- License available at https://creativecommons.org/licenses/by/4.0/ -->\n",
1907            &TextDetectionOptions::default(),
1908        );
1909
1910        assert!(
1911            scanned
1912                .detected_license_expression
1913                .as_deref()
1914                .is_some_and(|expression| {
1915                    expression.contains("cc-by-4.0") || expression.contains("CC-BY-4.0")
1916                }),
1917            "license expression: {:?}",
1918            scanned.detected_license_expression
1919        );
1920    }
1921
1922    #[test]
1923    fn scanner_detects_mit_license_from_shields_badge_markdown() {
1924        let scanned = scan_single_file_with_license_engine(
1925            "README.md",
1926            "[![](https://img.shields.io/badge/license-MIT-blue.svg)](https://opensource.org/licenses/MIT)\n",
1927            &TextDetectionOptions::default(),
1928        );
1929
1930        assert!(
1931            scanned
1932                .detected_license_expression
1933                .as_deref()
1934                .is_some_and(|expression| {
1935                    expression.contains("mit") || expression.contains("MIT")
1936                }),
1937            "license expression: {:?}",
1938            scanned.detected_license_expression
1939        );
1940    }
1941
1942    #[test]
1943    fn scanner_detects_apache_license_from_markdown_readme_phrase() {
1944        let scanned = scan_single_file_with_license_engine(
1945            "README.md",
1946            "This crate is distributed under the terms of the Apache License (Version 2.0).\n",
1947            &TextDetectionOptions::default(),
1948        );
1949
1950        assert!(
1951            scanned
1952                .detected_license_expression
1953                .as_deref()
1954                .is_some_and(|expression| {
1955                    expression.contains("apache-2.0") || expression.contains("Apache-2.0")
1956                }),
1957            "license expression: {:?}",
1958            scanned.detected_license_expression
1959        );
1960    }
1961
1962    #[test]
1963    fn scanner_prefers_dual_license_readme_expression_over_supplemental_mentions() {
1964        let scanned = scan_single_file_with_license_engine(
1965            "README.md",
1966            concat!(
1967                "## License\n\n",
1968                "Licensed under either of:\n\n",
1969                " * [Apache License, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0)\n",
1970                " * [MIT license](https://opensource.org/licenses/MIT)\n\n",
1971                "at your option.\n\n",
1972                "### Contribution\n\n",
1973                "Unless you explicitly state otherwise, any contribution intentionally submitted\n",
1974                "for inclusion in the work by you, as defined in the Apache-2.0 license, shall be\n",
1975                "dual licensed as above, without any additional terms or conditions.\n",
1976            ),
1977            &TextDetectionOptions::default(),
1978        );
1979
1980        assert!(
1981            matches!(
1982                scanned.detected_license_expression.as_deref(),
1983                Some("Apache-2.0 OR MIT") | Some("MIT OR Apache-2.0")
1984            ),
1985            "license expression: {:?}",
1986            scanned.detected_license_expression
1987        );
1988        assert!(
1989            !scanned
1990                .license_detections
1991                .iter()
1992                .any(|detection| detection.license_expression_spdx == "Apache-2.0"),
1993            "detections: {:?}",
1994            scanned.license_detections
1995        );
1996    }
1997
1998    #[test]
1999    fn scanner_drops_redundant_conjunctive_readme_detection_when_or_notice_exists() {
2000        let scanned = scan_single_file_with_license_engine(
2001            "README.md",
2002            concat!(
2003                "## License\n\n",
2004                "Licensed under either of:\n\n",
2005                " * [Apache License, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0)\n",
2006                " * [MIT license](https://opensource.org/licenses/MIT)\n\n",
2007                "at your option.\n\n",
2008                "### Contribution\n\n",
2009                "Unless you explicitly state otherwise, any contribution intentionally submitted\n",
2010                "for inclusion in the work by you, as defined in the Apache-2.0 license, shall be\n",
2011                "dual licensed as above, without any additional terms or conditions.\n\n",
2012                "[license-image]: https://img.shields.io/badge/license-Apache2.0/MIT-blue.svg\n",
2013            ),
2014            &TextDetectionOptions::default(),
2015        );
2016
2017        assert!(
2018            !scanned
2019                .license_detections
2020                .iter()
2021                .any(|detection| { detection.license_expression_spdx == "Apache-2.0 AND MIT" })
2022        );
2023    }
2024
2025    #[test]
2026    fn scanner_drops_unknown_placeholder_from_dual_license_readme_notice() {
2027        let scanned = scan_single_file_with_license_engine(
2028            "README.md",
2029            concat!(
2030                "## License\n\n",
2031                "This project is dual-licensed under MIT and Apache 2.0.\n",
2032            ),
2033            &TextDetectionOptions::default(),
2034        );
2035
2036        assert!(
2037            matches!(
2038                scanned.detected_license_expression.as_deref(),
2039                Some("Apache-2.0 OR MIT") | Some("MIT OR Apache-2.0")
2040            ),
2041            "license expression: {:?}",
2042            scanned.detected_license_expression
2043        );
2044        assert!(scanned.license_detections.iter().any(|detection| {
2045            detection
2046                .license_expression_spdx
2047                .contains("Apache-2.0 OR MIT")
2048                || detection
2049                    .license_expression_spdx
2050                    .contains("MIT OR Apache-2.0")
2051        }));
2052        assert!(!scanned.license_detections.iter().any(|detection| {
2053            detection.license_expression_spdx == "LicenseRef-scancode-unknown-license-reference"
2054        }));
2055        assert!(
2056            scanned
2057                .license_detections
2058                .iter()
2059                .any(|detection| detection.license_expression_spdx == "MIT"),
2060            "detections: {:?}",
2061            scanned.license_detections
2062        );
2063    }
2064
2065    #[test]
2066    fn scanner_sets_is_source_only_when_info_enabled() {
2067        let without_info = TextDetectionOptions {
2068            collect_info: false,
2069            detect_packages: false,
2070            detect_application_packages: false,
2071            detect_system_packages: false,
2072            detect_packages_in_compiled: false,
2073            detect_copyrights: false,
2074            detect_generated: false,
2075            detect_emails: false,
2076            detect_urls: false,
2077            max_emails: 50,
2078            max_urls: 50,
2079            timeout_seconds: 120.0,
2080        };
2081        let with_info = TextDetectionOptions {
2082            collect_info: true,
2083            ..without_info.clone()
2084        };
2085
2086        let scanned_without_info = scan_single_file("main.rs", "fn main() {}\n", &without_info);
2087        let scanned_with_info = scan_single_file("main.rs", "fn main() {}\n", &with_info);
2088
2089        assert_eq!(scanned_without_info.is_source, None);
2090        assert_eq!(scanned_with_info.is_source, Some(true));
2091    }
2092
2093    #[test]
2094    fn directory_omits_info_fields_when_info_disabled() {
2095        let temp_dir = TempDir::new().expect("create temp dir");
2096        fs::create_dir_all(temp_dir.path().join("nested")).expect("create nested dir");
2097
2098        let collected = collect_paths(temp_dir.path(), 0, &[]);
2099        let result = process_collected(
2100            &collected,
2101            Arc::new(ScanProgress::new(ProgressMode::Quiet)),
2102            None,
2103            LicenseScanOptions::default(),
2104            &TextDetectionOptions {
2105                collect_info: false,
2106                detect_packages: false,
2107                detect_application_packages: false,
2108                detect_system_packages: false,
2109                detect_packages_in_compiled: false,
2110                detect_copyrights: false,
2111                detect_generated: false,
2112                detect_emails: false,
2113                detect_urls: false,
2114                max_emails: 50,
2115                max_urls: 50,
2116                timeout_seconds: 120.0,
2117            },
2118        );
2119
2120        let directory = result
2121            .files
2122            .into_iter()
2123            .find(|entry| entry.file_type == FileType::Directory && entry.path.ends_with("nested"))
2124            .expect("directory entry");
2125
2126        assert!(directory.date.is_none());
2127        assert!(directory.file_type_label.is_none());
2128        assert!(directory.is_binary.is_none());
2129        assert!(directory.is_text.is_none());
2130        assert!(directory.is_archive.is_none());
2131        assert!(directory.is_media.is_none());
2132        assert!(directory.is_source.is_none());
2133        assert!(directory.is_script.is_none());
2134    }
2135
2136    #[test]
2137    fn directory_includes_info_fields_when_info_enabled() {
2138        let temp_dir = TempDir::new().expect("create temp dir");
2139        fs::create_dir_all(temp_dir.path().join("nested")).expect("create nested dir");
2140
2141        let collected = collect_paths(temp_dir.path(), 0, &[]);
2142        let result = process_collected(
2143            &collected,
2144            Arc::new(ScanProgress::new(ProgressMode::Quiet)),
2145            None,
2146            LicenseScanOptions::default(),
2147            &TextDetectionOptions {
2148                collect_info: true,
2149                detect_packages: false,
2150                detect_application_packages: false,
2151                detect_system_packages: false,
2152                detect_packages_in_compiled: false,
2153                detect_copyrights: false,
2154                detect_generated: false,
2155                detect_emails: false,
2156                detect_urls: false,
2157                max_emails: 50,
2158                max_urls: 50,
2159                timeout_seconds: 120.0,
2160            },
2161        );
2162
2163        let directory = result
2164            .files
2165            .into_iter()
2166            .find(|entry| entry.file_type == FileType::Directory && entry.path.ends_with("nested"))
2167            .expect("directory entry");
2168
2169        assert!(directory.date.is_none());
2170        assert!(directory.file_type_label.is_none());
2171        assert_eq!(directory.is_binary, Some(false));
2172        assert_eq!(directory.is_text, Some(false));
2173        assert_eq!(directory.is_archive, Some(false));
2174        assert_eq!(directory.is_media, Some(false));
2175        assert_eq!(directory.is_source, Some(false));
2176        assert_eq!(directory.is_script, Some(false));
2177        assert_eq!(directory.files_count, Some(0));
2178        assert_eq!(directory.dirs_count, Some(0));
2179        assert_eq!(directory.size_count, Some(0));
2180    }
2181
2182    #[test]
2183    fn collect_paths_includes_root_directory_entry() {
2184        let temp_dir = TempDir::new().expect("create temp dir");
2185        fs::create_dir_all(temp_dir.path().join("src")).expect("create nested dir");
2186        fs::write(temp_dir.path().join("src").join("main.rs"), "fn main() {}")
2187            .expect("write nested file");
2188
2189        let collected = collect_paths(temp_dir.path(), 0, &[]);
2190
2191        assert!(
2192            collected
2193                .directories
2194                .iter()
2195                .any(|(path, _)| path == temp_dir.path())
2196        );
2197    }
2198
2199    #[test]
2200    fn collect_paths_supports_single_file_input() {
2201        let temp_dir = TempDir::new().expect("create temp dir");
2202        let file_path = temp_dir.path().join("main.rs");
2203        fs::write(&file_path, "fn main() {}\n").expect("write file");
2204
2205        let collected = collect_paths(&file_path, 0, &[]);
2206
2207        assert_eq!(collected.files.len(), 1);
2208        assert!(collected.directories.is_empty());
2209        assert_eq!(collected.files[0].0, file_path);
2210    }
2211
2212    #[cfg(unix)]
2213    #[test]
2214    fn collect_selected_paths_does_not_walk_unselected_siblings() {
2215        use std::os::unix::fs::PermissionsExt;
2216
2217        let temp_dir = TempDir::new().expect("create temp dir");
2218        let root = temp_dir.path();
2219        fs::create_dir_all(root.join("selected/docs")).expect("create selected dir");
2220        fs::create_dir_all(root.join("blocked/secret")).expect("create blocked dir");
2221        fs::write(root.join("selected/docs/guide.md"), "# guide\n").expect("write guide");
2222
2223        let blocked = root.join("blocked");
2224        let mut perms = fs::metadata(&blocked)
2225            .expect("blocked metadata")
2226            .permissions();
2227        perms.set_mode(0o000);
2228        fs::set_permissions(&blocked, perms).expect("remove blocked permissions");
2229
2230        let collected = collect_selected_paths(
2231            root,
2232            &[CollectionFrontier {
2233                path: PathBuf::from("selected"),
2234                recurse: true,
2235            }],
2236            0,
2237            &[],
2238        );
2239
2240        let mut restore = fs::metadata(&blocked)
2241            .expect("blocked metadata")
2242            .permissions();
2243        restore.set_mode(0o755);
2244        fs::set_permissions(&blocked, restore).expect("restore blocked permissions");
2245
2246        assert!(
2247            collected.collection_errors.is_empty(),
2248            "{:#?}",
2249            collected.collection_errors
2250        );
2251        assert!(
2252            collected
2253                .files
2254                .iter()
2255                .any(|(path, _)| path == &root.join("selected/docs/guide.md"))
2256        );
2257        assert!(
2258            collected
2259                .files
2260                .iter()
2261                .all(|(path, _): &(PathBuf, fs::Metadata)| !path.starts_with(&blocked))
2262        );
2263    }
2264
2265    #[test]
2266    fn collect_selected_paths_respects_excluded_ancestor_directories() {
2267        let temp_dir = TempDir::new().expect("create temp dir");
2268        let root = temp_dir.path();
2269        fs::create_dir_all(root.join(".git")).expect("create git dir");
2270        fs::write(
2271            root.join(".git/config"),
2272            "[core]\nrepositoryformatversion = 0\n",
2273        )
2274        .expect("write git config");
2275
2276        let exclude_patterns =
2277            build_collection_exclude_patterns(root, &root.join(".provenant-cache"));
2278        let collected = collect_selected_paths(
2279            root,
2280            &[CollectionFrontier {
2281                path: PathBuf::from(".git/config"),
2282                recurse: false,
2283            }],
2284            0,
2285            &exclude_patterns,
2286        );
2287
2288        assert!(collected.files.is_empty());
2289        assert!(collected.directories.iter().all(|(path, _)| path == root));
2290        assert_eq!(collected.excluded_count, 1);
2291    }
2292
2293    #[test]
2294    fn process_collected_with_memory_limit_preserves_results_when_spilling() {
2295        let temp_dir = TempDir::new().expect("create temp dir");
2296        fs::write(temp_dir.path().join("a.txt"), "hello").expect("write first file");
2297        fs::write(temp_dir.path().join("b.txt"), "world").expect("write second file");
2298
2299        let collected = collect_paths(temp_dir.path(), 0, &[]);
2300        let result = process_collected_with_memory_limit(
2301            &collected,
2302            Arc::new(ScanProgress::new(ProgressMode::Quiet)),
2303            None,
2304            LicenseScanOptions::default(),
2305            &TextDetectionOptions {
2306                collect_info: false,
2307                detect_packages: false,
2308                detect_application_packages: false,
2309                detect_system_packages: false,
2310                detect_packages_in_compiled: false,
2311                detect_copyrights: false,
2312                detect_generated: false,
2313                detect_emails: false,
2314                detect_urls: false,
2315                max_emails: 50,
2316                max_urls: 50,
2317                timeout_seconds: 120.0,
2318            },
2319            MemoryMode::Limit(1),
2320        );
2321
2322        assert_eq!(result.files.len(), 3);
2323    }
2324
2325    #[test]
2326    fn process_collected_with_negative_one_uses_disk_only_mode() {
2327        let temp_dir = TempDir::new().expect("create temp dir");
2328        fs::write(temp_dir.path().join("a.txt"), "hello").expect("write first file");
2329
2330        let collected = collect_paths(temp_dir.path(), 0, &[]);
2331        let result = process_collected_with_memory_limit(
2332            &collected,
2333            Arc::new(ScanProgress::new(ProgressMode::Quiet)),
2334            None,
2335            LicenseScanOptions::default(),
2336            &TextDetectionOptions {
2337                collect_info: false,
2338                detect_packages: false,
2339                detect_application_packages: false,
2340                detect_system_packages: false,
2341                detect_packages_in_compiled: false,
2342                detect_copyrights: false,
2343                detect_generated: false,
2344                detect_emails: false,
2345                detect_urls: false,
2346                max_emails: 50,
2347                max_urls: 50,
2348                timeout_seconds: 120.0,
2349            },
2350            MemoryMode::StreamUnlimited,
2351        );
2352
2353        assert_eq!(result.files.len(), 2);
2354    }
2355}