1use derive_builder::Builder;
7use packageurl::PackageUrl;
8use serde::{Deserialize, Serialize};
9use sha1::{Digest, Sha1};
10use std::collections::HashMap;
11use std::fmt;
12use std::str::FromStr;
13
14use super::DatasourceId;
15use super::DependencyUid;
16use super::DiagnosticSeverity;
17use super::GitSha1;
18use super::LineNumber;
19use super::MatchScore;
20use super::Md5Digest;
21use super::PackageType;
22use super::PackageUid;
23use super::ScanDiagnostic;
24use super::Sha1Digest;
25use super::Sha256Digest;
26use super::Sha512Digest;
27
28use crate::license_detection::MatcherKind;
29use crate::license_detection::tokenize::tokenize_without_stopwords;
30use crate::models::output::Tallies;
31use crate::utils::spdx::combine_license_expressions;
32
33#[derive(Debug, Builder, Serialize, Deserialize, Clone)]
34#[builder(build_fn(skip))]
35pub struct FileInfo {
37 pub name: String,
38 pub base_name: String,
39 pub extension: String,
40 pub path: String,
41 pub file_type: FileType,
42 #[builder(default)]
43 #[serde(default)]
44 pub mime_type: Option<String>,
45 #[builder(default)]
46 #[serde(default)]
47 pub file_type_label: Option<String>,
48 pub size: u64,
49 #[builder(default)]
50 #[serde(default)]
51 pub date: Option<String>,
52 #[builder(default)]
53 #[serde(default)]
54 pub sha1: Option<Sha1Digest>,
55 #[builder(default)]
56 #[serde(default)]
57 pub md5: Option<Md5Digest>,
58 #[builder(default)]
59 #[serde(default)]
60 pub sha256: Option<Sha256Digest>,
61 #[builder(default)]
62 #[serde(default)]
63 pub sha1_git: Option<GitSha1>,
64 #[builder(default)]
65 #[serde(default)]
66 pub programming_language: Option<String>,
67 #[builder(default)]
68 #[serde(default)]
69 pub package_data: Vec<PackageData>,
70 #[builder(default)]
71 #[serde(default)]
72 pub detected_license_expression: Option<String>,
73 #[builder(default)]
74 #[serde(default)]
75 pub license_detections: Vec<LicenseDetection>,
76 #[builder(default)]
77 #[serde(default)]
78 pub license_clues: Vec<Match>,
79 #[builder(default)]
80 #[serde(default)]
81 pub percentage_of_license_text: Option<f64>,
82 #[builder(default)]
83 #[serde(default)]
84 pub copyrights: Vec<Copyright>,
85 #[builder(default)]
86 #[serde(default)]
87 pub holders: Vec<Holder>,
88 #[builder(default)]
89 #[serde(default)]
90 pub authors: Vec<Author>,
91 #[builder(default)]
92 #[serde(default)]
93 pub emails: Vec<OutputEmail>,
94 #[builder(default)]
95 #[serde(default)]
96 pub urls: Vec<OutputURL>,
97 #[builder(default)]
98 #[serde(default)]
99 pub for_packages: Vec<PackageUid>,
100 #[builder(default)]
101 #[serde(default)]
102 pub scan_diagnostics: Vec<ScanDiagnostic>,
103 #[builder(default)]
104 #[serde(default)]
105 pub license_policy: Option<Vec<LicensePolicyEntry>>,
106 #[builder(default)]
107 #[serde(default)]
108 pub is_generated: Option<bool>,
109 #[builder(default)]
110 #[serde(default)]
111 pub is_binary: Option<bool>,
112 #[builder(default)]
113 #[serde(default)]
114 pub is_text: Option<bool>,
115 #[builder(default)]
116 #[serde(default)]
117 pub is_archive: Option<bool>,
118 #[builder(default)]
119 #[serde(default)]
120 pub is_media: Option<bool>,
121 #[builder(default)]
122 #[serde(default)]
123 pub is_source: Option<bool>,
124 #[builder(default)]
125 #[serde(default)]
126 pub is_script: Option<bool>,
127 #[builder(default)]
128 #[serde(default)]
129 pub files_count: Option<usize>,
130 #[builder(default)]
131 #[serde(default)]
132 pub dirs_count: Option<usize>,
133 #[builder(default)]
134 #[serde(default)]
135 pub size_count: Option<u64>,
136 #[builder(default)]
137 #[serde(default)]
138 pub source_count: Option<usize>,
139 #[builder(default)]
140 #[serde(default)]
141 pub is_legal: bool,
142 #[builder(default)]
143 #[serde(default)]
144 pub is_manifest: bool,
145 #[builder(default)]
146 #[serde(default)]
147 pub is_readme: bool,
148 #[builder(default)]
149 #[serde(default)]
150 pub is_top_level: bool,
151 #[builder(default)]
152 #[serde(default)]
153 pub is_key_file: bool,
154 #[builder(default)]
155 #[serde(default)]
156 pub is_referenced: bool,
157 #[builder(default)]
158 #[serde(default)]
159 pub is_community: bool,
160 #[builder(default)]
161 #[serde(default)]
162 pub facets: Vec<String>,
163 #[builder(default)]
164 #[serde(default)]
165 pub tallies: Option<Tallies>,
166}
167
168impl FileInfoBuilder {
169 pub fn build(&self) -> Result<FileInfo, String> {
171 let mut file_info = FileInfo::new(
172 self.name.clone().ok_or("Missing field: name")?,
173 self.base_name.clone().ok_or("Missing field: base_name")?,
174 self.extension.clone().ok_or("Missing field: extension")?,
175 self.path.clone().ok_or("Missing field: path")?,
176 self.file_type.clone().ok_or("Missing field: file_type")?,
177 self.mime_type.clone().flatten(),
178 self.file_type_label.clone().flatten(),
179 self.size.ok_or("Missing field: size")?,
180 self.date.clone().flatten(),
181 self.sha1.flatten(),
182 self.md5.flatten(),
183 self.sha256.flatten(),
184 self.programming_language.clone().flatten(),
185 self.package_data.clone().unwrap_or_default(),
186 self.detected_license_expression.clone().flatten(),
187 self.license_detections.clone().unwrap_or_default(),
188 self.license_clues.clone().unwrap_or_default(),
189 self.copyrights.clone().unwrap_or_default(),
190 self.holders.clone().unwrap_or_default(),
191 self.authors.clone().unwrap_or_default(),
192 self.emails.clone().unwrap_or_default(),
193 self.urls.clone().unwrap_or_default(),
194 self.for_packages.clone().unwrap_or_default(),
195 self.scan_diagnostics.clone().unwrap_or_default(),
196 );
197 file_info.license_policy = self.license_policy.clone().flatten();
198 file_info.sha1_git = self.sha1_git.flatten();
199 file_info.is_binary = self.is_binary.flatten();
200 file_info.is_text = self.is_text.flatten();
201 file_info.is_archive = self.is_archive.flatten();
202 file_info.is_media = self.is_media.flatten();
203 file_info.is_script = self.is_script.flatten();
204 file_info.files_count = self.files_count.flatten();
205 file_info.dirs_count = self.dirs_count.flatten();
206 file_info.size_count = self.size_count.flatten();
207 file_info.is_referenced = self.is_referenced.unwrap_or_default();
208 Ok(file_info)
209 }
210}
211
212impl FileInfo {
213 #[allow(clippy::too_many_arguments)]
216 pub fn new(
217 name: String,
218 base_name: String,
219 extension: String,
220 path: String,
221 file_type: FileType,
222 mime_type: Option<String>,
223 file_type_label: Option<String>,
224 size: u64,
225 date: Option<String>,
226 sha1: Option<Sha1Digest>,
227 md5: Option<Md5Digest>,
228 sha256: Option<Sha256Digest>,
229 programming_language: Option<String>,
230 package_data: Vec<PackageData>,
231 mut detected_license_expression: Option<String>,
232 mut license_detections: Vec<LicenseDetection>,
233 license_clues: Vec<Match>,
234 copyrights: Vec<Copyright>,
235 holders: Vec<Holder>,
236 authors: Vec<Author>,
237 emails: Vec<OutputEmail>,
238 urls: Vec<OutputURL>,
239 for_packages: Vec<PackageUid>,
240 scan_diagnostics: Vec<ScanDiagnostic>,
241 ) -> Self {
242 let mut package_data = package_data;
243 for package in &mut package_data {
244 enrich_package_data_license_provenance(package, &path);
245 }
246
247 detected_license_expression = detected_license_expression.or_else(|| {
249 let expressions = package_data
250 .iter()
251 .filter_map(|pkg| pkg.get_license_expression());
252 combine_license_expressions(expressions)
253 });
254
255 if license_detections.is_empty() {
257 for pkg in &package_data {
258 license_detections.extend(pkg.license_detections.clone());
259 }
260 }
261
262 if detected_license_expression.is_none() && !license_detections.is_empty() {
264 let expressions = license_detections
265 .iter()
266 .map(|detection| detection.license_expression.clone());
267 let expressions: Vec<String> = expressions.collect();
268 detected_license_expression = crate::utils::spdx::select_primary_license_expression(
269 expressions.clone(),
270 )
271 .or_else(|| {
272 crate::utils::spdx::combine_license_expressions_preserving_structure(expressions)
273 });
274 }
275
276 let mut file_info = FileInfo {
277 name,
278 base_name,
279 extension,
280 path,
281 file_type,
282 mime_type,
283 file_type_label,
284 size,
285 date,
286 sha1,
287 md5,
288 sha256,
289 sha1_git: None,
290 programming_language,
291 package_data,
292 detected_license_expression,
293 license_detections,
294 license_clues,
295 percentage_of_license_text: None,
296 copyrights,
297 holders,
298 authors,
299 emails,
300 urls,
301 for_packages,
302 scan_diagnostics,
303 license_policy: None,
304 is_generated: None,
305 is_binary: None,
306 is_text: None,
307 is_archive: None,
308 is_media: None,
309 is_source: None,
310 is_script: None,
311 files_count: None,
312 dirs_count: None,
313 size_count: None,
314 source_count: None,
315 is_legal: false,
316 is_manifest: false,
317 is_readme: false,
318 is_top_level: false,
319 is_key_file: false,
320 is_referenced: false,
321 is_community: false,
322 facets: vec![],
323 tallies: None,
324 };
325
326 file_info.backfill_license_provenance();
327 file_info
328 }
329
330 pub fn backfill_license_provenance(&mut self) {
331 for detection in &mut self.license_detections {
332 enrich_license_detection_provenance(detection, &self.path);
333 }
334
335 for package in &mut self.package_data {
336 enrich_package_data_license_provenance(package, &self.path);
337 }
338 }
339}
340
341impl FileInfo {
342 pub fn warning_diagnostics(&self) -> impl Iterator<Item = &ScanDiagnostic> {
343 self.scan_diagnostics
344 .iter()
345 .filter(|diagnostic| diagnostic.severity == DiagnosticSeverity::Warning)
346 }
347
348 pub fn error_diagnostics(&self) -> impl Iterator<Item = &ScanDiagnostic> {
349 self.scan_diagnostics.iter().filter(|diagnostic| {
350 diagnostic.severity == DiagnosticSeverity::Error
351 || diagnostic.severity == DiagnosticSeverity::Timeout
352 })
353 }
354}
355
356fn enrich_package_data_license_provenance(package_data: &mut PackageData, path: &str) {
357 for detection in &mut package_data.license_detections {
358 enrich_license_detection_provenance(detection, path);
359 }
360 for detection in &mut package_data.other_license_detections {
361 enrich_license_detection_provenance(detection, path);
362 }
363}
364
365pub(crate) fn enrich_license_detection_provenance(detection: &mut LicenseDetection, path: &str) {
366 for detection_match in &mut detection.matches {
367 if detection_match.from_file.is_none() {
368 detection_match.from_file = Some(path.to_string());
369 }
370
371 if detection_match.rule_identifier.is_empty() {
372 detection_match.rule_identifier = detection_match.matcher.to_string();
373 }
374 }
375
376 if detection.identifier.is_empty() {
377 detection.identifier = compute_public_detection_identifier(detection);
378 }
379}
380
381fn compute_public_detection_identifier(detection: &LicenseDetection) -> String {
382 let expression = python_safe_name(&detection.license_expression);
383 let mut hasher = Sha1::new();
384 hasher.update(format_public_detection_content(detection).as_bytes());
385 let hex_str = hex::encode(hasher.finalize());
386 let uuid_hex = &hex_str[..32];
387 let content_uuid = uuid::Uuid::parse_str(uuid_hex)
388 .map(|uuid| uuid.to_string())
389 .unwrap_or_else(|_| uuid_hex.to_string());
390
391 format!("{}-{}", expression, content_uuid)
392}
393
394fn format_public_detection_content(detection: &LicenseDetection) -> String {
395 let mut result = String::from("(");
396
397 for (index, detection_match) in detection.matches.iter().enumerate() {
398 if index > 0 {
399 result.push_str(", ");
400 }
401 result.push_str(&format!(
402 "({}, {}, {})",
403 python_str_repr(if detection_match.rule_identifier.is_empty() {
404 detection_match.matcher.as_str()
405 } else {
406 detection_match.rule_identifier.as_str()
407 }),
408 detection_match.score.value() as f32,
409 python_token_tuple_repr(&tokenize_without_stopwords(
410 detection_match.matched_text.as_deref().unwrap_or_default(),
411 )),
412 ));
413 }
414
415 if detection.matches.len() == 1 {
416 result.push(',');
417 }
418 result.push(')');
419 result
420}
421
422fn python_safe_name(value: &str) -> String {
423 let mut result = String::new();
424 let mut prev_underscore = false;
425
426 for character in value.chars() {
427 if character.is_alphanumeric() {
428 result.push(character);
429 prev_underscore = false;
430 } else if !prev_underscore {
431 result.push('_');
432 prev_underscore = true;
433 }
434 }
435
436 let trimmed = result.trim_matches('_');
437 if trimmed.is_empty() {
438 String::new()
439 } else {
440 trimmed.to_string()
441 }
442}
443
444fn python_str_repr(value: &str) -> String {
445 if value.contains('\'') && !value.contains('"') {
446 format!("\"{}\"", value.replace('\\', "\\\\").replace('"', "\\\""))
447 } else {
448 format!("'{}'", value.replace('\\', "\\\\").replace('\'', "\\\'"))
449 }
450}
451
452fn python_token_tuple_repr(tokens: &[String]) -> String {
453 if tokens.is_empty() {
454 return String::from("()");
455 }
456
457 let mut result = String::from("(");
458 for (index, token) in tokens.iter().enumerate() {
459 if index > 0 {
460 result.push_str(", ");
461 }
462 result.push_str(&python_str_repr(token));
463 }
464
465 if tokens.len() == 1 {
466 result.push(',');
467 }
468 result.push(')');
469 result
470}
471
472#[derive(Serialize, Deserialize, Debug, Clone, Default)]
478pub struct PackageData {
479 pub package_type: Option<PackageType>,
480 pub namespace: Option<String>,
481 pub name: Option<String>,
482 pub version: Option<String>,
483 #[serde(default)]
484 pub qualifiers: Option<HashMap<String, String>>,
485 pub subpath: Option<String>,
486 pub primary_language: Option<String>,
487 pub description: Option<String>,
488 pub release_date: Option<String>,
489 #[serde(default)]
490 pub parties: Vec<Party>,
491 #[serde(default)]
492 pub keywords: Vec<String>,
493 pub homepage_url: Option<String>,
494 pub download_url: Option<String>,
495 pub size: Option<u64>,
496 pub sha1: Option<Sha1Digest>,
497 pub md5: Option<Md5Digest>,
498 pub sha256: Option<Sha256Digest>,
499 pub sha512: Option<Sha512Digest>,
500 pub bug_tracking_url: Option<String>,
501 pub code_view_url: Option<String>,
502 pub vcs_url: Option<String>,
503 pub copyright: Option<String>,
504 pub holder: Option<String>,
505 pub declared_license_expression: Option<String>,
506 pub declared_license_expression_spdx: Option<String>,
507 #[serde(default)]
508 pub license_detections: Vec<LicenseDetection>,
509 pub other_license_expression: Option<String>,
510 pub other_license_expression_spdx: Option<String>,
511 #[serde(default)]
512 pub other_license_detections: Vec<LicenseDetection>,
513 pub extracted_license_statement: Option<String>,
514 pub notice_text: Option<String>,
515 #[serde(default)]
516 pub source_packages: Vec<String>,
517 #[serde(default)]
518 pub file_references: Vec<FileReference>,
519 #[serde(default)]
520 pub is_private: bool,
521 #[serde(default)]
522 pub is_virtual: bool,
523 #[serde(default, with = "super::json_value_map")]
524 pub extra_data: Option<HashMap<String, serde_json::Value>>,
525 #[serde(default)]
526 pub dependencies: Vec<Dependency>,
527 pub repository_homepage_url: Option<String>,
528 pub repository_download_url: Option<String>,
529 pub api_data_url: Option<String>,
530 pub datasource_id: Option<DatasourceId>,
531 pub purl: Option<String>,
532}
533
534impl PackageData {
535 pub fn get_license_expression(&self) -> Option<String> {
538 if self.license_detections.is_empty() {
539 return None;
540 }
541
542 let expressions = self
543 .license_detections
544 .iter()
545 .map(|detection| detection.license_expression.clone());
546 combine_license_expressions(expressions)
547 }
548}
549
550#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
554pub struct LicenseDetection {
555 pub license_expression: String,
556 pub license_expression_spdx: String,
557 pub matches: Vec<Match>,
558 #[serde(default)]
559 pub detection_log: Vec<String>,
560 #[serde(default = "String::new")]
561 pub identifier: String,
562}
563
564#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
568pub struct Match {
569 pub license_expression: String,
570 pub license_expression_spdx: String,
571 pub from_file: Option<String>,
572 pub start_line: LineNumber,
573 pub end_line: LineNumber,
574 #[serde(default)]
575 pub matcher: MatcherKind,
576 pub score: MatchScore,
577 pub matched_length: Option<usize>,
578 pub match_coverage: Option<f64>,
579 pub rule_relevance: Option<u8>,
580 #[serde(default = "String::new")]
581 pub rule_identifier: String,
582 pub rule_url: Option<String>,
583 pub matched_text: Option<String>,
584 pub matched_text_diagnostics: Option<String>,
585 #[serde(default)]
586 pub referenced_filenames: Option<Vec<String>>,
587}
588
589#[derive(Serialize, Deserialize, Debug, Clone)]
590pub struct Copyright {
591 pub copyright: String,
592 #[serde(default)]
593 pub normalized_copyright: Option<String>,
594 pub start_line: LineNumber,
595 pub end_line: LineNumber,
596}
597
598impl Copyright {
599 pub fn normalized_text(&self) -> &str {
600 self.normalized_copyright
601 .as_deref()
602 .unwrap_or(self.copyright.as_str())
603 }
604}
605
606#[derive(Serialize, Deserialize, Debug, Clone)]
607pub struct Holder {
608 pub holder: String,
609 pub start_line: LineNumber,
610 pub end_line: LineNumber,
611}
612
613#[derive(Serialize, Deserialize, Debug, Clone)]
614pub struct Author {
615 pub author: String,
616 pub start_line: LineNumber,
617 pub end_line: LineNumber,
618}
619
620#[derive(Serialize, Deserialize, Debug, Clone)]
625pub struct Dependency {
626 pub purl: Option<String>,
627 pub extracted_requirement: Option<String>,
628 pub scope: Option<String>,
629 pub is_runtime: Option<bool>,
630 pub is_optional: Option<bool>,
631 pub is_pinned: Option<bool>,
632 pub is_direct: Option<bool>,
633 pub resolved_package: Option<Box<ResolvedPackage>>,
634 #[serde(default, with = "super::json_value_map")]
635 pub extra_data: Option<HashMap<String, serde_json::Value>>,
636}
637
638#[derive(Serialize, Deserialize, Debug, Clone)]
639pub struct ResolvedPackage {
640 pub package_type: PackageType,
641 pub namespace: String,
642 pub name: String,
643 pub version: String,
644 #[serde(default)]
645 pub qualifiers: Option<HashMap<String, String>>,
646 pub subpath: Option<String>,
647 pub primary_language: Option<String>,
648 pub description: Option<String>,
649 pub release_date: Option<String>,
650 #[serde(default)]
651 pub parties: Vec<Party>,
652 #[serde(default)]
653 pub keywords: Vec<String>,
654 pub homepage_url: Option<String>,
655 pub download_url: Option<String>,
656 pub size: Option<u64>,
657 pub sha1: Option<Sha1Digest>,
658 pub md5: Option<Md5Digest>,
659 pub sha256: Option<Sha256Digest>,
660 pub sha512: Option<Sha512Digest>,
661 pub bug_tracking_url: Option<String>,
662 pub code_view_url: Option<String>,
663 pub vcs_url: Option<String>,
664 pub copyright: Option<String>,
665 pub holder: Option<String>,
666 pub declared_license_expression: Option<String>,
667 pub declared_license_expression_spdx: Option<String>,
668 #[serde(default)]
669 pub license_detections: Vec<LicenseDetection>,
670 pub other_license_expression: Option<String>,
671 pub other_license_expression_spdx: Option<String>,
672 #[serde(default)]
673 pub other_license_detections: Vec<LicenseDetection>,
674 pub extracted_license_statement: Option<String>,
675 pub notice_text: Option<String>,
676 #[serde(default)]
677 pub source_packages: Vec<String>,
678 #[serde(default)]
679 pub file_references: Vec<FileReference>,
680 #[serde(default)]
681 pub is_private: bool,
682 #[serde(default)]
683 pub is_virtual: bool,
684 #[serde(default, with = "super::json_value_map")]
685 pub extra_data: Option<HashMap<String, serde_json::Value>>,
686 #[serde(default)]
687 pub dependencies: Vec<Dependency>,
688 pub repository_homepage_url: Option<String>,
689 pub repository_download_url: Option<String>,
690 pub api_data_url: Option<String>,
691 pub datasource_id: Option<DatasourceId>,
692 pub purl: Option<String>,
693}
694
695impl ResolvedPackage {
696 pub fn new(
697 package_type: PackageType,
698 namespace: String,
699 name: String,
700 version: String,
701 ) -> Self {
702 Self {
703 package_type,
704 namespace,
705 name,
706 version,
707 qualifiers: None,
708 subpath: None,
709 primary_language: None,
710 description: None,
711 release_date: None,
712 parties: vec![],
713 keywords: vec![],
714 homepage_url: None,
715 download_url: None,
716 size: None,
717 sha1: None,
718 md5: None,
719 sha256: None,
720 sha512: None,
721 bug_tracking_url: None,
722 code_view_url: None,
723 vcs_url: None,
724 copyright: None,
725 holder: None,
726 declared_license_expression: None,
727 declared_license_expression_spdx: None,
728 license_detections: vec![],
729 other_license_expression: None,
730 other_license_expression_spdx: None,
731 other_license_detections: vec![],
732 extracted_license_statement: None,
733 notice_text: None,
734 source_packages: vec![],
735 file_references: vec![],
736 is_private: false,
737 is_virtual: false,
738 extra_data: None,
739 dependencies: vec![],
740 repository_homepage_url: None,
741 repository_download_url: None,
742 api_data_url: None,
743 datasource_id: None,
744 purl: None,
745 }
746 }
747
748 pub fn from_package_data(package_data: &PackageData, fallback_type: PackageType) -> Self {
749 Self {
750 package_type: package_data.package_type.unwrap_or(fallback_type),
751 namespace: package_data.namespace.clone().unwrap_or_default(),
752 name: package_data.name.clone().unwrap_or_default(),
753 version: package_data.version.clone().unwrap_or_default(),
754 qualifiers: package_data.qualifiers.clone(),
755 subpath: package_data.subpath.clone(),
756 primary_language: package_data.primary_language.clone(),
757 description: package_data.description.clone(),
758 release_date: package_data.release_date.clone(),
759 parties: package_data.parties.clone(),
760 keywords: package_data.keywords.clone(),
761 homepage_url: package_data.homepage_url.clone(),
762 download_url: package_data.download_url.clone(),
763 size: package_data.size,
764 sha1: package_data.sha1,
765 md5: package_data.md5,
766 sha256: package_data.sha256,
767 sha512: package_data.sha512,
768 bug_tracking_url: package_data.bug_tracking_url.clone(),
769 code_view_url: package_data.code_view_url.clone(),
770 vcs_url: package_data.vcs_url.clone(),
771 copyright: package_data.copyright.clone(),
772 holder: package_data.holder.clone(),
773 declared_license_expression: package_data.declared_license_expression.clone(),
774 declared_license_expression_spdx: package_data.declared_license_expression_spdx.clone(),
775 license_detections: package_data.license_detections.clone(),
776 other_license_expression: package_data.other_license_expression.clone(),
777 other_license_expression_spdx: package_data.other_license_expression_spdx.clone(),
778 other_license_detections: package_data.other_license_detections.clone(),
779 extracted_license_statement: package_data.extracted_license_statement.clone(),
780 notice_text: package_data.notice_text.clone(),
781 source_packages: package_data.source_packages.clone(),
782 file_references: package_data.file_references.clone(),
783 is_private: package_data.is_private,
784 is_virtual: package_data.is_virtual,
785 extra_data: package_data.extra_data.clone(),
786 dependencies: package_data.dependencies.clone(),
787 repository_homepage_url: package_data.repository_homepage_url.clone(),
788 repository_download_url: package_data.repository_download_url.clone(),
789 api_data_url: package_data.api_data_url.clone(),
790 datasource_id: package_data.datasource_id,
791 purl: package_data.purl.clone(),
792 }
793 }
794}
795
796#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
797pub enum PartyType {
798 Person,
799 Organization,
800}
801
802impl fmt::Display for PartyType {
803 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
804 match self {
805 PartyType::Person => write!(f, "person"),
806 PartyType::Organization => write!(f, "organization"),
807 }
808 }
809}
810
811impl FromStr for PartyType {
812 type Err = String;
813 fn from_str(s: &str) -> Result<Self, Self::Err> {
814 match s {
815 "person" => Ok(PartyType::Person),
816 "organization" => Ok(PartyType::Organization),
817 other => Err(format!("unknown party type: {other}")),
818 }
819 }
820}
821
822#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
826pub struct Party {
827 pub r#type: Option<PartyType>,
828 pub role: Option<String>,
829 pub name: Option<String>,
830 pub email: Option<String>,
831 pub url: Option<String>,
832 pub organization: Option<String>,
833 pub organization_url: Option<String>,
834 pub timezone: Option<String>,
835}
836
837impl Party {
838 pub(crate) fn person(role: &str, name: Option<String>, email: Option<String>) -> Self {
839 Self {
840 r#type: Some(PartyType::Person),
841 role: Some(role.to_string()),
842 name,
843 email,
844 url: None,
845 organization: None,
846 organization_url: None,
847 timezone: None,
848 }
849 }
850}
851
852#[derive(Serialize, Deserialize, Debug, Clone)]
856pub struct FileReference {
857 pub path: String,
858 pub size: Option<u64>,
859 pub sha1: Option<Sha1Digest>,
860 pub md5: Option<Md5Digest>,
861 pub sha256: Option<Sha256Digest>,
862 pub sha512: Option<Sha512Digest>,
863 #[serde(with = "super::json_value_map")]
864 pub extra_data: Option<std::collections::HashMap<String, serde_json::Value>>,
865}
866
867impl FileReference {
868 pub(crate) fn from_path(path: String) -> Self {
869 Self {
870 path,
871 size: None,
872 sha1: None,
873 md5: None,
874 sha256: None,
875 sha512: None,
876 extra_data: None,
877 }
878 }
879}
880
881#[derive(Serialize, Deserialize, Debug, Clone)]
891pub struct Package {
892 pub package_type: Option<PackageType>,
893 pub namespace: Option<String>,
894 pub name: Option<String>,
895 pub version: Option<String>,
896 #[serde(default)]
897 pub qualifiers: Option<HashMap<String, String>>,
898 pub subpath: Option<String>,
899 pub primary_language: Option<String>,
900 pub description: Option<String>,
901 pub release_date: Option<String>,
902 #[serde(default)]
903 pub parties: Vec<Party>,
904 #[serde(default)]
905 pub keywords: Vec<String>,
906 pub homepage_url: Option<String>,
907 pub download_url: Option<String>,
908 pub size: Option<u64>,
909 pub sha1: Option<Sha1Digest>,
910 pub md5: Option<Md5Digest>,
911 pub sha256: Option<Sha256Digest>,
912 pub sha512: Option<Sha512Digest>,
913 pub bug_tracking_url: Option<String>,
914 pub code_view_url: Option<String>,
915 pub vcs_url: Option<String>,
916 pub copyright: Option<String>,
917 pub holder: Option<String>,
918 pub declared_license_expression: Option<String>,
919 pub declared_license_expression_spdx: Option<String>,
920 #[serde(default)]
921 pub license_detections: Vec<LicenseDetection>,
922 pub other_license_expression: Option<String>,
923 pub other_license_expression_spdx: Option<String>,
924 #[serde(default)]
925 pub other_license_detections: Vec<LicenseDetection>,
926 pub extracted_license_statement: Option<String>,
927 pub notice_text: Option<String>,
928 #[serde(default)]
929 pub source_packages: Vec<String>,
930 #[serde(default)]
931 pub is_private: bool,
932 #[serde(default)]
933 pub is_virtual: bool,
934 #[serde(default, with = "super::json_value_map")]
935 pub extra_data: Option<HashMap<String, serde_json::Value>>,
936 pub repository_homepage_url: Option<String>,
937 pub repository_download_url: Option<String>,
938 pub api_data_url: Option<String>,
939 pub purl: Option<String>,
940 pub package_uid: PackageUid,
942 pub datafile_paths: Vec<String>,
944 pub datasource_ids: Vec<DatasourceId>,
946}
947
948impl Package {
949 pub fn from_package_data(package_data: &PackageData, datafile_path: String) -> Self {
955 let mut package_data = package_data.clone();
956 enrich_package_data_license_provenance(&mut package_data, &datafile_path);
957
958 let mut package = Package {
959 package_type: package_data.package_type,
960 namespace: package_data.namespace.clone(),
961 name: package_data.name.clone(),
962 version: package_data.version.clone(),
963 qualifiers: package_data.qualifiers.clone(),
964 subpath: package_data.subpath.clone(),
965 primary_language: package_data.primary_language.clone(),
966 description: package_data.description.clone(),
967 release_date: package_data.release_date.clone(),
968 parties: package_data.parties.clone(),
969 keywords: package_data.keywords.clone(),
970 homepage_url: package_data.homepage_url.clone(),
971 download_url: package_data.download_url.clone(),
972 size: package_data.size,
973 sha1: package_data.sha1,
974 md5: package_data.md5,
975 sha256: package_data.sha256,
976 sha512: package_data.sha512,
977 bug_tracking_url: package_data.bug_tracking_url.clone(),
978 code_view_url: package_data.code_view_url.clone(),
979 vcs_url: package_data.vcs_url.clone(),
980 copyright: package_data.copyright.clone(),
981 holder: package_data.holder.clone(),
982 declared_license_expression: package_data.declared_license_expression.clone(),
983 declared_license_expression_spdx: package_data.declared_license_expression_spdx.clone(),
984 license_detections: package_data.license_detections.clone(),
985 other_license_expression: package_data.other_license_expression.clone(),
986 other_license_expression_spdx: package_data.other_license_expression_spdx.clone(),
987 other_license_detections: package_data.other_license_detections.clone(),
988 extracted_license_statement: package_data.extracted_license_statement.clone(),
989 notice_text: package_data.notice_text.clone(),
990 source_packages: package_data.source_packages.clone(),
991 is_private: package_data.is_private,
992 is_virtual: package_data.is_virtual,
993 extra_data: package_data.extra_data.clone(),
994 repository_homepage_url: package_data.repository_homepage_url.clone(),
995 repository_download_url: package_data.repository_download_url.clone(),
996 api_data_url: package_data.api_data_url.clone(),
997 purl: package_data.purl.clone(),
998 package_uid: PackageUid::empty(),
999 datafile_paths: vec![datafile_path],
1000 datasource_ids: if let Some(dsid) = package_data.datasource_id {
1001 vec![dsid]
1002 } else {
1003 vec![]
1004 },
1005 };
1006
1007 package.refresh_identity();
1008 if package.package_uid.is_empty() {
1009 package.package_uid = package.fallback_package_uid();
1010 }
1011
1012 package
1013 }
1014
1015 pub fn update(&mut self, package_data: &PackageData, datafile_path: String) {
1021 let mut package_data = package_data.clone();
1022 enrich_package_data_license_provenance(&mut package_data, &datafile_path);
1023
1024 if let Some(dsid) = package_data.datasource_id {
1025 self.datasource_ids.push(dsid);
1026 }
1027 self.datafile_paths.push(datafile_path);
1028
1029 macro_rules! fill_if_empty {
1030 ($field:ident) => {
1031 if self.$field.is_none() {
1032 self.$field = package_data.$field;
1033 }
1034 };
1035 }
1036
1037 fill_if_empty!(package_type);
1038 fill_if_empty!(name);
1039 fill_if_empty!(namespace);
1040 fill_if_empty!(version);
1041 fill_if_empty!(qualifiers);
1042 fill_if_empty!(subpath);
1043 fill_if_empty!(primary_language);
1044 fill_if_empty!(description);
1045 fill_if_empty!(release_date);
1046 fill_if_empty!(homepage_url);
1047 fill_if_empty!(download_url);
1048 fill_if_empty!(size);
1049 fill_if_empty!(sha1);
1050 fill_if_empty!(md5);
1051 fill_if_empty!(sha256);
1052 fill_if_empty!(sha512);
1053 fill_if_empty!(bug_tracking_url);
1054 fill_if_empty!(code_view_url);
1055 fill_if_empty!(vcs_url);
1056 fill_if_empty!(copyright);
1057 fill_if_empty!(holder);
1058 fill_if_empty!(declared_license_expression);
1059 fill_if_empty!(declared_license_expression_spdx);
1060 fill_if_empty!(other_license_expression);
1061 fill_if_empty!(other_license_expression_spdx);
1062 fill_if_empty!(extracted_license_statement);
1063 fill_if_empty!(notice_text);
1064 match (&mut self.extra_data, &package_data.extra_data) {
1065 (None, Some(extra_data)) => {
1066 self.extra_data = Some(extra_data.clone());
1067 }
1068 (Some(existing), Some(incoming)) => {
1069 for (key, value) in incoming {
1070 existing.entry(key.clone()).or_insert_with(|| value.clone());
1071 }
1072 }
1073 _ => {}
1074 }
1075 fill_if_empty!(repository_homepage_url);
1076 fill_if_empty!(repository_download_url);
1077 fill_if_empty!(api_data_url);
1078
1079 for party in &package_data.parties {
1080 if let Some(existing) = self.parties.iter_mut().find(|p| {
1081 p.role == party.role
1082 && ((p.name.is_some() && p.name == party.name)
1083 || (p.email.is_some() && p.email == party.email))
1084 }) {
1085 if existing.name.is_none() {
1086 existing.name = party.name.clone();
1087 }
1088 if existing.email.is_none() {
1089 existing.email = party.email.clone();
1090 }
1091 } else {
1092 self.parties.push(party.clone());
1093 }
1094 }
1095
1096 for keyword in &package_data.keywords {
1097 if !self.keywords.contains(keyword) {
1098 self.keywords.push(keyword.clone());
1099 }
1100 }
1101
1102 for detection in &package_data.license_detections {
1103 self.license_detections.push(detection.clone());
1104 }
1105
1106 for detection in &package_data.other_license_detections {
1107 self.other_license_detections.push(detection.clone());
1108 }
1109
1110 for source_pkg in &package_data.source_packages {
1111 if !self.source_packages.contains(source_pkg) {
1112 self.source_packages.push(source_pkg.clone());
1113 }
1114 }
1115
1116 self.refresh_identity();
1117 }
1118
1119 pub fn backfill_license_provenance(&mut self) {
1120 let Some(datafile_path) = self.datafile_paths.first().cloned() else {
1121 return;
1122 };
1123
1124 for detection in &mut self.license_detections {
1125 enrich_license_detection_provenance(detection, &datafile_path);
1126 }
1127 for detection in &mut self.other_license_detections {
1128 enrich_license_detection_provenance(detection, &datafile_path);
1129 }
1130 }
1131
1132 fn refresh_identity(&mut self) {
1133 let Some(next_purl) = self.build_current_purl() else {
1134 return;
1135 };
1136 let next_purl = crate::models::normalize_purl(&next_purl);
1140
1141 if self.purl.as_deref() != Some(next_purl.as_str()) || self.package_uid.is_empty() {
1142 self.package_uid = PackageUid::new(&next_purl);
1143 }
1144
1145 self.purl = Some(next_purl);
1146 }
1147
1148 fn fallback_package_uid(&self) -> PackageUid {
1149 let name = self
1150 .name
1151 .as_deref()
1152 .map(str::trim)
1153 .filter(|value| !value.is_empty())
1154 .unwrap_or("unknown");
1155 let version = self
1156 .version
1157 .as_deref()
1158 .map(str::trim)
1159 .filter(|value| !value.is_empty())
1160 .unwrap_or("unknown");
1161 let datasource = self
1162 .datasource_ids
1163 .first()
1164 .map(DatasourceId::as_str)
1165 .unwrap_or("unknown");
1166
1167 PackageUid::new_opaque(&format!("generated-package:{datasource}/{name}@{version}"))
1168 }
1169
1170 fn build_current_purl(&self) -> Option<String> {
1171 if let Some(existing_purl) = self.purl.as_deref() {
1172 let mut purl = PackageUrl::from_str(existing_purl).ok()?;
1173
1174 if let Some(version) = self
1175 .version
1176 .as_deref()
1177 .filter(|value| !value.trim().is_empty())
1178 {
1179 purl.with_version(version).ok()?;
1180 } else {
1181 purl.without_version();
1182 }
1183
1184 return Some(purl.to_string());
1185 }
1186
1187 if let (Some(package_type), Some(name)) = (
1188 self.package_type.as_ref(),
1189 self.name
1190 .as_deref()
1191 .filter(|value| !value.trim().is_empty()),
1192 ) {
1193 let purl_type = match package_type {
1194 PackageType::Deno => "generic",
1195 _ => package_type.as_str(),
1196 };
1197
1198 let has_namespace = self
1199 .namespace
1200 .as_deref()
1201 .is_some_and(|value| !value.trim().is_empty());
1202
1203 if purl_type == "swift" && !has_namespace {
1206 return None;
1207 }
1208
1209 let mut purl = PackageUrl::new(purl_type, name).ok()?;
1210
1211 if let Some(namespace) = self
1212 .namespace
1213 .as_deref()
1214 .filter(|value| !value.trim().is_empty())
1215 {
1216 purl.with_namespace(namespace).ok()?;
1217 }
1218
1219 if let Some(version) = self
1220 .version
1221 .as_deref()
1222 .filter(|value| !value.trim().is_empty())
1223 {
1224 purl.with_version(version).ok()?;
1225 }
1226
1227 if let Some(qualifiers) = &self.qualifiers {
1228 for (key, value) in qualifiers {
1229 purl.add_qualifier(key.as_str(), value.as_str()).ok()?;
1230 }
1231 }
1232
1233 if let Some(subpath) = self
1234 .subpath
1235 .as_deref()
1236 .filter(|value| !value.trim().is_empty())
1237 {
1238 purl.with_subpath(subpath).ok()?;
1239 }
1240
1241 return Some(purl.to_string());
1242 }
1243 None
1244 }
1245}
1246
1247#[cfg(test)]
1248mod tests {
1249 use super::*;
1250
1251 #[test]
1252 fn file_info_new_backfills_package_detection_provenance() {
1253 let package_data = PackageData {
1254 package_type: Some(PackageType::Npm),
1255 license_detections: vec![LicenseDetection {
1256 license_expression: "mit".to_string(),
1257 license_expression_spdx: "MIT".to_string(),
1258 matches: vec![Match {
1259 license_expression: "mit".to_string(),
1260 license_expression_spdx: "MIT".to_string(),
1261 from_file: None,
1262 start_line: LineNumber::ONE,
1263 end_line: LineNumber::ONE,
1264 matcher: MatcherKind::Declared,
1265 score: MatchScore::MAX,
1266 matched_length: Some(1),
1267 match_coverage: Some(100.0),
1268 rule_relevance: Some(100),
1269 rule_identifier: String::new(),
1270 rule_url: None,
1271 matched_text: Some("MIT".to_string()),
1272 referenced_filenames: None,
1273 matched_text_diagnostics: None,
1274 }],
1275 detection_log: vec![],
1276 identifier: String::new(),
1277 }],
1278 ..PackageData::default()
1279 };
1280
1281 let file_info = FileInfo::new(
1282 "package.json".to_string(),
1283 "package".to_string(),
1284 ".json".to_string(),
1285 "project/package.json".to_string(),
1286 FileType::File,
1287 None,
1288 None,
1289 1,
1290 None,
1291 None,
1292 None,
1293 None,
1294 None,
1295 vec![package_data],
1296 None,
1297 vec![],
1298 vec![],
1299 vec![],
1300 vec![],
1301 vec![],
1302 vec![],
1303 vec![],
1304 vec![],
1305 vec![],
1306 );
1307
1308 assert_eq!(file_info.license_detections.len(), 1);
1309 assert_eq!(
1310 file_info.license_detections[0].matches[0]
1311 .from_file
1312 .as_deref(),
1313 Some("project/package.json")
1314 );
1315 assert!(!file_info.license_detections[0].identifier.is_empty());
1316 assert_eq!(
1317 file_info.package_data[0].license_detections[0].matches[0]
1318 .from_file
1319 .as_deref(),
1320 Some("project/package.json")
1321 );
1322 assert_eq!(
1323 file_info.package_data[0].license_detections[0].matches[0].rule_identifier,
1324 "parser-declared-license"
1325 );
1326 assert!(
1327 !file_info.package_data[0].license_detections[0]
1328 .identifier
1329 .is_empty()
1330 );
1331 }
1332
1333 #[test]
1334 fn package_from_package_data_backfills_detection_provenance() {
1335 let package_data = PackageData {
1336 package_type: Some(PackageType::Npm),
1337 license_detections: vec![LicenseDetection {
1338 license_expression: "mit".to_string(),
1339 license_expression_spdx: "MIT".to_string(),
1340 matches: vec![Match {
1341 license_expression: "mit".to_string(),
1342 license_expression_spdx: "MIT".to_string(),
1343 from_file: None,
1344 start_line: LineNumber::ONE,
1345 end_line: LineNumber::ONE,
1346 matcher: MatcherKind::Declared,
1347 score: MatchScore::MAX,
1348 matched_length: Some(1),
1349 match_coverage: Some(100.0),
1350 rule_relevance: Some(100),
1351 rule_identifier: String::new(),
1352 rule_url: None,
1353 matched_text: Some("MIT".to_string()),
1354 referenced_filenames: None,
1355 matched_text_diagnostics: None,
1356 }],
1357 detection_log: vec![],
1358 identifier: String::new(),
1359 }],
1360 ..PackageData::default()
1361 };
1362
1363 let package = Package::from_package_data(&package_data, "project/package.json".to_string());
1364
1365 assert_eq!(
1366 package.license_detections[0].matches[0]
1367 .from_file
1368 .as_deref(),
1369 Some("project/package.json")
1370 );
1371 assert_eq!(
1372 package.license_detections[0].matches[0].rule_identifier,
1373 "parser-declared-license"
1374 );
1375 assert!(!package.license_detections[0].identifier.is_empty());
1376 }
1377
1378 #[test]
1379 fn package_from_package_data_preserves_existing_purl_qualifiers() {
1380 let package_data = PackageData {
1381 package_type: Some(PackageType::Apk),
1382 namespace: Some("alpine".to_string()),
1383 name: Some("busybox".to_string()),
1384 version: Some("1.35.0-r17".to_string()),
1385 purl: Some("pkg:alpine/busybox@1.35.0-r17?arch=x86_64".to_string()),
1386 ..PackageData::default()
1387 };
1388
1389 let package = Package::from_package_data(&package_data, "lib/apk/db/installed".to_string());
1390
1391 assert_eq!(
1392 package.purl.as_deref(),
1393 Some("pkg:alpine/busybox@1.35.0-r17?arch=x86_64")
1394 );
1395 assert!(
1396 package
1397 .package_uid
1398 .starts_with("pkg:alpine/busybox@1.35.0-r17?arch=x86_64&uuid=")
1399 );
1400 }
1401}
1402
1403#[derive(Serialize, Deserialize, Debug, Clone)]
1408pub struct TopLevelDependency {
1409 pub purl: Option<String>,
1410 pub extracted_requirement: Option<String>,
1411 pub scope: Option<String>,
1412 pub is_runtime: Option<bool>,
1413 pub is_optional: Option<bool>,
1414 pub is_pinned: Option<bool>,
1415 pub is_direct: Option<bool>,
1416 pub resolved_package: Option<Box<ResolvedPackage>>,
1417 #[serde(default, with = "super::json_value_map")]
1418 pub extra_data: Option<HashMap<String, serde_json::Value>>,
1419 pub dependency_uid: DependencyUid,
1421 pub for_package_uid: Option<PackageUid>,
1423 pub datafile_path: String,
1425 pub datasource_id: DatasourceId,
1427 pub namespace: Option<String>,
1429}
1430
1431impl TopLevelDependency {
1432 pub fn from_dependency(
1434 dep: &Dependency,
1435 datafile_path: String,
1436 datasource_id: DatasourceId,
1437 for_package_uid: Option<PackageUid>,
1438 ) -> Self {
1439 let dependency_uid = dep
1440 .purl
1441 .as_ref()
1442 .map(|p| DependencyUid::new(p))
1443 .unwrap_or_else(DependencyUid::empty);
1444
1445 TopLevelDependency {
1446 purl: dep.purl.clone(),
1447 extracted_requirement: dep.extracted_requirement.clone(),
1448 scope: dep.scope.clone(),
1449 is_runtime: dep.is_runtime,
1450 is_optional: dep.is_optional,
1451 is_pinned: dep.is_pinned,
1452 is_direct: dep.is_direct,
1453 resolved_package: dep.resolved_package.clone(),
1454 extra_data: dep.extra_data.clone(),
1455 dependency_uid,
1456 for_package_uid,
1457 datafile_path,
1458 datasource_id,
1459 namespace: None,
1460 }
1461 }
1462}
1463
1464#[derive(Serialize, Deserialize, Debug, Clone)]
1465pub struct OutputEmail {
1466 pub email: String,
1467 pub start_line: LineNumber,
1468 pub end_line: LineNumber,
1469}
1470
1471#[derive(Serialize, Deserialize, Debug, Clone)]
1472pub struct OutputURL {
1473 pub url: String,
1474 pub start_line: LineNumber,
1475 pub end_line: LineNumber,
1476}
1477
1478#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
1479pub struct LicensePolicyEntry {
1480 pub license_key: String,
1481 pub label: String,
1482 pub color_code: String,
1483 pub icon: String,
1484}
1485
1486#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
1487pub enum FileType {
1488 File,
1489 Directory,
1490}