1use derive_builder::Builder;
8use packageurl::PackageUrl;
9use serde::{Deserialize, Serialize};
10use sha1::{Digest, Sha1};
11use std::collections::HashMap;
12use std::fmt;
13use std::str::FromStr;
14
15use super::DatasourceId;
16use super::DependencyUid;
17use super::DiagnosticSeverity;
18use super::GitSha1;
19use super::LineNumber;
20use super::MatchScore;
21use super::Md5Digest;
22use super::PackageType;
23use super::PackageUid;
24use super::ScanDiagnostic;
25use super::Sha1Digest;
26use super::Sha256Digest;
27use super::Sha512Digest;
28
29use crate::license_detection::MatcherKind;
30use crate::license_detection::tokenize::tokenize_without_stopwords;
31use crate::models::output::Tallies;
32use crate::utils::spdx::combine_license_expressions;
33
34#[derive(Debug, Builder, Serialize, Deserialize, Clone)]
35#[builder(build_fn(skip))]
36pub struct FileInfo {
38 pub name: String,
39 pub base_name: String,
40 pub extension: String,
41 pub path: String,
42 pub file_type: FileType,
43 #[builder(default)]
44 #[serde(default)]
45 pub mime_type: Option<String>,
46 #[builder(default)]
47 #[serde(default)]
48 pub file_type_label: Option<String>,
49 pub size: u64,
50 #[builder(default)]
51 #[serde(default)]
52 pub date: Option<String>,
53 #[builder(default)]
54 #[serde(default)]
55 pub sha1: Option<Sha1Digest>,
56 #[builder(default)]
57 #[serde(default)]
58 pub md5: Option<Md5Digest>,
59 #[builder(default)]
60 #[serde(default)]
61 pub sha256: Option<Sha256Digest>,
62 #[builder(default)]
63 #[serde(default)]
64 pub sha1_git: Option<GitSha1>,
65 #[builder(default)]
66 #[serde(default)]
67 pub programming_language: Option<String>,
68 #[builder(default)]
69 #[serde(default)]
70 pub package_data: Vec<PackageData>,
71 #[builder(default)]
72 #[serde(default)]
73 pub detected_license_expression: Option<String>,
74 #[builder(default)]
75 #[serde(default)]
76 pub license_detections: Vec<LicenseDetection>,
77 #[builder(default)]
78 #[serde(default)]
79 pub license_clues: Vec<Match>,
80 #[builder(default)]
81 #[serde(default)]
82 pub percentage_of_license_text: Option<f64>,
83 #[builder(default)]
84 #[serde(default)]
85 pub copyrights: Vec<Copyright>,
86 #[builder(default)]
87 #[serde(default)]
88 pub holders: Vec<Holder>,
89 #[builder(default)]
90 #[serde(default)]
91 pub authors: Vec<Author>,
92 #[builder(default)]
93 #[serde(default)]
94 pub emails: Vec<OutputEmail>,
95 #[builder(default)]
96 #[serde(default)]
97 pub urls: Vec<OutputURL>,
98 #[builder(default)]
99 #[serde(default)]
100 pub for_packages: Vec<PackageUid>,
101 #[builder(default)]
102 #[serde(default)]
103 pub scan_diagnostics: Vec<ScanDiagnostic>,
104 #[builder(default)]
105 #[serde(default)]
106 pub license_policy: Option<Vec<LicensePolicyEntry>>,
107 #[builder(default)]
108 #[serde(default)]
109 pub is_generated: Option<bool>,
110 #[builder(default)]
111 #[serde(default)]
112 pub is_binary: Option<bool>,
113 #[builder(default)]
114 #[serde(default)]
115 pub is_text: Option<bool>,
116 #[builder(default)]
117 #[serde(default)]
118 pub is_archive: Option<bool>,
119 #[builder(default)]
120 #[serde(default)]
121 pub is_media: Option<bool>,
122 #[builder(default)]
123 #[serde(default)]
124 pub is_source: Option<bool>,
125 #[builder(default)]
126 #[serde(default)]
127 pub is_script: Option<bool>,
128 #[builder(default)]
129 #[serde(default)]
130 pub files_count: Option<usize>,
131 #[builder(default)]
132 #[serde(default)]
133 pub dirs_count: Option<usize>,
134 #[builder(default)]
135 #[serde(default)]
136 pub size_count: Option<u64>,
137 #[builder(default)]
138 #[serde(default)]
139 pub source_count: Option<usize>,
140 #[builder(default)]
141 #[serde(default)]
142 pub is_legal: bool,
143 #[builder(default)]
144 #[serde(default)]
145 pub is_manifest: bool,
146 #[builder(default)]
147 #[serde(default)]
148 pub is_readme: bool,
149 #[builder(default)]
150 #[serde(default)]
151 pub is_top_level: bool,
152 #[builder(default)]
153 #[serde(default)]
154 pub is_key_file: bool,
155 #[builder(default)]
156 #[serde(default)]
157 pub is_referenced: bool,
158 #[builder(default)]
159 #[serde(default)]
160 pub is_community: bool,
161 #[builder(default)]
162 #[serde(default)]
163 pub facets: Vec<String>,
164 #[builder(default)]
165 #[serde(default)]
166 pub tallies: Option<Tallies>,
167}
168
169impl FileInfoBuilder {
170 pub fn build(&self) -> Result<FileInfo, String> {
172 let mut file_info = FileInfo::new(
173 self.name.clone().ok_or("Missing field: name")?,
174 self.base_name.clone().ok_or("Missing field: base_name")?,
175 self.extension.clone().ok_or("Missing field: extension")?,
176 self.path.clone().ok_or("Missing field: path")?,
177 self.file_type.clone().ok_or("Missing field: file_type")?,
178 self.mime_type.clone().flatten(),
179 self.file_type_label.clone().flatten(),
180 self.size.ok_or("Missing field: size")?,
181 self.date.clone().flatten(),
182 self.sha1.flatten(),
183 self.md5.flatten(),
184 self.sha256.flatten(),
185 self.programming_language.clone().flatten(),
186 self.package_data.clone().unwrap_or_default(),
187 self.detected_license_expression.clone().flatten(),
188 self.license_detections.clone().unwrap_or_default(),
189 self.license_clues.clone().unwrap_or_default(),
190 self.copyrights.clone().unwrap_or_default(),
191 self.holders.clone().unwrap_or_default(),
192 self.authors.clone().unwrap_or_default(),
193 self.emails.clone().unwrap_or_default(),
194 self.urls.clone().unwrap_or_default(),
195 self.for_packages.clone().unwrap_or_default(),
196 self.scan_diagnostics.clone().unwrap_or_default(),
197 );
198 file_info.license_policy = self.license_policy.clone().flatten();
199 file_info.sha1_git = self.sha1_git.flatten();
200 file_info.is_binary = self.is_binary.flatten();
201 file_info.is_text = self.is_text.flatten();
202 file_info.is_archive = self.is_archive.flatten();
203 file_info.is_media = self.is_media.flatten();
204 file_info.is_script = self.is_script.flatten();
205 file_info.files_count = self.files_count.flatten();
206 file_info.dirs_count = self.dirs_count.flatten();
207 file_info.size_count = self.size_count.flatten();
208 file_info.is_referenced = self.is_referenced.unwrap_or_default();
209 Ok(file_info)
210 }
211}
212
213impl FileInfo {
214 #[allow(clippy::too_many_arguments)]
217 pub fn new(
218 name: String,
219 base_name: String,
220 extension: String,
221 path: String,
222 file_type: FileType,
223 mime_type: Option<String>,
224 file_type_label: Option<String>,
225 size: u64,
226 date: Option<String>,
227 sha1: Option<Sha1Digest>,
228 md5: Option<Md5Digest>,
229 sha256: Option<Sha256Digest>,
230 programming_language: Option<String>,
231 package_data: Vec<PackageData>,
232 mut detected_license_expression: Option<String>,
233 mut license_detections: Vec<LicenseDetection>,
234 license_clues: Vec<Match>,
235 copyrights: Vec<Copyright>,
236 holders: Vec<Holder>,
237 authors: Vec<Author>,
238 emails: Vec<OutputEmail>,
239 urls: Vec<OutputURL>,
240 for_packages: Vec<PackageUid>,
241 scan_diagnostics: Vec<ScanDiagnostic>,
242 ) -> Self {
243 let mut package_data = package_data;
244 for package in &mut package_data {
245 enrich_package_data_license_provenance(package, &path);
246 }
247
248 detected_license_expression = detected_license_expression.or_else(|| {
250 let expressions = package_data
251 .iter()
252 .filter_map(|pkg| pkg.get_license_expression());
253 combine_license_expressions(expressions)
254 });
255
256 if license_detections.is_empty() {
258 for pkg in &package_data {
259 license_detections.extend(pkg.license_detections.clone());
260 }
261 }
262
263 if detected_license_expression.is_none() && !license_detections.is_empty() {
265 let expressions = license_detections
266 .iter()
267 .map(|detection| detection.license_expression.clone());
268 let expressions: Vec<String> = expressions.collect();
269 detected_license_expression = crate::utils::spdx::select_primary_license_expression(
270 expressions.clone(),
271 )
272 .or_else(|| {
273 crate::utils::spdx::combine_license_expressions_preserving_structure(expressions)
274 });
275 }
276
277 let mut file_info = FileInfo {
278 name,
279 base_name,
280 extension,
281 path,
282 file_type,
283 mime_type,
284 file_type_label,
285 size,
286 date,
287 sha1,
288 md5,
289 sha256,
290 sha1_git: None,
291 programming_language,
292 package_data,
293 detected_license_expression,
294 license_detections,
295 license_clues,
296 percentage_of_license_text: None,
297 copyrights,
298 holders,
299 authors,
300 emails,
301 urls,
302 for_packages,
303 scan_diagnostics,
304 license_policy: None,
305 is_generated: None,
306 is_binary: None,
307 is_text: None,
308 is_archive: None,
309 is_media: None,
310 is_source: None,
311 is_script: None,
312 files_count: None,
313 dirs_count: None,
314 size_count: None,
315 source_count: None,
316 is_legal: false,
317 is_manifest: false,
318 is_readme: false,
319 is_top_level: false,
320 is_key_file: false,
321 is_referenced: false,
322 is_community: false,
323 facets: vec![],
324 tallies: None,
325 };
326
327 file_info.backfill_license_provenance();
328 file_info
329 }
330
331 pub fn backfill_license_provenance(&mut self) {
332 for detection in &mut self.license_detections {
333 enrich_license_detection_provenance(detection, &self.path);
334 }
335
336 for package in &mut self.package_data {
337 enrich_package_data_license_provenance(package, &self.path);
338 }
339 }
340}
341
342impl FileInfo {
343 pub fn warning_diagnostics(&self) -> impl Iterator<Item = &ScanDiagnostic> {
344 self.scan_diagnostics
345 .iter()
346 .filter(|diagnostic| diagnostic.severity == DiagnosticSeverity::Warning)
347 }
348
349 pub fn error_diagnostics(&self) -> impl Iterator<Item = &ScanDiagnostic> {
350 self.scan_diagnostics.iter().filter(|diagnostic| {
351 diagnostic.severity == DiagnosticSeverity::Error
352 || diagnostic.severity == DiagnosticSeverity::Timeout
353 })
354 }
355}
356
357fn enrich_package_data_license_provenance(package_data: &mut PackageData, path: &str) {
358 for detection in &mut package_data.license_detections {
359 enrich_license_detection_provenance(detection, path);
360 }
361 for detection in &mut package_data.other_license_detections {
362 enrich_license_detection_provenance(detection, path);
363 }
364}
365
366pub(crate) fn enrich_license_detection_provenance(detection: &mut LicenseDetection, path: &str) {
367 for detection_match in &mut detection.matches {
368 if detection_match.from_file.is_none() {
369 detection_match.from_file = Some(path.to_string());
370 }
371
372 if detection_match.rule_identifier.is_empty() {
373 detection_match.rule_identifier = detection_match.matcher.to_string();
374 }
375 }
376
377 if detection.identifier.is_empty() {
378 detection.identifier = compute_public_detection_identifier(detection);
379 }
380}
381
382fn compute_public_detection_identifier(detection: &LicenseDetection) -> String {
383 let expression = python_safe_name(&detection.license_expression);
384 let mut hasher = Sha1::new();
385 hasher.update(format_public_detection_content(detection).as_bytes());
386 let hex_str = hex::encode(hasher.finalize());
387 let uuid_hex = &hex_str[..32];
388 let content_uuid = uuid::Uuid::parse_str(uuid_hex)
389 .map(|uuid| uuid.to_string())
390 .unwrap_or_else(|_| uuid_hex.to_string());
391
392 format!("{}-{}", expression, content_uuid)
393}
394
395fn format_public_detection_content(detection: &LicenseDetection) -> String {
396 let mut result = String::from("(");
397
398 for (index, detection_match) in detection.matches.iter().enumerate() {
399 if index > 0 {
400 result.push_str(", ");
401 }
402 result.push_str(&format!(
403 "({}, {}, {})",
404 python_str_repr(if detection_match.rule_identifier.is_empty() {
405 detection_match.matcher.as_str()
406 } else {
407 detection_match.rule_identifier.as_str()
408 }),
409 detection_match.score.value() as f32,
410 python_token_tuple_repr(&tokenize_without_stopwords(
411 detection_match.matched_text.as_deref().unwrap_or_default(),
412 )),
413 ));
414 }
415
416 if detection.matches.len() == 1 {
417 result.push(',');
418 }
419 result.push(')');
420 result
421}
422
423fn python_safe_name(value: &str) -> String {
424 let mut result = String::new();
425 let mut prev_underscore = false;
426
427 for character in value.chars() {
428 if character.is_alphanumeric() {
429 result.push(character);
430 prev_underscore = false;
431 } else if !prev_underscore {
432 result.push('_');
433 prev_underscore = true;
434 }
435 }
436
437 let trimmed = result.trim_matches('_');
438 if trimmed.is_empty() {
439 String::new()
440 } else {
441 trimmed.to_string()
442 }
443}
444
445fn python_str_repr(value: &str) -> String {
446 if value.contains('\'') && !value.contains('"') {
447 format!("\"{}\"", value.replace('\\', "\\\\").replace('"', "\\\""))
448 } else {
449 format!("'{}'", value.replace('\\', "\\\\").replace('\'', "\\\'"))
450 }
451}
452
453fn python_token_tuple_repr(tokens: &[String]) -> String {
454 if tokens.is_empty() {
455 return String::from("()");
456 }
457
458 let mut result = String::from("(");
459 for (index, token) in tokens.iter().enumerate() {
460 if index > 0 {
461 result.push_str(", ");
462 }
463 result.push_str(&python_str_repr(token));
464 }
465
466 if tokens.len() == 1 {
467 result.push(',');
468 }
469 result.push(')');
470 result
471}
472
473#[derive(Serialize, Deserialize, Debug, Clone, Default)]
479pub struct PackageData {
480 pub package_type: Option<PackageType>,
481 pub namespace: Option<String>,
482 pub name: Option<String>,
483 pub version: Option<String>,
484 #[serde(default)]
485 pub qualifiers: Option<HashMap<String, String>>,
486 pub subpath: Option<String>,
487 pub primary_language: Option<String>,
488 pub description: Option<String>,
489 pub release_date: Option<String>,
490 #[serde(default)]
491 pub parties: Vec<Party>,
492 #[serde(default)]
493 pub keywords: Vec<String>,
494 pub homepage_url: Option<String>,
495 pub download_url: Option<String>,
496 pub size: Option<u64>,
497 pub sha1: Option<Sha1Digest>,
498 pub md5: Option<Md5Digest>,
499 pub sha256: Option<Sha256Digest>,
500 pub sha512: Option<Sha512Digest>,
501 pub bug_tracking_url: Option<String>,
502 pub code_view_url: Option<String>,
503 pub vcs_url: Option<String>,
504 pub copyright: Option<String>,
505 pub holder: Option<String>,
506 pub declared_license_expression: Option<String>,
507 pub declared_license_expression_spdx: Option<String>,
508 #[serde(default)]
509 pub license_detections: Vec<LicenseDetection>,
510 pub other_license_expression: Option<String>,
511 pub other_license_expression_spdx: Option<String>,
512 #[serde(default)]
513 pub other_license_detections: Vec<LicenseDetection>,
514 pub extracted_license_statement: Option<String>,
515 pub notice_text: Option<String>,
516 #[serde(default)]
517 pub source_packages: Vec<String>,
518 #[serde(default)]
519 pub file_references: Vec<FileReference>,
520 #[serde(default)]
521 pub is_private: bool,
522 #[serde(default)]
523 pub is_virtual: bool,
524 #[serde(default, with = "super::json_value_map")]
525 pub extra_data: Option<HashMap<String, serde_json::Value>>,
526 #[serde(default)]
527 pub dependencies: Vec<Dependency>,
528 pub repository_homepage_url: Option<String>,
529 pub repository_download_url: Option<String>,
530 pub api_data_url: Option<String>,
531 pub datasource_id: Option<DatasourceId>,
532 pub purl: Option<String>,
533}
534
535impl PackageData {
536 pub fn get_license_expression(&self) -> Option<String> {
539 if self.license_detections.is_empty() {
540 return None;
541 }
542
543 let expressions = self
544 .license_detections
545 .iter()
546 .map(|detection| detection.license_expression.clone());
547 combine_license_expressions(expressions)
548 }
549}
550
551#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
555pub struct LicenseDetection {
556 pub license_expression: String,
557 pub license_expression_spdx: String,
558 pub matches: Vec<Match>,
559 #[serde(default)]
560 pub detection_log: Vec<String>,
561 #[serde(default = "String::new")]
562 pub identifier: String,
563}
564
565#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
569pub struct Match {
570 pub license_expression: String,
571 pub license_expression_spdx: String,
572 pub from_file: Option<String>,
573 pub start_line: LineNumber,
574 pub end_line: LineNumber,
575 #[serde(default)]
576 pub matcher: MatcherKind,
577 pub score: MatchScore,
578 pub matched_length: Option<usize>,
579 pub match_coverage: Option<f64>,
580 pub rule_relevance: Option<u8>,
581 #[serde(default = "String::new")]
582 pub rule_identifier: String,
583 pub rule_url: Option<String>,
584 pub matched_text: Option<String>,
585 pub matched_text_diagnostics: Option<String>,
586 #[serde(default)]
587 pub referenced_filenames: Option<Vec<String>>,
588}
589
590#[derive(Serialize, Deserialize, Debug, Clone)]
591pub struct Copyright {
592 pub copyright: String,
593 #[serde(default)]
594 pub normalized_copyright: Option<String>,
595 pub start_line: LineNumber,
596 pub end_line: LineNumber,
597}
598
599impl Copyright {
600 pub fn normalized_text(&self) -> &str {
601 self.normalized_copyright
602 .as_deref()
603 .unwrap_or(self.copyright.as_str())
604 }
605}
606
607#[derive(Serialize, Deserialize, Debug, Clone)]
608pub struct Holder {
609 pub holder: String,
610 pub start_line: LineNumber,
611 pub end_line: LineNumber,
612}
613
614#[derive(Serialize, Deserialize, Debug, Clone)]
615pub struct Author {
616 pub author: String,
617 pub start_line: LineNumber,
618 pub end_line: LineNumber,
619}
620
621#[derive(Serialize, Deserialize, Debug, Clone)]
626pub struct Dependency {
627 pub purl: Option<String>,
628 pub extracted_requirement: Option<String>,
629 pub scope: Option<String>,
630 pub is_runtime: Option<bool>,
631 pub is_optional: Option<bool>,
632 pub is_pinned: Option<bool>,
633 pub is_direct: Option<bool>,
634 pub resolved_package: Option<Box<ResolvedPackage>>,
635 #[serde(default, with = "super::json_value_map")]
636 pub extra_data: Option<HashMap<String, serde_json::Value>>,
637}
638
639#[derive(Serialize, Deserialize, Debug, Clone)]
640pub struct ResolvedPackage {
641 pub package_type: PackageType,
642 pub namespace: String,
643 pub name: String,
644 pub version: String,
645 #[serde(default)]
646 pub qualifiers: Option<HashMap<String, String>>,
647 pub subpath: Option<String>,
648 pub primary_language: Option<String>,
649 pub description: Option<String>,
650 pub release_date: Option<String>,
651 #[serde(default)]
652 pub parties: Vec<Party>,
653 #[serde(default)]
654 pub keywords: Vec<String>,
655 pub homepage_url: Option<String>,
656 pub download_url: Option<String>,
657 pub size: Option<u64>,
658 pub sha1: Option<Sha1Digest>,
659 pub md5: Option<Md5Digest>,
660 pub sha256: Option<Sha256Digest>,
661 pub sha512: Option<Sha512Digest>,
662 pub bug_tracking_url: Option<String>,
663 pub code_view_url: Option<String>,
664 pub vcs_url: Option<String>,
665 pub copyright: Option<String>,
666 pub holder: Option<String>,
667 pub declared_license_expression: Option<String>,
668 pub declared_license_expression_spdx: Option<String>,
669 #[serde(default)]
670 pub license_detections: Vec<LicenseDetection>,
671 pub other_license_expression: Option<String>,
672 pub other_license_expression_spdx: Option<String>,
673 #[serde(default)]
674 pub other_license_detections: Vec<LicenseDetection>,
675 pub extracted_license_statement: Option<String>,
676 pub notice_text: Option<String>,
677 #[serde(default)]
678 pub source_packages: Vec<String>,
679 #[serde(default)]
680 pub file_references: Vec<FileReference>,
681 #[serde(default)]
682 pub is_private: bool,
683 #[serde(default)]
684 pub is_virtual: bool,
685 #[serde(default, with = "super::json_value_map")]
686 pub extra_data: Option<HashMap<String, serde_json::Value>>,
687 #[serde(default)]
688 pub dependencies: Vec<Dependency>,
689 pub repository_homepage_url: Option<String>,
690 pub repository_download_url: Option<String>,
691 pub api_data_url: Option<String>,
692 pub datasource_id: Option<DatasourceId>,
693 pub purl: Option<String>,
694}
695
696impl ResolvedPackage {
697 pub fn new(
698 package_type: PackageType,
699 namespace: String,
700 name: String,
701 version: String,
702 ) -> Self {
703 Self {
704 package_type,
705 namespace,
706 name,
707 version,
708 qualifiers: None,
709 subpath: None,
710 primary_language: None,
711 description: None,
712 release_date: None,
713 parties: vec![],
714 keywords: vec![],
715 homepage_url: None,
716 download_url: None,
717 size: None,
718 sha1: None,
719 md5: None,
720 sha256: None,
721 sha512: None,
722 bug_tracking_url: None,
723 code_view_url: None,
724 vcs_url: None,
725 copyright: None,
726 holder: None,
727 declared_license_expression: None,
728 declared_license_expression_spdx: None,
729 license_detections: vec![],
730 other_license_expression: None,
731 other_license_expression_spdx: None,
732 other_license_detections: vec![],
733 extracted_license_statement: None,
734 notice_text: None,
735 source_packages: vec![],
736 file_references: vec![],
737 is_private: false,
738 is_virtual: false,
739 extra_data: None,
740 dependencies: vec![],
741 repository_homepage_url: None,
742 repository_download_url: None,
743 api_data_url: None,
744 datasource_id: None,
745 purl: None,
746 }
747 }
748
749 pub fn from_package_data(package_data: &PackageData, fallback_type: PackageType) -> Self {
750 Self {
751 package_type: package_data.package_type.unwrap_or(fallback_type),
752 namespace: package_data.namespace.clone().unwrap_or_default(),
753 name: package_data.name.clone().unwrap_or_default(),
754 version: package_data.version.clone().unwrap_or_default(),
755 qualifiers: package_data.qualifiers.clone(),
756 subpath: package_data.subpath.clone(),
757 primary_language: package_data.primary_language.clone(),
758 description: package_data.description.clone(),
759 release_date: package_data.release_date.clone(),
760 parties: package_data.parties.clone(),
761 keywords: package_data.keywords.clone(),
762 homepage_url: package_data.homepage_url.clone(),
763 download_url: package_data.download_url.clone(),
764 size: package_data.size,
765 sha1: package_data.sha1,
766 md5: package_data.md5,
767 sha256: package_data.sha256,
768 sha512: package_data.sha512,
769 bug_tracking_url: package_data.bug_tracking_url.clone(),
770 code_view_url: package_data.code_view_url.clone(),
771 vcs_url: package_data.vcs_url.clone(),
772 copyright: package_data.copyright.clone(),
773 holder: package_data.holder.clone(),
774 declared_license_expression: package_data.declared_license_expression.clone(),
775 declared_license_expression_spdx: package_data.declared_license_expression_spdx.clone(),
776 license_detections: package_data.license_detections.clone(),
777 other_license_expression: package_data.other_license_expression.clone(),
778 other_license_expression_spdx: package_data.other_license_expression_spdx.clone(),
779 other_license_detections: package_data.other_license_detections.clone(),
780 extracted_license_statement: package_data.extracted_license_statement.clone(),
781 notice_text: package_data.notice_text.clone(),
782 source_packages: package_data.source_packages.clone(),
783 file_references: package_data.file_references.clone(),
784 is_private: package_data.is_private,
785 is_virtual: package_data.is_virtual,
786 extra_data: package_data.extra_data.clone(),
787 dependencies: package_data.dependencies.clone(),
788 repository_homepage_url: package_data.repository_homepage_url.clone(),
789 repository_download_url: package_data.repository_download_url.clone(),
790 api_data_url: package_data.api_data_url.clone(),
791 datasource_id: package_data.datasource_id,
792 purl: package_data.purl.clone(),
793 }
794 }
795}
796
797#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
798pub enum PartyType {
799 Person,
800 Organization,
801}
802
803impl fmt::Display for PartyType {
804 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
805 match self {
806 PartyType::Person => write!(f, "person"),
807 PartyType::Organization => write!(f, "organization"),
808 }
809 }
810}
811
812impl FromStr for PartyType {
813 type Err = String;
814 fn from_str(s: &str) -> Result<Self, Self::Err> {
815 match s {
816 "person" => Ok(PartyType::Person),
817 "organization" => Ok(PartyType::Organization),
818 other => Err(format!("unknown party type: {other}")),
819 }
820 }
821}
822
823#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
827pub struct Party {
828 pub r#type: Option<PartyType>,
829 pub role: Option<String>,
830 pub name: Option<String>,
831 pub email: Option<String>,
832 pub url: Option<String>,
833 pub organization: Option<String>,
834 pub organization_url: Option<String>,
835 pub timezone: Option<String>,
836}
837
838impl Party {
839 pub(crate) fn person(role: &str, name: Option<String>, email: Option<String>) -> Self {
840 Self {
841 r#type: Some(PartyType::Person),
842 role: Some(role.to_string()),
843 name,
844 email,
845 url: None,
846 organization: None,
847 organization_url: None,
848 timezone: None,
849 }
850 }
851}
852
853#[derive(Serialize, Deserialize, Debug, Clone)]
857pub struct FileReference {
858 pub path: String,
859 pub size: Option<u64>,
860 pub sha1: Option<Sha1Digest>,
861 pub md5: Option<Md5Digest>,
862 pub sha256: Option<Sha256Digest>,
863 pub sha512: Option<Sha512Digest>,
864 #[serde(with = "super::json_value_map")]
865 pub extra_data: Option<std::collections::HashMap<String, serde_json::Value>>,
866}
867
868impl FileReference {
869 pub(crate) fn from_path(path: String) -> Self {
870 Self {
871 path,
872 size: None,
873 sha1: None,
874 md5: None,
875 sha256: None,
876 sha512: None,
877 extra_data: None,
878 }
879 }
880}
881
882#[derive(Serialize, Deserialize, Debug, Clone)]
892pub struct Package {
893 pub package_type: Option<PackageType>,
894 pub namespace: Option<String>,
895 pub name: Option<String>,
896 pub version: Option<String>,
897 #[serde(default)]
898 pub qualifiers: Option<HashMap<String, String>>,
899 pub subpath: Option<String>,
900 pub primary_language: Option<String>,
901 pub description: Option<String>,
902 pub release_date: Option<String>,
903 #[serde(default)]
904 pub parties: Vec<Party>,
905 #[serde(default)]
906 pub keywords: Vec<String>,
907 pub homepage_url: Option<String>,
908 pub download_url: Option<String>,
909 pub size: Option<u64>,
910 pub sha1: Option<Sha1Digest>,
911 pub md5: Option<Md5Digest>,
912 pub sha256: Option<Sha256Digest>,
913 pub sha512: Option<Sha512Digest>,
914 pub bug_tracking_url: Option<String>,
915 pub code_view_url: Option<String>,
916 pub vcs_url: Option<String>,
917 pub copyright: Option<String>,
918 pub holder: Option<String>,
919 pub declared_license_expression: Option<String>,
920 pub declared_license_expression_spdx: Option<String>,
921 #[serde(default)]
922 pub license_detections: Vec<LicenseDetection>,
923 pub other_license_expression: Option<String>,
924 pub other_license_expression_spdx: Option<String>,
925 #[serde(default)]
926 pub other_license_detections: Vec<LicenseDetection>,
927 pub extracted_license_statement: Option<String>,
928 pub notice_text: Option<String>,
929 #[serde(default)]
930 pub source_packages: Vec<String>,
931 #[serde(default)]
932 pub is_private: bool,
933 #[serde(default)]
934 pub is_virtual: bool,
935 #[serde(default, with = "super::json_value_map")]
936 pub extra_data: Option<HashMap<String, serde_json::Value>>,
937 pub repository_homepage_url: Option<String>,
938 pub repository_download_url: Option<String>,
939 pub api_data_url: Option<String>,
940 pub purl: Option<String>,
941 pub package_uid: PackageUid,
943 pub datafile_paths: Vec<String>,
945 pub datasource_ids: Vec<DatasourceId>,
947}
948
949impl Package {
950 pub fn from_package_data(package_data: &PackageData, datafile_path: String) -> Self {
956 let mut package_data = package_data.clone();
957 enrich_package_data_license_provenance(&mut package_data, &datafile_path);
958
959 let mut package = Package {
960 package_type: package_data.package_type,
961 namespace: package_data.namespace.clone(),
962 name: package_data.name.clone(),
963 version: package_data.version.clone(),
964 qualifiers: package_data.qualifiers.clone(),
965 subpath: package_data.subpath.clone(),
966 primary_language: package_data.primary_language.clone(),
967 description: package_data.description.clone(),
968 release_date: package_data.release_date.clone(),
969 parties: package_data.parties.clone(),
970 keywords: package_data.keywords.clone(),
971 homepage_url: package_data.homepage_url.clone(),
972 download_url: package_data.download_url.clone(),
973 size: package_data.size,
974 sha1: package_data.sha1,
975 md5: package_data.md5,
976 sha256: package_data.sha256,
977 sha512: package_data.sha512,
978 bug_tracking_url: package_data.bug_tracking_url.clone(),
979 code_view_url: package_data.code_view_url.clone(),
980 vcs_url: package_data.vcs_url.clone(),
981 copyright: package_data.copyright.clone(),
982 holder: package_data.holder.clone(),
983 declared_license_expression: package_data.declared_license_expression.clone(),
984 declared_license_expression_spdx: package_data.declared_license_expression_spdx.clone(),
985 license_detections: package_data.license_detections.clone(),
986 other_license_expression: package_data.other_license_expression.clone(),
987 other_license_expression_spdx: package_data.other_license_expression_spdx.clone(),
988 other_license_detections: package_data.other_license_detections.clone(),
989 extracted_license_statement: package_data.extracted_license_statement.clone(),
990 notice_text: package_data.notice_text.clone(),
991 source_packages: package_data.source_packages.clone(),
992 is_private: package_data.is_private,
993 is_virtual: package_data.is_virtual,
994 extra_data: package_data.extra_data.clone(),
995 repository_homepage_url: package_data.repository_homepage_url.clone(),
996 repository_download_url: package_data.repository_download_url.clone(),
997 api_data_url: package_data.api_data_url.clone(),
998 purl: package_data.purl.clone(),
999 package_uid: PackageUid::empty(),
1000 datafile_paths: vec![datafile_path],
1001 datasource_ids: if let Some(dsid) = package_data.datasource_id {
1002 vec![dsid]
1003 } else {
1004 vec![]
1005 },
1006 };
1007
1008 package.refresh_identity();
1009 if package.package_uid.is_empty() {
1010 package.package_uid = package.fallback_package_uid();
1011 }
1012
1013 package
1014 }
1015
1016 pub fn update(&mut self, package_data: &PackageData, datafile_path: String) {
1022 let mut package_data = package_data.clone();
1023 enrich_package_data_license_provenance(&mut package_data, &datafile_path);
1024
1025 if let Some(dsid) = package_data.datasource_id {
1026 self.datasource_ids.push(dsid);
1027 }
1028 self.datafile_paths.push(datafile_path);
1029
1030 macro_rules! fill_if_empty {
1031 ($field:ident) => {
1032 if self.$field.is_none() {
1033 self.$field = package_data.$field;
1034 }
1035 };
1036 }
1037
1038 fill_if_empty!(package_type);
1039 fill_if_empty!(name);
1040 fill_if_empty!(namespace);
1041 fill_if_empty!(version);
1042 fill_if_empty!(qualifiers);
1043 fill_if_empty!(subpath);
1044 fill_if_empty!(primary_language);
1045 fill_if_empty!(description);
1046 fill_if_empty!(release_date);
1047 fill_if_empty!(homepage_url);
1048 fill_if_empty!(download_url);
1049 fill_if_empty!(size);
1050 fill_if_empty!(sha1);
1051 fill_if_empty!(md5);
1052 fill_if_empty!(sha256);
1053 fill_if_empty!(sha512);
1054 fill_if_empty!(bug_tracking_url);
1055 fill_if_empty!(code_view_url);
1056 fill_if_empty!(vcs_url);
1057 fill_if_empty!(copyright);
1058 fill_if_empty!(holder);
1059 fill_if_empty!(declared_license_expression);
1060 fill_if_empty!(declared_license_expression_spdx);
1061 fill_if_empty!(other_license_expression);
1062 fill_if_empty!(other_license_expression_spdx);
1063 fill_if_empty!(extracted_license_statement);
1064 fill_if_empty!(notice_text);
1065 match (&mut self.extra_data, &package_data.extra_data) {
1066 (None, Some(extra_data)) => {
1067 self.extra_data = Some(extra_data.clone());
1068 }
1069 (Some(existing), Some(incoming)) => {
1070 for (key, value) in incoming {
1071 existing.entry(key.clone()).or_insert_with(|| value.clone());
1072 }
1073 }
1074 _ => {}
1075 }
1076 fill_if_empty!(repository_homepage_url);
1077 fill_if_empty!(repository_download_url);
1078 fill_if_empty!(api_data_url);
1079
1080 for party in &package_data.parties {
1081 if let Some(existing) = self.parties.iter_mut().find(|p| {
1082 p.role == party.role
1083 && ((p.name.is_some() && p.name == party.name)
1084 || (p.email.is_some() && p.email == party.email))
1085 }) {
1086 if existing.name.is_none() {
1087 existing.name = party.name.clone();
1088 }
1089 if existing.email.is_none() {
1090 existing.email = party.email.clone();
1091 }
1092 } else {
1093 self.parties.push(party.clone());
1094 }
1095 }
1096
1097 for keyword in &package_data.keywords {
1098 if !self.keywords.contains(keyword) {
1099 self.keywords.push(keyword.clone());
1100 }
1101 }
1102
1103 for detection in &package_data.license_detections {
1104 self.license_detections.push(detection.clone());
1105 }
1106
1107 for detection in &package_data.other_license_detections {
1108 self.other_license_detections.push(detection.clone());
1109 }
1110
1111 for source_pkg in &package_data.source_packages {
1112 if !self.source_packages.contains(source_pkg) {
1113 self.source_packages.push(source_pkg.clone());
1114 }
1115 }
1116
1117 self.refresh_identity();
1118 }
1119
1120 pub fn backfill_license_provenance(&mut self) {
1121 let Some(datafile_path) = self.datafile_paths.first().cloned() else {
1122 return;
1123 };
1124
1125 for detection in &mut self.license_detections {
1126 enrich_license_detection_provenance(detection, &datafile_path);
1127 }
1128 for detection in &mut self.other_license_detections {
1129 enrich_license_detection_provenance(detection, &datafile_path);
1130 }
1131 }
1132
1133 fn refresh_identity(&mut self) {
1134 let Some(next_purl) = self.build_current_purl() else {
1135 return;
1136 };
1137 let next_purl = crate::models::normalize_purl(&next_purl);
1141
1142 if self.purl.as_deref() != Some(next_purl.as_str()) || self.package_uid.is_empty() {
1143 self.package_uid = PackageUid::new(&next_purl);
1144 }
1145
1146 self.purl = Some(next_purl);
1147 }
1148
1149 fn fallback_package_uid(&self) -> PackageUid {
1150 let name = self
1151 .name
1152 .as_deref()
1153 .map(str::trim)
1154 .filter(|value| !value.is_empty())
1155 .unwrap_or("unknown");
1156 let version = self
1157 .version
1158 .as_deref()
1159 .map(str::trim)
1160 .filter(|value| !value.is_empty())
1161 .unwrap_or("unknown");
1162 let datasource = self
1163 .datasource_ids
1164 .first()
1165 .map(DatasourceId::as_str)
1166 .unwrap_or("unknown");
1167
1168 PackageUid::new_opaque(&format!("generated-package:{datasource}/{name}@{version}"))
1169 }
1170
1171 fn build_current_purl(&self) -> Option<String> {
1172 if let Some(existing_purl) = self.purl.as_deref() {
1173 let mut purl = PackageUrl::from_str(existing_purl).ok()?;
1174
1175 if let Some(version) = self
1176 .version
1177 .as_deref()
1178 .filter(|value| !value.trim().is_empty())
1179 {
1180 purl.with_version(version).ok()?;
1181 } else {
1182 purl.without_version();
1183 }
1184
1185 return Some(purl.to_string());
1186 }
1187
1188 if let (Some(package_type), Some(name)) = (
1189 self.package_type.as_ref(),
1190 self.name
1191 .as_deref()
1192 .filter(|value| !value.trim().is_empty()),
1193 ) {
1194 let purl_type = match package_type {
1195 PackageType::Deno => "generic",
1196 _ => package_type.as_str(),
1197 };
1198
1199 let has_namespace = self
1200 .namespace
1201 .as_deref()
1202 .is_some_and(|value| !value.trim().is_empty());
1203
1204 if purl_type == "swift" && !has_namespace {
1207 return None;
1208 }
1209
1210 let mut purl = PackageUrl::new(purl_type, name).ok()?;
1211
1212 if let Some(namespace) = self
1213 .namespace
1214 .as_deref()
1215 .filter(|value| !value.trim().is_empty())
1216 {
1217 purl.with_namespace(namespace).ok()?;
1218 }
1219
1220 if let Some(version) = self
1221 .version
1222 .as_deref()
1223 .filter(|value| !value.trim().is_empty())
1224 {
1225 purl.with_version(version).ok()?;
1226 }
1227
1228 if let Some(qualifiers) = &self.qualifiers {
1229 for (key, value) in qualifiers {
1230 purl.add_qualifier(key.as_str(), value.as_str()).ok()?;
1231 }
1232 }
1233
1234 if let Some(subpath) = self
1235 .subpath
1236 .as_deref()
1237 .filter(|value| !value.trim().is_empty())
1238 {
1239 purl.with_subpath(subpath).ok()?;
1240 }
1241
1242 return Some(purl.to_string());
1243 }
1244 None
1245 }
1246}
1247
1248#[cfg(test)]
1249mod tests {
1250 use super::*;
1251
1252 #[test]
1253 fn file_info_new_backfills_package_detection_provenance() {
1254 let package_data = PackageData {
1255 package_type: Some(PackageType::Npm),
1256 license_detections: vec![LicenseDetection {
1257 license_expression: "mit".to_string(),
1258 license_expression_spdx: "MIT".to_string(),
1259 matches: vec![Match {
1260 license_expression: "mit".to_string(),
1261 license_expression_spdx: "MIT".to_string(),
1262 from_file: None,
1263 start_line: LineNumber::ONE,
1264 end_line: LineNumber::ONE,
1265 matcher: MatcherKind::Declared,
1266 score: MatchScore::MAX,
1267 matched_length: Some(1),
1268 match_coverage: Some(100.0),
1269 rule_relevance: Some(100),
1270 rule_identifier: String::new(),
1271 rule_url: None,
1272 matched_text: Some("MIT".to_string()),
1273 referenced_filenames: None,
1274 matched_text_diagnostics: None,
1275 }],
1276 detection_log: vec![],
1277 identifier: String::new(),
1278 }],
1279 ..PackageData::default()
1280 };
1281
1282 let file_info = FileInfo::new(
1283 "package.json".to_string(),
1284 "package".to_string(),
1285 ".json".to_string(),
1286 "project/package.json".to_string(),
1287 FileType::File,
1288 None,
1289 None,
1290 1,
1291 None,
1292 None,
1293 None,
1294 None,
1295 None,
1296 vec![package_data],
1297 None,
1298 vec![],
1299 vec![],
1300 vec![],
1301 vec![],
1302 vec![],
1303 vec![],
1304 vec![],
1305 vec![],
1306 vec![],
1307 );
1308
1309 assert_eq!(file_info.license_detections.len(), 1);
1310 assert_eq!(
1311 file_info.license_detections[0].matches[0]
1312 .from_file
1313 .as_deref(),
1314 Some("project/package.json")
1315 );
1316 assert!(!file_info.license_detections[0].identifier.is_empty());
1317 assert_eq!(
1318 file_info.package_data[0].license_detections[0].matches[0]
1319 .from_file
1320 .as_deref(),
1321 Some("project/package.json")
1322 );
1323 assert_eq!(
1324 file_info.package_data[0].license_detections[0].matches[0].rule_identifier,
1325 "parser-declared-license"
1326 );
1327 assert!(
1328 !file_info.package_data[0].license_detections[0]
1329 .identifier
1330 .is_empty()
1331 );
1332 }
1333
1334 #[test]
1335 fn package_from_package_data_backfills_detection_provenance() {
1336 let package_data = PackageData {
1337 package_type: Some(PackageType::Npm),
1338 license_detections: vec![LicenseDetection {
1339 license_expression: "mit".to_string(),
1340 license_expression_spdx: "MIT".to_string(),
1341 matches: vec![Match {
1342 license_expression: "mit".to_string(),
1343 license_expression_spdx: "MIT".to_string(),
1344 from_file: None,
1345 start_line: LineNumber::ONE,
1346 end_line: LineNumber::ONE,
1347 matcher: MatcherKind::Declared,
1348 score: MatchScore::MAX,
1349 matched_length: Some(1),
1350 match_coverage: Some(100.0),
1351 rule_relevance: Some(100),
1352 rule_identifier: String::new(),
1353 rule_url: None,
1354 matched_text: Some("MIT".to_string()),
1355 referenced_filenames: None,
1356 matched_text_diagnostics: None,
1357 }],
1358 detection_log: vec![],
1359 identifier: String::new(),
1360 }],
1361 ..PackageData::default()
1362 };
1363
1364 let package = Package::from_package_data(&package_data, "project/package.json".to_string());
1365
1366 assert_eq!(
1367 package.license_detections[0].matches[0]
1368 .from_file
1369 .as_deref(),
1370 Some("project/package.json")
1371 );
1372 assert_eq!(
1373 package.license_detections[0].matches[0].rule_identifier,
1374 "parser-declared-license"
1375 );
1376 assert!(!package.license_detections[0].identifier.is_empty());
1377 }
1378
1379 #[test]
1380 fn package_from_package_data_preserves_existing_purl_qualifiers() {
1381 let package_data = PackageData {
1382 package_type: Some(PackageType::Apk),
1383 namespace: Some("alpine".to_string()),
1384 name: Some("busybox".to_string()),
1385 version: Some("1.35.0-r17".to_string()),
1386 purl: Some("pkg:alpine/busybox@1.35.0-r17?arch=x86_64".to_string()),
1387 ..PackageData::default()
1388 };
1389
1390 let package = Package::from_package_data(&package_data, "lib/apk/db/installed".to_string());
1391
1392 assert_eq!(
1393 package.purl.as_deref(),
1394 Some("pkg:alpine/busybox@1.35.0-r17?arch=x86_64")
1395 );
1396 assert!(
1397 package
1398 .package_uid
1399 .starts_with("pkg:alpine/busybox@1.35.0-r17?arch=x86_64&uuid=")
1400 );
1401 }
1402}
1403
1404#[derive(Serialize, Deserialize, Debug, Clone)]
1409pub struct TopLevelDependency {
1410 pub purl: Option<String>,
1411 pub extracted_requirement: Option<String>,
1412 pub scope: Option<String>,
1413 pub is_runtime: Option<bool>,
1414 pub is_optional: Option<bool>,
1415 pub is_pinned: Option<bool>,
1416 pub is_direct: Option<bool>,
1417 pub resolved_package: Option<Box<ResolvedPackage>>,
1418 #[serde(default, with = "super::json_value_map")]
1419 pub extra_data: Option<HashMap<String, serde_json::Value>>,
1420 pub dependency_uid: DependencyUid,
1422 pub for_package_uid: Option<PackageUid>,
1424 pub datafile_path: String,
1426 pub datasource_id: DatasourceId,
1428 pub namespace: Option<String>,
1430}
1431
1432impl TopLevelDependency {
1433 pub fn from_dependency(
1435 dep: &Dependency,
1436 datafile_path: String,
1437 datasource_id: DatasourceId,
1438 for_package_uid: Option<PackageUid>,
1439 ) -> Self {
1440 let dependency_uid = dep
1441 .purl
1442 .as_ref()
1443 .map(|p| DependencyUid::new(p))
1444 .unwrap_or_else(DependencyUid::empty);
1445
1446 TopLevelDependency {
1447 purl: dep.purl.clone(),
1448 extracted_requirement: dep.extracted_requirement.clone(),
1449 scope: dep.scope.clone(),
1450 is_runtime: dep.is_runtime,
1451 is_optional: dep.is_optional,
1452 is_pinned: dep.is_pinned,
1453 is_direct: dep.is_direct,
1454 resolved_package: dep.resolved_package.clone(),
1455 extra_data: dep.extra_data.clone(),
1456 dependency_uid,
1457 for_package_uid,
1458 datafile_path,
1459 datasource_id,
1460 namespace: None,
1461 }
1462 }
1463}
1464
1465#[derive(Serialize, Deserialize, Debug, Clone)]
1466pub struct OutputEmail {
1467 pub email: String,
1468 pub start_line: LineNumber,
1469 pub end_line: LineNumber,
1470}
1471
1472#[derive(Serialize, Deserialize, Debug, Clone)]
1473pub struct OutputURL {
1474 pub url: String,
1475 pub start_line: LineNumber,
1476 pub end_line: LineNumber,
1477}
1478
1479#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
1480pub struct LicensePolicyEntry {
1481 pub license_key: String,
1482 pub label: String,
1483 pub color_code: String,
1484 pub icon: String,
1485}
1486
1487#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
1488pub enum FileType {
1489 File,
1490 Directory,
1491}