provenant/license_detection/
dataset.rs

1// SPDX-FileCopyrightText: Provenant contributors
2// SPDX-License-Identifier: Apache-2.0
3
4use std::fmt::Write as _;
5use std::path::Path;
6
7use anyhow::{Context, Result, anyhow};
8use rayon::prelude::*;
9use serde::{Deserialize, Serialize};
10
11use crate::cache::{create_dir_all_private, write_bytes_atomically, write_bytes_owner_only};
12use crate::license_detection::embedded::index::load_loader_snapshot_from_bytes;
13use crate::license_detection::embedded::schema::EmbeddedArtifactMetadata;
14use crate::license_detection::license_cache::compute_rules_fingerprint;
15use crate::license_detection::models::{LoadedLicense, LoadedRule, RuleKind};
16use crate::license_detection::rules::{parse_license_to_loaded, parse_rule_to_loaded};
17use crate::models::Sha256Digest;
18use crate::version::BUILD_VERSION;
19
20pub const LICENSE_DATASET_RULES_DIR: &str = "rules";
21pub const LICENSE_DATASET_LICENSES_DIR: &str = "licenses";
22pub const LICENSE_DATASET_MANIFEST_FILE: &str = "manifest.json";
23pub const LICENSE_DATASET_README_FILE: &str = "README.md";
24pub const CUSTOM_LICENSE_DATASET_SOURCE: &str = "custom-license-dataset";
25const LICENSE_DATASET_SCHEMA_VERSION: u32 = 1;
26
27#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
28pub struct LicenseDatasetManifest {
29    pub schema_version: u32,
30    pub spdx_license_list_version: String,
31    pub dataset_fingerprint: String,
32    pub exported_from_source: String,
33    pub exported_by_version: String,
34}
35
36#[derive(Debug, Clone)]
37pub struct LoadedLicenseDataset {
38    pub manifest: LicenseDatasetManifest,
39    pub rules: Vec<LoadedRule>,
40    pub licenses: Vec<LoadedLicense>,
41}
42
43/// Result of a dataset export: the manifest plus the number of `.RULE`/`.LICENSE`
44/// files written (excluding the manifest and README), for a caller's summary.
45#[derive(Debug, Clone)]
46pub struct ExportOutcome {
47    pub manifest: LicenseDatasetManifest,
48    pub files_written: usize,
49}
50
51pub fn export_embedded_license_dataset(target_root: &Path) -> Result<ExportOutcome> {
52    let artifact_bytes = include_bytes!("../../resources/license_detection/license_index.zst");
53    let snapshot = load_loader_snapshot_from_bytes(artifact_bytes)
54        .map_err(|error| anyhow!("Failed to load embedded license dataset: {}", error))?;
55
56    export_license_dataset_to_root(
57        target_root,
58        &snapshot.rules,
59        &snapshot.licenses,
60        &snapshot.metadata,
61    )
62}
63
64pub fn export_license_dataset_to_root(
65    target_root: &Path,
66    rules: &[LoadedRule],
67    licenses: &[LoadedLicense],
68    metadata: &EmbeddedArtifactMetadata,
69) -> Result<ExportOutcome> {
70    ensure_export_target_is_empty(target_root)?;
71    // Reject duplicate output names up front: the parallel writers would
72    // otherwise race two workers onto the same path (truncate-and-write), and a
73    // dataset with colliding rule identifiers / license keys is malformed anyway.
74    ensure_unique_export_names(rules, licenses)?;
75
76    let manifest = LicenseDatasetManifest {
77        schema_version: LICENSE_DATASET_SCHEMA_VERSION,
78        spdx_license_list_version: metadata.spdx_license_list_version.clone(),
79        dataset_fingerprint: compute_dataset_fingerprint_string(rules, licenses)?,
80        exported_from_source: metadata.license_index_provenance.source.clone(),
81        exported_by_version: BUILD_VERSION.to_string(),
82    };
83
84    write_rule_files(target_root, rules)?;
85    write_license_files(target_root, licenses)?;
86    write_dataset_readme(target_root, &manifest)?;
87    // Write the manifest last: the loader requires `manifest.json`, so its
88    // presence marks a complete export. If the process is interrupted mid-write
89    // the manifest is absent and the loader rejects the partial directory rather
90    // than silently loading an incomplete license index.
91    write_dataset_manifest(target_root, &manifest)?;
92
93    Ok(ExportOutcome {
94        manifest,
95        files_written: rules.len() + licenses.len(),
96    })
97}
98
99fn ensure_unique_export_names(rules: &[LoadedRule], licenses: &[LoadedLicense]) -> Result<()> {
100    let mut rule_ids = std::collections::HashSet::with_capacity(rules.len());
101    for rule in rules {
102        if !rule_ids.insert(rule.identifier.as_str()) {
103            return Err(anyhow!(
104                "Duplicate rule identifier in license dataset export: {}",
105                rule.identifier
106            ));
107        }
108    }
109    let mut license_keys = std::collections::HashSet::with_capacity(licenses.len());
110    for license in licenses {
111        if !license_keys.insert(license.key.as_str()) {
112            return Err(anyhow!(
113                "Duplicate license key in license dataset export: {}",
114                license.key
115            ));
116        }
117    }
118    Ok(())
119}
120
121pub fn load_license_dataset_from_root(root: &Path) -> Result<LoadedLicenseDataset> {
122    let rules_dir = root.join(LICENSE_DATASET_RULES_DIR);
123    let licenses_dir = root.join(LICENSE_DATASET_LICENSES_DIR);
124
125    if !root.is_dir() {
126        return Err(anyhow!(
127            "License dataset root does not exist or is not a directory: {}",
128            root.display()
129        ));
130    }
131    if !rules_dir.is_dir() {
132        return Err(anyhow!(
133            "License dataset is missing required rules/ directory: {}",
134            rules_dir.display()
135        ));
136    }
137    if !licenses_dir.is_dir() {
138        return Err(anyhow!(
139            "License dataset is missing required licenses/ directory: {}",
140            licenses_dir.display()
141        ));
142    }
143
144    let manifest_path = root.join(LICENSE_DATASET_MANIFEST_FILE);
145    let manifest_text = std::fs::read_to_string(&manifest_path).with_context(|| {
146        format!(
147            "License dataset is missing required manifest.json at {}",
148            manifest_path.display()
149        )
150    })?;
151    let manifest: LicenseDatasetManifest =
152        serde_json::from_str(&manifest_text).with_context(|| {
153            format!(
154                "Failed to parse license dataset manifest at {}",
155                manifest_path.display()
156            )
157        })?;
158
159    if manifest.schema_version != LICENSE_DATASET_SCHEMA_VERSION {
160        return Err(anyhow!(
161            "Unsupported license dataset schema version {} in {} (expected {})",
162            manifest.schema_version,
163            manifest_path.display(),
164            LICENSE_DATASET_SCHEMA_VERSION
165        ));
166    }
167
168    let rules = load_strict_loaded_rules_from_directory(&rules_dir)?;
169    let licenses = load_strict_loaded_licenses_from_directory(&licenses_dir)?;
170
171    Ok(LoadedLicenseDataset {
172        manifest,
173        rules,
174        licenses,
175    })
176}
177
178pub fn compute_dataset_fingerprint_string(
179    rules: &[LoadedRule],
180    licenses: &[LoadedLicense],
181) -> Result<String> {
182    Ok(Sha256Digest::from_bytes(compute_rules_fingerprint(rules, licenses)?).to_string())
183}
184
185fn ensure_export_target_is_empty(target_root: &Path) -> Result<()> {
186    if target_root.exists() {
187        let mut entries = std::fs::read_dir(target_root)
188            .with_context(|| format!("Failed to read export target {}", target_root.display()))?;
189        if entries.next().is_some() {
190            return Err(anyhow!(
191                "Refusing to export into non-empty directory {}",
192                target_root.display()
193            ));
194        }
195    } else {
196        std::fs::create_dir_all(target_root)
197            .with_context(|| format!("Failed to create export target {}", target_root.display()))?;
198    }
199
200    Ok(())
201}
202
203fn write_dataset_manifest(root: &Path, manifest: &LicenseDatasetManifest) -> Result<()> {
204    let payload = serde_json::to_vec_pretty(manifest).context("Serialize dataset manifest")?;
205    write_bytes_atomically(&root.join(LICENSE_DATASET_MANIFEST_FILE), &payload)
206        .context("Write dataset manifest")?;
207    Ok(())
208}
209
210fn write_dataset_readme(root: &Path, manifest: &LicenseDatasetManifest) -> Result<()> {
211    let text = format!(
212        "# Exported Provenant license dataset\n\nThis directory contains the effective `.RULE` and `.LICENSE` files used by Provenant.\n\n- Reuse it with `provenant --license-dataset-path <DIR> --license ...`\n- Edit files under `rules/` and `licenses/` to customize scan behavior\n- `manifest.json` records the exported dataset fingerprint and SPDX license list version\n- The fingerprint in `manifest.json` is informational; if you edit files, Provenant computes the active dataset fingerprint from current file contents\n\nExport metadata:\n\n- schema_version: {}\n- spdx_license_list_version: {}\n- dataset_fingerprint: {}\n- exported_from_source: {}\n- exported_by_version: {}\n",
213        manifest.schema_version,
214        manifest.spdx_license_list_version,
215        manifest.dataset_fingerprint,
216        manifest.exported_from_source,
217        manifest.exported_by_version,
218    );
219    write_bytes_atomically(&root.join(LICENSE_DATASET_README_FILE), text.as_bytes())
220        .context("Write dataset README")?;
221    Ok(())
222}
223
224fn write_rule_files(root: &Path, rules: &[LoadedRule]) -> Result<()> {
225    // Create the target directory once, then write files directly (no per-file
226    // fsync/temp-rename): the target is a freshly emptied export dir, so per-file
227    // durability is unnecessary and its fsync cost dominates at this file count.
228    let rules_dir = root.join(LICENSE_DATASET_RULES_DIR);
229    create_dir_all_private(&rules_dir)?;
230
231    rules.par_iter().try_for_each(|rule| -> Result<()> {
232        validate_dataset_filename_component(&rule.identifier, "rule identifier")?;
233        let rendered = render_rule(rule)?;
234        let output_path = rules_dir.join(&rule.identifier);
235        write_bytes_owner_only(&output_path, rendered.as_bytes())
236            .with_context(|| format!("Write rule dataset file {}", output_path.display()))
237    })
238}
239
240fn write_license_files(root: &Path, licenses: &[LoadedLicense]) -> Result<()> {
241    let licenses_dir = root.join(LICENSE_DATASET_LICENSES_DIR);
242    create_dir_all_private(&licenses_dir)?;
243
244    licenses.par_iter().try_for_each(|license| -> Result<()> {
245        validate_dataset_filename_component(&license.key, "license key")?;
246        let rendered = render_license(license)?;
247        let output_path = licenses_dir.join(format!("{}.LICENSE", license.key));
248        write_bytes_owner_only(&output_path, rendered.as_bytes())
249            .with_context(|| format!("Write license dataset file {}", output_path.display()))
250    })
251}
252
253fn load_strict_loaded_rules_from_directory(dir: &Path) -> Result<Vec<LoadedRule>> {
254    let mut rules = Vec::new();
255    let entries = std::fs::read_dir(dir)
256        .with_context(|| format!("Failed to read rules directory: {}", dir.display()))?;
257
258    for entry in entries {
259        let entry = entry
260            .with_context(|| format!("Failed to read directory entry in: {}", dir.display()))?;
261        let path = entry.path();
262        if path.is_file() && path.extension().and_then(|s| s.to_str()) == Some("RULE") {
263            rules.push(parse_rule_to_loaded(&path).with_context(|| {
264                format!("Failed to parse dataset rule file {}", path.display())
265            })?);
266        }
267    }
268
269    Ok(rules)
270}
271
272fn load_strict_loaded_licenses_from_directory(dir: &Path) -> Result<Vec<LoadedLicense>> {
273    let mut licenses = Vec::new();
274    let entries = std::fs::read_dir(dir)
275        .with_context(|| format!("Failed to read licenses directory: {}", dir.display()))?;
276
277    for entry in entries {
278        let entry = entry
279            .with_context(|| format!("Failed to read directory entry in: {}", dir.display()))?;
280        let path = entry.path();
281        if path.is_file() && path.extension().and_then(|s| s.to_str()) == Some("LICENSE") {
282            licenses.push(parse_license_to_loaded(&path).with_context(|| {
283                format!("Failed to parse dataset license file {}", path.display())
284            })?);
285        }
286    }
287
288    Ok(licenses)
289}
290
291fn validate_dataset_filename_component(value: &str, kind: &str) -> Result<()> {
292    if value.is_empty()
293        || value.contains('/')
294        || value.contains('\\')
295        || value.contains("..")
296        || Path::new(value).components().count() != 1
297    {
298        return Err(anyhow!(
299            "Invalid {} for exported license dataset: {}",
300            kind,
301            value
302        ));
303    }
304
305    Ok(())
306}
307
308fn render_rule(rule: &LoadedRule) -> Result<String> {
309    let mut rendered = String::from("---\n");
310    push_yaml_string(
311        &mut rendered,
312        "license_expression",
313        Some(&rule.license_expression),
314    )?;
315    push_rule_kind(&mut rendered, rule.rule_kind);
316    push_yaml_bool(&mut rendered, "is_false_positive", rule.is_false_positive);
317    push_yaml_bool(&mut rendered, "is_required_phrase", rule.is_required_phrase);
318    push_yaml_bool(
319        &mut rendered,
320        "skip_for_required_phrase_generation",
321        rule.skip_for_required_phrase_generation,
322    );
323    push_yaml_u8(&mut rendered, "relevance", rule.relevance);
324    if rule.has_stored_minimum_coverage {
325        push_yaml_u8(&mut rendered, "minimum_coverage", rule.minimum_coverage);
326    }
327    push_yaml_bool(&mut rendered, "is_continuous", rule.is_continuous);
328    push_yaml_bool(&mut rendered, "is_deprecated", rule.is_deprecated);
329    push_yaml_list(
330        &mut rendered,
331        "referenced_filenames",
332        rule.referenced_filenames.as_deref(),
333    )?;
334    push_yaml_list(&mut rendered, "replaced_by", Some(&rule.replaced_by))?;
335    push_yaml_list(
336        &mut rendered,
337        "ignorable_urls",
338        rule.ignorable_urls.as_deref(),
339    )?;
340    push_yaml_list(
341        &mut rendered,
342        "ignorable_emails",
343        rule.ignorable_emails.as_deref(),
344    )?;
345    push_yaml_string(&mut rendered, "notes", rule.notes.as_deref())?;
346    push_yaml_list(
347        &mut rendered,
348        "ignorable_copyrights",
349        rule.ignorable_copyrights.as_deref(),
350    )?;
351    push_yaml_list(
352        &mut rendered,
353        "ignorable_holders",
354        rule.ignorable_holders.as_deref(),
355    )?;
356    push_yaml_list(
357        &mut rendered,
358        "ignorable_authors",
359        rule.ignorable_authors.as_deref(),
360    )?;
361    push_yaml_string(&mut rendered, "language", rule.language.as_deref())?;
362    rendered.push_str("---\n\n");
363    rendered.push_str(&rule.text);
364    rendered.push('\n');
365    Ok(rendered)
366}
367
368fn render_license(license: &LoadedLicense) -> Result<String> {
369    let mut rendered = String::from("---\n");
370    push_yaml_string(&mut rendered, "key", Some(&license.key))?;
371    push_yaml_string(&mut rendered, "short_name", license.short_name.as_deref())?;
372    push_yaml_string(&mut rendered, "name", Some(&license.name))?;
373    push_yaml_string(
374        &mut rendered,
375        "spdx_license_key",
376        license.spdx_license_key.as_deref(),
377    )?;
378    push_yaml_list(
379        &mut rendered,
380        "other_spdx_license_keys",
381        Some(&license.other_spdx_license_keys),
382    )?;
383    push_yaml_string(&mut rendered, "category", license.category.as_deref())?;
384    push_yaml_string(&mut rendered, "owner", license.owner.as_deref())?;
385    push_yaml_string(
386        &mut rendered,
387        "homepage_url",
388        license.homepage_url.as_deref(),
389    )?;
390    push_yaml_string(
391        &mut rendered,
392        "osi_license_key",
393        license.osi_license_key.as_deref(),
394    )?;
395    push_yaml_list(&mut rendered, "text_urls", Some(&license.text_urls))?;
396    push_yaml_string(&mut rendered, "osi_url", license.osi_url.as_deref())?;
397    push_yaml_string(&mut rendered, "faq_url", license.faq_url.as_deref())?;
398    push_yaml_list(&mut rendered, "other_urls", Some(&license.other_urls))?;
399    push_yaml_string(&mut rendered, "notes", license.notes.as_deref())?;
400    push_yaml_bool(&mut rendered, "is_deprecated", license.is_deprecated);
401    push_yaml_bool(&mut rendered, "is_exception", license.is_exception);
402    push_yaml_bool(&mut rendered, "is_unknown", license.is_unknown);
403    push_yaml_bool(&mut rendered, "is_generic", license.is_generic);
404    push_yaml_list(&mut rendered, "replaced_by", Some(&license.replaced_by))?;
405    push_yaml_u8(&mut rendered, "minimum_coverage", license.minimum_coverage);
406    push_yaml_string(
407        &mut rendered,
408        "standard_notice",
409        license.standard_notice.as_deref(),
410    )?;
411    push_yaml_list(
412        &mut rendered,
413        "ignorable_copyrights",
414        license.ignorable_copyrights.as_deref(),
415    )?;
416    push_yaml_list(
417        &mut rendered,
418        "ignorable_holders",
419        license.ignorable_holders.as_deref(),
420    )?;
421    push_yaml_list(
422        &mut rendered,
423        "ignorable_authors",
424        license.ignorable_authors.as_deref(),
425    )?;
426    push_yaml_list(
427        &mut rendered,
428        "ignorable_urls",
429        license.ignorable_urls.as_deref(),
430    )?;
431    push_yaml_list(
432        &mut rendered,
433        "ignorable_emails",
434        license.ignorable_emails.as_deref(),
435    )?;
436    rendered.push_str("---\n\n");
437    rendered.push_str(&license.text);
438    rendered.push('\n');
439    Ok(rendered)
440}
441
442fn push_rule_kind(rendered: &mut String, rule_kind: RuleKind) {
443    let key = match rule_kind {
444        RuleKind::None => return,
445        RuleKind::Text => "is_license_text",
446        RuleKind::Notice => "is_license_notice",
447        RuleKind::Reference => "is_license_reference",
448        RuleKind::Tag => "is_license_tag",
449        RuleKind::Intro => "is_license_intro",
450        RuleKind::Clue => "is_license_clue",
451    };
452    let _ = writeln!(rendered, "{key}: true");
453}
454
455fn push_yaml_bool(rendered: &mut String, key: &str, value: bool) {
456    if value {
457        let _ = writeln!(rendered, "{key}: true");
458    }
459}
460
461fn push_yaml_u8(rendered: &mut String, key: &str, value: Option<u8>) {
462    if let Some(value) = value {
463        let _ = writeln!(rendered, "{key}: {value}");
464    }
465}
466
467fn push_yaml_string(rendered: &mut String, key: &str, value: Option<&str>) -> Result<()> {
468    let Some(value) = value else {
469        return Ok(());
470    };
471    let quoted = serde_json::to_string(value).context("serialize yaml string")?;
472    let _ = writeln!(rendered, "{key}: {quoted}");
473    Ok(())
474}
475
476fn push_yaml_list(rendered: &mut String, key: &str, values: Option<&[String]>) -> Result<()> {
477    let Some(values) = values else {
478        return Ok(());
479    };
480    if values.is_empty() {
481        return Ok(());
482    }
483
484    let _ = writeln!(rendered, "{key}:");
485    for value in values {
486        let quoted = serde_json::to_string(value).context("serialize yaml list entry")?;
487        let _ = writeln!(rendered, "  - {quoted}");
488    }
489    Ok(())
490}
491
492#[cfg(test)]
493mod tests {
494    use super::*;
495    use crate::license_detection::models::RuleKind;
496    use crate::license_detection::rules::{parse_license_str_to_loaded, parse_rule_str_to_loaded};
497    use tempfile::TempDir;
498
499    fn create_loaded_rule() -> LoadedRule {
500        LoadedRule {
501            identifier: "example.RULE".to_string(),
502            license_expression: "mit OR apache-2.0".to_string(),
503            text: "Example rule text".to_string(),
504            rule_kind: RuleKind::Notice,
505            is_false_positive: false,
506            is_required_phrase: true,
507            skip_for_required_phrase_generation: true,
508            relevance: Some(100),
509            minimum_coverage: Some(75),
510            has_stored_minimum_coverage: true,
511            is_continuous: true,
512            referenced_filenames: Some(vec!["LICENSE".to_string()]),
513            ignorable_urls: Some(vec!["https://example.com".to_string()]),
514            ignorable_emails: Some(vec!["legal@example.com".to_string()]),
515            ignorable_copyrights: Some(vec!["Copyright Example".to_string()]),
516            ignorable_holders: Some(vec!["Example Org".to_string()]),
517            ignorable_authors: Some(vec!["Jane Doe".to_string()]),
518            language: Some("en".to_string()),
519            notes: Some("Example note".to_string()),
520            is_deprecated: true,
521            replaced_by: vec!["replacement.RULE".to_string()],
522        }
523    }
524
525    fn create_loaded_license() -> LoadedLicense {
526        LoadedLicense {
527            key: "example-license".to_string(),
528            short_name: Some("Example".to_string()),
529            name: "Example License".to_string(),
530            language: Some("en".to_string()),
531            spdx_license_key: Some("MIT".to_string()),
532            other_spdx_license_keys: vec!["Apache-2.0".to_string()],
533            category: Some("Permissive".to_string()),
534            owner: Some("Example Org".to_string()),
535            homepage_url: Some("https://example.com".to_string()),
536            text: "Example license text".to_string(),
537            reference_urls: vec![
538                "https://example.com/text".to_string(),
539                "https://example.com/other".to_string(),
540                "https://opensource.org/licenses/MIT".to_string(),
541                "https://example.com/faq".to_string(),
542                "https://example.com".to_string(),
543            ],
544            osi_license_key: Some("MIT".to_string()),
545            text_urls: vec!["https://example.com/text".to_string()],
546            osi_url: Some("https://opensource.org/licenses/MIT".to_string()),
547            faq_url: Some("https://example.com/faq".to_string()),
548            other_urls: vec!["https://example.com/other".to_string()],
549            notes: Some("Example note".to_string()),
550            is_deprecated: true,
551            is_exception: true,
552            is_unknown: true,
553            is_generic: true,
554            replaced_by: vec!["replacement".to_string()],
555            minimum_coverage: Some(55),
556            standard_notice: Some("Standard notice".to_string()),
557            ignorable_copyrights: Some(vec!["Copyright Example".to_string()]),
558            ignorable_holders: Some(vec!["Example Org".to_string()]),
559            ignorable_authors: Some(vec!["Jane Doe".to_string()]),
560            ignorable_urls: Some(vec!["https://example.com".to_string()]),
561            ignorable_emails: Some(vec!["legal@example.com".to_string()]),
562        }
563    }
564
565    #[test]
566    fn render_rule_roundtrips_through_loader() {
567        let rule = create_loaded_rule();
568        let rendered = render_rule(&rule).expect("render rule");
569        let reparsed = parse_rule_str_to_loaded(&rule.identifier, &rendered).expect("reparse rule");
570        assert_eq!(reparsed, rule);
571    }
572
573    #[test]
574    fn render_license_roundtrips_through_loader() {
575        let license = create_loaded_license();
576        let rendered = render_license(&license).expect("render license");
577        let reparsed =
578            parse_license_str_to_loaded("example-license.LICENSE", &rendered).expect("reparse");
579        assert_eq!(reparsed, license);
580    }
581
582    #[test]
583    fn load_license_dataset_requires_manifest_and_expected_dirs() {
584        let temp = TempDir::new().expect("temp dir");
585        std::fs::create_dir_all(temp.path().join("rules")).expect("rules dir");
586        std::fs::create_dir_all(temp.path().join("licenses")).expect("licenses dir");
587
588        let error = load_license_dataset_from_root(temp.path()).expect_err("missing manifest");
589        assert!(error.to_string().contains("manifest.json"));
590    }
591
592    #[test]
593    fn load_license_dataset_fails_on_invalid_rule_file() {
594        let temp = TempDir::new().expect("temp dir");
595        let root = temp.path();
596        std::fs::create_dir_all(root.join("rules")).expect("rules dir");
597        std::fs::create_dir_all(root.join("licenses")).expect("licenses dir");
598        std::fs::write(
599            root.join("manifest.json"),
600            serde_json::json!({
601                "schema_version": 1,
602                "spdx_license_list_version": "3.27",
603                "dataset_fingerprint": "abc",
604                "exported_from_source": "embedded-artifact",
605                "exported_by_version": "test",
606            })
607            .to_string(),
608        )
609        .expect("manifest");
610        std::fs::write(root.join("rules").join("broken.RULE"), "not-frontmatter")
611            .expect("broken rule");
612        std::fs::write(
613            root.join("licenses").join("mit.LICENSE"),
614            "---\nkey: \"mit\"\nname: \"MIT License\"\n---\n\nMIT text\n",
615        )
616        .expect("license");
617
618        let error = load_license_dataset_from_root(root).expect_err("invalid rule should fail");
619        assert!(
620            error
621                .to_string()
622                .contains("Failed to parse dataset rule file")
623        );
624    }
625
626    #[test]
627    fn export_license_dataset_rejects_path_like_rule_identifier() {
628        let manifest = EmbeddedArtifactMetadata {
629            spdx_license_list_version: "3.27".to_string(),
630            license_index_provenance: crate::models::LicenseIndexProvenance {
631                source: "embedded-artifact".to_string(),
632                dataset_fingerprint: "abc123".to_string(),
633                ignored_rules: vec![],
634                ignored_licenses: vec![],
635                ignored_rules_due_to_licenses: vec![],
636                added_rules: vec![],
637                replaced_rules: vec![],
638                added_licenses: vec![],
639                replaced_licenses: vec![],
640            },
641        };
642        let temp = TempDir::new().expect("temp dir");
643
644        let error = export_license_dataset_to_root(
645            temp.path(),
646            &[LoadedRule {
647                identifier: "nested/path.RULE".to_string(),
648                ..create_loaded_rule()
649            }],
650            &[create_loaded_license()],
651            &manifest,
652        )
653        .expect_err("path-like identifiers should be rejected");
654
655        assert!(
656            error
657                .to_string()
658                .contains("Invalid rule identifier for exported license dataset")
659        );
660    }
661
662    #[test]
663    fn export_reports_file_count_and_roundtrips() {
664        let metadata = EmbeddedArtifactMetadata {
665            spdx_license_list_version: "3.27".to_string(),
666            license_index_provenance: crate::models::LicenseIndexProvenance {
667                source: "embedded-artifact".to_string(),
668                dataset_fingerprint: "abc123".to_string(),
669                ignored_rules: vec![],
670                ignored_licenses: vec![],
671                ignored_rules_due_to_licenses: vec![],
672                added_rules: vec![],
673                replaced_rules: vec![],
674                added_licenses: vec![],
675                replaced_licenses: vec![],
676            },
677        };
678        let rules = vec![
679            LoadedRule {
680                identifier: "first.RULE".to_string(),
681                ..create_loaded_rule()
682            },
683            LoadedRule {
684                identifier: "second.RULE".to_string(),
685                ..create_loaded_rule()
686            },
687        ];
688        let licenses = vec![create_loaded_license()];
689        let temp = TempDir::new().expect("temp dir");
690
691        let outcome = export_license_dataset_to_root(temp.path(), &rules, &licenses, &metadata)
692            .expect("export should succeed");
693
694        // Counts the rule and license files written (manifest/README excluded).
695        assert_eq!(outcome.files_written, 3);
696
697        // The written tree must reload through the dataset loader unchanged.
698        let reloaded =
699            load_license_dataset_from_root(temp.path()).expect("reload exported dataset");
700        assert_eq!(reloaded.rules.len(), 2);
701        assert_eq!(reloaded.licenses.len(), 1);
702    }
703
704    #[test]
705    fn export_rejects_duplicate_rule_identifiers() {
706        let metadata = EmbeddedArtifactMetadata {
707            spdx_license_list_version: "3.27".to_string(),
708            license_index_provenance: crate::models::LicenseIndexProvenance {
709                source: "embedded-artifact".to_string(),
710                dataset_fingerprint: "abc123".to_string(),
711                ignored_rules: vec![],
712                ignored_licenses: vec![],
713                ignored_rules_due_to_licenses: vec![],
714                added_rules: vec![],
715                replaced_rules: vec![],
716                added_licenses: vec![],
717                replaced_licenses: vec![],
718            },
719        };
720        let rules = vec![create_loaded_rule(), create_loaded_rule()];
721        let temp = TempDir::new().expect("temp dir");
722
723        let error = export_license_dataset_to_root(temp.path(), &rules, &[], &metadata)
724            .expect_err("duplicate rule identifiers should be rejected before any parallel write");
725        assert!(
726            error.to_string().contains("Duplicate rule identifier"),
727            "unexpected error: {error}"
728        );
729    }
730}
provenant/license_detection/dataset.rs

provenant/license_detection/
dataset.rs
