provenant/assembly/
assemblers.rs

1// SPDX-FileCopyrightText: nexB Inc. and others
2// ScanCode is a trademark of nexB Inc.
3// SPDX-FileCopyrightText: Provenant contributors
4// SPDX-License-Identifier: Apache-2.0
5// Derived from ScanCode Toolkit (Apache-2.0); modified. See NOTICE.
6
7use std::collections::HashSet;
8
9use crate::models::PackageType;
10use crate::models::{DatasourceId, FileInfo, Package, TopLevelDependency};
11use strum::EnumIter;
12
13use super::{
14    AssemblerConfig, AssemblyMode, DirectoryMergeOutput, bazel_merge, bazel_prune,
15    cargo_resource_assign, composer_resource_assign, conda_rootfs_merge, debian_source_merge,
16    file_ref_resolve, hackage_merge, nix_flake_compat_merge, npm_resource_assign,
17    nuget_cpm_resolve, python_requirements_assign, ruby_resource_assign, swift_merge, topology,
18};
19
20#[derive(Clone, Copy)]
21pub(super) enum SpecialDirectoryMergerKind {
22    Skip,
23    Bazel,
24    DebianSource,
25    Hackage,
26    Huggingface,
27    WindowsUpdate,
28}
29
30#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, EnumIter)]
31pub(super) enum PostAssemblyPassKind {
32    SwiftMerge,
33    CondaRootfsMerge,
34    NpmResourceAssign,
35    PythonRequirementsAssign,
36    FileReferenceResolve,
37    RpmYumdbMerge,
38    NpmWorkspaceMerge,
39    CargoWorkspaceMerge,
40    NugetCpmResolve,
41    CargoResourceAssign,
42    ComposerResourceAssign,
43    RubyResourceAssign,
44    NixFlakeCompatMerge,
45    BazelPrune,
46}
47
48pub(super) fn special_directory_merger_for(
49    config_key: DatasourceId,
50) -> Option<SpecialDirectoryMergerKind> {
51    match config_key {
52        DatasourceId::BazelBuild => Some(SpecialDirectoryMergerKind::Bazel),
53        DatasourceId::DebianControlInSource => Some(SpecialDirectoryMergerKind::DebianSource),
54        DatasourceId::HackageCabal => Some(SpecialDirectoryMergerKind::Hackage),
55        DatasourceId::HuggingfaceModelCard => Some(SpecialDirectoryMergerKind::Huggingface),
56        DatasourceId::MicrosoftUpdateManifestMum => Some(SpecialDirectoryMergerKind::WindowsUpdate),
57        DatasourceId::SwiftPackageManifestJson => Some(SpecialDirectoryMergerKind::Skip),
58        _ => None,
59    }
60}
61
62pub(super) static POST_ASSEMBLY_PASSES: &[PostAssemblyPassKind] = &[
63    PostAssemblyPassKind::SwiftMerge,
64    PostAssemblyPassKind::CondaRootfsMerge,
65    PostAssemblyPassKind::NpmResourceAssign,
66    PostAssemblyPassKind::PythonRequirementsAssign,
67    PostAssemblyPassKind::FileReferenceResolve,
68    PostAssemblyPassKind::RpmYumdbMerge,
69    PostAssemblyPassKind::NpmWorkspaceMerge,
70    PostAssemblyPassKind::CargoWorkspaceMerge,
71    PostAssemblyPassKind::NugetCpmResolve,
72    PostAssemblyPassKind::CargoResourceAssign,
73    PostAssemblyPassKind::ComposerResourceAssign,
74    PostAssemblyPassKind::RubyResourceAssign,
75    PostAssemblyPassKind::NixFlakeCompatMerge,
76    PostAssemblyPassKind::BazelPrune,
77];
78
79const SWIFT_POST_ASSEMBLY_DATASOURCE_IDS: &[DatasourceId] = &[
80    DatasourceId::SwiftPackageManifestJson,
81    DatasourceId::SwiftPackageResolved,
82    DatasourceId::SwiftPackageShowDependencies,
83];
84
85const CONDA_ROOTFS_POST_ASSEMBLY_DATASOURCE_IDS: &[DatasourceId] =
86    &[DatasourceId::CondaMetaJson, DatasourceId::CondaMetaYaml];
87
88const RPM_INSTALLED_DATABASE_DATASOURCE_IDS: &[DatasourceId] = &[
89    DatasourceId::RpmInstalledDatabaseBdb,
90    DatasourceId::RpmInstalledDatabaseNdb,
91    DatasourceId::RpmInstalledDatabaseSqlite,
92];
93
94const NUGET_CPM_CONFIG_DATASOURCE_IDS: &[DatasourceId] = &[
95    DatasourceId::NugetDirectoryBuildProps,
96    DatasourceId::NugetDirectoryPackagesProps,
97];
98
99const NUGET_CPM_PROJECT_DATASOURCE_IDS: &[DatasourceId] = &[
100    DatasourceId::NugetCsproj,
101    DatasourceId::NugetFsproj,
102    DatasourceId::NugetVbproj,
103];
104
105#[derive(Default)]
106struct PostAssemblyInputs {
107    package_types: HashSet<PackageType>,
108    file_datasource_ids: HashSet<DatasourceId>,
109    has_npm_workspace_markers: bool,
110    has_cargo_workspace_markers: bool,
111}
112
113pub(super) fn run_post_assembly_passes(
114    files: &mut [FileInfo],
115    packages: &mut Vec<Package>,
116    dependencies: &mut Vec<TopLevelDependency>,
117    topology_plan: &topology::TopologyPlan,
118) {
119    let inputs = PostAssemblyInputs::collect(files, packages);
120
121    for pass in POST_ASSEMBLY_PASSES {
122        if !pass.should_run(&inputs) {
123            continue;
124        }
125
126        pass.run(files, packages, dependencies, topology_plan);
127    }
128}
129
130impl PostAssemblyInputs {
131    fn collect(files: &[FileInfo], packages: &[Package]) -> Self {
132        let mut inputs = Self {
133            package_types: packages
134                .iter()
135                .filter_map(|package| package.package_type)
136                .collect(),
137            ..Self::default()
138        };
139
140        for file in files {
141            for package_data in &file.package_data {
142                let Some(datasource_id) = package_data.datasource_id else {
143                    continue;
144                };
145
146                inputs.file_datasource_ids.insert(datasource_id);
147
148                if matches!(
149                    datasource_id,
150                    DatasourceId::NpmPackageJson | DatasourceId::PnpmWorkspaceYaml
151                ) && package_data
152                    .extra_data
153                    .as_ref()
154                    .is_some_and(|extra_data| extra_data.contains_key("workspaces"))
155                {
156                    inputs.has_npm_workspace_markers = true;
157                }
158
159                if datasource_id == DatasourceId::CargoToml
160                    && package_data
161                        .extra_data
162                        .as_ref()
163                        .and_then(|extra_data| extra_data.get("workspace"))
164                        .and_then(|workspace| workspace.get("members"))
165                        .and_then(|members| members.as_array())
166                        .is_some_and(|members| !members.is_empty())
167                {
168                    inputs.has_cargo_workspace_markers = true;
169                }
170            }
171        }
172
173        inputs
174    }
175
176    fn has_package_type(&self, package_type: PackageType) -> bool {
177        self.package_types.contains(&package_type)
178    }
179
180    fn has_any_file_datasource(&self, datasource_ids: &[DatasourceId]) -> bool {
181        datasource_ids
182            .iter()
183            .any(|datasource_id| self.file_datasource_ids.contains(datasource_id))
184    }
185
186    fn has_all_file_datasources(&self, datasource_ids: &[DatasourceId]) -> bool {
187        datasource_ids
188            .iter()
189            .all(|datasource_id| self.file_datasource_ids.contains(datasource_id))
190    }
191}
192
193impl SpecialDirectoryMergerKind {
194    pub(super) fn run(
195        self,
196        config: &AssemblerConfig,
197        files: &[FileInfo],
198        file_indices: &[usize],
199    ) -> Vec<DirectoryMergeOutput> {
200        match self {
201            Self::Skip => Vec::new(),
202            Self::Bazel => bazel_merge::assemble_bazel_packages(config, files, file_indices),
203            Self::DebianSource => {
204                debian_source_merge::assemble_debian_source_packages(config, files, file_indices)
205            }
206            Self::Hackage => hackage_merge::assemble_hackage_packages(files, file_indices),
207            Self::Huggingface => {
208                super::huggingface_merge::assemble_huggingface_packages(files, file_indices)
209            }
210            Self::WindowsUpdate => super::windows_update_merge::assemble_windows_update_packages(
211                config,
212                files,
213                file_indices,
214            ),
215        }
216    }
217}
218
219impl PostAssemblyPassKind {
220    fn should_run(self, inputs: &PostAssemblyInputs) -> bool {
221        match self {
222            Self::SwiftMerge => inputs.has_any_file_datasource(SWIFT_POST_ASSEMBLY_DATASOURCE_IDS),
223            Self::CondaRootfsMerge => {
224                inputs.has_all_file_datasources(CONDA_ROOTFS_POST_ASSEMBLY_DATASOURCE_IDS)
225            }
226            Self::NpmResourceAssign => inputs.has_package_type(PackageType::Npm),
227            Self::PythonRequirementsAssign => {
228                inputs.has_package_type(PackageType::Pypi)
229                    && inputs.has_any_file_datasource(&[DatasourceId::PipRequirements])
230            }
231            Self::FileReferenceResolve => {
232                file_ref_resolve::has_relevant_file_reference_datasource_ids(
233                    &inputs.file_datasource_ids,
234                )
235            }
236            Self::RpmYumdbMerge => {
237                inputs.has_any_file_datasource(&[DatasourceId::RpmYumdb])
238                    && inputs.has_any_file_datasource(RPM_INSTALLED_DATABASE_DATASOURCE_IDS)
239            }
240            Self::NpmWorkspaceMerge => inputs.has_npm_workspace_markers,
241            Self::CargoWorkspaceMerge => inputs.has_cargo_workspace_markers,
242            Self::NugetCpmResolve => {
243                inputs.has_any_file_datasource(NUGET_CPM_CONFIG_DATASOURCE_IDS)
244                    && inputs.has_any_file_datasource(NUGET_CPM_PROJECT_DATASOURCE_IDS)
245            }
246            Self::CargoResourceAssign => inputs.has_package_type(PackageType::Cargo),
247            Self::ComposerResourceAssign => inputs.has_package_type(PackageType::Composer),
248            Self::RubyResourceAssign => inputs.has_package_type(PackageType::Gem),
249            Self::NixFlakeCompatMerge => {
250                inputs.has_any_file_datasource(&[DatasourceId::NixDefaultNix])
251                    && inputs.has_any_file_datasource(&[
252                        DatasourceId::NixFlakeNix,
253                        DatasourceId::NixFlakeLock,
254                    ])
255            }
256            Self::BazelPrune => inputs.has_package_type(PackageType::Bazel),
257        }
258    }
259
260    fn run(
261        self,
262        files: &mut [FileInfo],
263        packages: &mut Vec<Package>,
264        dependencies: &mut Vec<TopLevelDependency>,
265        topology_plan: &topology::TopologyPlan,
266    ) {
267        match self {
268            Self::SwiftMerge => swift_merge::assemble_swift_packages(files, packages, dependencies),
269            Self::CondaRootfsMerge => {
270                conda_rootfs_merge::merge_conda_rootfs_metadata(files, packages, dependencies)
271            }
272            Self::NpmResourceAssign => {
273                npm_resource_assign::assign_npm_package_resources(files, packages)
274            }
275            Self::PythonRequirementsAssign => {
276                python_requirements_assign::assign_python_requirements_to_projects(
277                    files,
278                    packages,
279                    dependencies,
280                )
281            }
282            Self::FileReferenceResolve => {
283                file_ref_resolve::resolve_file_references(files, packages, dependencies)
284            }
285            Self::RpmYumdbMerge => file_ref_resolve::merge_rpm_yumdb_metadata(files, packages),
286            Self::NpmWorkspaceMerge => {
287                topology_plan.apply_npm_workspace_domains(files, packages, dependencies)
288            }
289            Self::CargoWorkspaceMerge => {
290                topology_plan.apply_cargo_workspace_domains(files, packages, dependencies)
291            }
292            Self::NugetCpmResolve => {
293                nuget_cpm_resolve::resolve_nuget_cpm_versions(files, dependencies)
294            }
295            Self::CargoResourceAssign => {
296                cargo_resource_assign::assign_cargo_package_resources(files, packages)
297            }
298            Self::ComposerResourceAssign => {
299                composer_resource_assign::assign_composer_package_resources(files, packages)
300            }
301            Self::RubyResourceAssign => {
302                ruby_resource_assign::assign_ruby_package_resources(files, packages)
303            }
304            Self::NixFlakeCompatMerge => {
305                nix_flake_compat_merge::attach_flake_compat_default_files(files, packages)
306            }
307            Self::BazelPrune => {
308                bazel_prune::prune_unused_bazel_packages(files, packages, dependencies)
309            }
310        }
311    }
312}
313
314pub static ASSEMBLERS: &[AssemblerConfig] = &[
315    // ── Sibling-merge assemblers ──
316    //
317    // npm ecosystem: package.json + lockfiles in same directory.
318    // NOTE: npm-shrinkwrap.json emits "npm_package_lock_json" as its datasource_id,
319    // so "npm_shrinkwrap_json" is NOT a real datasource_id.
320    AssemblerConfig {
321        datasource_ids: &[
322            DatasourceId::BunLock,
323            DatasourceId::BunLockb,
324            DatasourceId::NpmPackageJson,
325            DatasourceId::NpmPackageLockJson,
326            DatasourceId::YarnLock,
327            DatasourceId::YarnLockV1,
328            DatasourceId::YarnLockV2,
329            DatasourceId::YarnPnpCjs,
330            DatasourceId::PnpmLockYaml,
331            DatasourceId::PnpmWorkspaceYaml,
332        ],
333        sibling_file_patterns: &[
334            "package.json",
335            "bun.lock",
336            "bun.lockb",
337            ".package-lock.json",
338            "package-lock.json",
339            ".npm-shrinkwrap.json",
340            "npm-shrinkwrap.json",
341            "yarn.lock",
342            ".pnp.cjs",
343            "pnpm-lock.yaml",
344            "shrinkwrap.yaml",
345            "pnpm-workspace.yaml",
346        ],
347        mode: AssemblyMode::SiblingMerge,
348    },
349    // Rust/Cargo ecosystem
350    AssemblerConfig {
351        datasource_ids: &[DatasourceId::CargoToml, DatasourceId::CargoLock],
352        sibling_file_patterns: &["Cargo.toml", "Cargo.lock"],
353        mode: AssemblyMode::SiblingMerge,
354    },
355    // Julia ecosystem
356    AssemblerConfig {
357        datasource_ids: &[
358            DatasourceId::JuliaProjectToml,
359            DatasourceId::JuliaManifestToml,
360        ],
361        sibling_file_patterns: &["Project.toml", "Manifest.toml"],
362        mode: AssemblyMode::SiblingMerge,
363    },
364    // Erlang/OTP Rebar ecosystem
365    AssemblerConfig {
366        datasource_ids: &[DatasourceId::RebarConfig, DatasourceId::RebarLock],
367        sibling_file_patterns: &["rebar.config", "rebar.lock"],
368        mode: AssemblyMode::SiblingMerge,
369    },
370    // Carthage ecosystem
371    AssemblerConfig {
372        datasource_ids: &[
373            DatasourceId::CarthageCartfile,
374            DatasourceId::CarthageCartfileResolved,
375        ],
376        sibling_file_patterns: &["Cartfile", "Cartfile.private", "Cartfile.resolved"],
377        mode: AssemblyMode::SiblingMerge,
378    },
379    // CocoaPods ecosystem
380    AssemblerConfig {
381        datasource_ids: &[
382            DatasourceId::CocoapodsPodspec,
383            DatasourceId::CocoapodsPodspecJson,
384            DatasourceId::CocoapodsPodfile,
385            DatasourceId::CocoapodsPodfileLock,
386        ],
387        sibling_file_patterns: &["*.podspec", "*.podspec.json", "Podfile", "Podfile.lock"],
388        mode: AssemblyMode::SiblingMerge,
389    },
390    // PHP Composer ecosystem
391    AssemblerConfig {
392        datasource_ids: &[DatasourceId::PhpComposerJson, DatasourceId::PhpComposerLock],
393        sibling_file_patterns: &[
394            "*composer.json",
395            "composer.*.json",
396            "*composer.lock",
397            "composer.*.lock",
398        ],
399        mode: AssemblyMode::SiblingMerge,
400    },
401    // Go ecosystem (includes legacy Godeps)
402    AssemblerConfig {
403        datasource_ids: &[
404            DatasourceId::GoMod,
405            DatasourceId::GoModGraph,
406            DatasourceId::GoSum,
407            DatasourceId::GoWork,
408            DatasourceId::Godeps,
409        ],
410        sibling_file_patterns: &[
411            "go.mod",
412            "go.work",
413            "go.mod.graph",
414            "go.modgraph",
415            "go.sum",
416            "Godeps.json",
417        ],
418        mode: AssemblyMode::SiblingMerge,
419    },
420    // Dart/Flutter ecosystem
421    AssemblerConfig {
422        datasource_ids: &[DatasourceId::PubspecYaml, DatasourceId::PubspecLock],
423        sibling_file_patterns: &["pubspec.yaml", "pubspec.lock"],
424        mode: AssemblyMode::SiblingMerge,
425    },
426    // Pixi ecosystem
427    AssemblerConfig {
428        datasource_ids: &[DatasourceId::PixiToml, DatasourceId::PixiLock],
429        sibling_file_patterns: &["pixi.toml", "pixi.lock"],
430        mode: AssemblyMode::SiblingMerge,
431    },
432    AssemblerConfig {
433        datasource_ids: &[DatasourceId::NixFlakeNix, DatasourceId::NixFlakeLock],
434        sibling_file_patterns: &["flake.nix", "flake.lock"],
435        mode: AssemblyMode::SiblingMerge,
436    },
437    AssemblerConfig {
438        datasource_ids: &[DatasourceId::NixDefaultNix],
439        sibling_file_patterns: &["default.nix"],
440        mode: AssemblyMode::OnePerPackageData,
441    },
442    // Helm chart ecosystem
443    AssemblerConfig {
444        datasource_ids: &[DatasourceId::HelmChartYaml, DatasourceId::HelmChartLock],
445        sibling_file_patterns: &["Chart.yaml", "Chart.lock"],
446        mode: AssemblyMode::SiblingMerge,
447    },
448    AssemblerConfig {
449        datasource_ids: &[
450            DatasourceId::HackageCabal,
451            DatasourceId::HackageCabalProject,
452            DatasourceId::HackageStackYaml,
453        ],
454        sibling_file_patterns: &["*.cabal", "cabal.project", "stack.yaml"],
455        mode: AssemblyMode::SiblingMerge,
456    },
457    // Chef ecosystem
458    AssemblerConfig {
459        datasource_ids: &[
460            DatasourceId::ChefCookbookMetadataJson,
461            DatasourceId::ChefCookbookMetadataRb,
462        ],
463        sibling_file_patterns: &["metadata.json", "metadata.rb"],
464        mode: AssemblyMode::SiblingMerge,
465    },
466    // Conan (C/C++) ecosystem
467    AssemblerConfig {
468        datasource_ids: &[
469            DatasourceId::ConanConanFilePy,
470            DatasourceId::ConanConanFileTxt,
471            DatasourceId::ConanLock,
472            DatasourceId::ConanConanDataYml,
473        ],
474        sibling_file_patterns: &[
475            "conanfile.py",
476            "conanfile.txt",
477            "conan.lock",
478            "conandata.yml",
479        ],
480        mode: AssemblyMode::SiblingMerge,
481    },
482    // Maven/Java ecosystem (nested merge via META-INF)
483    AssemblerConfig {
484        datasource_ids: &[
485            DatasourceId::MavenPom,
486            DatasourceId::MavenPomProperties,
487            DatasourceId::JavaJarManifest,
488            DatasourceId::JavaOsgiManifest,
489        ],
490        sibling_file_patterns: &[
491            "pom.xml",
492            "*.pom",
493            "pom.properties",
494            "**/META-INF/MANIFEST.MF",
495        ],
496        mode: AssemblyMode::SiblingMerge,
497    },
498    AssemblerConfig {
499        datasource_ids: &[DatasourceId::PypiWheel, DatasourceId::PypiPipOriginJson],
500        sibling_file_patterns: &["*.whl", "origin.json"],
501        mode: AssemblyMode::SiblingMerge,
502    },
503    // Python/PyPI ecosystem
504    AssemblerConfig {
505        datasource_ids: &[
506            DatasourceId::PypiPyprojectToml,
507            DatasourceId::PypiPoetryPyprojectToml,
508            DatasourceId::PypiSetupPy,
509            DatasourceId::PypiSetupCfg,
510            DatasourceId::PypiWheelMetadata,
511            DatasourceId::PypiEgg,
512            DatasourceId::PypiEggPkginfo,
513            DatasourceId::PypiEditableEggPkginfo,
514            DatasourceId::PypiJson,
515            DatasourceId::PypiSdist,
516            DatasourceId::PypiSdistPkginfo,
517            DatasourceId::PypiInspectDeplock,
518            DatasourceId::PipRequirements,
519            DatasourceId::PypiPoetryLock,
520            DatasourceId::PypiPylockToml,
521            DatasourceId::PypiUvLock,
522            DatasourceId::Pipfile,
523            DatasourceId::PipfileLock,
524        ],
525        sibling_file_patterns: &[
526            "pyproject.toml",
527            "setup.py",
528            "setup.cfg",
529            "PKG-INFO",
530            "METADATA",
531            "pypi.json",
532            "pip-inspect.deplock",
533            "*.tar.gz",
534            "*.tgz",
535            "*.tar.bz2",
536            "*.tar.xz",
537            "*.zip",
538            "requirements*.txt",
539            "Pipfile",
540            "Pipfile.lock",
541            "poetry.lock",
542            "pylock.toml",
543            "pylock.*.toml",
544            "uv.lock",
545        ],
546        mode: AssemblyMode::SiblingMerge,
547    },
548    AssemblerConfig {
549        datasource_ids: &[DatasourceId::DenoJson, DatasourceId::DenoLock],
550        sibling_file_patterns: &["deno.json", "deno.jsonc", "deno.lock"],
551        mode: AssemblyMode::SiblingMerge,
552    },
553    // Ruby/RubyGems ecosystem
554    AssemblerConfig {
555        datasource_ids: &[
556            DatasourceId::GemArchiveExtracted,
557            DatasourceId::Gemspec,
558            DatasourceId::GemspecExtracted,
559            DatasourceId::Gemfile,
560            DatasourceId::GemfileExtracted,
561            DatasourceId::GemfileLock,
562            DatasourceId::GemfileLockExtracted,
563        ],
564        sibling_file_patterns: &[
565            "metadata.gz-extract",
566            "**/data.gz-extract/*.gemspec",
567            "**/data.gz-extract/Gemfile",
568            "**/data.gz-extract/Gemfile.lock",
569            "*.gemspec",
570            "Gemfile",
571            "Gemfile.lock",
572        ],
573        mode: AssemblyMode::SiblingMerge,
574    },
575    AssemblerConfig {
576        datasource_ids: &[DatasourceId::GemArchive],
577        sibling_file_patterns: &["*.gem"],
578        mode: AssemblyMode::OnePerPackageData,
579    },
580    // Conda ecosystem
581    AssemblerConfig {
582        datasource_ids: &[
583            DatasourceId::CondaMetaYaml,
584            DatasourceId::CondaYaml,
585            DatasourceId::CondaMetaJson,
586        ],
587        sibling_file_patterns: &[
588            "meta.yaml",
589            "meta.yml",
590            "recipe.yaml",
591            "recipe.yml",
592            "environment.yml",
593            "environment.yaml",
594            "conda.yaml",
595            "conda.yml",
596            "*conda*.yaml",
597            "*conda*.yml",
598            "env.yaml",
599            "env.yml",
600            "*env*.yaml",
601            "*env*.yml",
602            "*environment*.yaml",
603            "*environment*.yml",
604            "*.json",
605        ],
606        mode: AssemblyMode::SiblingMerge,
607    },
608    // RPM specfile (source packages)
609    AssemblerConfig {
610        datasource_ids: &[DatasourceId::RpmSpecfile],
611        sibling_file_patterns: &["*.spec"],
612        mode: AssemblyMode::OnePerPackageData,
613    },
614    // Debian source packages (nested merge via debian/ directory)
615    AssemblerConfig {
616        datasource_ids: &[
617            DatasourceId::DebianControlInSource,
618            DatasourceId::DebianCopyrightInSource,
619        ],
620        sibling_file_patterns: &["control", "copyright"],
621        mode: AssemblyMode::SiblingMerge,
622    },
623    // Gradle/Android ecosystem
624    AssemblerConfig {
625        datasource_ids: &[DatasourceId::BuildGradle, DatasourceId::GradleLockfile],
626        sibling_file_patterns: &["build.gradle", "build.gradle.kts", "gradle.lockfile"],
627        mode: AssemblyMode::SiblingMerge,
628    },
629    AssemblerConfig {
630        datasource_ids: &[DatasourceId::GradleModule],
631        sibling_file_patterns: &["*.module"],
632        mode: AssemblyMode::OnePerPackageData,
633    },
634    // Hugging Face model/dataset metadata. The model-card README.md, Transformers
635    // config.json, and Diffusers model_index.json in one repository directory
636    // describe a single logical model, so they are merged into one package by a
637    // dedicated directory merger (see huggingface_merge). Only files the Hugging
638    // Face parsers actually claim (carrying a huggingface datasource id)
639    // participate, so a generic README.md/config.json never triggers a merge.
640    AssemblerConfig {
641        datasource_ids: &[
642            DatasourceId::HuggingfaceModelCard,
643            DatasourceId::HuggingfaceConfigJson,
644            DatasourceId::HuggingfaceModelIndexJson,
645        ],
646        sibling_file_patterns: &["README.md", "config.json", "model_index.json"],
647        mode: AssemblyMode::SiblingMerge,
648    },
649    // CPAN/Perl ecosystem
650    AssemblerConfig {
651        datasource_ids: &[
652            DatasourceId::CpanMetaJson,
653            DatasourceId::CpanMetaYml,
654            DatasourceId::CpanManifest,
655            DatasourceId::CpanDistIni,
656            DatasourceId::CpanMakefile,
657        ],
658        sibling_file_patterns: &[
659            "META.json",
660            "META.yml",
661            "MANIFEST",
662            "dist.ini",
663            "Makefile.PL",
664        ],
665        mode: AssemblyMode::SiblingMerge,
666    },
667    // NuGet/.NET ecosystem
668    AssemblerConfig {
669        datasource_ids: &[
670            DatasourceId::NugetCsproj,
671            DatasourceId::NugetFsproj,
672            DatasourceId::NugetNuspec,
673            DatasourceId::NugetNupkg,
674            DatasourceId::NugetProjectJson,
675            DatasourceId::NugetProjectLockJson,
676            DatasourceId::NugetPackagesConfig,
677            DatasourceId::NugetPackagesLock,
678            DatasourceId::NugetVbproj,
679        ],
680        sibling_file_patterns: &[
681            "*.csproj",
682            "*.fsproj",
683            "*.nuspec",
684            "*.nupkg",
685            "project.json",
686            "project.lock.json",
687            "packages.config",
688            "packages.lock.json",
689            "*.packages.lock.json",
690            "*.vbproj",
691        ],
692        mode: AssemblyMode::SiblingMerge,
693    },
694    AssemblerConfig {
695        datasource_ids: &[DatasourceId::NugetDepsJson],
696        sibling_file_patterns: &["*.deps.json"],
697        mode: AssemblyMode::OnePerPackageData,
698    },
699    // Swift/SPM ecosystem
700    AssemblerConfig {
701        datasource_ids: &[
702            DatasourceId::SwiftPackageManifestJson,
703            DatasourceId::SwiftPackageResolved,
704            DatasourceId::SwiftPackageShowDependencies,
705        ],
706        sibling_file_patterns: &[
707            "Package.swift.json",
708            "Package.swift.deplock",
709            "Package.resolved",
710            ".package.resolved",
711            "swift-show-dependencies.deplock",
712        ],
713        mode: AssemblyMode::SiblingMerge,
714    },
715    // ── Standalone assemblers (single file → single package) ──
716    //
717    // These ecosystems have only one manifest file type with no sibling merging.
718    // They still need configs so their datasource_ids are recognized by the assembler.
719    //
720    // Bower (JavaScript)
721    AssemblerConfig {
722        datasource_ids: &[DatasourceId::BowerJson],
723        sibling_file_patterns: &["bower.json"],
724        mode: AssemblyMode::SiblingMerge,
725    },
726    // CRAN (R language)
727    AssemblerConfig {
728        datasource_ids: &[DatasourceId::CranDescription],
729        sibling_file_patterns: &["DESCRIPTION"],
730        mode: AssemblyMode::SiblingMerge,
731    },
732    // FreeBSD packages
733    AssemblerConfig {
734        datasource_ids: &[DatasourceId::FreebsdCompactManifest],
735        sibling_file_patterns: &["+COMPACT_MANIFEST"],
736        mode: AssemblyMode::SiblingMerge,
737    },
738    // Haxe ecosystem
739    AssemblerConfig {
740        datasource_ids: &[DatasourceId::HaxelibJson],
741        sibling_file_patterns: &["haxelib.json"],
742        mode: AssemblyMode::SiblingMerge,
743    },
744    AssemblerConfig {
745        datasource_ids: &[DatasourceId::Gitmodules],
746        sibling_file_patterns: &[".gitmodules"],
747        mode: AssemblyMode::SiblingMerge,
748    },
749    // OCaml/opam ecosystem
750    AssemblerConfig {
751        datasource_ids: &[DatasourceId::OpamFile],
752        sibling_file_patterns: &["opam", "*.opam"],
753        mode: AssemblyMode::SiblingMerge,
754    },
755    AssemblerConfig {
756        datasource_ids: &[DatasourceId::RpmYumdb],
757        sibling_file_patterns: &["**/var/lib/yum/yumdb/*/*/from_repo"],
758        mode: AssemblyMode::OnePerPackageData,
759    },
760    // Microsoft Update Manifest
761    AssemblerConfig {
762        datasource_ids: &[DatasourceId::MicrosoftUpdateManifestMum],
763        sibling_file_patterns: &["*.mum"],
764        mode: AssemblyMode::SiblingMerge,
765    },
766    // Autotools (C/C++ build system)
767    AssemblerConfig {
768        datasource_ids: &[DatasourceId::AutotoolsConfigure],
769        sibling_file_patterns: &["configure", "configure.ac"],
770        mode: AssemblyMode::SiblingMerge,
771    },
772    // Bazel (build system)
773    AssemblerConfig {
774        datasource_ids: &[DatasourceId::BazelBuild],
775        sibling_file_patterns: &["BUILD"],
776        mode: AssemblyMode::SiblingMerge,
777    },
778    AssemblerConfig {
779        datasource_ids: &[DatasourceId::BazelModule],
780        sibling_file_patterns: &["MODULE.bazel"],
781        mode: AssemblyMode::OnePerPackageData,
782    },
783    // Buck (build system)
784    AssemblerConfig {
785        datasource_ids: &[DatasourceId::BuckFile, DatasourceId::BuckMetadata],
786        sibling_file_patterns: &["BUCK", "METADATA.bzl", ".buckconfig"],
787        mode: AssemblyMode::SiblingMerge,
788    },
789    // Ant/Ivy (Java dependency management)
790    AssemblerConfig {
791        datasource_ids: &[DatasourceId::AntIvyXml],
792        sibling_file_patterns: &["ivy.xml"],
793        mode: AssemblyMode::SiblingMerge,
794    },
795    // JVM archives introspected in place (one archive == one package).
796    AssemblerConfig {
797        datasource_ids: &[
798            DatasourceId::JavaJar,
799            DatasourceId::JavaWarArchive,
800            DatasourceId::AndroidAarLibrary,
801        ],
802        sibling_file_patterns: &["*.jar", "*.war", "*.aar"],
803        mode: AssemblyMode::OnePerPackageData,
804    },
805    // Meteor (JavaScript platform)
806    AssemblerConfig {
807        datasource_ids: &[DatasourceId::MeteorPackage],
808        sibling_file_patterns: &["package.js"],
809        mode: AssemblyMode::SiblingMerge,
810    },
811    // ── One-per-PackageData assemblers (database files with many packages) ──
812    //
813    // Alpine installed package database
814    AssemblerConfig {
815        datasource_ids: &[DatasourceId::AlpineInstalledDb],
816        sibling_file_patterns: &["installed"],
817        mode: AssemblyMode::OnePerPackageData,
818    },
819    AssemblerConfig {
820        datasource_ids: &[DatasourceId::AlpineApkbuild],
821        sibling_file_patterns: &["APKBUILD"],
822        mode: AssemblyMode::SiblingMerge,
823    },
824    // RPM installed package databases (BDB, NDB, SQLite)
825    AssemblerConfig {
826        datasource_ids: &[
827            DatasourceId::RpmInstalledDatabaseBdb,
828            DatasourceId::RpmInstalledDatabaseNdb,
829            DatasourceId::RpmInstalledDatabaseSqlite,
830            DatasourceId::RpmMarinerManifest,
831        ],
832        sibling_file_patterns: &[
833            "Packages",
834            "Packages.db",
835            "rpmdb.sqlite",
836            "container-manifest-2",
837        ],
838        mode: AssemblyMode::OnePerPackageData,
839    },
840    AssemblerConfig {
841        datasource_ids: &[DatasourceId::RpmArchive],
842        sibling_file_patterns: &["*.rpm", "*.srpm"],
843        mode: AssemblyMode::OnePerPackageData,
844    },
845    // Debian installed package databases
846    AssemblerConfig {
847        datasource_ids: &[DatasourceId::DebianDeb],
848        sibling_file_patterns: &["*.deb"],
849        mode: AssemblyMode::OnePerPackageData,
850    },
851    AssemblerConfig {
852        datasource_ids: &[
853            DatasourceId::DebianInstalledStatusDb,
854            DatasourceId::DebianDistrolessInstalledDb,
855        ],
856        sibling_file_patterns: &["status"],
857        mode: AssemblyMode::OnePerPackageData,
858    },
859    AssemblerConfig {
860        datasource_ids: &[
861            DatasourceId::DebianControlExtractedDeb,
862            DatasourceId::DebianMd5SumsInExtractedDeb,
863        ],
864        sibling_file_patterns: &["control", "md5sums"],
865        mode: AssemblyMode::SiblingMerge,
866    },
867    AssemblerConfig {
868        datasource_ids: &[DatasourceId::DebianSourceControlDsc],
869        sibling_file_patterns: &["*.dsc"],
870        mode: AssemblyMode::OnePerPackageData,
871    },
872    AssemblerConfig {
873        datasource_ids: &[DatasourceId::AboutFile],
874        sibling_file_patterns: &["*.ABOUT"],
875        mode: AssemblyMode::OnePerPackageData,
876    },
877    AssemblerConfig {
878        datasource_ids: &[
879            DatasourceId::BitbakeRecipe,
880            DatasourceId::BitbakeRecipeAppend,
881        ],
882        sibling_file_patterns: &["*.bb", "*.bbappend"],
883        mode: AssemblyMode::SiblingMerge,
884    },
885];
886
887// Datasource IDs intentionally excluded from package assembly.
888//
889// This list is runtime-significant: files with these datasource IDs may remain
890// unowned by any Package, while their dependencies are still eligible for
891// top-level hoisting. Tests also use it to enforce explicit assembly accounting.
892pub static UNASSEMBLED_DATASOURCE_IDS: &[DatasourceId] = &[
893    // Non-package metadata
894    DatasourceId::Readme,
895    DatasourceId::EtcOsRelease,
896    // Binary archives (require external extraction via ExtractCode before scanning)
897    DatasourceId::AlpineApkArchive,
898    DatasourceId::AndroidAab,
899    DatasourceId::AndroidApk,
900    DatasourceId::AndroidManifestXml,
901    DatasourceId::AndroidSoongMetadata,
902    DatasourceId::AppleDmg,
903    DatasourceId::Axis2Mar,
904    DatasourceId::ChromeCrx,
905    DatasourceId::DebianOriginalSourceTarball,
906    DatasourceId::DebianSourceMetadataTarball,
907    DatasourceId::InstallshieldInstaller,
908    DatasourceId::IosIpa,
909    DatasourceId::IsoDiskImage,
910    DatasourceId::JavaEarArchive,
911    DatasourceId::JbossSar,
912    DatasourceId::MicrosoftCabinet,
913    DatasourceId::MozillaXpi,
914    DatasourceId::NsisInstaller,
915    DatasourceId::SharShellArchive,
916    DatasourceId::SquashfsDiskImage,
917    // Supplementary metadata (not primary package definitions)
918    DatasourceId::ArchAurinfo,
919    DatasourceId::ArchPkginfo,
920    DatasourceId::ArchSrcinfo,
921    DatasourceId::Axis2ModuleXml,
922    DatasourceId::ClojureDepsEdn,
923    DatasourceId::ClojureProjectClj,
924    DatasourceId::DebianInstalledFilesList,
925    DatasourceId::DebianInstalledMd5Sums,
926    DatasourceId::DebianCopyright,
927    DatasourceId::DebianCopyrightInPackage,
928    DatasourceId::DebianCopyrightStandalone,
929    DatasourceId::GoBinary,
930    DatasourceId::WindowsExecutable,
931    DatasourceId::Dockerfile,
932    DatasourceId::OciImageIndex,
933    DatasourceId::OciImageManifest,
934    DatasourceId::ErlangOtpAppSrc,
935    DatasourceId::HexMixLock,
936    DatasourceId::JavaEarApplicationXml,
937    DatasourceId::JavaWarWebXml,
938    DatasourceId::JbossServiceXml,
939    DatasourceId::MesonBuild,
940    DatasourceId::GemGemspecInstalledSpecifications,
941    DatasourceId::NugetDirectoryBuildProps,
942    DatasourceId::NugetDirectoryPackagesProps,
943    DatasourceId::CitationCff,
944    DatasourceId::PubliccodeYaml,
945    DatasourceId::RpmPackageLicenses,
946    DatasourceId::RustBinary,
947    DatasourceId::SbtBuildSbt,
948    DatasourceId::VcpkgJson,
949];
950
951#[cfg(test)]
952mod tests {
953    use super::*;
954    use std::collections::HashSet;
955    use strum::IntoEnumIterator;
956
957    #[test]
958    fn test_every_datasource_id_is_accounted_for() {
959        let mut assembled: HashSet<DatasourceId> = HashSet::new();
960        for config in ASSEMBLERS {
961            for &dsid in config.datasource_ids {
962                assembled.insert(dsid);
963            }
964        }
965
966        let unassembled: HashSet<DatasourceId> =
967            UNASSEMBLED_DATASOURCE_IDS.iter().copied().collect();
968
969        let overlap: Vec<_> = assembled.intersection(&unassembled).collect();
970        assert!(
971            overlap.is_empty(),
972            "Datasource IDs in BOTH ASSEMBLERS and UNASSEMBLED: {overlap:?}"
973        );
974
975        let missing: Vec<_> = DatasourceId::iter()
976            .filter(|dsid| !assembled.contains(dsid) && !unassembled.contains(dsid))
977            .collect();
978
979        assert!(
980            missing.is_empty(),
981            "Datasource IDs in NEITHER ASSEMBLERS nor UNASSEMBLED: {missing:?}\n\
982             Add each to an AssemblerConfig in ASSEMBLERS, or to UNASSEMBLED_DATASOURCE_IDS."
983        );
984    }
985
986    #[test]
987    fn test_post_assembly_passes_are_unique() {
988        let unique: HashSet<PostAssemblyPassKind> = POST_ASSEMBLY_PASSES.iter().copied().collect();
989
990        assert_eq!(
991            unique.len(),
992            POST_ASSEMBLY_PASSES.len(),
993            "POST_ASSEMBLY_PASSES contains duplicate entries"
994        );
995    }
996
997    #[test]
998    fn test_every_post_assembly_pass_kind_is_registered_once() {
999        let registered: HashSet<PostAssemblyPassKind> =
1000            POST_ASSEMBLY_PASSES.iter().copied().collect();
1001
1002        let missing: Vec<_> = PostAssemblyPassKind::iter()
1003            .filter(|pass| !registered.contains(pass))
1004            .collect();
1005
1006        assert!(
1007            missing.is_empty(),
1008            "Post-assembly pass variants not registered in POST_ASSEMBLY_PASSES: {missing:?}"
1009        );
1010
1011        for pass in PostAssemblyPassKind::iter() {
1012            let count = POST_ASSEMBLY_PASSES
1013                .iter()
1014                .filter(|registered| **registered == pass)
1015                .count();
1016            assert_eq!(
1017                count, 1,
1018                "Post-assembly pass {pass:?} should be registered exactly once"
1019            );
1020        }
1021    }
1022
1023    #[test]
1024    fn test_post_assembly_passes_skip_irrelevant_inputs() {
1025        let inputs = PostAssemblyInputs::default();
1026
1027        for pass in PostAssemblyPassKind::iter() {
1028            assert!(
1029                !pass.should_run(&inputs),
1030                "{pass:?} should skip when no relevant inputs are present"
1031            );
1032        }
1033    }
1034
1035    #[test]
1036    fn test_npm_workspace_inputs_only_run_npm_passes() {
1037        let inputs = PostAssemblyInputs {
1038            package_types: HashSet::from([PackageType::Npm]),
1039            file_datasource_ids: HashSet::from([DatasourceId::NpmPackageJson]),
1040            has_npm_workspace_markers: true,
1041            has_cargo_workspace_markers: false,
1042        };
1043
1044        let runnable: HashSet<_> = PostAssemblyPassKind::iter()
1045            .filter(|pass| pass.should_run(&inputs))
1046            .collect();
1047
1048        assert_eq!(
1049            runnable,
1050            HashSet::from([
1051                PostAssemblyPassKind::NpmResourceAssign,
1052                PostAssemblyPassKind::NpmWorkspaceMerge,
1053            ])
1054        );
1055    }
1056
1057    #[test]
1058    fn test_cargo_workspace_merge_requires_workspace_markers() {
1059        let without_markers = PostAssemblyInputs {
1060            package_types: HashSet::from([PackageType::Cargo]),
1061            file_datasource_ids: HashSet::from([DatasourceId::CargoToml]),
1062            has_npm_workspace_markers: false,
1063            has_cargo_workspace_markers: false,
1064        };
1065
1066        assert!(!PostAssemblyPassKind::CargoWorkspaceMerge.should_run(&without_markers));
1067
1068        let with_markers = PostAssemblyInputs {
1069            has_cargo_workspace_markers: true,
1070            ..without_markers
1071        };
1072
1073        assert!(PostAssemblyPassKind::CargoWorkspaceMerge.should_run(&with_markers));
1074    }
1075}
provenant/assembly/assemblers.rs

provenant/assembly/
assemblers.rs
