provenant/assembly/
assemblers.rs

1// SPDX-FileCopyrightText: nexB Inc. and others
2// SPDX-FileCopyrightText: Provenant contributors
3// SPDX-License-Identifier: Apache-2.0
4// Derived from ScanCode Toolkit (Apache-2.0); modified. See NOTICE.
5
6use std::collections::HashSet;
7
8use crate::models::PackageType;
9use crate::models::{DatasourceId, FileInfo, Package, TopLevelDependency};
10use strum::EnumIter;
11
12use super::{
13    AssemblerConfig, AssemblyMode, DirectoryMergeOutput, bazel_merge, bazel_prune,
14    cargo_resource_assign, composer_resource_assign, conda_rootfs_merge, debian_source_merge,
15    file_ref_resolve, hackage_merge, nix_flake_compat_merge, npm_resource_assign,
16    nuget_cpm_resolve, python_requirements_assign, ruby_resource_assign, swift_merge, topology,
17};
18
19#[derive(Clone, Copy)]
20pub(super) enum SpecialDirectoryMergerKind {
21    Skip,
22    Bazel,
23    DebianSource,
24    Hackage,
25    Huggingface,
26    WindowsUpdate,
27}
28
29#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, EnumIter)]
30pub(super) enum PostAssemblyPassKind {
31    SwiftMerge,
32    CondaRootfsMerge,
33    NpmResourceAssign,
34    PythonRequirementsAssign,
35    FileReferenceResolve,
36    RpmYumdbMerge,
37    NpmWorkspaceMerge,
38    CargoWorkspaceMerge,
39    NugetCpmResolve,
40    CargoResourceAssign,
41    ComposerResourceAssign,
42    RubyResourceAssign,
43    NixFlakeCompatMerge,
44    BazelPrune,
45}
46
47pub(super) fn special_directory_merger_for(
48    config_key: DatasourceId,
49) -> Option<SpecialDirectoryMergerKind> {
50    match config_key {
51        DatasourceId::BazelBuild => Some(SpecialDirectoryMergerKind::Bazel),
52        DatasourceId::DebianControlInSource => Some(SpecialDirectoryMergerKind::DebianSource),
53        DatasourceId::HackageCabal => Some(SpecialDirectoryMergerKind::Hackage),
54        DatasourceId::HuggingfaceModelCard => Some(SpecialDirectoryMergerKind::Huggingface),
55        DatasourceId::MicrosoftUpdateManifestMum => Some(SpecialDirectoryMergerKind::WindowsUpdate),
56        DatasourceId::SwiftPackageManifestJson => Some(SpecialDirectoryMergerKind::Skip),
57        _ => None,
58    }
59}
60
61pub(super) static POST_ASSEMBLY_PASSES: &[PostAssemblyPassKind] = &[
62    PostAssemblyPassKind::SwiftMerge,
63    PostAssemblyPassKind::CondaRootfsMerge,
64    PostAssemblyPassKind::NpmResourceAssign,
65    PostAssemblyPassKind::PythonRequirementsAssign,
66    PostAssemblyPassKind::FileReferenceResolve,
67    PostAssemblyPassKind::RpmYumdbMerge,
68    PostAssemblyPassKind::NpmWorkspaceMerge,
69    PostAssemblyPassKind::CargoWorkspaceMerge,
70    PostAssemblyPassKind::NugetCpmResolve,
71    PostAssemblyPassKind::CargoResourceAssign,
72    PostAssemblyPassKind::ComposerResourceAssign,
73    PostAssemblyPassKind::RubyResourceAssign,
74    PostAssemblyPassKind::NixFlakeCompatMerge,
75    PostAssemblyPassKind::BazelPrune,
76];
77
78const SWIFT_POST_ASSEMBLY_DATASOURCE_IDS: &[DatasourceId] = &[
79    DatasourceId::SwiftPackageManifestJson,
80    DatasourceId::SwiftPackageResolved,
81    DatasourceId::SwiftPackageShowDependencies,
82];
83
84const CONDA_ROOTFS_POST_ASSEMBLY_DATASOURCE_IDS: &[DatasourceId] =
85    &[DatasourceId::CondaMetaJson, DatasourceId::CondaMetaYaml];
86
87const RPM_INSTALLED_DATABASE_DATASOURCE_IDS: &[DatasourceId] = &[
88    DatasourceId::RpmInstalledDatabaseBdb,
89    DatasourceId::RpmInstalledDatabaseNdb,
90    DatasourceId::RpmInstalledDatabaseSqlite,
91];
92
93const NUGET_CPM_CONFIG_DATASOURCE_IDS: &[DatasourceId] = &[
94    DatasourceId::NugetDirectoryBuildProps,
95    DatasourceId::NugetDirectoryPackagesProps,
96];
97
98const NUGET_CPM_PROJECT_DATASOURCE_IDS: &[DatasourceId] = &[
99    DatasourceId::NugetCsproj,
100    DatasourceId::NugetFsproj,
101    DatasourceId::NugetVbproj,
102];
103
104#[derive(Default)]
105struct PostAssemblyInputs {
106    package_types: HashSet<PackageType>,
107    file_datasource_ids: HashSet<DatasourceId>,
108    has_npm_workspace_markers: bool,
109    has_cargo_workspace_markers: bool,
110}
111
112pub(super) fn run_post_assembly_passes(
113    files: &mut [FileInfo],
114    packages: &mut Vec<Package>,
115    dependencies: &mut Vec<TopLevelDependency>,
116    topology_plan: &topology::TopologyPlan,
117) {
118    let inputs = PostAssemblyInputs::collect(files, packages);
119
120    for pass in POST_ASSEMBLY_PASSES {
121        if !pass.should_run(&inputs) {
122            continue;
123        }
124
125        pass.run(files, packages, dependencies, topology_plan);
126    }
127}
128
129impl PostAssemblyInputs {
130    fn collect(files: &[FileInfo], packages: &[Package]) -> Self {
131        let mut inputs = Self {
132            package_types: packages
133                .iter()
134                .filter_map(|package| package.package_type)
135                .collect(),
136            ..Self::default()
137        };
138
139        for file in files {
140            for package_data in &file.package_data {
141                let Some(datasource_id) = package_data.datasource_id else {
142                    continue;
143                };
144
145                inputs.file_datasource_ids.insert(datasource_id);
146
147                if matches!(
148                    datasource_id,
149                    DatasourceId::NpmPackageJson | DatasourceId::PnpmWorkspaceYaml
150                ) && package_data
151                    .extra_data
152                    .as_ref()
153                    .is_some_and(|extra_data| extra_data.contains_key("workspaces"))
154                {
155                    inputs.has_npm_workspace_markers = true;
156                }
157
158                if datasource_id == DatasourceId::CargoToml
159                    && package_data
160                        .extra_data
161                        .as_ref()
162                        .and_then(|extra_data| extra_data.get("workspace"))
163                        .and_then(|workspace| workspace.get("members"))
164                        .and_then(|members| members.as_array())
165                        .is_some_and(|members| !members.is_empty())
166                {
167                    inputs.has_cargo_workspace_markers = true;
168                }
169            }
170        }
171
172        inputs
173    }
174
175    fn has_package_type(&self, package_type: PackageType) -> bool {
176        self.package_types.contains(&package_type)
177    }
178
179    fn has_any_file_datasource(&self, datasource_ids: &[DatasourceId]) -> bool {
180        datasource_ids
181            .iter()
182            .any(|datasource_id| self.file_datasource_ids.contains(datasource_id))
183    }
184
185    fn has_all_file_datasources(&self, datasource_ids: &[DatasourceId]) -> bool {
186        datasource_ids
187            .iter()
188            .all(|datasource_id| self.file_datasource_ids.contains(datasource_id))
189    }
190}
191
192impl SpecialDirectoryMergerKind {
193    pub(super) fn run(
194        self,
195        config: &AssemblerConfig,
196        files: &[FileInfo],
197        file_indices: &[usize],
198    ) -> Vec<DirectoryMergeOutput> {
199        match self {
200            Self::Skip => Vec::new(),
201            Self::Bazel => bazel_merge::assemble_bazel_packages(config, files, file_indices),
202            Self::DebianSource => {
203                debian_source_merge::assemble_debian_source_packages(config, files, file_indices)
204            }
205            Self::Hackage => hackage_merge::assemble_hackage_packages(files, file_indices),
206            Self::Huggingface => {
207                super::huggingface_merge::assemble_huggingface_packages(files, file_indices)
208            }
209            Self::WindowsUpdate => super::windows_update_merge::assemble_windows_update_packages(
210                config,
211                files,
212                file_indices,
213            ),
214        }
215    }
216}
217
218impl PostAssemblyPassKind {
219    fn should_run(self, inputs: &PostAssemblyInputs) -> bool {
220        match self {
221            Self::SwiftMerge => inputs.has_any_file_datasource(SWIFT_POST_ASSEMBLY_DATASOURCE_IDS),
222            Self::CondaRootfsMerge => {
223                inputs.has_all_file_datasources(CONDA_ROOTFS_POST_ASSEMBLY_DATASOURCE_IDS)
224            }
225            Self::NpmResourceAssign => inputs.has_package_type(PackageType::Npm),
226            Self::PythonRequirementsAssign => {
227                inputs.has_package_type(PackageType::Pypi)
228                    && inputs.has_any_file_datasource(&[DatasourceId::PipRequirements])
229            }
230            Self::FileReferenceResolve => {
231                file_ref_resolve::has_relevant_file_reference_datasource_ids(
232                    &inputs.file_datasource_ids,
233                )
234            }
235            Self::RpmYumdbMerge => {
236                inputs.has_any_file_datasource(&[DatasourceId::RpmYumdb])
237                    && inputs.has_any_file_datasource(RPM_INSTALLED_DATABASE_DATASOURCE_IDS)
238            }
239            Self::NpmWorkspaceMerge => inputs.has_npm_workspace_markers,
240            Self::CargoWorkspaceMerge => inputs.has_cargo_workspace_markers,
241            Self::NugetCpmResolve => {
242                inputs.has_any_file_datasource(NUGET_CPM_CONFIG_DATASOURCE_IDS)
243                    && inputs.has_any_file_datasource(NUGET_CPM_PROJECT_DATASOURCE_IDS)
244            }
245            Self::CargoResourceAssign => inputs.has_package_type(PackageType::Cargo),
246            Self::ComposerResourceAssign => inputs.has_package_type(PackageType::Composer),
247            Self::RubyResourceAssign => inputs.has_package_type(PackageType::Gem),
248            Self::NixFlakeCompatMerge => {
249                inputs.has_any_file_datasource(&[DatasourceId::NixDefaultNix])
250                    && inputs.has_any_file_datasource(&[
251                        DatasourceId::NixFlakeNix,
252                        DatasourceId::NixFlakeLock,
253                    ])
254            }
255            Self::BazelPrune => inputs.has_package_type(PackageType::Bazel),
256        }
257    }
258
259    fn run(
260        self,
261        files: &mut [FileInfo],
262        packages: &mut Vec<Package>,
263        dependencies: &mut Vec<TopLevelDependency>,
264        topology_plan: &topology::TopologyPlan,
265    ) {
266        match self {
267            Self::SwiftMerge => swift_merge::assemble_swift_packages(files, packages, dependencies),
268            Self::CondaRootfsMerge => {
269                conda_rootfs_merge::merge_conda_rootfs_metadata(files, packages, dependencies)
270            }
271            Self::NpmResourceAssign => {
272                npm_resource_assign::assign_npm_package_resources(files, packages)
273            }
274            Self::PythonRequirementsAssign => {
275                python_requirements_assign::assign_python_requirements_to_projects(
276                    files,
277                    packages,
278                    dependencies,
279                )
280            }
281            Self::FileReferenceResolve => {
282                file_ref_resolve::resolve_file_references(files, packages, dependencies)
283            }
284            Self::RpmYumdbMerge => file_ref_resolve::merge_rpm_yumdb_metadata(files, packages),
285            Self::NpmWorkspaceMerge => {
286                topology_plan.apply_npm_workspace_domains(files, packages, dependencies)
287            }
288            Self::CargoWorkspaceMerge => {
289                topology_plan.apply_cargo_workspace_domains(files, packages, dependencies)
290            }
291            Self::NugetCpmResolve => {
292                nuget_cpm_resolve::resolve_nuget_cpm_versions(files, dependencies)
293            }
294            Self::CargoResourceAssign => {
295                cargo_resource_assign::assign_cargo_package_resources(files, packages)
296            }
297            Self::ComposerResourceAssign => {
298                composer_resource_assign::assign_composer_package_resources(files, packages)
299            }
300            Self::RubyResourceAssign => {
301                ruby_resource_assign::assign_ruby_package_resources(files, packages)
302            }
303            Self::NixFlakeCompatMerge => {
304                nix_flake_compat_merge::attach_flake_compat_default_files(files, packages)
305            }
306            Self::BazelPrune => {
307                bazel_prune::prune_unused_bazel_packages(files, packages, dependencies)
308            }
309        }
310    }
311}
312
313pub static ASSEMBLERS: &[AssemblerConfig] = &[
314    // ── Sibling-merge assemblers ──
315    //
316    // npm ecosystem: package.json + lockfiles in same directory.
317    // NOTE: npm-shrinkwrap.json emits "npm_package_lock_json" as its datasource_id,
318    // so "npm_shrinkwrap_json" is NOT a real datasource_id.
319    AssemblerConfig {
320        datasource_ids: &[
321            DatasourceId::BunLock,
322            DatasourceId::BunLockb,
323            DatasourceId::NpmPackageJson,
324            DatasourceId::NpmPackageLockJson,
325            DatasourceId::YarnLock,
326            DatasourceId::YarnLockV1,
327            DatasourceId::YarnLockV2,
328            DatasourceId::YarnPnpCjs,
329            DatasourceId::PnpmLockYaml,
330            DatasourceId::PnpmWorkspaceYaml,
331        ],
332        sibling_file_patterns: &[
333            "package.json",
334            "bun.lock",
335            "bun.lockb",
336            ".package-lock.json",
337            "package-lock.json",
338            ".npm-shrinkwrap.json",
339            "npm-shrinkwrap.json",
340            "yarn.lock",
341            ".pnp.cjs",
342            "pnpm-lock.yaml",
343            "shrinkwrap.yaml",
344            "pnpm-workspace.yaml",
345        ],
346        mode: AssemblyMode::SiblingMerge,
347    },
348    // Rust/Cargo ecosystem
349    AssemblerConfig {
350        datasource_ids: &[DatasourceId::CargoToml, DatasourceId::CargoLock],
351        sibling_file_patterns: &["Cargo.toml", "Cargo.lock"],
352        mode: AssemblyMode::SiblingMerge,
353    },
354    // Julia ecosystem
355    AssemblerConfig {
356        datasource_ids: &[
357            DatasourceId::JuliaProjectToml,
358            DatasourceId::JuliaManifestToml,
359        ],
360        sibling_file_patterns: &["Project.toml", "Manifest.toml"],
361        mode: AssemblyMode::SiblingMerge,
362    },
363    // Erlang/OTP Rebar ecosystem
364    AssemblerConfig {
365        datasource_ids: &[DatasourceId::RebarConfig, DatasourceId::RebarLock],
366        sibling_file_patterns: &["rebar.config", "rebar.lock"],
367        mode: AssemblyMode::SiblingMerge,
368    },
369    // Carthage ecosystem
370    AssemblerConfig {
371        datasource_ids: &[
372            DatasourceId::CarthageCartfile,
373            DatasourceId::CarthageCartfileResolved,
374        ],
375        sibling_file_patterns: &["Cartfile", "Cartfile.private", "Cartfile.resolved"],
376        mode: AssemblyMode::SiblingMerge,
377    },
378    // CocoaPods ecosystem
379    AssemblerConfig {
380        datasource_ids: &[
381            DatasourceId::CocoapodsPodspec,
382            DatasourceId::CocoapodsPodspecJson,
383            DatasourceId::CocoapodsPodfile,
384            DatasourceId::CocoapodsPodfileLock,
385        ],
386        sibling_file_patterns: &["*.podspec", "*.podspec.json", "Podfile", "Podfile.lock"],
387        mode: AssemblyMode::SiblingMerge,
388    },
389    // PHP Composer ecosystem
390    AssemblerConfig {
391        datasource_ids: &[DatasourceId::PhpComposerJson, DatasourceId::PhpComposerLock],
392        sibling_file_patterns: &[
393            "*composer.json",
394            "composer.*.json",
395            "*composer.lock",
396            "composer.*.lock",
397        ],
398        mode: AssemblyMode::SiblingMerge,
399    },
400    // Go ecosystem (includes legacy Godeps)
401    AssemblerConfig {
402        datasource_ids: &[
403            DatasourceId::GoMod,
404            DatasourceId::GoModGraph,
405            DatasourceId::GoSum,
406            DatasourceId::GoWork,
407            DatasourceId::Godeps,
408        ],
409        sibling_file_patterns: &[
410            "go.mod",
411            "go.work",
412            "go.mod.graph",
413            "go.modgraph",
414            "go.sum",
415            "Godeps.json",
416        ],
417        mode: AssemblyMode::SiblingMerge,
418    },
419    // Dart/Flutter ecosystem
420    AssemblerConfig {
421        datasource_ids: &[DatasourceId::PubspecYaml, DatasourceId::PubspecLock],
422        sibling_file_patterns: &["pubspec.yaml", "pubspec.lock"],
423        mode: AssemblyMode::SiblingMerge,
424    },
425    // Pixi ecosystem
426    AssemblerConfig {
427        datasource_ids: &[DatasourceId::PixiToml, DatasourceId::PixiLock],
428        sibling_file_patterns: &["pixi.toml", "pixi.lock"],
429        mode: AssemblyMode::SiblingMerge,
430    },
431    AssemblerConfig {
432        datasource_ids: &[DatasourceId::NixFlakeNix, DatasourceId::NixFlakeLock],
433        sibling_file_patterns: &["flake.nix", "flake.lock"],
434        mode: AssemblyMode::SiblingMerge,
435    },
436    AssemblerConfig {
437        datasource_ids: &[DatasourceId::NixDefaultNix],
438        sibling_file_patterns: &["default.nix"],
439        mode: AssemblyMode::OnePerPackageData,
440    },
441    // Helm chart ecosystem
442    AssemblerConfig {
443        datasource_ids: &[DatasourceId::HelmChartYaml, DatasourceId::HelmChartLock],
444        sibling_file_patterns: &["Chart.yaml", "Chart.lock"],
445        mode: AssemblyMode::SiblingMerge,
446    },
447    AssemblerConfig {
448        datasource_ids: &[
449            DatasourceId::HackageCabal,
450            DatasourceId::HackageCabalProject,
451            DatasourceId::HackageStackYaml,
452        ],
453        sibling_file_patterns: &["*.cabal", "cabal.project", "stack.yaml"],
454        mode: AssemblyMode::SiblingMerge,
455    },
456    // Chef ecosystem
457    AssemblerConfig {
458        datasource_ids: &[
459            DatasourceId::ChefCookbookMetadataJson,
460            DatasourceId::ChefCookbookMetadataRb,
461        ],
462        sibling_file_patterns: &["metadata.json", "metadata.rb"],
463        mode: AssemblyMode::SiblingMerge,
464    },
465    // Conan (C/C++) ecosystem
466    AssemblerConfig {
467        datasource_ids: &[
468            DatasourceId::ConanConanFilePy,
469            DatasourceId::ConanConanFileTxt,
470            DatasourceId::ConanLock,
471            DatasourceId::ConanConanDataYml,
472        ],
473        sibling_file_patterns: &[
474            "conanfile.py",
475            "conanfile.txt",
476            "conan.lock",
477            "conandata.yml",
478        ],
479        mode: AssemblyMode::SiblingMerge,
480    },
481    // Maven/Java ecosystem (nested merge via META-INF)
482    AssemblerConfig {
483        datasource_ids: &[
484            DatasourceId::MavenPom,
485            DatasourceId::MavenPomProperties,
486            DatasourceId::JavaJarManifest,
487            DatasourceId::JavaOsgiManifest,
488        ],
489        sibling_file_patterns: &[
490            "pom.xml",
491            "*.pom",
492            "pom.properties",
493            "**/META-INF/MANIFEST.MF",
494        ],
495        mode: AssemblyMode::SiblingMerge,
496    },
497    AssemblerConfig {
498        datasource_ids: &[DatasourceId::PypiWheel, DatasourceId::PypiPipOriginJson],
499        sibling_file_patterns: &["*.whl", "origin.json"],
500        mode: AssemblyMode::SiblingMerge,
501    },
502    // Python/PyPI ecosystem
503    AssemblerConfig {
504        datasource_ids: &[
505            DatasourceId::PypiPyprojectToml,
506            DatasourceId::PypiPoetryPyprojectToml,
507            DatasourceId::PypiSetupPy,
508            DatasourceId::PypiSetupCfg,
509            DatasourceId::PypiWheelMetadata,
510            DatasourceId::PypiEgg,
511            DatasourceId::PypiEggPkginfo,
512            DatasourceId::PypiEditableEggPkginfo,
513            DatasourceId::PypiJson,
514            DatasourceId::PypiSdist,
515            DatasourceId::PypiSdistPkginfo,
516            DatasourceId::PypiInspectDeplock,
517            DatasourceId::PipRequirements,
518            DatasourceId::PypiPoetryLock,
519            DatasourceId::PypiPylockToml,
520            DatasourceId::PypiUvLock,
521            DatasourceId::Pipfile,
522            DatasourceId::PipfileLock,
523        ],
524        sibling_file_patterns: &[
525            "pyproject.toml",
526            "setup.py",
527            "setup.cfg",
528            "PKG-INFO",
529            "METADATA",
530            "pypi.json",
531            "pip-inspect.deplock",
532            "*.tar.gz",
533            "*.tgz",
534            "*.tar.bz2",
535            "*.tar.xz",
536            "*.zip",
537            "requirements*.txt",
538            "Pipfile",
539            "Pipfile.lock",
540            "poetry.lock",
541            "pylock.toml",
542            "pylock.*.toml",
543            "uv.lock",
544        ],
545        mode: AssemblyMode::SiblingMerge,
546    },
547    AssemblerConfig {
548        datasource_ids: &[DatasourceId::DenoJson, DatasourceId::DenoLock],
549        sibling_file_patterns: &["deno.json", "deno.jsonc", "deno.lock"],
550        mode: AssemblyMode::SiblingMerge,
551    },
552    // Ruby/RubyGems ecosystem
553    AssemblerConfig {
554        datasource_ids: &[
555            DatasourceId::GemArchiveExtracted,
556            DatasourceId::Gemspec,
557            DatasourceId::GemspecExtracted,
558            DatasourceId::Gemfile,
559            DatasourceId::GemfileExtracted,
560            DatasourceId::GemfileLock,
561            DatasourceId::GemfileLockExtracted,
562        ],
563        sibling_file_patterns: &[
564            "metadata.gz-extract",
565            "**/data.gz-extract/*.gemspec",
566            "**/data.gz-extract/Gemfile",
567            "**/data.gz-extract/Gemfile.lock",
568            "*.gemspec",
569            "Gemfile",
570            "Gemfile.lock",
571        ],
572        mode: AssemblyMode::SiblingMerge,
573    },
574    AssemblerConfig {
575        datasource_ids: &[DatasourceId::GemArchive],
576        sibling_file_patterns: &["*.gem"],
577        mode: AssemblyMode::OnePerPackageData,
578    },
579    // Conda ecosystem
580    AssemblerConfig {
581        datasource_ids: &[
582            DatasourceId::CondaMetaYaml,
583            DatasourceId::CondaYaml,
584            DatasourceId::CondaMetaJson,
585        ],
586        sibling_file_patterns: &[
587            "meta.yaml",
588            "meta.yml",
589            "recipe.yaml",
590            "recipe.yml",
591            "environment.yml",
592            "environment.yaml",
593            "conda.yaml",
594            "conda.yml",
595            "*conda*.yaml",
596            "*conda*.yml",
597            "env.yaml",
598            "env.yml",
599            "*env*.yaml",
600            "*env*.yml",
601            "*environment*.yaml",
602            "*environment*.yml",
603            "*.json",
604        ],
605        mode: AssemblyMode::SiblingMerge,
606    },
607    // RPM specfile (source packages)
608    AssemblerConfig {
609        datasource_ids: &[DatasourceId::RpmSpecfile],
610        sibling_file_patterns: &["*.spec"],
611        mode: AssemblyMode::OnePerPackageData,
612    },
613    // Debian source packages (nested merge via debian/ directory)
614    AssemblerConfig {
615        datasource_ids: &[
616            DatasourceId::DebianControlInSource,
617            DatasourceId::DebianCopyrightInSource,
618        ],
619        sibling_file_patterns: &["control", "copyright"],
620        mode: AssemblyMode::SiblingMerge,
621    },
622    // Gradle/Android ecosystem
623    AssemblerConfig {
624        datasource_ids: &[DatasourceId::BuildGradle, DatasourceId::GradleLockfile],
625        sibling_file_patterns: &["build.gradle", "build.gradle.kts", "gradle.lockfile"],
626        mode: AssemblyMode::SiblingMerge,
627    },
628    AssemblerConfig {
629        datasource_ids: &[DatasourceId::GradleModule],
630        sibling_file_patterns: &["*.module"],
631        mode: AssemblyMode::OnePerPackageData,
632    },
633    // Hugging Face model/dataset metadata. The model-card README.md, Transformers
634    // config.json, and Diffusers model_index.json in one repository directory
635    // describe a single logical model, so they are merged into one package by a
636    // dedicated directory merger (see huggingface_merge). Only files the Hugging
637    // Face parsers actually claim (carrying a huggingface datasource id)
638    // participate, so a generic README.md/config.json never triggers a merge.
639    AssemblerConfig {
640        datasource_ids: &[
641            DatasourceId::HuggingfaceModelCard,
642            DatasourceId::HuggingfaceConfigJson,
643            DatasourceId::HuggingfaceModelIndexJson,
644        ],
645        sibling_file_patterns: &["README.md", "config.json", "model_index.json"],
646        mode: AssemblyMode::SiblingMerge,
647    },
648    // CPAN/Perl ecosystem
649    AssemblerConfig {
650        datasource_ids: &[
651            DatasourceId::CpanMetaJson,
652            DatasourceId::CpanMetaYml,
653            DatasourceId::CpanManifest,
654            DatasourceId::CpanDistIni,
655            DatasourceId::CpanMakefile,
656        ],
657        sibling_file_patterns: &[
658            "META.json",
659            "META.yml",
660            "MANIFEST",
661            "dist.ini",
662            "Makefile.PL",
663        ],
664        mode: AssemblyMode::SiblingMerge,
665    },
666    // NuGet/.NET ecosystem
667    AssemblerConfig {
668        datasource_ids: &[
669            DatasourceId::NugetCsproj,
670            DatasourceId::NugetFsproj,
671            DatasourceId::NugetNuspec,
672            DatasourceId::NugetNupkg,
673            DatasourceId::NugetProjectJson,
674            DatasourceId::NugetProjectLockJson,
675            DatasourceId::NugetPackagesConfig,
676            DatasourceId::NugetPackagesLock,
677            DatasourceId::NugetVbproj,
678        ],
679        sibling_file_patterns: &[
680            "*.csproj",
681            "*.fsproj",
682            "*.nuspec",
683            "*.nupkg",
684            "project.json",
685            "project.lock.json",
686            "packages.config",
687            "packages.lock.json",
688            "*.packages.lock.json",
689            "*.vbproj",
690        ],
691        mode: AssemblyMode::SiblingMerge,
692    },
693    AssemblerConfig {
694        datasource_ids: &[DatasourceId::NugetDepsJson],
695        sibling_file_patterns: &["*.deps.json"],
696        mode: AssemblyMode::OnePerPackageData,
697    },
698    // Swift/SPM ecosystem
699    AssemblerConfig {
700        datasource_ids: &[
701            DatasourceId::SwiftPackageManifestJson,
702            DatasourceId::SwiftPackageResolved,
703            DatasourceId::SwiftPackageShowDependencies,
704        ],
705        sibling_file_patterns: &[
706            "Package.swift.json",
707            "Package.swift.deplock",
708            "Package.resolved",
709            ".package.resolved",
710            "swift-show-dependencies.deplock",
711        ],
712        mode: AssemblyMode::SiblingMerge,
713    },
714    // ── Standalone assemblers (single file → single package) ──
715    //
716    // These ecosystems have only one manifest file type with no sibling merging.
717    // They still need configs so their datasource_ids are recognized by the assembler.
718    //
719    // Bower (JavaScript)
720    AssemblerConfig {
721        datasource_ids: &[DatasourceId::BowerJson],
722        sibling_file_patterns: &["bower.json"],
723        mode: AssemblyMode::SiblingMerge,
724    },
725    // CRAN (R language)
726    AssemblerConfig {
727        datasource_ids: &[DatasourceId::CranDescription],
728        sibling_file_patterns: &["DESCRIPTION"],
729        mode: AssemblyMode::SiblingMerge,
730    },
731    // FreeBSD packages
732    AssemblerConfig {
733        datasource_ids: &[DatasourceId::FreebsdCompactManifest],
734        sibling_file_patterns: &["+COMPACT_MANIFEST"],
735        mode: AssemblyMode::SiblingMerge,
736    },
737    // Haxe ecosystem
738    AssemblerConfig {
739        datasource_ids: &[DatasourceId::HaxelibJson],
740        sibling_file_patterns: &["haxelib.json"],
741        mode: AssemblyMode::SiblingMerge,
742    },
743    AssemblerConfig {
744        datasource_ids: &[DatasourceId::Gitmodules],
745        sibling_file_patterns: &[".gitmodules"],
746        mode: AssemblyMode::SiblingMerge,
747    },
748    // OCaml/opam ecosystem
749    AssemblerConfig {
750        datasource_ids: &[DatasourceId::OpamFile],
751        sibling_file_patterns: &["opam", "*.opam"],
752        mode: AssemblyMode::SiblingMerge,
753    },
754    AssemblerConfig {
755        datasource_ids: &[DatasourceId::RpmYumdb],
756        sibling_file_patterns: &["**/var/lib/yum/yumdb/*/*/from_repo"],
757        mode: AssemblyMode::OnePerPackageData,
758    },
759    // Microsoft Update Manifest
760    AssemblerConfig {
761        datasource_ids: &[DatasourceId::MicrosoftUpdateManifestMum],
762        sibling_file_patterns: &["*.mum"],
763        mode: AssemblyMode::SiblingMerge,
764    },
765    // Autotools (C/C++ build system)
766    AssemblerConfig {
767        datasource_ids: &[DatasourceId::AutotoolsConfigure],
768        sibling_file_patterns: &["configure", "configure.ac"],
769        mode: AssemblyMode::SiblingMerge,
770    },
771    // Bazel (build system)
772    AssemblerConfig {
773        datasource_ids: &[DatasourceId::BazelBuild],
774        sibling_file_patterns: &["BUILD"],
775        mode: AssemblyMode::SiblingMerge,
776    },
777    AssemblerConfig {
778        datasource_ids: &[DatasourceId::BazelModule],
779        sibling_file_patterns: &["MODULE.bazel"],
780        mode: AssemblyMode::OnePerPackageData,
781    },
782    // Buck (build system)
783    AssemblerConfig {
784        datasource_ids: &[DatasourceId::BuckFile, DatasourceId::BuckMetadata],
785        sibling_file_patterns: &["BUCK", "METADATA.bzl", ".buckconfig"],
786        mode: AssemblyMode::SiblingMerge,
787    },
788    // Ant/Ivy (Java dependency management)
789    AssemblerConfig {
790        datasource_ids: &[DatasourceId::AntIvyXml],
791        sibling_file_patterns: &["ivy.xml"],
792        mode: AssemblyMode::SiblingMerge,
793    },
794    // JVM archives introspected in place (one archive == one package).
795    AssemblerConfig {
796        datasource_ids: &[
797            DatasourceId::JavaJar,
798            DatasourceId::JavaWarArchive,
799            DatasourceId::AndroidAarLibrary,
800        ],
801        sibling_file_patterns: &["*.jar", "*.war", "*.aar"],
802        mode: AssemblyMode::OnePerPackageData,
803    },
804    // Meteor (JavaScript platform)
805    AssemblerConfig {
806        datasource_ids: &[DatasourceId::MeteorPackage],
807        sibling_file_patterns: &["package.js"],
808        mode: AssemblyMode::SiblingMerge,
809    },
810    // ── One-per-PackageData assemblers (database files with many packages) ──
811    //
812    // Alpine installed package database
813    AssemblerConfig {
814        datasource_ids: &[DatasourceId::AlpineInstalledDb],
815        sibling_file_patterns: &["installed"],
816        mode: AssemblyMode::OnePerPackageData,
817    },
818    AssemblerConfig {
819        datasource_ids: &[DatasourceId::AlpineApkbuild],
820        sibling_file_patterns: &["APKBUILD"],
821        mode: AssemblyMode::SiblingMerge,
822    },
823    // RPM installed package databases (BDB, NDB, SQLite)
824    AssemblerConfig {
825        datasource_ids: &[
826            DatasourceId::RpmInstalledDatabaseBdb,
827            DatasourceId::RpmInstalledDatabaseNdb,
828            DatasourceId::RpmInstalledDatabaseSqlite,
829            DatasourceId::RpmMarinerManifest,
830        ],
831        sibling_file_patterns: &[
832            "Packages",
833            "Packages.db",
834            "rpmdb.sqlite",
835            "container-manifest-2",
836        ],
837        mode: AssemblyMode::OnePerPackageData,
838    },
839    AssemblerConfig {
840        datasource_ids: &[DatasourceId::RpmArchive],
841        sibling_file_patterns: &["*.rpm", "*.srpm"],
842        mode: AssemblyMode::OnePerPackageData,
843    },
844    // Debian installed package databases
845    AssemblerConfig {
846        datasource_ids: &[DatasourceId::DebianDeb],
847        sibling_file_patterns: &["*.deb"],
848        mode: AssemblyMode::OnePerPackageData,
849    },
850    AssemblerConfig {
851        datasource_ids: &[
852            DatasourceId::DebianInstalledStatusDb,
853            DatasourceId::DebianDistrolessInstalledDb,
854        ],
855        sibling_file_patterns: &["status"],
856        mode: AssemblyMode::OnePerPackageData,
857    },
858    AssemblerConfig {
859        datasource_ids: &[
860            DatasourceId::DebianControlExtractedDeb,
861            DatasourceId::DebianMd5SumsInExtractedDeb,
862        ],
863        sibling_file_patterns: &["control", "md5sums"],
864        mode: AssemblyMode::SiblingMerge,
865    },
866    AssemblerConfig {
867        datasource_ids: &[DatasourceId::DebianSourceControlDsc],
868        sibling_file_patterns: &["*.dsc"],
869        mode: AssemblyMode::OnePerPackageData,
870    },
871    AssemblerConfig {
872        datasource_ids: &[DatasourceId::AboutFile],
873        sibling_file_patterns: &["*.ABOUT"],
874        mode: AssemblyMode::OnePerPackageData,
875    },
876    AssemblerConfig {
877        datasource_ids: &[
878            DatasourceId::BitbakeRecipe,
879            DatasourceId::BitbakeRecipeAppend,
880        ],
881        sibling_file_patterns: &["*.bb", "*.bbappend"],
882        mode: AssemblyMode::SiblingMerge,
883    },
884];
885
886// Datasource IDs intentionally excluded from package assembly.
887//
888// This list is runtime-significant: files with these datasource IDs may remain
889// unowned by any Package, while their dependencies are still eligible for
890// top-level hoisting. Tests also use it to enforce explicit assembly accounting.
891pub static UNASSEMBLED_DATASOURCE_IDS: &[DatasourceId] = &[
892    // Non-package metadata
893    DatasourceId::Readme,
894    DatasourceId::EtcOsRelease,
895    // Binary archives (require external extraction via ExtractCode before scanning)
896    DatasourceId::AlpineApkArchive,
897    DatasourceId::AndroidAab,
898    DatasourceId::AndroidApk,
899    DatasourceId::AndroidManifestXml,
900    DatasourceId::AndroidSoongMetadata,
901    DatasourceId::AppleDmg,
902    DatasourceId::Axis2Mar,
903    DatasourceId::ChromeCrx,
904    DatasourceId::DebianOriginalSourceTarball,
905    DatasourceId::DebianSourceMetadataTarball,
906    DatasourceId::InstallshieldInstaller,
907    DatasourceId::IosIpa,
908    DatasourceId::IsoDiskImage,
909    DatasourceId::JavaEarArchive,
910    DatasourceId::JbossSar,
911    DatasourceId::MicrosoftCabinet,
912    DatasourceId::MozillaXpi,
913    DatasourceId::NsisInstaller,
914    DatasourceId::SharShellArchive,
915    DatasourceId::SquashfsDiskImage,
916    // Supplementary metadata (not primary package definitions)
917    DatasourceId::ArchAurinfo,
918    DatasourceId::ArchPkginfo,
919    DatasourceId::ArchSrcinfo,
920    DatasourceId::Axis2ModuleXml,
921    DatasourceId::ClojureDepsEdn,
922    DatasourceId::ClojureProjectClj,
923    DatasourceId::DebianInstalledFilesList,
924    DatasourceId::DebianInstalledMd5Sums,
925    DatasourceId::DebianCopyright,
926    DatasourceId::DebianCopyrightInPackage,
927    DatasourceId::DebianCopyrightStandalone,
928    DatasourceId::GoBinary,
929    DatasourceId::WindowsExecutable,
930    DatasourceId::Dockerfile,
931    DatasourceId::OciImageIndex,
932    DatasourceId::OciImageManifest,
933    DatasourceId::ErlangOtpAppSrc,
934    DatasourceId::HexMixLock,
935    DatasourceId::JavaEarApplicationXml,
936    DatasourceId::JavaWarWebXml,
937    DatasourceId::JbossServiceXml,
938    DatasourceId::MesonBuild,
939    DatasourceId::GemGemspecInstalledSpecifications,
940    DatasourceId::NugetDirectoryBuildProps,
941    DatasourceId::NugetDirectoryPackagesProps,
942    DatasourceId::CitationCff,
943    DatasourceId::PubliccodeYaml,
944    DatasourceId::RpmPackageLicenses,
945    DatasourceId::RustBinary,
946    DatasourceId::SbtBuildSbt,
947    DatasourceId::VcpkgJson,
948];
949
950#[cfg(test)]
951mod tests {
952    use super::*;
953    use std::collections::HashSet;
954    use strum::IntoEnumIterator;
955
956    #[test]
957    fn test_every_datasource_id_is_accounted_for() {
958        let mut assembled: HashSet<DatasourceId> = HashSet::new();
959        for config in ASSEMBLERS {
960            for &dsid in config.datasource_ids {
961                assembled.insert(dsid);
962            }
963        }
964
965        let unassembled: HashSet<DatasourceId> =
966            UNASSEMBLED_DATASOURCE_IDS.iter().copied().collect();
967
968        let overlap: Vec<_> = assembled.intersection(&unassembled).collect();
969        assert!(
970            overlap.is_empty(),
971            "Datasource IDs in BOTH ASSEMBLERS and UNASSEMBLED: {overlap:?}"
972        );
973
974        let missing: Vec<_> = DatasourceId::iter()
975            .filter(|dsid| !assembled.contains(dsid) && !unassembled.contains(dsid))
976            .collect();
977
978        assert!(
979            missing.is_empty(),
980            "Datasource IDs in NEITHER ASSEMBLERS nor UNASSEMBLED: {missing:?}\n\
981             Add each to an AssemblerConfig in ASSEMBLERS, or to UNASSEMBLED_DATASOURCE_IDS."
982        );
983    }
984
985    #[test]
986    fn test_post_assembly_passes_are_unique() {
987        let unique: HashSet<PostAssemblyPassKind> = POST_ASSEMBLY_PASSES.iter().copied().collect();
988
989        assert_eq!(
990            unique.len(),
991            POST_ASSEMBLY_PASSES.len(),
992            "POST_ASSEMBLY_PASSES contains duplicate entries"
993        );
994    }
995
996    #[test]
997    fn test_every_post_assembly_pass_kind_is_registered_once() {
998        let registered: HashSet<PostAssemblyPassKind> =
999            POST_ASSEMBLY_PASSES.iter().copied().collect();
1000
1001        let missing: Vec<_> = PostAssemblyPassKind::iter()
1002            .filter(|pass| !registered.contains(pass))
1003            .collect();
1004
1005        assert!(
1006            missing.is_empty(),
1007            "Post-assembly pass variants not registered in POST_ASSEMBLY_PASSES: {missing:?}"
1008        );
1009
1010        for pass in PostAssemblyPassKind::iter() {
1011            let count = POST_ASSEMBLY_PASSES
1012                .iter()
1013                .filter(|registered| **registered == pass)
1014                .count();
1015            assert_eq!(
1016                count, 1,
1017                "Post-assembly pass {pass:?} should be registered exactly once"
1018            );
1019        }
1020    }
1021
1022    #[test]
1023    fn test_post_assembly_passes_skip_irrelevant_inputs() {
1024        let inputs = PostAssemblyInputs::default();
1025
1026        for pass in PostAssemblyPassKind::iter() {
1027            assert!(
1028                !pass.should_run(&inputs),
1029                "{pass:?} should skip when no relevant inputs are present"
1030            );
1031        }
1032    }
1033
1034    #[test]
1035    fn test_npm_workspace_inputs_only_run_npm_passes() {
1036        let inputs = PostAssemblyInputs {
1037            package_types: HashSet::from([PackageType::Npm]),
1038            file_datasource_ids: HashSet::from([DatasourceId::NpmPackageJson]),
1039            has_npm_workspace_markers: true,
1040            has_cargo_workspace_markers: false,
1041        };
1042
1043        let runnable: HashSet<_> = PostAssemblyPassKind::iter()
1044            .filter(|pass| pass.should_run(&inputs))
1045            .collect();
1046
1047        assert_eq!(
1048            runnable,
1049            HashSet::from([
1050                PostAssemblyPassKind::NpmResourceAssign,
1051                PostAssemblyPassKind::NpmWorkspaceMerge,
1052            ])
1053        );
1054    }
1055
1056    #[test]
1057    fn test_cargo_workspace_merge_requires_workspace_markers() {
1058        let without_markers = PostAssemblyInputs {
1059            package_types: HashSet::from([PackageType::Cargo]),
1060            file_datasource_ids: HashSet::from([DatasourceId::CargoToml]),
1061            has_npm_workspace_markers: false,
1062            has_cargo_workspace_markers: false,
1063        };
1064
1065        assert!(!PostAssemblyPassKind::CargoWorkspaceMerge.should_run(&without_markers));
1066
1067        let with_markers = PostAssemblyInputs {
1068            has_cargo_workspace_markers: true,
1069            ..without_markers
1070        };
1071
1072        assert!(PostAssemblyPassKind::CargoWorkspaceMerge.should_run(&with_markers));
1073    }
1074}
provenant/assembly/assemblers.rs

provenant/assembly/
assemblers.rs
