miden_assembly/
project.rs

1use alloc::{boxed::Box, collections::BTreeMap, format, string::ToString, sync::Arc, vec::Vec};
2use std::{
3    fs,
4    path::{Path as FsPath, PathBuf},
5};
6
7use miden_assembly_syntax::{
8    ModuleParser,
9    ast::{self, ModuleKind, Path as MasmPath},
10    diagnostics::Report,
11};
12use miden_core::serde::{Deserializable, Serializable};
13use miden_mast_package::{Package as MastPackage, Section, SectionId, TargetType};
14use miden_package_registry::{PackageId, PackageStore, Version as PackageVersion};
15use miden_project::{
16    Linkage, Package as ProjectPackage, Profile, ProjectDependencyNodeProvenance, ProjectSource,
17    ProjectSourceOrigin, Target,
18};
19
20use crate::{Assembler, assembler::debuginfo::DebugInfoSections, ast::Module};
21
22mod build_provenance;
23mod dependency_graph;
24mod package_ext;
25mod runtime_dependencies;
26mod target_selector;
27
28use build_provenance::PackageBuildProvenance;
29use dependency_graph::DependencyGraph;
30use package_ext::ProjectPackageExt;
31use runtime_dependencies::RuntimeDependencies;
32pub use target_selector::ProjectTargetSelector;
33
34#[cfg(test)]
35mod tests;
36
37// ASSEMBLER EXTENSIONS
38// ================================================================================================
39
40impl Assembler {
41    /// Get a [ProjectAssembler] configured for the project whose manifest is at `manifest_path`.
42    pub fn for_project_at_path<'a, S>(
43        self,
44        manifest_path: impl AsRef<FsPath>,
45        store: &'a mut S,
46    ) -> Result<ProjectAssembler<'a, S>, Report>
47    where
48        S: PackageStore + ?Sized,
49    {
50        let manifest_path = manifest_path.as_ref();
51        let source_manager = self.source_manager();
52        let project = miden_project::Project::load(manifest_path, &source_manager)?;
53        let package = project.package();
54        let dependency_graph =
55            DependencyGraph::from_project_path(manifest_path, store, source_manager)?;
56
57        Ok(ProjectAssembler {
58            assembler: self,
59            project: package,
60            dependency_graph,
61            store,
62        })
63    }
64
65    /// Get a [ProjectAssembler] configured for `project`
66    pub fn for_project<'a, S>(
67        self,
68        project: Arc<ProjectPackage>,
69        store: &'a mut S,
70    ) -> Result<ProjectAssembler<'a, S>, Report>
71    where
72        S: PackageStore + ?Sized,
73    {
74        let source_manager = self.source_manager();
75        let dependency_graph =
76            DependencyGraph::from_project(project.clone(), store, source_manager)?;
77        Ok(ProjectAssembler {
78            assembler: self,
79            project,
80            dependency_graph,
81            store,
82        })
83    }
84}
85
86// PROJECT ASSEMBLER
87// ================================================================================================
88
89pub struct ProjectSourceInputs {
90    pub root: Box<Module>,
91    pub support: Vec<Box<Module>>,
92}
93
94pub struct ProjectAssembler<'a, S: PackageStore + ?Sized> {
95    assembler: Assembler,
96    project: Arc<ProjectPackage>,
97    dependency_graph: DependencyGraph,
98    store: &'a mut S,
99}
100
101impl<'a, S> ProjectAssembler<'a, S>
102where
103    S: PackageStore + ?Sized,
104{
105    pub fn project(&self) -> &ProjectPackage {
106        self.project.as_ref()
107    }
108
109    pub fn assemble(
110        &mut self,
111        target: ProjectTargetSelector<'_>,
112        profile: &str,
113    ) -> Result<Arc<MastPackage>, Report> {
114        self.assemble_impl(target, profile, None)
115    }
116
117    pub fn assemble_with_sources(
118        &mut self,
119        target: ProjectTargetSelector<'_>,
120        profile: &str,
121        sources: ProjectSourceInputs,
122    ) -> Result<Arc<MastPackage>, Report> {
123        self.assemble_impl(target, profile, Some(sources))
124    }
125
126    fn assemble_impl(
127        &mut self,
128        target_selector: ProjectTargetSelector<'_>,
129        profile_name: &str,
130        sources: Option<ProjectSourceInputs>,
131    ) -> Result<Arc<MastPackage>, Report> {
132        let target = target_selector.select_target(self.project.as_ref())?;
133
134        // When building an executable target from a project with a library target, we require
135        // that the executable target be linked statically against the library target
136        let mut cache = BTreeMap::new();
137        let root_id = self.dependency_graph.root().clone();
138        let required_lib = if target.is_executable()
139            && let Some(library_target) =
140                self.project.library_target().map(|target| target.inner().clone())
141        {
142            Some(self.assemble_source_package(
143                root_id.clone(),
144                Arc::clone(&self.project),
145                &library_target,
146                profile_name,
147                None,
148                None,
149                &mut cache,
150            )?)
151        } else {
152            None
153        };
154
155        self.assemble_source_package(
156            root_id,
157            Arc::clone(&self.project),
158            &target,
159            profile_name,
160            required_lib,
161            sources,
162            &mut cache,
163        )
164        .map(|resolved| resolved.package)
165    }
166
167    fn assemble_source_package(
168        &mut self,
169        package_id: PackageId,
170        project: Arc<ProjectPackage>,
171        target: &Target,
172        profile_name: &str,
173        required_lib: Option<ResolvedPackage>,
174        sources: Option<ProjectSourceInputs>,
175        cache: &mut BTreeMap<PackageId, ResolvedPackage>,
176    ) -> Result<ResolvedPackage, Report> {
177        let cache_key = project.target_package_name(target);
178        if sources.is_none()
179            && let Some(package) = cache.get(&cache_key).cloned()
180        {
181            assert_eq!(package.package.kind, target.ty);
182            return Ok(package);
183        }
184
185        let profile = project.resolve_profile(profile_name)?;
186        let mut assembler = self
187            .assembler
188            .clone()
189            .with_emit_debug_info(profile.should_emit_debug_info())
190            .with_trim_paths(profile.should_trim_paths());
191        let mut runtime_dependencies = RuntimeDependencies::default();
192        match required_lib {
193            Some(required_lib) if required_lib.package.is_kernel() => {
194                assembler.link_package(required_lib.package.clone(), Linkage::Dynamic)?;
195                runtime_dependencies.record_linked_kernel_dependency(required_lib.package)?;
196            },
197            Some(required_lib) => {
198                assembler.link_package(required_lib.package.clone(), Linkage::Static)?;
199                if let Some(kernel_package) = required_lib.linked_kernel_package {
200                    runtime_dependencies.record_linked_kernel_dependency(kernel_package)?;
201                }
202            },
203            None => (),
204        }
205
206        let node = self.dependency_graph.get(&package_id)?;
207        let dependencies = node.dependencies.clone();
208        for edge in dependencies.iter() {
209            let dependency_package =
210                self.resolve_dependency_package(&edge.dependency, profile_name, cache)?;
211            if !dependency_package.package.is_library() {
212                return Err(Report::msg(format!(
213                    "dependency '{}' resolved to executable package '{}', but only library-like packages can be linked",
214                    edge.dependency, dependency_package.package.name
215                )));
216            }
217
218            assembler.link_package(dependency_package.package.clone(), edge.linkage)?;
219            runtime_dependencies.merge_package(dependency_package, edge.linkage)?;
220        }
221
222        let has_provided_sources = sources.is_some();
223        let LoadedTargetSources { root, support } = match sources {
224            Some(sources) => self.normalize_provided_sources(target, sources)?,
225            None => self.load_target_sources(project.as_ref(), target)?,
226        };
227
228        let product = match target.ty {
229            TargetType::Executable => assembler.assemble_executable_modules(root, support)?,
230            TargetType::Kernel => {
231                if !support.is_empty() {
232                    assembler.compile_and_statically_link_all(support)?;
233                }
234                assembler.assemble_kernel_module(root)?
235            },
236            _ if target.ty.is_library() => {
237                let mut modules = Vec::with_capacity(support.len() + 1);
238                modules.push(root);
239                modules.extend(support);
240                assembler.assemble_library_modules(modules, target.ty)?
241            },
242            _ => unreachable!("non-exhaustive target type"),
243        };
244
245        let manifest = product
246            .manifest()
247            .clone()
248            .with_dependencies(runtime_dependencies.deps.into_values())
249            .expect("assembled package manifest should have unique runtime dependencies");
250        let debug_info = product.debug_info().cloned();
251
252        // Emit custom sections
253        let mut sections = Vec::new();
254
255        // Section: build provenance
256        if let Some(provenance) = self.dependency_graph.build_source_provenance(
257            &package_id,
258            project.as_ref(),
259            target,
260            profile_name,
261            has_provided_sources,
262        )? {
263            sections.push(provenance.to_section());
264        }
265
266        // Section: embedded kernel package
267        if target.ty.is_executable()
268            && let Some(kernel_package) = runtime_dependencies.kernel.clone()
269        {
270            sections.push(linked_kernel_package_section(kernel_package.as_ref()));
271        }
272
273        // Section: debug info
274        if let Some(DebugInfoSections {
275            debug_sources_section,
276            debug_functions_section,
277            debug_types_section,
278        }) = debug_info.as_ref()
279        {
280            sections.push(Section::new(SectionId::DEBUG_SOURCES, debug_sources_section.to_bytes()));
281            sections
282                .push(Section::new(SectionId::DEBUG_FUNCTIONS, debug_functions_section.to_bytes()));
283            sections.push(Section::new(SectionId::DEBUG_TYPES, debug_types_section.to_bytes()));
284        }
285
286        let package = Arc::new(MastPackage {
287            name: project.target_package_name(target),
288            version: project.version().into_inner().clone(),
289            description: project.description().map(|description| description.to_string()),
290            kind: product.kind(),
291            mast: product.into_artifact(),
292            manifest,
293            sections,
294        });
295
296        let resolved = ResolvedPackage {
297            package: Arc::clone(&package),
298            linked_kernel_package: runtime_dependencies.kernel,
299        };
300        if !has_provided_sources {
301            cache.insert(package_id, resolved.clone());
302        }
303
304        Ok(resolved)
305    }
306
307    fn resolve_dependency_package(
308        &mut self,
309        package_id: &PackageId,
310        profile_name: &str,
311        cache: &mut BTreeMap<PackageId, ResolvedPackage>,
312    ) -> Result<ResolvedPackage, Report> {
313        if let Some(package) = cache.get(package_id).cloned() {
314            return Ok(package);
315        }
316
317        let node = self.dependency_graph.get(package_id)?;
318        let node_version = node.version.clone();
319
320        let package = match &node.provenance {
321            ProjectDependencyNodeProvenance::Source(ProjectSource::Virtual { .. }) => {
322                return Err(Report::msg(format!(
323                    "package '{package_id}' is missing a manifest path",
324                )));
325            },
326            ProjectDependencyNodeProvenance::Source(ProjectSource::Real {
327                manifest_path,
328                origin,
329                library_path: Some(_),
330                workspace_root,
331                ..
332            }) => {
333                let project = ProjectPackage::load_package(
334                    self.assembler.source_manager(),
335                    package_id,
336                    manifest_path,
337                )?;
338                let target = project
339                    .library_target()
340                    .map(|target| target.inner().clone())
341                    .ok_or_else(|| {
342                        Report::msg(format!(
343                            "dependency '{}' does not define a library target",
344                            package_id
345                        ))
346                    })?;
347                match self.try_reuse_registered_source_package(
348                    package_id,
349                    &node_version,
350                    &project,
351                    &target,
352                    profile_name,
353                    origin,
354                    manifest_path,
355                    workspace_root.as_deref(),
356                )? {
357                    RegisteredSourcePackage::Loaded(package) => ResolvedPackage {
358                        linked_kernel_package: self
359                            .resolve_linked_kernel_package(package.clone())?,
360                        package,
361                    },
362                    reuse => {
363                        let package = self.assemble_source_package(
364                            package_id.clone(),
365                            project,
366                            &target,
367                            profile_name,
368                            None,
369                            None,
370                            cache,
371                        )?;
372                        match reuse {
373                            RegisteredSourcePackage::Missing => {
374                                self.publish_source_dependency(package.package.clone())?;
375                            },
376                            RegisteredSourcePackage::IndexedButUnreadable(expected) => {
377                                let actual = PackageVersion::new(
378                                    package.package.version.clone(),
379                                    package.package.digest(),
380                                );
381                                if actual != expected {
382                                    return Err(Report::msg(format!(
383                                        "package '{}' version '{}' is already registered as '{}', but the canonical artifact could not be loaded and rebuilding from source produced '{}'; bump the semantic version or repair the package store",
384                                        package_id, node_version, expected, actual
385                                    )));
386                                }
387                            },
388                            RegisteredSourcePackage::Loaded(_) => unreachable!(),
389                        }
390                        package
391                    },
392                }
393            },
394            ProjectDependencyNodeProvenance::Source(_) => {
395                let package =
396                    self.load_canonical_package(package_id, &node_version)?.ok_or_else(|| {
397                        Report::msg(format!(
398                            "dependency '{}' version '{}' was not found in the package registry",
399                            package_id, node_version
400                        ))
401                    })?;
402                ResolvedPackage {
403                    linked_kernel_package: self.resolve_linked_kernel_package(package.clone())?,
404                    package,
405                }
406            },
407            ProjectDependencyNodeProvenance::Registry { selected, .. } => {
408                let package = self.store.load_package(package_id, selected)?;
409                ResolvedPackage {
410                    linked_kernel_package: self.resolve_linked_kernel_package(package.clone())?,
411                    package,
412                }
413            },
414            ProjectDependencyNodeProvenance::Preassembled { path, selected } => {
415                let package = load_selected_preassembled_package(path, package_id, selected)?;
416                ResolvedPackage {
417                    linked_kernel_package: self.resolve_linked_kernel_package(package.clone())?,
418                    package,
419                }
420            },
421        };
422
423        cache.insert(package_id.clone(), package.clone());
424        Ok(package)
425    }
426
427    fn resolve_linked_kernel_package(
428        &self,
429        package: Arc<MastPackage>,
430    ) -> Result<Option<Arc<MastPackage>>, Report> {
431        if package.is_kernel() {
432            return Ok(Some(package));
433        }
434
435        let Some(kernel_dependency) = package.kernel_runtime_dependency()? else {
436            return Ok(None);
437        };
438
439        let version =
440            PackageVersion::new(kernel_dependency.version.clone(), kernel_dependency.digest);
441        if self.store.get_exact_version(&kernel_dependency.name, &version).is_some() {
442            match self.store.load_package(&kernel_dependency.name, &version) {
443                Ok(kernel_package) => {
444                    if !kernel_package.is_kernel() {
445                        return Err(Report::msg(format!(
446                            "runtime kernel dependency '{}@{}#{}' resolved to non-kernel package '{}'",
447                            kernel_dependency.name,
448                            kernel_dependency.version,
449                            kernel_dependency.digest,
450                            kernel_package.name
451                        )));
452                    }
453                    return Ok(Some(kernel_package));
454                },
455                Err(load_error) => {
456                    if let Some(kernel_package) = package
457                        .try_embedded_kernel_package()
458                        .map(|kernel_package| kernel_package.map(Arc::new))?
459                    {
460                        return Ok(Some(kernel_package));
461                    }
462                    return Err(load_error);
463                },
464            }
465        }
466
467        package
468            .try_embedded_kernel_package()
469            .map(|kernel_package| kernel_package.map(Arc::new))
470    }
471
472    fn load_canonical_package(
473        &self,
474        package_id: &PackageId,
475        version: &miden_project::SemVer,
476    ) -> Result<Option<Arc<MastPackage>>, Report> {
477        let Some(record) = self.store.get_by_semver(package_id, version) else {
478            return Ok(None);
479        };
480        self.store.load_package(package_id, record.version()).map(Some)
481    }
482
483    fn try_reuse_registered_source_package(
484        &self,
485        package_id: &PackageId,
486        version: &miden_project::SemVer,
487        project: &ProjectPackage,
488        target: &Target,
489        profile_name: &str,
490        origin: &ProjectSourceOrigin,
491        manifest_path: &FsPath,
492        workspace_root: Option<&FsPath>,
493    ) -> Result<RegisteredSourcePackage, Report> {
494        let Some(record) = self.store.get_by_semver(package_id, version) else {
495            return Ok(RegisteredSourcePackage::Missing);
496        };
497        let package = match self.store.load_package(package_id, record.version()) {
498            Ok(package) => package,
499            Err(_) => {
500                return Ok(RegisteredSourcePackage::IndexedButUnreadable(record.version().clone()));
501            },
502        };
503
504        let expected = self.dependency_graph.expected_source_provenance(
505            package_id,
506            project,
507            target,
508            profile_name,
509            origin,
510            manifest_path,
511            workspace_root,
512        )?;
513
514        match PackageBuildProvenance::from_package(&package)? {
515            Some(actual) if actual == expected => Ok(()),
516            Some(actual) => Err(Report::msg(format!(
517                "package '{}' version '{}' is already registered with different source provenance (expected {}, found {}); bump the semantic version",
518                package_id,
519                version,
520                expected.describe(),
521                actual.describe(),
522            ))),
523            None => Err(Report::msg(format!(
524                "package '{}' version '{}' is already registered, but the canonical artifact is missing source provenance; bump the semantic version",
525                package_id, version
526            ))),
527        }?;
528
529        Ok(RegisteredSourcePackage::Loaded(package))
530    }
531
532    fn publish_source_dependency(&mut self, package: Arc<MastPackage>) -> Result<(), Report> {
533        self.store
534            .publish_package(package)
535            .map(|_| ())
536            .map_err(|error| Report::msg(error.to_string()))
537    }
538
539    fn normalize_provided_sources(
540        &self,
541        target: &Target,
542        sources: ProjectSourceInputs,
543    ) -> Result<LoadedTargetSources, Report> {
544        let mut root = sources.root;
545        root.set_kind(target_root_module_kind(target.ty));
546        root.set_path(target.namespace.inner().as_ref());
547
548        let support = sources
549            .support
550            .into_iter()
551            .map(|mut module| {
552                module.set_kind(ModuleKind::Library);
553                Ok(module)
554            })
555            .collect::<Result<Vec<_>, Report>>()?;
556
557        Ok(LoadedTargetSources { root, support })
558    }
559
560    fn load_target_sources(
561        &self,
562        project: &ProjectPackage,
563        target: &Target,
564    ) -> Result<LoadedTargetSources, Report> {
565        let source_paths = project.resolve_target_source_paths(target)?;
566        let root = self.parse_module_file(
567            &source_paths.root,
568            target_root_module_kind(target.ty),
569            target.namespace.inner().as_ref(),
570        )?;
571        let support = source_paths
572            .support
573            .iter()
574            .map(|path| {
575                let relative = path.strip_prefix(&source_paths.root_dir).map_err(|error| {
576                    Report::msg(format!(
577                        "failed to derive module path for '{}': {error}",
578                        path.display()
579                    ))
580                })?;
581                let module_path = module_path_from_relative(target.namespace.inner(), relative)?;
582                self.parse_module_file(path, ModuleKind::Library, module_path.as_ref())
583            })
584            .collect::<Result<Vec<_>, Report>>()?;
585
586        Ok(LoadedTargetSources { root, support })
587    }
588
589    fn parse_module_file(
590        &self,
591        path: &FsPath,
592        kind: ModuleKind,
593        module_path: &MasmPath,
594    ) -> Result<Box<Module>, Report> {
595        let mut parser = ModuleParser::new(kind);
596        parser.set_warnings_as_errors(self.assembler.warnings_as_errors());
597        parser.parse_file(module_path, path, self.assembler.source_manager())
598    }
599}
600
601// ================================================================================================
602
603#[derive(Clone)]
604struct ResolvedPackage {
605    package: Arc<MastPackage>,
606    linked_kernel_package: Option<Arc<MastPackage>>,
607}
608
609enum RegisteredSourcePackage {
610    Missing,
611    Loaded(Arc<MastPackage>),
612    IndexedButUnreadable(PackageVersion),
613}
614
615struct LoadedTargetSources {
616    root: Box<Module>,
617    #[allow(clippy::vec_box)]
618    support: Vec<Box<Module>>,
619}
620
621#[derive(Debug)]
622struct TargetSourcePaths {
623    root: PathBuf,
624    root_dir: PathBuf,
625    support: Vec<PathBuf>,
626}
627
628#[derive(Debug, Clone, PartialEq, Eq)]
629struct PackageBuildSettings {
630    emit_debug_info: bool,
631    trim_paths: bool,
632}
633
634impl PackageBuildSettings {
635    fn legacy() -> Self {
636        Self { emit_debug_info: true, trim_paths: false }
637    }
638
639    fn from_profile(profile: &Profile) -> Self {
640        Self {
641            emit_debug_info: profile.should_emit_debug_info(),
642            trim_paths: profile.should_trim_paths(),
643        }
644    }
645
646    fn is_legacy(&self) -> bool {
647        *self == Self::legacy()
648    }
649}
650
651// HELPER FUNCTIONS
652// ================================================================================================
653
654fn target_root_module_kind(ty: TargetType) -> ModuleKind {
655    match ty {
656        TargetType::Executable => ModuleKind::Executable,
657        TargetType::Kernel => ModuleKind::Kernel,
658        _ => ModuleKind::Library,
659    }
660}
661
662fn linked_kernel_package_section(package: &MastPackage) -> Section {
663    Section::new(SectionId::KERNEL, package.to_bytes())
664}
665
666fn module_path_from_relative(
667    namespace: &MasmPath,
668    relative: &FsPath,
669) -> Result<Arc<MasmPath>, Report> {
670    let mut module_path = namespace.to_path_buf();
671    let stem = relative.with_extension("");
672    let mut components = stem
673        .iter()
674        .map(|component| {
675            component.to_str().ok_or_else(|| {
676                Report::msg(format!("module path '{}' contains invalid UTF-8", relative.display()))
677            })
678        })
679        .collect::<Result<Vec<_>, Report>>()?;
680
681    if components.last().is_some_and(|component| *component == ast::Module::ROOT) {
682        components.pop();
683    }
684
685    for component in components {
686        MasmPath::validate(component).map_err(|error| Report::msg(error.to_string()))?;
687        module_path.push(component);
688    }
689
690    Ok(module_path.into())
691}
692
693fn load_selected_preassembled_package(
694    path: &FsPath,
695    expected_name: &PackageId,
696    selected: &PackageVersion,
697) -> Result<Arc<MastPackage>, Report> {
698    let package = load_package_from_path(path)?;
699    if &package.name != expected_name {
700        return Err(Report::msg(format!(
701            "preassembled dependency '{}' at '{}' resolved to package '{}'",
702            expected_name,
703            path.display(),
704            package.name
705        )));
706    }
707
708    let actual = PackageVersion::new(package.version.clone(), package.digest());
709    if &actual != selected {
710        return Err(Report::msg(format!(
711            "preassembled dependency '{}@{}' at '{}' no longer matches the dependency graph selection '{}'",
712            expected_name,
713            actual,
714            path.display(),
715            selected
716        )));
717    }
718
719    Ok(package)
720}
721
722fn load_package_from_path(path: &FsPath) -> Result<Arc<MastPackage>, Report> {
723    let bytes = fs::read(path)
724        .map_err(|error| Report::msg(format!("failed to read '{}': {error}", path.display())))?;
725    let package = MastPackage::read_from_bytes(&bytes).map_err(|error| {
726        Report::msg(format!("failed to decode package '{}': {error}", path.display()))
727    })?;
728    Ok(Arc::new(package))
729}
miden_assembly/project.rs

miden_assembly/
project.rs
