veryl_metadata/

metadata.rs

use crate::build::{Build, Target};
use crate::build_info::BuildInfo;
use crate::component::Component;
use crate::doc::Doc;
use crate::format::Format;
use crate::git::Git;
use crate::lint::Lint;
use crate::lockfile::Lockfile;
use crate::project::Project;
use crate::pubfile::{Pubfile, Release};
use crate::publish::Publish;
use crate::synth::Synth;
use crate::test::Test;
use crate::{FilelistType, MetadataError, SourceMapTarget};
use log::{debug, info, warn};
use once_cell::sync::Lazy;
use regex::Regex;
use semver::VersionReq;
use serde::{Deserialize, Serialize};
use spdx::Expression;
use std::collections::{BTreeMap, HashMap};
use std::env;
use std::fmt;
use std::fs;
use std::path::{Path, PathBuf};
use std::str::FromStr;
use std::time::SystemTime;
use url::Url;
use veryl_path::{PathSet, ignore_already_exists};

#[derive(Clone, Copy, Debug)]
pub enum BumpKind {
    Major,
    Minor,
    Patch,
}

#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Metadata {
    pub project: Project,
    #[serde(default)]
    pub build: Build,
    #[serde(default)]
    pub format: Format,
    #[serde(default)]
    pub lint: Lint,
    #[serde(default)]
    pub publish: Publish,
    #[serde(default)]
    pub doc: Doc,
    #[serde(default)]
    pub test: Test,
    #[serde(default)]
    pub synth: Synth,
    #[serde(default)]
    pub properties: BTreeMap<String, ProjectProperty>,
    #[serde(default)]
    pub components: Vec<Component>,
    #[serde(default)]
    pub dependencies: HashMap<String, Dependency>,
    #[serde(default)]
    pub metadata: HashMap<String, toml::Value>,
    #[serde(skip)]
    pub metadata_path: PathBuf,
    #[serde(skip)]
    pub pubfile_path: PathBuf,
    #[serde(skip)]
    pub pubfile: Pubfile,
    #[serde(skip)]
    pub lockfile_path: PathBuf,
    #[serde(skip)]
    pub lockfile: Lockfile,
    #[serde(skip)]
    pub build_info: BuildInfo,
    /// Output directory override (e.g. `veryl build --out-dir`).
    /// When set, build outputs are emitted relative to this directory
    /// instead of the project path. Never read from Veryl.toml.
    #[serde(skip)]
    pub output_dir_override: Option<PathBuf>,
}

#[derive(Clone, Debug, Serialize, Deserialize, Hash, PartialEq, Eq, PartialOrd, Ord)]
#[serde(untagged)]
pub enum ProjectProperty {
    Int(i64),
    Bool(bool),
}

impl ProjectProperty {
    pub fn is_compatible(&self, other: &ProjectProperty) -> bool {
        matches!(
            (self, other),
            (ProjectProperty::Int(_), ProjectProperty::Int(_))
                | (ProjectProperty::Bool(_), ProjectProperty::Bool(_))
        )
    }

    pub fn type_name(&self) -> String {
        match self {
            ProjectProperty::Int(_) => "int".to_string(),
            ProjectProperty::Bool(_) => "bool".to_string(),
        }
    }

    pub fn value_string(&self) -> String {
        match self {
            ProjectProperty::Int(x) => x.to_string(),
            ProjectProperty::Bool(x) => x.to_string(),
        }
    }

    pub fn verilog_value_string(&self) -> String {
        match self {
            ProjectProperty::Int(x) => x.to_string(),
            ProjectProperty::Bool(x) => (if *x { "1'b1" } else { "1'b0" }).to_string(),
        }
    }
}

#[derive(Clone, Debug, Serialize, Deserialize, Hash, PartialEq, Eq, PartialOrd, Ord)]
#[serde(untagged)]
pub enum UrlPath {
    Url(Url),
    Path(PathBuf),
}

impl fmt::Display for UrlPath {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            UrlPath::Url(x) => x.fmt(f),
            UrlPath::Path(x) => {
                let text = x.to_string_lossy();
                text.fmt(f)
            }
        }
    }
}

static VALID_PROJECT_NAME: Lazy<Regex> =
    Lazy::new(|| Regex::new(r"^[a-zA-Z_][0-9a-zA-Z_]*$").unwrap());

/// Validates a project or component name: identifiers only, `__` reserved.
pub fn check_project_name(name: &str) -> Result<(), MetadataError> {
    if !VALID_PROJECT_NAME.is_match(name) {
        return Err(MetadataError::InvalidProjectName(name.to_string()));
    }
    if name.starts_with("__") {
        return Err(MetadataError::ReservedProjectName(name.to_string()));
    }
    Ok(())
}

impl Metadata {
    pub fn search_from_current() -> Result<PathBuf, MetadataError> {
        Metadata::search_from(
            env::current_dir().map_err(|x| MetadataError::file_io(x, &PathBuf::from(".")))?,
        )
    }

    pub fn search_from<T: AsRef<Path>>(from: T) -> Result<PathBuf, MetadataError> {
        for path in from.as_ref().ancestors() {
            let path = path.join("Veryl.toml");
            if path.is_file() {
                return Ok(path);
            }
        }

        Err(MetadataError::FileNotFound(from.as_ref().to_path_buf()))
    }

    pub fn load<T: AsRef<Path>>(path: T) -> Result<Self, MetadataError> {
        let path = path
            .as_ref()
            .canonicalize()
            .map_err(|x| MetadataError::file_io(x, path.as_ref()))?;
        let text = fs::read_to_string(&path).map_err(|x| MetadataError::file_io(x, &path))?;
        let mut metadata: Metadata = Self::from_str(&text)?;
        metadata.metadata_path.clone_from(&path);
        metadata.pubfile_path = path.with_file_name("Veryl.pub");
        metadata.lockfile_path = path.with_file_name("Veryl.lock");
        metadata.check()?;

        if metadata.pubfile_path.exists() {
            metadata.pubfile = Pubfile::load(&metadata.pubfile_path)?;
        }

        let dot_build = metadata.project_dot_build_path();
        if !dot_build.exists() {
            ignore_already_exists(fs::create_dir(&dot_build))
                .map_err(|x| MetadataError::file_io(x, &dot_build))?;
        }

        let build_info = metadata.project_build_info_path();
        if build_info.exists() {
            if let Ok(info) = BuildInfo::load(&build_info) {
                metadata.build_info = info;
            } else {
                // ignore failure of loading BuildInfo
                info!("Discarded incompatible .build/info.toml");
            }
        }

        debug!(
            "Loaded metadata ({})",
            metadata.metadata_path.to_string_lossy()
        );
        Ok(metadata)
    }

    pub fn publish(&mut self) -> Result<(), MetadataError> {
        let prj_path = self.project_path();
        let git = Git::open(&prj_path)?;
        if !git.is_clean()? {
            return Err(MetadataError::ModifiedProject(prj_path.to_path_buf()));
        }

        let version = self
            .project
            .version
            .clone()
            .ok_or(MetadataError::MissingVersion)?;

        for release in &self.pubfile.releases {
            if release.version == version {
                return Err(MetadataError::PublishedVersion(version));
            }
        }

        let revision = git.get_revision()?;

        info!("Publishing release ({version} @ {revision})");

        let release = Release { version, revision };

        self.pubfile.releases.push(release);

        self.pubfile.save(&self.pubfile_path)?;
        info!("Writing metadata ({})", self.pubfile_path.to_string_lossy());

        if self.publish.publish_commit {
            git.add(&self.pubfile_path)?;
            git.commit(&self.publish.publish_commit_message)?;
            info!(
                "Committing metadata ({})",
                self.pubfile_path.to_string_lossy()
            );
        } else {
            warn!(
                "Please git add and commit Veryl.pub (set `publish_commit = true` in [publish] to automate this)"
            );
        }

        Ok(())
    }

    pub fn check(&self) -> Result<(), MetadataError> {
        check_project_name(&self.project.name)?;

        if let Some(ref license) = self.project.license {
            let _ = Expression::parse(license)?;
        }

        Ok(())
    }

    pub fn bump_version(&mut self, kind: BumpKind) -> Result<(), MetadataError> {
        let prj_path = self.project_path();
        let git = Git::open(&prj_path)?;

        let current_version = self
            .project
            .version
            .as_ref()
            .ok_or(MetadataError::MissingVersion)?;

        let mut bumped_version = current_version.clone();

        match kind {
            BumpKind::Major => {
                bumped_version.major += 1;
                bumped_version.minor = 0;
                bumped_version.patch = 0;
            }
            BumpKind::Minor => {
                bumped_version.minor += 1;
                bumped_version.patch = 0;
            }
            BumpKind::Patch => bumped_version.patch += 1,
        }
        info!(
            "Bumping version ({} -> {})",
            current_version, bumped_version
        );

        self.project.version = Some(bumped_version.clone());

        let toml = fs::read_to_string(&self.metadata_path)
            .map_err(|x| MetadataError::file_io(x, &self.metadata_path))?;
        let re = Regex::new(r#"version\s+=\s+"([^"]*)""#).unwrap();
        let caps = re
            .captures(&toml)
            .expect("safely unwrap because metadata is valid");
        let bumped_field = caps[0].replace(&caps[1], &bumped_version.to_string());
        let bumped_toml = re.replace(&toml, bumped_field);
        fs::write(&self.metadata_path, bumped_toml.as_bytes())
            .map_err(|x| MetadataError::file_io(x, &self.metadata_path))?;
        info!(
            "Updating version field ({})",
            self.metadata_path.to_string_lossy()
        );

        if self.publish.bump_commit {
            git.add(&self.metadata_path)?;
            git.commit(&self.publish.bump_commit_message)?;
            info!(
                "Committing metadata ({})",
                self.metadata_path.to_string_lossy()
            );
        }

        Ok(())
    }

    pub fn update_lockfile(&mut self) -> Result<(), MetadataError> {
        let modified = if self.lockfile_path.exists() {
            let mut lockfile = Lockfile::load(self)?;
            let modified = lockfile.update(self, false)?;
            self.lockfile = lockfile;
            modified
        } else {
            self.lockfile = Lockfile::new(self)?;
            true
        };
        if modified {
            self.lockfile.save(&self.lockfile_path)?;
        }
        Ok(())
    }

    pub fn save_build_info(&mut self) -> Result<(), MetadataError> {
        let build_info = self.project_build_info_path();
        self.build_info.save(&build_info)
    }

    pub fn add_generated_file(&mut self, path: PathBuf) {
        self.build_info
            .generated_files
            .insert(path, SystemTime::now());
    }

    pub fn paths<T: AsRef<Path>>(
        &mut self,
        files: &[T],
        symlink: bool,
        include_dependencies: bool,
    ) -> Result<Vec<PathSet>, MetadataError> {
        let sources = if self.build.source.iter().count() > 0 {
            warn!(
                "[Veryl.toml] \"source\" field is deprecated. Replace it with \"sources\" field."
            );
            vec![self.build.source.clone()]
        } else {
            self.build.sources.clone()
        };

        let base = self.project_path();
        // Build outputs may be redirected (e.g. `veryl build --out-dir`);
        // sources are always resolved against the project path.
        let out_base = self.output_dir();
        let mut ret = Vec::new();

        // Pre-canonicalize explicit file args once so we can route each to
        // the source dir it actually belongs to (without re-processing the
        // same file for every configured source dir).
        let canonical_files = if files.is_empty() {
            None
        } else {
            let mut v = Vec::new();
            for file in files {
                v.push(
                    fs::canonicalize(file.as_ref())
                        .map_err(|x| MetadataError::file_io(x, file.as_ref()))?,
                );
            }
            Some(v)
        };
        let mut explicit_routed = canonical_files.as_ref().map(|v| vec![false; v.len()]);

        // `examples/` is reserved; dependency source collection
        // (`Lockfile::paths`) skips it entirely.
        let examples_base = base.join("examples");
        if let Some(source) = sources
            .iter()
            .find(|x| base.join(x).starts_with(&examples_base))
        {
            return Err(MetadataError::ReservedSourceDir(base.join(source)));
        }

        let mut source_dirs: Vec<(PathBuf, bool)> =
            sources.iter().map(|x| (base.join(x), false)).collect();
        if examples_base.exists() {
            source_dirs.push((examples_base.clone(), true));
        }

        for (src_base, is_example) in source_dirs {
            let src_files = if let Some(cf) = canonical_files.as_ref() {
                // Only keep files that live under this source dir; other
                // source dirs in `sources` will pick them up. Files under
                // `examples/` belong to the examples dir only, even when a
                // source dir contains it.
                let mut ret = Vec::new();
                for (i, path) in cf.iter().enumerate() {
                    if path.starts_with(&src_base)
                        && (is_example || !path.starts_with(&examples_base))
                    {
                        ret.push(path.clone());
                        if let Some(ref mut flags) = explicit_routed {
                            flags[i] = true;
                        }
                    }
                }
                ret
            } else {
                let mut files =
                    veryl_path::gather_files_with_extension(&src_base, "veryl", symlink)?;
                if !is_example {
                    files.retain(|x| !x.starts_with(&examples_base));
                }
                files
            };

            for src in src_files {
                let Ok(src_relative) = src.strip_prefix(&src_base) else {
                    return Err(MetadataError::InvalidSourceLocation(src));
                };
                let dst = match self.build.target {
                    Target::Source => {
                        if self.output_dir_override.is_some() {
                            // Redirected source-target builds keep the
                            // source-relative layout under the override.
                            out_base.join(src_relative.with_extension("sv"))
                        } else {
                            src.with_extension("sv")
                        }
                    }
                    Target::Directory { ref path } => {
                        out_base.join(path.join(src_relative.with_extension("sv")))
                    }
                    Target::Bundle { .. } => out_base.join(
                        PathBuf::from("target").join(src.with_extension("sv").file_name().unwrap()),
                    ),
                };
                let map = match &self.build.sourcemap_target {
                    SourceMapTarget::Directory { path } => {
                        if let Target::Directory { .. } = self.build.target {
                            out_base.join(path.join(src_relative.with_extension("sv.map")))
                        } else {
                            let dst = dst.strip_prefix(&out_base).unwrap();
                            out_base.join(path.join(dst.with_extension("sv.map")))
                        }
                    }
                    _ => {
                        let mut map = dst.clone();
                        map.set_extension("sv.map");
                        map
                    }
                };
                ret.push(PathSet {
                    prj: self.project.name.clone(),
                    src: src.to_path_buf(),
                    dst,
                    map,
                    example: is_example,
                });
            }
        }

        // Any explicit file that wasn't claimed by a configured source dir
        // is outside the project — preserve the original error semantics.
        if let (Some(cf), Some(flags)) = (canonical_files.as_ref(), explicit_routed.as_ref())
            && let Some(pos) = flags.iter().position(|f| !f)
        {
            return Err(MetadataError::InvalidSourceLocation(cf[pos].clone()));
        }

        let base_dst = self.output_dependencies_path();
        if !base_dst.exists() {
            ignore_already_exists(fs::create_dir_all(&base_dst))
                .map_err(|x| MetadataError::file_io(x, &base_dst))?;
        }

        if include_dependencies {
            if !self.build.exclude_std {
                veryl_std::expand()?;
                ret.append(&mut veryl_std::paths(&base_dst)?);
            }

            self.update_lockfile()?;

            let mut deps = self.lockfile.paths(&base_dst)?;
            ret.append(&mut deps);
        }

        Ok(ret)
    }

    pub fn create_default_toml(name: &str) -> Result<String, MetadataError> {
        check_project_name(name)?;

        Ok(format!(
            r###"[project]
name = "{name}"
version = "0.1.0"
[build]
sources = ["src"]
target = {{type = "directory", path = "target"}}"###
        ))
    }

    pub fn create_default(name: &str) -> Result<Metadata, MetadataError> {
        let metadata: Metadata = toml::from_str(&Self::create_default_toml(name)?)?;
        Ok(metadata)
    }

    pub fn create_default_gitignore() -> &'static str {
        r#"# Build output
.build/
/target
/dependencies
*.f

# Verilator
obj_dir/
"#
    }

    pub fn project_path(&self) -> PathBuf {
        self.metadata_path.parent().unwrap().to_path_buf()
    }

    pub fn output_dir(&self) -> PathBuf {
        self.output_dir_override
            .clone()
            .unwrap_or_else(|| self.project_path())
    }

    pub fn project_dependencies_path(&self) -> PathBuf {
        self.project_path().join("dependencies")
    }

    pub fn output_dependencies_path(&self) -> PathBuf {
        self.output_dir().join("dependencies")
    }

    pub fn project_dot_build_path(&self) -> PathBuf {
        self.project_path().join(".build")
    }

    pub fn project_build_info_path(&self) -> PathBuf {
        self.project_dot_build_path().join("info.toml")
    }

    pub fn filelist_path(&self) -> PathBuf {
        let filelist_name = match self.build.filelist_type {
            FilelistType::Absolute => format!("{}.f", self.project.name),
            FilelistType::Relative => format!("{}.f", self.project.name),
            FilelistType::Flgen => format!("{}.list.rb", self.project.name),
        };

        self.output_dir().join(filelist_name)
    }

    pub fn doc_path(&self) -> PathBuf {
        self.metadata_path.parent().unwrap().join(&self.doc.path)
    }

    /// Collects `[[components]]` declared by direct dependencies. Requires a
    /// loaded lockfile; the dependency checkouts are already present when
    /// this is called from the build/test flow.
    pub fn collect_dependency_components(
        &self,
    ) -> Result<Vec<crate::lockfile::DependencyComponents>, MetadataError> {
        self.lockfile.collect_components()
    }

    /// Collects the interface manifests of this project's and its direct
    /// dependencies' components, keyed like the `$comp` symbols
    /// (`<name>` / `<project>::<name>`). The names are the packages'
    /// `veryl_component_export!` export names, enumerated per
    /// `[[components]]` entry (see [`Component::collect_manifests`] for
    /// the source priority). On a name collision the earlier entry wins.
    pub fn collect_component_manifests(
        &self,
    ) -> Vec<(String, crate::component_manifest::ComponentManifest)> {
        // Enumerates one project's entries with first-declaration-wins
        // dedup; the same policy must hold wherever exports are collected
        // (`veryl test` mirrors it when registering libraries).
        fn collect_project(
            components: &[crate::Component],
            root: &Path,
            target_dir: &Path,
            project: Option<&str>,
            ret: &mut Vec<(String, crate::component_manifest::ComponentManifest)>,
        ) {
            let mut seen = std::collections::HashSet::new();
            for def in components {
                for (name, manifest) in def.collect_manifests(root, target_dir) {
                    if seen.insert(name.clone()) {
                        let key = match project {
                            Some(project) => format!("{project}::{name}"),
                            None => name,
                        };
                        ret.push((key, manifest));
                    } else {
                        let scope = project
                            .map(|p| format!(" of dependency `{p}`"))
                            .unwrap_or_default();
                        log::warn!(
                            "component `{name}` is exported by more than one [[components]] package{scope}; the first declaration wins"
                        );
                    }
                }
            }
        }

        let mut ret = vec![];
        // An in-memory metadata (no backing Veryl.toml) has no project
        // directory to read manifests from.
        if self.metadata_path.as_os_str().is_empty() {
            return ret;
        }
        let root = self.project_path();
        let target_dir = root.join("target/veryl-components");
        collect_project(&self.components, &root, &target_dir, None, &mut ret);
        if let Ok(deps) = self.collect_dependency_components() {
            for dep in &deps {
                collect_project(
                    &dep.components,
                    &dep.root,
                    &dep.target_dir,
                    Some(&dep.project),
                    &mut ret,
                );
            }
        }
        ret
    }
}

impl FromStr for Metadata {
    type Err = MetadataError;

    fn from_str(s: &str) -> Result<Self, Self::Err> {
        let metadata: Metadata = toml::from_str(s)?;
        Ok(metadata)
    }
}

#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(untagged)]
#[serde(deny_unknown_fields)]
pub enum Dependency {
    Version(VersionReq),
    Entry(Box<DependencyEntry>),
}

#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct DependencyEntry {
    pub version: Option<VersionReq>,
    pub git: Option<UrlPath>,
    pub github: Option<String>,
    pub project: Option<String>,
    pub path: Option<PathBuf>,
    #[serde(default)]
    pub properties: HashMap<String, ProjectProperty>,
}