rsconstruct 0.9.157

Rust based fast build system
//! iprotobuf generator: Protocol Buffer definitions compiled in-process.
//!
//! The Rust alternative to the `protobuf` generator (which runs `protoc`):
//! the `.proto` files under `src_dirs` are compiled with protox, a Rust
//! implementation of the protobuf compiler, and prost-build turns the
//! result into Rust code, one `<package>.rs` per protobuf package in the
//! output directory. Optionally the compiled descriptor set is written as
//! well, the file `protoc --descriptor_set_out` produces.
//!
//! All the files of a stanza are compiled together as one product, since
//! a package's generated file gathers every `.proto` that declares it.

use std::collections::BTreeSet;
use std::path::{Path, PathBuf};
use std::sync::LazyLock;

use anyhow::{Context, Result, anyhow};
use regex::Regex;
use serde::{Deserialize, Serialize};

use crate::config::{StandardConfig, output_config_hash};
use crate::file_index::FileIndex;
use crate::graph::{BuildGraph, Product};
use crate::processor::{Processor, resolve_dep_inputs};

#[derive(Debug, Deserialize, Serialize, Clone, Default)]
pub struct IprotobufConfig {
    /// Directories `import` statements are resolved against. Empty: the
    /// `src_dirs`.
    #[serde(default)]
    pub include_paths: Vec<String>,
    /// Where to write the compiled `FileDescriptorSet` (what
    /// `protoc --include_imports --descriptor_set_out` writes). Empty: not
    /// written.
    #[serde(default)]
    pub descriptor_set: String,
    #[serde(flatten)]
    pub standard: StandardConfig,
}

pub struct IprotobufProcessor {
    config: IprotobufConfig,
}

impl IprotobufProcessor {
    pub const fn new(config: IprotobufConfig) -> Self {
        Self { config }
    }

    fn output_dir(&self) -> &str {
        &self.config.standard.output_dir
    }

    fn include_paths(&self) -> Vec<PathBuf> {
        let configured: Vec<PathBuf> = if self.config.include_paths.is_empty() {
            self.config
                .standard
                .src_dirs()
                .iter()
                .filter(|d| !d.is_empty())
                .map(PathBuf::from)
                .collect()
        } else {
            self.config
                .include_paths
                .iter()
                .map(PathBuf::from)
                .collect()
        };
        if configured.is_empty() {
            vec![PathBuf::from(".")]
        } else {
            configured
        }
    }

    /// The Rust file prost-build writes for the package a `.proto` declares:
    /// the package's dotted path with each component in snake case, or `_`
    /// for a file with no `package` statement.
    fn generated_file_name(proto: &Path) -> Result<String> {
        let text = proto_text(proto)?;
        let name = PACKAGE.captures(&text).map_or_else(
            || "_".to_string(),
            |c| {
                c[1].split('.')
                    .filter(|s| !s.is_empty())
                    .map(to_snake)
                    .collect::<Vec<_>>()
                    .join(".")
            },
        );
        Ok(format!("{name}.rs"))
    }

    /// The `.proto` files the `roots` import, directly or through other
    /// imports, resolved against the include paths; sorted, without the
    /// roots themselves. A file under `src_dirs` is a scanned input already,
    /// but an import resolved from a user-set `include_paths` outside them
    /// was compiled (`include_imports`) without being an input, so an edit
    /// to it rebuilt nothing. The well-known types (`google/protobuf/*`) are
    /// built into protox and need no file; any other import that no include
    /// path holds is an error naming the importer, as protox would report a
    /// moment later without saying which file asked for it.
    fn resolve_imports(&self, roots: &[PathBuf]) -> Result<Vec<PathBuf>> {
        let include_paths = self.include_paths();
        let mut seen: BTreeSet<PathBuf> = roots.iter().cloned().collect();
        let mut queue: Vec<PathBuf> = roots.to_vec();
        let mut imported = Vec::new();
        while let Some(file) = queue.pop() {
            let text = proto_text(&file)?;
            for cap in IMPORT.captures_iter(&text) {
                let name = &cap[1];
                let resolved = include_paths
                    .iter()
                    .map(|dir| {
                        if dir == Path::new(".") {
                            PathBuf::from(name)
                        } else {
                            dir.join(name)
                        }
                    })
                    .find(|p| p.is_file());
                let Some(path) = resolved else {
                    if name.starts_with("google/protobuf/") {
                        continue;
                    }
                    let searched: Vec<String> = include_paths
                        .iter()
                        .map(|p| p.display().to_string())
                        .collect();
                    anyhow::bail!(
                        "iprotobuf: {} imports \"{name}\", which none of the include paths hold ({})",
                        file.display(),
                        searched.join(", ")
                    );
                };
                if seen.insert(path.clone()) {
                    queue.push(path.clone());
                    imported.push(path);
                }
            }
        }
        imported.sort();
        Ok(imported)
    }

    fn compile(&self, files: &[PathBuf]) -> Result<()> {
        let out_dir = Path::new(self.output_dir());
        std::fs::create_dir_all(out_dir)
            .with_context(|| format!("Failed to create output directory {}", out_dir.display()))?;
        let mut compiler =
            protox::Compiler::new(self.include_paths()).map_err(|e| anyhow!("iprotobuf: {e}"))?;
        compiler.include_imports(true).include_source_info(true);
        compiler
            .open_files(files)
            .map_err(|e| anyhow!("iprotobuf: {e}"))?;
        let fds = compiler.file_descriptor_set();
        if !self.config.descriptor_set.is_empty() {
            // What `protoc --include_imports --descriptor_set_out` writes:
            // the source locations stay out unless asked for, and they are
            // only needed here for the comments in the generated code.
            let mut plain = fds.clone();
            for file in &mut plain.file {
                file.source_code_info = None;
            }
            let path = Path::new(&self.config.descriptor_set);
            crate::processor::ensure_output_dir(path)?;
            std::fs::write(path, prost::Message::encode_to_vec(&plain))
                .with_context(|| format!("Failed to write descriptor set {}", path.display()))?;
        }
        prost_build::Config::new()
            .out_dir(out_dir)
            .compile_fds(fds)
            .with_context(|| format!("Failed to generate Rust code into {}", out_dir.display()))
    }
}

/// `package a.b;` at the start of a line, in comment-free text.
static PACKAGE: LazyLock<Regex> = LazyLock::new(|| {
    Regex::new(r"(?m)^\s*package\s+([A-Za-z_][A-Za-z0-9_.]*)\s*;").expect("package regex")
});

/// `import "x.proto";`, with the optional `public` or `weak` modifier.
static IMPORT: LazyLock<Regex> = LazyLock::new(|| {
    Regex::new(r#"(?m)^\s*import\s+(?:public\s+|weak\s+)?"([^"]+)"\s*;"#).expect("import regex")
});

/// The text of a `.proto` file with its comments removed, for the
/// statement regexes above.
fn proto_text(proto: &Path) -> Result<String> {
    let text = std::fs::read_to_string(proto)
        .with_context(|| format!("Failed to read {}", proto.display()))?;
    Ok(strip_comments(&text))
}

/// `text` with every comment outside a string literal removed: `//` to the
/// end of the line, and `/* ... */` blocks, whose newlines are kept so the
/// `(?m)^` anchors of the statement regexes still see line starts. The
/// regexes used to run over the raw text, so a `package old.name;` left in
/// a block comment named the generated file.
fn strip_comments(text: &str) -> String {
    let mut out = String::with_capacity(text.len());
    let mut chars = text.chars().peekable();
    while let Some(c) = chars.next() {
        match c {
            '"' | '\'' => {
                // A string literal is copied through, escapes included.
                out.push(c);
                while let Some(s) = chars.next() {
                    out.push(s);
                    if s == '\\' {
                        if let Some(escaped) = chars.next() {
                            out.push(escaped);
                        }
                    } else if s == c {
                        break;
                    }
                }
            }
            '/' if chars.peek() == Some(&'/') => {
                for s in chars.by_ref() {
                    if s == '\n' {
                        out.push('\n');
                        break;
                    }
                }
            }
            '/' if chars.peek() == Some(&'*') => {
                chars.next();
                out.push(' ');
                let mut prev = ' ';
                for s in chars.by_ref() {
                    if s == '\n' {
                        out.push('\n');
                    }
                    if prev == '*' && s == '/' {
                        break;
                    }
                    prev = s;
                }
            }
            _ => out.push(c),
        }
    }
    out
}

/// heck's snake case, as prost-build applies it to package components.
fn to_snake(s: &str) -> String {
    let mut out = String::new();
    let chars: Vec<char> = s.chars().collect();
    for (i, &c) in chars.iter().enumerate() {
        if c.is_uppercase() {
            let prev_lower =
                i > 0 && (chars[i - 1].is_lowercase() || chars[i - 1].is_ascii_digit());
            let next_lower = chars.get(i + 1).is_some_and(|n| n.is_lowercase());
            let prev_upper = i > 0 && chars[i - 1].is_uppercase();
            if !out.is_empty() && !out.ends_with('_') && (prev_lower || (prev_upper && next_lower))
            {
                out.push('_');
            }
            out.extend(c.to_lowercase());
        } else {
            out.push(c);
        }
    }
    out
}

impl Processor for IprotobufProcessor {
    fn scan_config(&self) -> &crate::config::StandardConfig {
        &self.config.standard
    }

    /// One product over all the stanza's files.
    fn discovery(&self) -> crate::processor::Discovery {
        crate::processor::Discovery::WholeIndex
    }

    fn config_json(&self) -> Option<String> {
        crate::processor::ProcessorBase::config_json(&self.config)
    }

    fn clean(&self, product: &Product, verbose: bool) -> Result<usize> {
        crate::processor::ProcessorBase::clean(product, &product.processor, verbose)
    }

    fn auto_detect(&self, file_index: &FileIndex) -> bool {
        !file_index.scan(&self.config.standard, true).is_empty()
    }

    fn required_tools(&self) -> Vec<String> {
        Vec::new()
    }

    fn discover(
        &self,
        graph: &mut BuildGraph,
        file_index: &FileIndex,
        instance_name: &str,
    ) -> Result<()> {
        let files = file_index.scan(&self.config.standard, true);
        if files.is_empty() {
            return Ok(());
        }
        // Imported files are compiled too (`include_imports`), so they are
        // inputs, and prost-build writes their packages' files as well.
        let imported = self.resolve_imports(&files)?;
        let out_dir = Path::new(self.output_dir());
        let mut outputs: BTreeSet<PathBuf> = BTreeSet::new();
        for f in files.iter().chain(&imported) {
            outputs.insert(out_dir.join(Self::generated_file_name(f)?));
        }
        if !self.config.descriptor_set.is_empty() {
            outputs.insert(PathBuf::from(&self.config.descriptor_set));
        }
        let extra = resolve_dep_inputs(
            &self.config.standard.dep_inputs,
            &self.config.standard.dep_auto,
        )?;
        let mut inputs = files;
        inputs.extend(imported);
        inputs.extend_from_slice(&extra);
        graph.add_product(
            inputs,
            outputs.into_iter().collect(),
            instance_name,
            Some(output_config_hash(
                &self.config,
                &crate::config::checksum_fields_of(instance_name),
            )),
        )?;
        Ok(())
    }

    fn execute(&self, _ctx: &crate::build_context::BuildContext, product: &Product) -> Result<()> {
        let files: Vec<PathBuf> = product
            .inputs
            .iter()
            .filter(|p| p.extension().is_some_and(|e| e == "proto"))
            .cloned()
            .collect();
        self.compile(&files)
    }
}

fn plugin_create(toml: &toml::Value) -> anyhow::Result<Box<dyn crate::processor::Processor>> {
    crate::registries::deserialize_and_create(toml, |cfg| Box::new(IprotobufProcessor::new(cfg)))
}
inventory::submit! {
    crate::registries::RustAlternative { replaces: "processor.generator.protobuf", by: "processor.generator.iprotobuf" }
}
inventory::submit! {
    crate::registries::ProcessorPlugin {
        version: 1,
        name: "iprotobuf",
        processor_type: crate::processor::ProcessorType::Generator,
        create: plugin_create,
        fields: &[
            crate::config::FieldSpec { name: "include_paths", ty: crate::config::FieldType::StringArray,
                affects_output: true, required: false,
                doc: "Directories import statements are resolved against. Empty: the src_dirs" },
            crate::config::FieldSpec { name: "descriptor_set", ty: crate::config::FieldType::String,
                affects_output: true, required: false,
                doc: "Where to write the compiled FileDescriptorSet (protoc --descriptor_set_out). Empty: not written" },
        ],
        omit_standard_fields: &["command", "formats", "args", "required_tools", "batch"],
        scan_defaults: Some(crate::config::ScanDefaultsData { src_dirs: &[], src_extensions: &[".proto"], src_exclude_dirs: &[] }),
        defaults: Some(crate::config::ProcessorDefaults { output_dir: "out/processor.generator.iprotobuf", ..crate::config::ProcessorDefaults::EMPTY }),
        defconfig_json: crate::registries::default_config_json::<IprotobufConfig>,
        keywords: &["protobuf", "proto", "generator", "grpc", "serialization", "rust"],
        description: "Compile Protocol Buffer definitions to Rust (in-process)",
        is_builtin: true,
        is_rust: true,
        supports_batch: false,
        caches_outputs: true,
        sandboxable: true,
        max_jobs_cap: None,
    }
}