pub(crate) mod comments;
pub mod context;
pub(crate) mod defaults;
pub(crate) mod enumeration;
pub(crate) mod extension;
pub(crate) mod features;
#[doc(hidden)]
pub use buffa_descriptor::generated;
pub mod idents;
pub(crate) mod impl_message;
pub(crate) mod impl_text;
pub(crate) mod imports;
pub(crate) mod message;
pub(crate) mod oneof;
pub(crate) mod view;
use crate::generated::descriptor::FileDescriptorProto;
use proc_macro2::TokenStream;
use quote::{format_ident, quote};
pub const ALLOW_LINTS: &[&str] = &[
"non_camel_case_types",
"dead_code",
"unused_imports",
"clippy::derivable_impls",
"clippy::match_single_binding",
"clippy::uninlined_format_args",
"clippy::doc_lazy_continuation",
"clippy::module_inception",
];
pub fn allow_lints_attr() -> TokenStream {
let lints: Vec<TokenStream> = ALLOW_LINTS
.iter()
.map(|l| syn::parse_str(l).expect("lint name parses as path"))
.collect();
quote! { #[allow( #(#lints),* )] }
}
#[derive(Debug)]
pub struct GeneratedFile {
pub name: String,
pub package: String,
pub kind: GeneratedFileKind,
pub content: String,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum GeneratedFileKind {
Owned,
View,
Oneof,
ViewOneof,
Ext,
PackageMod,
Companion,
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct CodeGenConfig {
pub generate_views: bool,
pub preserve_unknown_fields: bool,
pub generate_json: bool,
pub generate_arbitrary: bool,
pub extern_paths: Vec<(String, String)>,
pub bytes_fields: Vec<String>,
pub strict_utf8_mapping: bool,
pub allow_message_set: bool,
pub generate_text: bool,
pub emit_register_fn: bool,
pub file_per_package: bool,
pub type_attributes: Vec<(String, String)>,
pub field_attributes: Vec<(String, String)>,
pub message_attributes: Vec<(String, String)>,
pub enum_attributes: Vec<(String, String)>,
}
impl Default for CodeGenConfig {
fn default() -> Self {
Self {
generate_views: true,
preserve_unknown_fields: true,
generate_json: false,
generate_arbitrary: false,
extern_paths: Vec::new(),
bytes_fields: Vec::new(),
strict_utf8_mapping: false,
allow_message_set: false,
generate_text: false,
emit_register_fn: true,
file_per_package: false,
type_attributes: Vec::new(),
field_attributes: Vec::new(),
message_attributes: Vec::new(),
enum_attributes: Vec::new(),
}
}
}
pub(crate) fn effective_extern_paths(
file_descriptors: &[FileDescriptorProto],
files_to_generate: &[String],
config: &CodeGenConfig,
) -> Vec<(String, String)> {
let mut paths = config.extern_paths.clone();
let has_wkt_mapping = paths.iter().any(|(proto, _)| proto == ".google.protobuf");
if !has_wkt_mapping {
let generating_wkts = file_descriptors
.iter()
.filter(|fd| {
fd.name
.as_deref()
.is_some_and(|n| files_to_generate.iter().any(|f| f == n))
})
.any(|fd| fd.package.as_deref() == Some("google.protobuf"));
if !generating_wkts {
paths.push((
".google.protobuf".to_string(),
"::buffa_types::google::protobuf".to_string(),
));
}
}
paths
}
pub fn generate(
file_descriptors: &[FileDescriptorProto],
files_to_generate: &[String],
config: &CodeGenConfig,
) -> Result<Vec<GeneratedFile>, CodeGenError> {
let ctx = context::CodeGenContext::for_generate(file_descriptors, files_to_generate, config);
let mut by_package: std::collections::BTreeMap<String, Vec<&FileDescriptorProto>> =
std::collections::BTreeMap::new();
for file_name in files_to_generate {
let file_desc = file_descriptors
.iter()
.find(|f| f.name.as_deref() == Some(file_name.as_str()))
.ok_or_else(|| CodeGenError::FileNotFound(file_name.clone()))?;
let pkg = file_desc.package.as_deref().unwrap_or("").to_string();
by_package.entry(pkg).or_default().push(file_desc);
}
let mut output = Vec::new();
for (package, files) in by_package {
generate_package(&ctx, &package, &files, &mut output)?;
}
Ok(output)
}
pub fn generate_module_tree<F: AsRef<str>, P: AsRef<str>>(
entries: &[(F, P)],
include_mode: IncludeMode<'_>,
emit_inner_allow: bool,
) -> String {
use std::collections::BTreeMap;
use std::fmt::Write;
use crate::idents::escape_mod_ident;
#[derive(Default)]
struct ModNode {
files: Vec<String>,
children: BTreeMap<String, Self>,
}
let mut root = ModNode::default();
for (file_name, package) in entries {
let package = package.as_ref();
let pkg_parts: Vec<&str> = if package.is_empty() {
vec![]
} else {
package.split('.').collect()
};
let mut node = &mut root;
for seg in &pkg_parts {
node = node.children.entry(seg.to_string()).or_default();
}
node.files.push(file_name.as_ref().to_string());
}
let lints = ALLOW_LINTS.join(", ");
let mut out = String::new();
let _ = writeln!(out, "// @generated by buffa-codegen. DO NOT EDIT.");
if emit_inner_allow {
let _ = writeln!(out, "#![allow({lints})]");
}
let _ = writeln!(out);
fn emit(out: &mut String, node: &ModNode, depth: usize, mode: IncludeMode<'_>, lints: &str) {
let indent = " ".repeat(depth);
for file in &node.files {
match mode {
IncludeMode::Relative(prefix) => {
let _ = writeln!(out, r#"{indent}include!("{prefix}{file}");"#);
}
IncludeMode::OutDir => {
let _ = writeln!(
out,
r#"{indent}include!(concat!(env!("OUT_DIR"), "/{file}"));"#
);
}
}
}
for (name, child) in &node.children {
let escaped = escape_mod_ident(name);
let _ = writeln!(out, "{indent}#[allow({lints})]");
let _ = writeln!(out, "{indent}pub mod {escaped} {{");
let _ = writeln!(out, "{indent} use super::*;");
emit(out, child, depth + 1, mode, lints);
let _ = writeln!(out, "{indent}}}");
}
}
emit(&mut out, &root, 0, include_mode, &lints);
out
}
#[derive(Debug, Clone, Copy)]
pub enum IncludeMode<'a> {
Relative(&'a str),
OutDir,
}
fn validate_file(file: &FileDescriptorProto) -> Result<(), CodeGenError> {
use std::collections::HashMap;
let sentinel = context::SENTINEL_MOD;
let package = file.package.as_deref().unwrap_or("");
if package.split('.').any(|seg| seg == sentinel) {
return Err(CodeGenError::ReservedModuleName {
name: sentinel.to_string(),
location: format!("package '{package}'"),
});
}
for enum_type in &file.enum_type {
let name = enum_type.name.as_deref().unwrap_or("");
if name == sentinel {
return Err(CodeGenError::ReservedModuleName {
name: sentinel.to_string(),
location: format!("enum '{package}.{name}'"),
});
}
}
fn walk(
messages: &[crate::generated::descriptor::DescriptorProto],
scope: &str,
sentinel: &str,
) -> Result<(), CodeGenError> {
let mut seen: HashMap<String, &str> = HashMap::new();
for msg in messages {
let name = msg.name.as_deref().unwrap_or("");
let fqn = if scope.is_empty() {
name.to_string()
} else {
format!("{scope}.{name}")
};
for field in &msg.field {
if let Some(fname) = &field.name {
if fname.starts_with("__buffa_") {
return Err(CodeGenError::ReservedFieldName {
message_name: fqn,
field_name: fname.clone(),
});
}
}
}
let module_name = crate::oneof::to_snake_case(name);
if module_name == sentinel {
return Err(CodeGenError::ReservedModuleName {
name: sentinel.to_string(),
location: format!("message '{fqn}'"),
});
}
if let Some(existing) = seen.get(&module_name) {
return Err(CodeGenError::ModuleNameConflict {
scope: scope.to_string(),
name_a: existing.to_string(),
name_b: name.to_string(),
module_name,
});
}
seen.insert(module_name, name);
walk(&msg.nested_type, &fqn, sentinel)?;
}
Ok(())
}
walk(&file.message_type, package, sentinel)
}
struct ProtoContent {
stem: String,
owned: TokenStream,
view: TokenStream,
oneof: TokenStream,
view_oneof: TokenStream,
ext: TokenStream,
root_reexports: Vec<message::ReexportCandidate>,
}
fn generate_proto_content(
ctx: &context::CodeGenContext,
current_package: &str,
file: &FileDescriptorProto,
reg: &mut message::RegistryPaths,
) -> Result<ProtoContent, CodeGenError> {
use crate::idents::make_field_ident;
use crate::message::MessageOutput;
validate_file(file)?;
let resolver = imports::ImportResolver::new();
let features = crate::features::for_file(file);
let mut owned = TokenStream::new();
let mut view = TokenStream::new();
let mut oneof = TokenStream::new();
let mut view_oneof = TokenStream::new();
let mut ext = TokenStream::new();
let mut root_reexports: Vec<message::ReexportCandidate> = Vec::new();
let sentinel = make_field_ident(context::SENTINEL_MOD);
for enum_type in &file.enum_type {
let enum_rust_name = enum_type.name.as_deref().unwrap_or("");
let enum_fqn = if current_package.is_empty() {
enum_rust_name.to_string()
} else {
format!("{}.{}", current_package, enum_rust_name)
};
owned.extend(enumeration::generate_enum(
ctx,
enum_type,
enum_rust_name,
&enum_fqn,
&features,
&resolver,
)?);
}
for message_type in &file.message_type {
let top_level_name = message_type.name.as_deref().unwrap_or("");
let proto_fqn = if current_package.is_empty() {
top_level_name.to_string()
} else {
format!("{}.{}", current_package, top_level_name)
};
let MessageOutput {
owned_top,
owned_mod,
oneof_tree: msg_oneof,
view_tree: msg_view,
view_oneof_tree: msg_view_oneof,
reg: msg_reg,
} = message::generate_message(
ctx,
message_type,
current_package,
top_level_name,
&proto_fqn,
&features,
&resolver,
)?;
owned.extend(owned_top);
let mod_ident = make_field_ident(&crate::oneof::to_snake_case(top_level_name));
for p in msg_reg.json_ext {
reg.json_ext.push(quote! { #mod_ident :: #p });
}
for p in msg_reg.text_ext {
reg.text_ext.push(quote! { #mod_ident :: #p });
}
reg.json_any.extend(msg_reg.json_any);
reg.text_any.extend(msg_reg.text_any);
if !owned_mod.is_empty() {
owned.extend(quote! {
pub mod #mod_ident {
#[allow(unused_imports)]
use super::*;
#owned_mod
}
});
}
oneof.extend(msg_oneof);
view.extend(msg_view);
view_oneof.extend(msg_view_oneof);
if ctx.config.generate_views {
let view_ident = format_ident!("{top_level_name}View");
root_reexports.push(message::ReexportCandidate {
name: view_ident.to_string(),
tokens: quote! {
#[doc(inline)]
pub use self :: #sentinel :: view :: #view_ident;
},
});
}
}
let (file_ext_tokens, file_ext_json, file_ext_text) = extension::generate_extensions(
ctx,
&file.extension,
current_package,
2,
&features,
current_package,
)?;
ext.extend(file_ext_tokens);
for id in file_ext_json {
reg.json_ext.push(quote! { #sentinel :: ext :: #id });
}
for id in file_ext_text {
reg.text_ext.push(quote! { #sentinel :: ext :: #id });
}
for ext_field in &file.extension {
let const_ident = extension::extension_const_ident(ext_field.name.as_deref().unwrap_or(""));
root_reexports.push(message::ReexportCandidate {
name: const_ident.to_string(),
tokens: quote! {
#[doc(inline)]
pub use self :: #sentinel :: ext :: #const_ident;
},
});
}
Ok(ProtoContent {
stem: proto_path_to_stem(file.name.as_deref().unwrap_or("")),
owned,
view,
oneof,
view_oneof,
ext,
root_reexports,
})
}
#[derive(Default)]
struct PackageSections {
owned: Vec<TokenStream>,
view: Vec<TokenStream>,
oneof: Vec<TokenStream>,
view_oneof: Vec<TokenStream>,
ext: Vec<TokenStream>,
}
impl PackageSections {
fn from_stems(stems: &[String]) -> Self {
let includes = |suffix: &str| -> Vec<TokenStream> {
stems
.iter()
.map(|stem| {
let path = format!("{stem}{suffix}.rs");
quote! { include!(#path); }
})
.collect()
};
Self {
owned: includes(""),
view: includes(".__view"),
oneof: includes(".__oneof"),
view_oneof: includes(".__view_oneof"),
ext: includes(".__ext"),
}
}
fn push_inline(&mut self, pc: ProtoContent) {
self.owned.push(pc.owned);
self.view.push(pc.view);
self.oneof.push(pc.oneof);
self.view_oneof.push(pc.view_oneof);
self.ext.push(pc.ext);
}
}
fn generate_package(
ctx: &context::CodeGenContext,
current_package: &str,
files: &[&FileDescriptorProto],
out: &mut Vec<GeneratedFile>,
) -> Result<(), CodeGenError> {
let mut reg = message::RegistryPaths::default();
let mut root_reexports: Vec<message::ReexportCandidate> = Vec::new();
let sections = if ctx.config.file_per_package {
let mut sections = PackageSections::default();
for file in files {
let mut pc = generate_proto_content(ctx, current_package, file, &mut reg)?;
root_reexports.append(&mut pc.root_reexports);
sections.push_inline(pc);
}
sections
} else {
let mut stems: Vec<String> = Vec::new();
for file in files {
let mut pc = generate_proto_content(ctx, current_package, file, &mut reg)?;
root_reexports.append(&mut pc.root_reexports);
let source = file.name.as_deref().unwrap_or("");
let push = |out: &mut Vec<GeneratedFile>,
suffix: &str,
kind: GeneratedFileKind,
tokens: TokenStream|
-> Result<(), CodeGenError> {
out.push(GeneratedFile {
name: format!("{}{suffix}.rs", pc.stem),
package: current_package.to_string(),
kind,
content: format_tokens(tokens, source)?,
});
Ok(())
};
push(out, "", GeneratedFileKind::Owned, pc.owned)?;
push(out, ".__view", GeneratedFileKind::View, pc.view)?;
push(out, ".__oneof", GeneratedFileKind::Oneof, pc.oneof)?;
push(
out,
".__view_oneof",
GeneratedFileKind::ViewOneof,
pc.view_oneof,
)?;
push(out, ".__ext", GeneratedFileKind::Ext, pc.ext)?;
stems.push(pc.stem);
}
PackageSections::from_stems(&stems)
};
let reexport_block = surviving_root_reexports(ctx, files, ®, root_reexports);
out.push(GeneratedFile {
name: if ctx.config.file_per_package {
package_to_filename(current_package)
} else {
package_to_mod_filename(current_package)
},
package: current_package.to_string(),
kind: GeneratedFileKind::PackageMod,
content: generate_package_mod(ctx, §ions, ®, &reexport_block)?,
});
Ok(())
}
fn surviving_root_reexports(
ctx: &context::CodeGenContext,
files: &[&FileDescriptorProto],
reg: &message::RegistryPaths,
mut candidates: Vec<message::ReexportCandidate>,
) -> TokenStream {
use crate::idents::make_field_ident;
use std::collections::BTreeSet;
let mut occupied: BTreeSet<String> = BTreeSet::new();
occupied.insert(context::SENTINEL_MOD.to_string());
for file in files {
for m in &file.message_type {
let name = m.name.as_deref().unwrap_or("");
occupied.insert(name.to_string());
occupied.insert(crate::oneof::to_snake_case(name));
}
for e in &file.enum_type {
occupied.insert(e.name.as_deref().unwrap_or("").to_string());
}
}
if ctx.config.emit_register_fn && !reg.is_empty() {
let sentinel = make_field_ident(context::SENTINEL_MOD);
candidates.push(message::ReexportCandidate {
name: "register_types".to_string(),
tokens: quote! {
#[doc(inline)]
pub use self :: #sentinel :: register_types;
},
});
}
message::emit_surviving_reexports(candidates, &occupied)
}
fn generate_package_mod(
ctx: &context::CodeGenContext,
sections: &PackageSections,
reg: &message::RegistryPaths,
root_reexports: &TokenStream,
) -> Result<String, CodeGenError> {
use crate::idents::make_field_ident;
let owned = §ions.owned;
let view = §ions.view;
let view_oneof = §ions.view_oneof;
let oneof = §ions.oneof;
let ext = §ions.ext;
let view_mod = if ctx.config.generate_views {
quote! {
pub mod view {
#[allow(unused_imports)]
use super::*;
#(#view)*
pub mod oneof {
#[allow(unused_imports)]
use super::*;
#(#view_oneof)*
}
}
}
} else {
TokenStream::new()
};
let register_fn = if ctx.config.emit_register_fn && !reg.is_empty() {
let json_any = ®.json_any;
let json_ext = ®.json_ext;
let text_any = ®.text_any;
let text_ext = ®.text_ext;
quote! {
pub fn register_types(reg: &mut ::buffa::type_registry::TypeRegistry) {
#( reg.register_json_any(super::#json_any); )*
#( reg.register_json_ext(super::#json_ext); )*
#( reg.register_text_any(super::#text_any); )*
#( reg.register_text_ext(super::#text_ext); )*
}
}
} else {
TokenStream::new()
};
let allow = allow_lints_attr();
let sentinel = make_field_ident(context::SENTINEL_MOD);
let tokens = quote! {
#(#owned)*
#allow
pub mod #sentinel {
#[allow(unused_imports)]
use super::*;
#view_mod
pub mod oneof {
#[allow(unused_imports)]
use super::*;
#(#oneof)*
}
pub mod ext {
#[allow(unused_imports)]
use super::*;
#(#ext)*
}
#register_fn
}
#root_reexports
};
format_tokens(tokens, "")
}
fn format_tokens(tokens: TokenStream, source: &str) -> Result<String, CodeGenError> {
let syntax_tree =
syn::parse2::<syn::File>(tokens).map_err(|e| CodeGenError::InvalidSyntax(e.to_string()))?;
let formatted = prettyplease::unparse(&syntax_tree);
let source_line = if source.is_empty() {
String::new()
} else {
format!("// source: {source}\n")
};
Ok(format!(
"// @generated by buffa-codegen. DO NOT EDIT.\n{source_line}\n{formatted}"
))
}
pub fn package_to_mod_filename(package: &str) -> String {
if package.is_empty() {
format!("{}.mod.rs", context::SENTINEL_MOD)
} else {
format!("{package}.mod.rs")
}
}
pub fn package_to_filename(package: &str) -> String {
if package.is_empty() {
format!("{}.rs", context::SENTINEL_MOD)
} else {
format!("{package}.rs")
}
}
pub fn proto_path_to_stem(proto_path: &str) -> String {
let without_ext = proto_path.strip_suffix(".proto").unwrap_or(proto_path);
without_ext.replace('/', ".")
}
pub fn apply_companions(files: &mut Vec<GeneratedFile>, companions: Vec<GeneratedFile>) {
for comp in &companions {
debug_assert!(
!comp.name.contains(['"', '\\', '/', '\n']),
"companion file name {:?} contains a character that would break \
the generated include!() literal or its bare-sibling resolution",
comp.name
);
if let Some(pkg_mod) = files
.iter_mut()
.find(|f| f.kind == GeneratedFileKind::PackageMod && f.package == comp.package)
{
pkg_mod
.content
.push_str(&format!("include!(\"{}\");\n", comp.name));
}
}
files.extend(companions);
}
#[derive(Debug, Clone, thiserror::Error)]
#[non_exhaustive]
pub enum CodeGenError {
#[error("missing required descriptor field: {0}")]
MissingField(&'static str),
#[error("invalid Rust type path: '{0}'")]
InvalidTypePath(String),
#[error("generated code failed to parse as Rust: {0}")]
InvalidSyntax(String),
#[error("file_to_generate '{0}' not found in descriptor set")]
FileNotFound(String),
#[error("codegen error: {0}")]
Other(String),
#[error(
"reserved field name '{field_name}' in message '{message_name}': \
proto field names starting with '__buffa_' conflict with buffa's \
internal fields"
)]
ReservedFieldName {
message_name: String,
field_name: String,
},
#[error(
"module name conflict in '{scope}': messages '{name_a}' and '{name_b}' \
both produce module '{module_name}'"
)]
ModuleNameConflict {
scope: String,
name_a: String,
name_b: String,
module_name: String,
},
#[error(
"reserved name '{name}' at {location}: this name is reserved for \
buffa's generated ancillary types (views, oneof enums, \
extensions). Rename the proto element."
)]
ReservedModuleName { name: String, location: String },
#[error(
"message '{message_name}' uses `option message_set_wire_format = true` \
but CodeGenConfig::allow_message_set is false; MessageSet is a legacy \
wire format — set allow_message_set(true) if this is intentional"
)]
MessageSetNotSupported { message_name: String },
#[error(
"invalid custom attribute for path '{path}': '{attribute}' is not a valid \
Rust attribute ({detail})"
)]
InvalidCustomAttribute {
path: String,
attribute: String,
detail: String,
},
}
#[cfg(test)]
mod tests;