#![warn(missing_docs)]
use std::collections::btree_map::Entry;
use std::collections::{BTreeMap, BTreeSet};
use std::env;
use std::path::{Component, Path, PathBuf};
use std::sync::{Mutex, PoisonError};
#[doc(hidden)]
pub const NATIVE_FALLBACK: Option<&str> = option_env!("RAPIDPROTO_BUILD_FALLBACK");
#[doc(hidden)]
pub struct SourceFile {
pub name: String,
pub body: String,
}
#[doc(hidden)]
pub struct Request {
pub entries: Vec<String>,
pub sources: Vec<SourceFile>,
pub modes: String,
pub unknown_present: bool,
pub arena: bool,
pub stream: bool,
pub serde: bool,
pub module_path: String,
}
impl Default for Request {
fn default() -> Self {
Request {
entries: Vec::new(),
sources: Vec::new(),
modes: String::new(),
unknown_present: false,
arena: true,
stream: false,
serde: false,
module_path: "crate".to_owned(),
}
}
}
#[derive(Debug)]
pub struct GenError {
#[doc(hidden)]
pub code: i32,
pub message: String,
}
impl std::fmt::Display for GenError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"generator failed (status {}): {}",
self.code, self.message
)
}
}
impl std::error::Error for GenError {}
mod frame {
use super::{GenError, Request};
fn put_len(buf: &mut Vec<u8>, n: usize) -> Result<(), GenError> {
let len = u32::try_from(n).map_err(|_| GenError {
code: -1,
message: format!("frame field too large: {n} bytes (max {})", u32::MAX),
})?;
buf.extend_from_slice(&len.to_le_bytes());
Ok(())
}
fn put_bytes(buf: &mut Vec<u8>, bytes: &[u8]) -> Result<(), GenError> {
put_len(buf, bytes.len())?;
buf.extend_from_slice(bytes);
Ok(())
}
pub(super) fn encode(request: &Request) -> Result<Vec<u8>, GenError> {
let mut buf = Vec::new();
buf.extend_from_slice(b"RPWF5\n");
put_len(&mut buf, request.entries.len())?;
for entry in &request.entries {
put_bytes(&mut buf, entry.as_bytes())?;
}
put_len(&mut buf, request.sources.len())?;
for source in &request.sources {
put_bytes(&mut buf, source.name.as_bytes())?;
put_bytes(&mut buf, source.body.as_bytes())?;
}
put_bytes(&mut buf, request.modes.as_bytes())?;
buf.push(u8::from(request.unknown_present));
buf.push(u8::from(request.arena));
buf.push(u8::from(request.stream));
buf.push(u8::from(request.serde));
put_bytes(&mut buf, request.module_path.as_bytes())?;
Ok(buf)
}
}
#[doc(hidden)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Backend {
#[cfg(rp_native)]
Native,
#[cfg(feature = "wasm")]
Wasm,
}
#[doc(hidden)]
pub const BACKENDS: &[Backend] = &[
#[cfg(rp_native)]
Backend::Native,
#[cfg(feature = "wasm")]
Backend::Wasm,
];
#[doc(hidden)]
pub fn generate_with(backend: Backend, request: &Request) -> Result<String, GenError> {
let frame = frame::encode(request)?;
let (status, text) = match backend {
#[cfg(rp_native)]
Backend::Native => native::run(&frame)?,
#[cfg(feature = "wasm")]
Backend::Wasm => wasm::run(frame)?,
};
if status == 0 {
Ok(String::from_utf8_lossy(&text).into_owned())
} else {
Err(GenError {
code: status,
message: String::from_utf8_lossy(&text).trim_end().to_owned(),
})
}
}
#[cfg(rp_native)]
mod native {
use std::ffi::{c_char, c_int, c_void};
use super::{internal, GenError};
type Sink = extern "C" fn(context: *mut c_void, data: *const c_char, len: usize);
extern "C" {
fn rapidproto_rustgen_run(
frame: *const c_char,
frame_len: usize,
sink: Sink,
context: *mut c_void,
) -> c_int;
}
const STACK_BYTES: usize = 8 << 20;
pub(super) fn run(frame: &[u8]) -> Result<(i32, Vec<u8>), GenError> {
std::thread::scope(|scope| {
std::thread::Builder::new()
.stack_size(STACK_BYTES)
.spawn_scoped(scope, || call(frame))
.map_err(internal)?
.join()
.map_err(|_| internal("the native generator panicked"))
})
}
fn call(frame: &[u8]) -> (i32, Vec<u8>) {
extern "C" fn sink(context: *mut c_void, data: *const c_char, len: usize) {
unsafe {
let out = &mut *context.cast::<Vec<u8>>();
out.extend_from_slice(std::slice::from_raw_parts(data.cast::<u8>(), len));
}
}
let mut text = Vec::new();
let status = unsafe {
rapidproto_rustgen_run(
frame.as_ptr().cast(),
frame.len(),
sink,
std::ptr::from_mut(&mut text).cast(),
)
};
if status == -1 {
let detail = String::from_utf8_lossy(&text);
text = format!("internal error in the native generator: {detail}").into_bytes();
}
(status, text)
}
}
#[cfg(feature = "wasm")]
mod wasm {
use std::io::Cursor;
use std::sync::OnceLock;
use wasmi::{Engine, Linker, Module, Store};
use wasmi_wasi::sync::WasiCtxBuilder;
use wasmi_wasi::wasi_common::pipe::{ReadPipe, WritePipe};
use wasmi_wasi::WasiCtx;
use super::{internal, GenError};
const RUSTGEN_WASM: &[u8] = include_bytes!("../wasm/rustgen.wasm");
pub(super) fn run(frame: Vec<u8>) -> Result<(i32, Vec<u8>), GenError> {
let (engine, module) = compiled();
let stdout = WritePipe::new_in_memory();
let stderr = WritePipe::new_in_memory();
let wasi = WasiCtxBuilder::new()
.stdin(Box::new(ReadPipe::from(frame)))
.stdout(Box::new(stdout.clone()))
.stderr(Box::new(stderr.clone()))
.build();
let mut store = Store::new(engine, wasi);
let mut linker: Linker<WasiCtx> = Linker::new(engine);
wasmi_wasi::sync::add_to_linker(&mut linker, |ctx| ctx).map_err(internal)?;
let instance = linker
.instantiate_and_start(&mut store, module)
.map_err(internal)?;
let start = instance
.get_typed_func::<(), ()>(&store, "_start")
.map_err(internal)?;
let run = start.call(&mut store, ());
drop(store);
let exit = match run {
Ok(()) => 0,
Err(error) => match error.i32_exit_status() {
Some(code) => code,
None => return Err(internal(error)),
},
};
Ok((exit, take(if exit == 0 { stdout } else { stderr })))
}
fn compiled() -> &'static (Engine, Module) {
static CELL: OnceLock<(Engine, Module)> = OnceLock::new();
CELL.get_or_init(|| {
let engine = Engine::default();
let module = Module::new(&engine, RUSTGEN_WASM)
.expect("the embedded generator is a valid module");
(engine, module)
})
}
fn take(pipe: WritePipe<Cursor<Vec<u8>>>) -> Vec<u8> {
pipe.try_into_inner()
.expect("pipe still shared after the store was dropped")
.into_inner()
}
}
fn internal(error: impl std::fmt::Display) -> GenError {
GenError {
code: -1,
message: error.to_string(),
}
}
#[derive(Debug)]
pub struct Config {
module_path: String,
includes: Vec<PathBuf>,
modes: String,
unknown_present: bool,
arena: bool,
stream: bool,
serde: bool,
backend: Backend,
}
impl Config {
pub fn new(module_path: impl Into<String>) -> Self {
Config {
module_path: module_path.into(),
includes: Vec::new(),
modes: String::new(),
unknown_present: false,
arena: true,
stream: false,
serde: false,
backend: BACKENDS[0],
}
}
pub fn include(mut self, dir: impl Into<PathBuf>) -> Self {
self.includes.push(dir.into());
self
}
pub fn field_modes(mut self, modes: impl Into<String>) -> Self {
self.modes = modes.into();
self
}
pub fn unknown_present(mut self, yes: bool) -> Self {
self.unknown_present = yes;
self
}
pub fn arena(mut self, yes: bool) -> Self {
self.arena = yes;
self
}
pub fn stream(mut self, yes: bool) -> Self {
self.stream = yes;
self
}
pub fn serde(mut self, yes: bool) -> Self {
self.serde = yes;
self
}
#[doc(hidden)]
pub fn backend(mut self, backend: Backend) -> Self {
self.backend = backend;
self
}
pub fn generate(&self, entries: &[impl AsRef<Path>]) -> Result<String, GenError> {
let includes: Vec<PathBuf> = self
.include_dirs()
.iter()
.map(|dir| {
dir.canonicalize()
.map_err(|e| internal(format!("include dir {}: {e}", dir.display())))
})
.collect::<Result<_, _>>()?;
let mut sources: BTreeMap<String, String> = BTreeMap::new();
for include in &includes {
collect_protos(include, include, &mut sources).map_err(internal)?;
}
let mut entry_names = Vec::with_capacity(entries.len());
for entry in entries {
let entry = entry.as_ref();
let canonical = entry
.canonicalize()
.map_err(|e| internal(format!("entry {}: {e}", entry.display())))?;
let name = import_relative_name(&canonical, &includes);
let body = read(&canonical).map_err(internal)?;
sources.insert(name.clone(), body);
entry_names.push(name);
}
let request = Request {
entries: entry_names,
sources: sources
.into_iter()
.map(|(name, body)| SourceFile { name, body })
.collect(),
modes: self.modes.clone(),
unknown_present: self.unknown_present,
arena: self.arena,
stream: self.stream,
serde: self.serde,
module_path: self.module_path.clone(),
};
generate_with(self.backend, &request)
}
pub fn compile(&self, entries: &[impl AsRef<Path>]) -> Result<PathBuf, GenError> {
if let Some(reason) = NATIVE_FALLBACK {
static WARNED: std::sync::Once = std::sync::Once::new();
WARNED.call_once(|| {
println!(
"cargo:warning=rapidproto-build uses the slower wasm generator: the native one \
did not build, {reason}. To use the wasm one on purpose, depend on \
rapidproto-build with `default-features = false, features = [\"wasm\"]`."
);
});
}
let out_dir = env::var_os("OUT_DIR").ok_or_else(|| GenError {
code: -1,
message: "compile(): OUT_DIR is unset -- call it from a cargo build script".to_owned(),
})?;
let stem = entries
.first()
.and_then(|e| e.as_ref().file_stem())
.map_or_else(|| "schema".to_owned(), |s| s.to_string_lossy().into_owned());
let out_path = PathBuf::from(out_dir).join(format!("{stem}.rs"));
static WRITTEN: Mutex<BTreeSet<PathBuf>> = Mutex::new(BTreeSet::new());
let first = WRITTEN
.lock()
.unwrap_or_else(PoisonError::into_inner)
.insert(out_path.clone());
if !first {
return Err(GenError {
code: -1,
message: format!(
"compile(): an earlier call in this build script also writes {stem}.rs, \
since each call names its file after its first entry. Use generate() and \
write the files under names of your own"
),
});
}
let code = self.generate(entries)?;
std::fs::write(&out_path, code).map_err(|e| GenError {
code: -1,
message: format!("compile(): cannot write {}: {e}", out_path.display()),
})?;
for entry in entries {
println!("cargo:rerun-if-changed={}", entry.as_ref().display());
}
for dir in &self.include_dirs() {
track_protos(dir);
}
Ok(out_path)
}
fn include_dirs(&self) -> Vec<PathBuf> {
if self.includes.is_empty() {
vec![PathBuf::from(".")]
} else {
self.includes.clone()
}
}
}
fn track_protos(dir: &Path) {
let Ok(entries) = std::fs::read_dir(dir) else {
return;
};
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
track_protos(&path);
} else if path.extension().is_some_and(|e| e == "proto") {
println!("cargo:rerun-if-changed={}", path.display());
}
}
}
fn import_relative_name(entry: &Path, includes: &[PathBuf]) -> String {
for include in includes {
if let Ok(rel) = entry.strip_prefix(include) {
return to_slash(rel);
}
}
entry
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_default()
}
fn collect_protos(
root: &Path,
dir: &Path,
out: &mut BTreeMap<String, String>,
) -> Result<(), String> {
let entries = std::fs::read_dir(dir)
.map_err(|e| format!("cannot read include dir {}: {e}", dir.display()))?;
for entry in entries.flatten() {
let path = entry.path();
let Ok(file_type) = entry.file_type() else {
continue;
};
if file_type.is_dir() {
let _ = collect_protos(root, &path, out); } else if path.extension().is_some_and(|e| e == "proto") {
let rel = path.strip_prefix(root).unwrap_or(&path);
if let Entry::Vacant(slot) = out.entry(to_slash(rel)) {
if let Ok(body) = std::fs::read_to_string(&path) {
slot.insert(body);
}
}
}
}
Ok(())
}
fn read(path: &Path) -> Result<String, String> {
std::fs::read_to_string(path).map_err(|e| format!("cannot read {}: {e}", path.display()))
}
fn to_slash(path: &Path) -> String {
path.components()
.filter_map(|c| match c {
Component::Normal(s) => Some(s.to_string_lossy()),
_ => None,
})
.collect::<Vec<_>>()
.join("/")
}