Skip to main content

tonneau_build/
lib.rs

1//! The code generator for [`tonneau`](https://crates.io/crates/tonneau).
2#![cfg_attr(feature = "docs", doc = "## Feature flags")]
3#![cfg_attr(feature = "docs", doc = document_features::document_features!())]
4//! ## Usage
5//!
6//! In your `build.rs`:
7//!
8//! ```rust,no_run
9//! # #[allow(clippy::needless_doctest_main)]
10//! fn main() {
11//!     tonneau_build::Config::prost()
12//!         .compile_protos(&["proto/test.proto"], &["proto"])
13//!         .unwrap();
14//! }
15//! ```
16//!
17//! Look at [`Config`] to see different options to configure the generator.
18//!
19//! ## License
20//!
21//! This project is licensed under the MIT or Apache-2.0 license.
22//! You can choose between one of them if you use this work.
23//!
24//! `SPDX-License-Identifier: MIT OR Apache-2.0`
25#![cfg_attr(docsrs, feature(doc_cfg))]
26#![cfg_attr(all(test, coverage_nightly), feature(coverage_attribute))]
27#![deny(missing_docs)]
28#![deny(unsafe_code)]
29#![deny(unreachable_pub)]
30#![deny(clippy::mod_module_files)]
31#![cfg_attr(not(feature = "prost"), allow(unused_variables, dead_code))]
32
33use std::io::ErrorKind;
34use std::path::{Path, PathBuf};
35
36use anyhow::Context;
37use extern_paths::ExternPaths;
38
39use crate::path_set::PathSet;
40
41mod codegen;
42mod extern_paths;
43mod path_set;
44
45#[cfg(feature = "prost")]
46mod prost_explore;
47
48mod types;
49
50/// The mode to use for the generator, currently we only support `prost` codegen.
51#[derive(Debug, Clone, Copy)]
52pub enum Mode {
53    /// Use `prost` to generate the protobuf structures
54    #[cfg(feature = "prost")]
55    Prost,
56}
57
58impl quote::ToTokens for Mode {
59    fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) {
60        match self {
61            #[cfg(feature = "prost")]
62            Mode::Prost => quote::quote!(prost).to_tokens(tokens),
63            #[cfg(not(feature = "prost"))]
64            _ => unreachable!(),
65        }
66    }
67}
68
69#[derive(Default, Debug)]
70struct PathConfigs {
71    btree_maps: Vec<String>,
72    bytes: Vec<String>,
73    boxed: Vec<String>,
74    floats_with_non_finite_vals: PathSet,
75}
76
77/// A config for configuring how tonneau builds / generates code.
78#[derive(Debug)]
79pub struct Config {
80    disable_tonneau_include: bool,
81    root_module: bool,
82    mode: Mode,
83    paths: PathConfigs,
84    extern_paths: ExternPaths,
85    out_dir: PathBuf,
86}
87
88impl Config {
89    /// New config with prost mode.
90    #[cfg(feature = "prost")]
91    pub fn prost() -> Self {
92        Self::new(Mode::Prost)
93    }
94
95    /// Make a new config with a given mode.
96    pub fn new(mode: Mode) -> Self {
97        Self::new_with_out_dir(mode, std::env::var_os("OUT_DIR").expect("OUT_DIR not set"))
98    }
99
100    /// Make a new config with a given mode.
101    pub fn new_with_out_dir(mode: Mode, out_dir: impl Into<PathBuf>) -> Self {
102        Self {
103            disable_tonneau_include: false,
104            mode,
105            paths: PathConfigs::default(),
106            extern_paths: ExternPaths::new(mode),
107            root_module: true,
108            out_dir: out_dir.into(),
109        }
110    }
111
112    /// Disable tonneau auto-include. By default tonneau will add its own
113    /// annotations into the include path of protoc.
114    pub fn disable_tonneau_include(&mut self) -> &mut Self {
115        self.disable_tonneau_include = true;
116        self
117    }
118
119    /// Disable the root module generation
120    /// which allows for `tonneau::include_protos!()` without
121    /// providing a package.
122    pub fn disable_root_module(&mut self) -> &mut Self {
123        self.root_module = false;
124        self
125    }
126
127    /// Specify a path to generate a `BTreeMap` instead of a `HashMap` for proto map.
128    pub fn btree_map(&mut self, path: impl std::fmt::Display) -> &mut Self {
129        self.paths.btree_maps.push(path.to_string());
130        self
131    }
132
133    /// Specify a path to generate `bytes::Bytes` instead of `Vec<u8>` for proto bytes.
134    pub fn bytes(&mut self, path: impl std::fmt::Display) -> &mut Self {
135        self.paths.bytes.push(path.to_string());
136        self
137    }
138
139    /// Specify a path to wrap around a `Box` instead of including it directly into the struct.
140    pub fn boxed(&mut self, path: impl std::fmt::Display) -> &mut Self {
141        self.paths.boxed.push(path.to_string());
142        self
143    }
144
145    /// Specify a path to float/double field (or derivative, like repeated float/double)
146    /// that must use serializer/deserializer with non-finite values support (NaN/Infinity).
147    pub fn float_with_non_finite_vals(&mut self, path: impl std::fmt::Display) -> &mut Self {
148        self.paths.floats_with_non_finite_vals.insert(path);
149        self
150    }
151
152    /// Compile and generate all the protos with the includes.
153    pub fn compile_protos(
154        &mut self,
155        protos: &[impl AsRef<Path>],
156        includes: &[impl AsRef<Path>],
157    ) -> anyhow::Result<()> {
158        match self.mode {
159            #[cfg(feature = "prost")]
160            Mode::Prost => self.compile_protos_prost(protos, includes),
161        }
162    }
163
164    /// Generate tonneau code based on a precompiled FileDescriptorSet.
165    pub fn load_fds(&mut self, fds: impl bytes::Buf) -> anyhow::Result<()> {
166        match self.mode {
167            #[cfg(feature = "prost")]
168            Mode::Prost => self.load_fds_prost(fds),
169        }
170    }
171
172    #[cfg(feature = "prost")]
173    fn compile_protos_prost(
174        &mut self,
175        protos: &[impl AsRef<Path>],
176        includes: &[impl AsRef<Path>],
177    ) -> anyhow::Result<()> {
178        let fd_path = self.out_dir.join("tonneau.fd.bin");
179
180        let mut config = prost_build::Config::new();
181        config.file_descriptor_set_path(&fd_path);
182
183        let mut includes = includes.iter().map(|i| i.as_ref()).collect::<Vec<_>>();
184
185        {
186            let tonneau_out = self.out_dir.join("tonneau");
187            std::fs::create_dir_all(&tonneau_out).context("failed to create tonneau directory")?;
188            std::fs::write(
189                tonneau_out.join("annotations.proto"),
190                tonneau_pb_prost::TONNEAU_ANNOTATIONS,
191            )
192            .context("failed to write tonneau_annotations.rs")?;
193            includes.push(&self.out_dir);
194        }
195
196        config
197            .load_fds(protos, &includes)
198            .context("failed to generate tonic fds")?;
199        let fds_bytes = std::fs::read(fd_path).context("failed to read tonic fds")?;
200        self.load_fds_prost(fds_bytes.as_slice())
201    }
202
203    #[cfg(feature = "prost")]
204    fn load_fds_prost(&mut self, fds: impl bytes::Buf) -> anyhow::Result<()> {
205        use std::collections::BTreeMap;
206
207        use codegen::prost_sanatize::to_snake;
208        use codegen::utils::get_common_import_path;
209        use proc_macro2::Span;
210        use prost::Message;
211        use prost_reflect::DescriptorPool;
212        use prost_types::FileDescriptorSet;
213        use quote::{ToTokens, quote};
214        use syn::parse_quote;
215        use types::{ProtoPath, ProtoTypeRegistry};
216
217        let pool = DescriptorPool::decode(fds).context("failed to add tonic fds")?;
218
219        let mut registry = ProtoTypeRegistry::new(
220            self.mode,
221            self.extern_paths.clone(),
222            self.paths.floats_with_non_finite_vals.clone(),
223        );
224
225        let mut config = prost_build::Config::new();
226
227        // This option is provided to make sure prost_build does not internally
228        // set extern_paths. We manage that via a re-export of prost_types in the
229        // tonneau crate.
230        config.compile_well_known_types();
231
232        config.btree_map(self.paths.btree_maps.iter());
233        self.paths.boxed.iter().for_each(|path| {
234            config.boxed(path);
235        });
236        config.bytes(self.paths.bytes.iter());
237
238        for (proto, rust) in self.extern_paths.paths() {
239            let proto = if proto.starts_with('.') {
240                proto.to_string()
241            } else {
242                format!(".{proto}")
243            };
244            config.extern_path(proto, rust.to_token_stream().to_string());
245        }
246
247        prost_explore::Extensions::new(&pool)
248            .process(&mut registry)
249            .context("failed to process extensions")?;
250
251        let mut packages = codegen::generate_modules(&registry)?;
252
253        packages.iter_mut().for_each(|(path, package)| {
254            if self.extern_paths.contains(path) {
255                return;
256            }
257
258            package.enum_configs().for_each(|(path, enum_config)| {
259                if self.extern_paths.contains(path) {
260                    return;
261                }
262
263                enum_config.attributes().for_each(|attribute| {
264                    config.enum_attribute(path, attribute.to_token_stream().to_string());
265                });
266                enum_config.variants().for_each(|variant| {
267                    let path = format!("{path}.{variant}");
268                    enum_config
269                        .variant_attributes(variant)
270                        .for_each(|attribute| {
271                            config.field_attribute(&path, attribute.to_token_stream().to_string());
272                        });
273                });
274            });
275
276            package
277                .message_configs()
278                .for_each(|(path, message_config)| {
279                    if self.extern_paths.contains(path) {
280                        return;
281                    }
282
283                    message_config.attributes().for_each(|attribute| {
284                        config.message_attribute(path, attribute.to_token_stream().to_string());
285                    });
286                    message_config.fields().for_each(|field| {
287                        let path = format!("{path}.{field}");
288                        message_config
289                            .field_attributes(field)
290                            .for_each(|attribute| {
291                                config.field_attribute(
292                                    &path,
293                                    attribute.to_token_stream().to_string(),
294                                );
295                            });
296                    });
297                    message_config
298                        .oneof_configs()
299                        .for_each(|(field, oneof_config)| {
300                            let path = format!("{path}.{field}");
301                            oneof_config.attributes().for_each(|attribute| {
302                                // In prost oneofs (container) are treated as enums
303                                config
304                                    .enum_attribute(&path, attribute.to_token_stream().to_string());
305                            });
306                            oneof_config.fields().for_each(|field| {
307                                let path = format!("{path}.{field}");
308                                oneof_config.field_attributes(field).for_each(|attribute| {
309                                    config.field_attribute(
310                                        &path,
311                                        attribute.to_token_stream().to_string(),
312                                    );
313                                });
314                            });
315                        });
316                });
317
318            package
319                .extra_items
320                .extend(package.services.iter().flat_map(|service| {
321                    let mut builder = tonic_build::CodeGenBuilder::new();
322
323                    builder.emit_package(true).build_transport(true);
324
325                    let make_service = |is_client: bool| {
326                        let mut builder = tonic_build::manual::Service::builder()
327                            .name(service.name())
328                            .package(&service.package);
329
330                        if !service.comments.is_empty() {
331                            builder = builder.comment(service.comments.to_string());
332                        }
333
334                        service
335                        .methods
336                        .iter()
337                        .fold(builder, |service_builder, (name, method)| {
338                            let codec_path = if let Some(Some(codec_path)) =
339                                (!is_client).then_some(method.codec_path.as_ref())
340                            {
341                                let path = get_common_import_path(&service.full_name, codec_path);
342                                quote!(#path::<::tonneau::reexports::tonic_prost::ProstCodec<_, _>>)
343                            } else {
344                                quote!(::tonneau::reexports::tonic_prost::ProstCodec)
345                            };
346
347                            let mut builder = tonic_build::manual::Method::builder()
348                                .input_type(
349                                    registry
350                                        .resolve_rust_path(
351                                            &service.full_name,
352                                            method.input.value_type().proto_path(),
353                                        )
354                                        .unwrap()
355                                        .to_token_stream()
356                                        .to_string(),
357                                )
358                                .output_type(
359                                    registry
360                                        .resolve_rust_path(
361                                            &service.full_name,
362                                            method.output.value_type().proto_path(),
363                                        )
364                                        .unwrap()
365                                        .to_token_stream()
366                                        .to_string(),
367                                )
368                                .codec_path(codec_path.to_string())
369                                .name(to_snake(name))
370                                .route_name(name);
371
372                            if method.input.is_stream() {
373                                builder = builder.client_streaming()
374                            }
375
376                            if method.output.is_stream() {
377                                builder = builder.server_streaming();
378                            }
379
380                            if !method.comments.is_empty() {
381                                builder = builder.comment(method.comments.to_string());
382                            }
383
384                            service_builder.method(builder.build())
385                        })
386                        .build()
387                    };
388
389                    let mut client: syn::ItemMod =
390                        syn::parse2(builder.generate_client(&make_service(true), "")).unwrap();
391                    client.content.as_mut().unwrap().1.insert(
392                        0,
393                        parse_quote!(
394                            use ::tonneau::reexports::tonic;
395                        ),
396                    );
397
398                    let mut server: syn::ItemMod =
399                        syn::parse2(builder.generate_server(&make_service(false), "")).unwrap();
400                    server.content.as_mut().unwrap().1.insert(
401                        0,
402                        parse_quote!(
403                            use ::tonneau::reexports::tonic;
404                        ),
405                    );
406
407                    [client.into(), server.into()]
408                }));
409        });
410
411        for package in packages.keys() {
412            match std::fs::remove_file(self.out_dir.join(format!("{package}.rs"))) {
413                Err(err) if err.kind() != ErrorKind::NotFound => {
414                    return Err(anyhow::anyhow!(err).context("remove"));
415                }
416                _ => {}
417            }
418        }
419
420        let fds = FileDescriptorSet {
421            file: pool.file_descriptor_protos().cloned().collect(),
422        };
423
424        let fd_path = self.out_dir.join("tonneau.fd.bin");
425        std::fs::write(fd_path, fds.encode_to_vec()).context("write fds")?;
426
427        config.compile_fds(fds).context("prost compile")?;
428
429        for (package, module) in &mut packages {
430            if self.extern_paths.contains(package) {
431                continue;
432            };
433
434            let path = self.out_dir.join(format!("{package}.rs"));
435            write_module(&path, std::mem::take(&mut module.extra_items))
436                .with_context(|| package.to_owned())?;
437        }
438
439        #[derive(Default)]
440        struct Module<'a> {
441            proto_path: Option<&'a ProtoPath>,
442            children: BTreeMap<&'a str, Module<'a>>,
443        }
444
445        impl ToTokens for Module<'_> {
446            fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) {
447                let include = self
448                    .proto_path
449                    .map(|p| p.as_ref())
450                    .map(|path| quote!(include!(concat!(#path, ".rs"));));
451                let children = self.children.iter().map(|(part, child)| {
452                    let ident = syn::Ident::new(&to_snake(part), Span::call_site());
453                    quote! {
454                        #[allow(clippy::all)]
455                        pub mod #ident {
456                            #child
457                        }
458                    }
459                });
460                quote! {
461                    #include
462                    #(#children)*
463                }
464                .to_tokens(tokens);
465            }
466        }
467
468        if self.root_module {
469            let mut module = Module::default();
470            for package in packages.keys() {
471                let mut module = &mut module;
472                for part in package.split('.') {
473                    module = module.children.entry(part).or_default();
474                }
475                module.proto_path = Some(package);
476            }
477
478            let file: syn::File = parse_quote!(#module);
479            std::fs::write(
480                self.out_dir.join("___root_module.rs"),
481                prettyplease::unparse(&file),
482            )
483            .context("write root module")?;
484        }
485
486        Ok(())
487    }
488}
489
490fn write_module(path: &std::path::Path, module: Vec<syn::Item>) -> anyhow::Result<()> {
491    let mut file = match std::fs::read_to_string(path) {
492        Ok(content) if !content.is_empty() => syn::parse_file(&content).context("parse")?,
493        Err(err) if err.kind() != ErrorKind::NotFound => {
494            return Err(anyhow::anyhow!(err).context("read"));
495        }
496        _ => syn::File {
497            attrs: Vec::new(),
498            items: Vec::new(),
499            shebang: None,
500        },
501    };
502
503    file.items.extend(module);
504    std::fs::write(path, prettyplease::unparse(&file)).context("write")?;
505
506    Ok(())
507}