prost_build/lib.rs
1#![doc(html_root_url = "https://docs.rs/prost-build/0.14.3")]
2
3//! `prost-build` compiles `.proto` files into Rust.
4//!
5//! `prost-build` is designed to be used for build-time code generation as part of a Cargo
6//! build-script.
7//!
8//! ## Example
9//!
10//! Let's create a small library crate, `snazzy`, that defines a collection of
11//! snazzy new items in a protobuf file.
12//!
13//! ```bash
14//! $ cargo new --lib snazzy && cd snazzy
15//! ```
16//!
17//! First, add `prost-build` and `prost` as dependencies to `Cargo.toml`:
18//!
19//! ```bash
20//! $ cargo add --build prost-build
21//! $ cargo add prost
22//! ```
23//!
24//! Next, add `src/items.proto` to the project:
25//!
26//! ```proto
27//! syntax = "proto3";
28//!
29//! package snazzy.items;
30//!
31//! // A snazzy new shirt!
32//! message Shirt {
33//! // Label sizes
34//! enum Size {
35//! SMALL = 0;
36//! MEDIUM = 1;
37//! LARGE = 2;
38//! }
39//!
40//! // The base color
41//! string color = 1;
42//! // The size as stated on the label
43//! Size size = 2;
44//! }
45//! ```
46//!
47//! To generate Rust code from `items.proto`, we use `prost-build` in the crate's
48//! `build.rs` build-script:
49//!
50//! ```rust,no_run
51//! use std::io::Result;
52//! fn main() -> Result<()> {
53//! prost_build::compile_protos(&["src/items.proto"], &["src/"])?;
54//! Ok(())
55//! }
56//! ```
57//!
58//! And finally, in `lib.rs`, include the generated code:
59//!
60//! ```rust,ignore
61//! // Include the `items` module, which is generated from items.proto.
62//! // It is important to maintain the same structure as in the proto.
63//! pub mod snazzy {
64//! pub mod items {
65//! include!(concat!(env!("OUT_DIR"), "/snazzy.items.rs"));
66//! }
67//! }
68//!
69//! use snazzy::items;
70//!
71//! /// Returns a large shirt of the specified color
72//! pub fn create_large_shirt(color: String) -> items::Shirt {
73//! let mut shirt = items::Shirt::default();
74//! shirt.color = color;
75//! shirt.set_size(items::shirt::Size::Large);
76//! shirt
77//! }
78//! ```
79//!
80//! That's it! Run `cargo doc` to see documentation for the generated code. The full
81//! example project can be found on [GitHub](https://github.com/danburkert/snazzy).
82//!
83//! ## Feature Flags
84//! - `format`: Format the generated output. This feature is enabled by default.
85//! - `cleanup-markdown`: Clean up Markdown in protobuf docs. Enable this to clean up protobuf files from third parties.
86//!
87//! ### Cleaning up Markdown in code docs
88//!
89//! If you are using protobuf files from third parties, where the author of the protobuf
90//! is not treating comments as Markdown, or is, but has codeblocks in their docs,
91//! then you may need to clean up the documentation in order that `cargo test --doc`
92//! will not fail spuriously, and that `cargo doc` doesn't attempt to render the
93//! codeblocks as Rust code.
94//!
95//! To do this, in your `Cargo.toml`, add `features = ["cleanup-markdown"]` to the inclusion
96//! of the `prost-build` crate and when your code is generated, the code docs will automatically
97//! be cleaned up a bit.
98//!
99//! ## Sourcing `protoc`
100//!
101//! `prost-build` depends on the Protocol Buffers compiler, `protoc`, to parse `.proto` files into
102//! a representation that can be transformed into Rust.
103//!
104//! The easiest way for `prost-build` to find `protoc` is to install it in your `PATH`.
105//! This can be done by following the [`protoc` install instructions]. `prost-build` will search
106//! the current path for `protoc` or `protoc.exe`.
107//!
108//! When `protoc` is installed in a different location, set `PROTOC` to the path of the executable.
109//! If set, `prost-build` uses the `PROTOC`
110//! for locating `protoc`. For example, on a macOS system where Protobuf is installed
111//! with Homebrew, set the environment variables to:
112//!
113//! ```bash
114//! PROTOC=/usr/local/bin/protoc
115//! ```
116//!
117//! Alternatively, the path to `protoc` executable can be explicitly set
118//! via [`Config::protoc_executable()`].
119//!
120//! If `prost-build` can not find `protoc`
121//! via these methods the `compile_protos` method will fail.
122//!
123//! [`protoc` install instructions]: https://github.com/protocolbuffers/protobuf#protocol-compiler-installation
124//!
125//! ### Compiling `protoc` from source
126//!
127//! To compile `protoc` from source you can use the `protobuf-src` crate and
128//! set the correct environment variables.
129//! ```no_run,ignore, rust
130//! std::env::set_var("PROTOC", protobuf_src::protoc());
131//!
132//! // Now compile your proto files via prost-build
133//! ```
134//!
135//! [`protobuf-src`]: https://docs.rs/protobuf-src
136
137use std::io::Result;
138use std::path::Path;
139
140use prost_types::FileDescriptorSet;
141
142mod ast;
143pub use crate::ast::{Comments, Method, Service};
144
145mod collections;
146pub(crate) use collections::{BytesType, MapType};
147
148mod code_generator;
149mod context;
150mod extern_paths;
151mod ident;
152mod message_graph;
153mod path;
154
155mod config;
156pub use config::{
157 error_message_protoc_not_found, protoc_from_env, protoc_include_from_env, Config,
158};
159
160mod module;
161pub use module::Module;
162
163/// A service generator takes a service descriptor and generates Rust code.
164///
165/// `ServiceGenerator` can be used to generate application-specific interfaces
166/// or implementations for Protobuf service definitions.
167///
168/// Service generators are registered with a code generator using the
169/// `Config::service_generator` method.
170///
171/// A viable scenario is that an RPC framework provides a service generator. It generates a trait
172/// describing methods of the service and some glue code to call the methods of the trait, defining
173/// details like how errors are handled or if it is asynchronous. Then the user provides an
174/// implementation of the generated trait in the application code and plugs it into the framework.
175///
176/// Such framework isn't part of Prost at present.
177pub trait ServiceGenerator {
178 /// Generates a Rust interface or implementation for a service, writing the
179 /// result to `buf`.
180 fn generate(&mut self, service: Service, buf: &mut String);
181
182 /// Finalizes the generation process.
183 ///
184 /// In case there's something that needs to be output at the end of the generation process, it
185 /// goes here. Similar to [`generate`](Self::generate), the output should be appended to
186 /// `buf`.
187 ///
188 /// An example can be a module or other thing that needs to appear just once, not for each
189 /// service generated.
190 ///
191 /// This still can be called multiple times in a lifetime of the service generator, because it
192 /// is called once per `.proto` file.
193 ///
194 /// The default implementation is empty and does nothing.
195 fn finalize(&mut self, _buf: &mut String) {}
196
197 /// Finalizes the generation process for an entire protobuf package.
198 ///
199 /// This differs from [`finalize`](Self::finalize) by where (and how often) it is called
200 /// during the service generator life cycle. This method is called once per protobuf package,
201 /// making it ideal for grouping services within a single package spread across multiple
202 /// `.proto` files.
203 ///
204 /// The default implementation is empty and does nothing.
205 fn finalize_package(&mut self, _package: &str, _buf: &mut String) {}
206}
207
208/// Compile `.proto` files into Rust files during a Cargo build.
209///
210/// The generated `.rs` files are written to the Cargo `OUT_DIR` directory, suitable for use with
211/// the [include!][1] macro. See the [Cargo `build.rs` code generation][2] example for more info.
212///
213/// This function should be called in a project's `build.rs`.
214///
215/// # Arguments
216///
217/// **`protos`** - Paths to `.proto` files to compile. Any transitively [imported][3] `.proto`
218/// files are automatically be included.
219///
220/// **`includes`** - Paths to directories in which to search for imports. Directories are searched
221/// in order. The `.proto` files passed in **`protos`** must be found in one of the provided
222/// include directories.
223///
224/// # Errors
225///
226/// This function can fail for a number of reasons:
227///
228/// - Failure to locate or download `protoc`.
229/// - Failure to parse the `.proto`s.
230/// - Failure to locate an imported `.proto`.
231/// - Failure to compile a `.proto` without a [package specifier][4].
232///
233/// It's expected that this function call be `unwrap`ed in a `build.rs`; there is typically no
234/// reason to gracefully recover from errors during a build.
235///
236/// # Example `build.rs`
237///
238/// ```rust,no_run
239/// # use std::io::Result;
240/// fn main() -> Result<()> {
241/// prost_build::compile_protos(&["src/frontend.proto", "src/backend.proto"], &["src"])?;
242/// Ok(())
243/// }
244/// ```
245///
246/// [1]: https://doc.rust-lang.org/std/macro.include.html
247/// [2]: https://doc.rust-lang.org/cargo/reference/build-script-examples.html
248/// [3]: https://protobuf.dev/programming-guides/proto3/#importing
249/// [4]: https://protobuf.dev/programming-guides/proto3/#packages
250pub fn compile_protos(protos: &[impl AsRef<Path>], includes: &[impl AsRef<Path>]) -> Result<()> {
251 Config::new().compile_protos(protos, includes)
252}
253
254/// Compile a [`FileDescriptorSet`] into Rust files during a Cargo build.
255///
256/// The generated `.rs` files are written to the Cargo `OUT_DIR` directory, suitable for use with
257/// the [include!][1] macro. See the [Cargo `build.rs` code generation][2] example for more info.
258///
259/// This function should be called in a project's `build.rs`.
260///
261/// This function can be combined with a crate like [`protox`] which outputs a
262/// [`FileDescriptorSet`] and is a pure Rust implementation of `protoc`.
263///
264/// # Example
265/// ```rust,no_run
266/// # use prost_types::FileDescriptorSet;
267/// # fn fds() -> FileDescriptorSet { todo!() }
268/// fn main() -> std::io::Result<()> {
269/// let file_descriptor_set = fds();
270///
271/// prost_build::compile_fds(file_descriptor_set)
272/// }
273/// ```
274///
275/// [`protox`]: https://github.com/andrewhickman/protox
276/// [1]: https://doc.rust-lang.org/std/macro.include.html
277/// [2]: https://doc.rust-lang.org/cargo/reference/build-script-examples.html
278pub fn compile_fds(fds: FileDescriptorSet) -> Result<()> {
279 Config::new().compile_fds(fds)
280}
281
282#[cfg(test)]
283mod tests {
284 use std::cell::RefCell;
285 use std::rc::Rc;
286
287 use super::*;
288
289 macro_rules! assert_eq_fixture_file {
290 ($expected_path:expr, $actual_path:expr) => {{
291 let actual = std::fs::read_to_string($actual_path).expect("Failed to read actual file");
292
293 // Normalizes windows and Linux-style EOL
294 let actual = actual.replace("\r\n", "\n");
295
296 assert_eq_fixture_contents!($expected_path, actual);
297 }};
298 }
299
300 macro_rules! assert_eq_fixture_contents {
301 ($expected_path:expr, $actual:expr) => {{
302 let expected =
303 std::fs::read_to_string($expected_path).expect("Failed to read expected file");
304
305 // Normalizes windows and Linux-style EOL
306 let expected = expected.replace("\r\n", "\n");
307
308 if expected != $actual {
309 std::fs::write($expected_path, &$actual).expect("Failed to write expected file");
310 }
311
312 assert_eq!(expected, $actual);
313 }};
314 }
315
316 /// An example service generator that generates a trait with methods corresponding to the
317 /// service methods.
318 struct ServiceTraitGenerator;
319
320 impl ServiceGenerator for ServiceTraitGenerator {
321 fn generate(&mut self, service: Service, buf: &mut String) {
322 // Generate a trait for the service.
323 service.comments.append_with_indent(0, buf);
324 buf.push_str(&format!("trait {} {{\n", &service.name));
325
326 // Generate the service methods.
327 for method in service.methods {
328 method.comments.append_with_indent(1, buf);
329 buf.push_str(&format!(
330 " fn {}(_: {}) -> {};\n",
331 method.name, method.input_type, method.output_type
332 ));
333 }
334
335 // Close out the trait.
336 buf.push_str("}\n");
337 }
338 fn finalize(&mut self, buf: &mut String) {
339 // Needs to be present only once, no matter how many services there are
340 buf.push_str("pub mod utils { }\n");
341 }
342 }
343
344 /// Implements `ServiceGenerator` and provides some state for assertions.
345 struct MockServiceGenerator {
346 state: Rc<RefCell<MockState>>,
347 }
348
349 /// Holds state for `MockServiceGenerator`
350 #[derive(Default)]
351 struct MockState {
352 service_names: Vec<String>,
353 package_names: Vec<String>,
354 finalized: u32,
355 }
356
357 impl MockServiceGenerator {
358 fn new(state: Rc<RefCell<MockState>>) -> Self {
359 Self { state }
360 }
361 }
362
363 impl ServiceGenerator for MockServiceGenerator {
364 fn generate(&mut self, service: Service, _buf: &mut String) {
365 let mut state = self.state.borrow_mut();
366 state.service_names.push(service.name);
367 }
368
369 fn finalize(&mut self, _buf: &mut String) {
370 let mut state = self.state.borrow_mut();
371 state.finalized += 1;
372 }
373
374 fn finalize_package(&mut self, package: &str, _buf: &mut String) {
375 let mut state = self.state.borrow_mut();
376 state.package_names.push(package.to_string());
377 }
378 }
379
380 #[test]
381 fn smoke_test() {
382 let _ = env_logger::try_init();
383 let tempdir = tempfile::tempdir().unwrap();
384
385 Config::new()
386 .service_generator(Box::new(ServiceTraitGenerator))
387 .out_dir(tempdir.path())
388 .compile_protos(&["src/fixtures/smoke_test/smoke_test.proto"], &["src"])
389 .unwrap();
390
391 // Check all generated files against fixture
392 for entry in std::fs::read_dir(tempdir.path()).unwrap() {
393 let file = entry.unwrap();
394 let file_name = file.file_name().into_string().unwrap();
395
396 assert_eq!(file_name, "smoke_test.rs");
397 assert_eq_fixture_file!(
398 if cfg!(feature = "format") {
399 "src/fixtures/smoke_test/_expected_smoke_test_formatted.rs"
400 } else {
401 "src/fixtures/smoke_test/_expected_smoke_test.rs"
402 },
403 file.path()
404 );
405 }
406 }
407
408 #[test]
409 fn finalize_package() {
410 let _ = env_logger::try_init();
411 let tempdir = tempfile::tempdir().unwrap();
412
413 let state = Rc::new(RefCell::new(MockState::default()));
414 let generator = MockServiceGenerator::new(Rc::clone(&state));
415
416 Config::new()
417 .service_generator(Box::new(generator))
418 .include_file("_protos.rs")
419 .out_dir(tempdir.path())
420 .compile_protos(
421 &[
422 "src/fixtures/helloworld/hello.proto",
423 "src/fixtures/helloworld/goodbye.proto",
424 ],
425 &["src/fixtures/helloworld"],
426 )
427 .unwrap();
428
429 let state = state.borrow();
430 assert_eq!(&state.service_names, &["Greeting", "Farewell"]);
431 assert_eq!(&state.package_names, &["helloworld"]);
432 assert_eq!(state.finalized, 3);
433 }
434
435 #[test]
436 fn test_generate_message_attributes() {
437 let _ = env_logger::try_init();
438 let tempdir = tempfile::tempdir().unwrap();
439
440 let mut config = Config::new();
441 config
442 .out_dir(tempdir.path())
443 // Add attributes to all messages and enums
444 .message_attribute(".", "#[derive(derive_builder::Builder)]")
445 .enum_attribute(".", "#[some_enum_attr(u8)]");
446
447 let fds = config
448 .load_fds(
449 &["src/fixtures/helloworld/hello.proto"],
450 &["src/fixtures/helloworld"],
451 )
452 .unwrap();
453
454 // Add custom attributes to messages that are service inputs or outputs.
455 for file in &fds.file {
456 for service in &file.service {
457 for method in &service.method {
458 if let Some(input) = &method.input_type {
459 config.message_attribute(input, "#[derive(custom_proto::Input)]");
460 }
461 if let Some(output) = &method.output_type {
462 config.message_attribute(output, "#[derive(custom_proto::Output)]");
463 }
464 }
465 }
466 }
467
468 config.compile_fds(fds).unwrap();
469
470 // Check all generated files against fixture
471 for entry in std::fs::read_dir(tempdir.path()).unwrap() {
472 let file = entry.unwrap();
473 let file_name = file.file_name().into_string().unwrap();
474
475 assert_eq_fixture_file!(
476 format!("src/fixtures/helloworld/_expected_{file_name}"),
477 file.path()
478 );
479 }
480 }
481
482 #[test]
483 fn test_generate_no_empty_outputs() {
484 let _ = env_logger::try_init();
485 let state = Rc::new(RefCell::new(MockState::default()));
486 let generator = MockServiceGenerator::new(Rc::clone(&state));
487 let include_file = "_include.rs";
488 let tempdir = tempfile::tempdir().unwrap();
489 let previously_empty_proto_path = tempdir.path().join(Path::new("google.protobuf.rs"));
490
491 Config::new()
492 .service_generator(Box::new(generator))
493 .include_file(include_file)
494 .out_dir(tempdir.path())
495 .compile_protos(
496 &["src/fixtures/imports_empty/imports_empty.proto"],
497 &["src/fixtures/imports_empty"],
498 )
499 .unwrap();
500
501 // Prior to PR introducing this test, the generated include file would have the file
502 // google.protobuf.rs which was an empty file. Now that file should only exist if it has content
503 assert!(!std::fs::exists(previously_empty_proto_path).unwrap());
504
505 // Check all generated files against fixture
506 for entry in std::fs::read_dir(tempdir.path()).unwrap() {
507 let file = entry.unwrap();
508 let file_name = file.file_name().into_string().unwrap();
509 if file_name == include_file {
510 // `google.protobuf.rs` wasn't generated so the result include file should not reference it
511 assert_eq_fixture_file!(
512 "src/fixtures/imports_empty/_expected_include.rs",
513 file.path()
514 );
515 } else if file_name == "com.prost_test.test.v1.rs" {
516 let content = std::fs::read_to_string(file.path()).unwrap();
517 assert!(content.contains("struct TestConfig"));
518 assert!(content.contains("struct GetTestResponse"));
519 } else {
520 panic!("Found unexpected file: {}", file_name);
521 }
522 }
523 }
524
525 #[test]
526 fn test_generate_field_attributes() {
527 let _ = env_logger::try_init();
528 let tempdir = tempfile::tempdir().unwrap();
529
530 Config::new()
531 .out_dir(tempdir.path())
532 .boxed("Container.data.foo")
533 .boxed("Bar.qux")
534 .compile_protos(
535 &["src/fixtures/field_attributes/field_attributes.proto"],
536 &["src/fixtures/field_attributes"],
537 )
538 .unwrap();
539
540 assert_eq_fixture_file!(
541 if cfg!(feature = "format") {
542 "src/fixtures/field_attributes/_expected_field_attributes_formatted.rs"
543 } else {
544 "src/fixtures/field_attributes/_expected_field_attributes.rs"
545 },
546 tempdir.path().join("field_attributes.rs")
547 );
548 }
549
550 #[test]
551 fn deterministic_include_file() {
552 let _ = env_logger::try_init();
553
554 for _ in 1..10 {
555 let state = Rc::new(RefCell::new(MockState::default()));
556 let generator = MockServiceGenerator::new(Rc::clone(&state));
557 let include_file = "_include.rs";
558 let tempdir = tempfile::tempdir().unwrap();
559
560 Config::new()
561 .service_generator(Box::new(generator))
562 .include_file(include_file)
563 .out_dir(tempdir.path())
564 .compile_protos(
565 &[
566 "src/fixtures/alphabet/a.proto",
567 "src/fixtures/alphabet/b.proto",
568 "src/fixtures/alphabet/c.proto",
569 "src/fixtures/alphabet/d.proto",
570 "src/fixtures/alphabet/e.proto",
571 "src/fixtures/alphabet/f.proto",
572 ],
573 &["src/fixtures/alphabet"],
574 )
575 .unwrap();
576
577 assert_eq_fixture_file!(
578 "src/fixtures/alphabet/_expected_include.rs",
579 tempdir.path().join(Path::new(include_file))
580 );
581 }
582 }
583
584 #[test]
585 fn write_includes() {
586 let modules = [
587 Module::from_protobuf_package_name("foo.bar.baz"),
588 Module::from_protobuf_package_name(""),
589 Module::from_protobuf_package_name("foo.bar"),
590 Module::from_protobuf_package_name("bar"),
591 Module::from_protobuf_package_name("foo"),
592 Module::from_protobuf_package_name("foo.bar.qux"),
593 Module::from_protobuf_package_name("foo.bar.a.b.c"),
594 ];
595
596 let file_names = modules
597 .iter()
598 .map(|m| (m.clone(), m.to_file_name_or("_.default")))
599 .collect();
600
601 let mut buf = Vec::new();
602 Config::new()
603 .default_package_filename("_.default")
604 .write_includes(modules.iter().collect(), &mut buf, None, &file_names)
605 .unwrap();
606 let actual = String::from_utf8(buf).unwrap();
607 assert_eq_fixture_contents!("src/fixtures/write_includes/_.includes.rs", actual);
608 }
609
610 #[test]
611 fn test_generate_deprecated() {
612 let _ = env_logger::try_init();
613 let tempdir = tempfile::tempdir().unwrap();
614
615 Config::new()
616 .out_dir(tempdir.path())
617 .compile_protos(
618 &["src/fixtures/deprecated/all_deprecated.proto"],
619 &["src/fixtures/deprecated"],
620 )
621 .unwrap();
622
623 assert_eq_fixture_file!(
624 "src/fixtures/deprecated/_all_deprecated.rs",
625 tempdir.path().join("all_deprecated.rs")
626 );
627 }
628}