protobuf_codegen_pure/
lib.rs

1//! # API to generate `.rs` files
2//!
3//! This API does not require `protoc` command present in `$PATH`.
4//!
5//! ```
6//! extern crate protoc_rust;
7//!
8//! fn main() {
9//!     protobuf_codegen_pure::Codegen::new()
10//!         .out_dir("src/protos")
11//!         .inputs(&["protos/a.proto", "protos/b.proto"])
12//!         .include("protos")
13//!         .run()
14//!         .expect("Codegen failed.");
15//! }
16//! ```
17//!
18//! And in `Cargo.toml`:
19//!
20//! ```toml
21//! [build-dependencies]
22//! protobuf-codegen-pure = "2"
23//! ```
24//!
25//! It is advisable that `protobuf-codegen-pure` build-dependecy version be the same as
26//! `protobuf` dependency.
27//!
28//! The alternative is to use [`protoc-rust`](https://docs.rs/protoc-rust/=2) crate
29//! which uses `protoc` command for parsing (so it uses the same parser
30//! Google is using in their protobuf implementations).
31//!
32//! # Version 2
33//!
34//! This is documentation for version 2 of the crate.
35//!
36//! In version 3, this API is moved to
37//! [`protobuf-codegen` crate](https://docs.rs/protobuf-codegen/%3E=3.0.0-alpha).
38
39#![deny(missing_docs)]
40#![deny(rustdoc::broken_intra_doc_links)]
41
42extern crate protobuf;
43extern crate protobuf_codegen;
44
45mod convert;
46
47use std::error::Error;
48use std::fmt;
49use std::fmt::Formatter;
50use std::fs;
51use std::io;
52use std::io::Read;
53use std::path::Path;
54use std::path::PathBuf;
55use std::path::StripPrefixError;
56
57mod linked_hash_map;
58mod model;
59mod parser;
60
61use linked_hash_map::LinkedHashMap;
62pub use protobuf_codegen::Customize;
63
64#[cfg(test)]
65mod test_against_protobuf_protos;
66
67/// Invoke pure rust codegen. See [crate docs](crate) for example.
68// TODO: merge with protoc-rust def
69#[derive(Debug, Default)]
70pub struct Codegen {
71    /// --lang_out= param
72    out_dir: PathBuf,
73    /// -I args
74    includes: Vec<PathBuf>,
75    /// List of .proto files to compile
76    inputs: Vec<PathBuf>,
77    /// Customize code generation
78    customize: Customize,
79}
80
81impl Codegen {
82    /// Fresh new codegen object.
83    pub fn new() -> Self {
84        Self::default()
85    }
86
87    /// Set the output directory for codegen.
88    pub fn out_dir(&mut self, out_dir: impl AsRef<Path>) -> &mut Self {
89        self.out_dir = out_dir.as_ref().to_owned();
90        self
91    }
92
93    /// Add an include directory.
94    pub fn include(&mut self, include: impl AsRef<Path>) -> &mut Self {
95        self.includes.push(include.as_ref().to_owned());
96        self
97    }
98
99    /// Add include directories.
100    pub fn includes(&mut self, includes: impl IntoIterator<Item = impl AsRef<Path>>) -> &mut Self {
101        for include in includes {
102            self.include(include);
103        }
104        self
105    }
106
107    /// Add an input (`.proto` file).
108    pub fn input(&mut self, input: impl AsRef<Path>) -> &mut Self {
109        self.inputs.push(input.as_ref().to_owned());
110        self
111    }
112
113    /// Add inputs (`.proto` files).
114    pub fn inputs(&mut self, inputs: impl IntoIterator<Item = impl AsRef<Path>>) -> &mut Self {
115        for input in inputs {
116            self.input(input);
117        }
118        self
119    }
120
121    /// Specify generated code [`Customize`] object.
122    pub fn customize(&mut self, customize: Customize) -> &mut Self {
123        self.customize = customize;
124        self
125    }
126
127    /// Like `protoc --rust_out=...` but without requiring `protoc` or `protoc-gen-rust`
128    /// commands in `$PATH`.
129    pub fn run(&self) -> io::Result<()> {
130        let includes: Vec<&Path> = self.includes.iter().map(|p| p.as_path()).collect();
131        let inputs: Vec<&Path> = self.inputs.iter().map(|p| p.as_path()).collect();
132        let p = parse_and_typecheck(&includes, &inputs)?;
133
134        protobuf_codegen::gen_and_write(
135            &p.file_descriptors,
136            &p.relative_paths,
137            &self.out_dir,
138            &self.customize,
139        )
140    }
141}
142
143/// Arguments for pure rust codegen invocation.
144// TODO: merge with protoc-rust def
145#[derive(Debug, Default)]
146#[deprecated(since = "2.14", note = "Use Codegen object instead")]
147pub struct Args<'a> {
148    /// --lang_out= param
149    pub out_dir: &'a str,
150    /// -I args
151    pub includes: &'a [&'a str],
152    /// List of .proto files to compile
153    pub input: &'a [&'a str],
154    /// Customize code generation
155    pub customize: Customize,
156}
157
158/// Convert OS path to protobuf path (with slashes)
159/// Function is `pub(crate)` for test.
160pub(crate) fn relative_path_to_protobuf_path(path: &Path) -> String {
161    assert!(path.is_relative());
162    let path = path.to_str().expect("not a valid UTF-8 name");
163    if cfg!(windows) {
164        path.replace('\\', "/")
165    } else {
166        path.to_owned()
167    }
168}
169
170#[derive(Clone)]
171struct FileDescriptorPair {
172    parsed: model::FileDescriptor,
173    descriptor: protobuf::descriptor::FileDescriptorProto,
174}
175
176#[derive(Debug)]
177enum CodegenError {
178    ParserErrorWithLocation(parser::ParserErrorWithLocation),
179    ConvertError(convert::ConvertError),
180}
181
182impl fmt::Display for CodegenError {
183    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
184        match self {
185            CodegenError::ParserErrorWithLocation(_) => write!(f, "Parse error"),
186            CodegenError::ConvertError(_) => write!(f, "Could not typecheck parsed file"),
187        }
188    }
189}
190
191impl From<parser::ParserErrorWithLocation> for CodegenError {
192    fn from(e: parser::ParserErrorWithLocation) -> Self {
193        CodegenError::ParserErrorWithLocation(e)
194    }
195}
196
197impl From<convert::ConvertError> for CodegenError {
198    fn from(e: convert::ConvertError) -> Self {
199        CodegenError::ConvertError(e)
200    }
201}
202
203#[derive(Debug)]
204struct WithFileError {
205    file: String,
206    error: CodegenError,
207}
208
209impl fmt::Display for WithFileError {
210    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
211        write!(f, "Could not write to {}: {}", self.file, self.error)
212    }
213}
214
215impl Error for WithFileError {
216    fn description(&self) -> &str {
217        "WithFileError"
218    }
219}
220
221struct Run<'a> {
222    parsed_files: LinkedHashMap<String, FileDescriptorPair>,
223    includes: &'a [&'a Path],
224}
225
226impl<'a> Run<'a> {
227    fn get_file_and_all_deps_already_parsed(
228        &self,
229        protobuf_path: &str,
230        result: &mut LinkedHashMap<String, FileDescriptorPair>,
231    ) {
232        if let Some(_) = result.get(protobuf_path) {
233            return;
234        }
235
236        let pair = self
237            .parsed_files
238            .get(protobuf_path)
239            .expect("must be already parsed");
240        result.insert(protobuf_path.to_owned(), pair.clone());
241
242        self.get_all_deps_already_parsed(&pair.parsed, result);
243    }
244
245    fn get_all_deps_already_parsed(
246        &self,
247        parsed: &model::FileDescriptor,
248        result: &mut LinkedHashMap<String, FileDescriptorPair>,
249    ) {
250        for import in &parsed.imports {
251            self.get_file_and_all_deps_already_parsed(&import.path, result);
252        }
253    }
254
255    fn add_file(&mut self, protobuf_path: &str, fs_path: &Path) -> io::Result<()> {
256        if let Some(_) = self.parsed_files.get(protobuf_path) {
257            return Ok(());
258        }
259
260        let mut content = String::new();
261        fs::File::open(fs_path)?.read_to_string(&mut content)?;
262
263        self.add_file_content(protobuf_path, fs_path, &content)
264    }
265
266    fn add_file_content(
267        &mut self,
268        protobuf_path: &str,
269        fs_path: &Path,
270        content: &str,
271    ) -> io::Result<()> {
272        let parsed = model::FileDescriptor::parse(content).map_err(|e| {
273            io::Error::new(
274                io::ErrorKind::Other,
275                WithFileError {
276                    file: format!("{}", fs_path.display()),
277                    error: e.into(),
278                },
279            )
280        })?;
281
282        for import_path in &parsed.imports {
283            self.add_imported_file(&import_path.path)?;
284        }
285
286        let mut this_file_deps = LinkedHashMap::new();
287        self.get_all_deps_already_parsed(&parsed, &mut this_file_deps);
288
289        let this_file_deps: Vec<_> = this_file_deps.into_iter().map(|(_, v)| v.parsed).collect();
290
291        let descriptor =
292            convert::file_descriptor(protobuf_path.to_owned(), &parsed, &this_file_deps).map_err(
293                |e| {
294                    io::Error::new(
295                        io::ErrorKind::Other,
296                        WithFileError {
297                            file: format!("{}", fs_path.display()),
298                            error: e.into(),
299                        },
300                    )
301                },
302            )?;
303
304        self.parsed_files.insert(
305            protobuf_path.to_owned(),
306            FileDescriptorPair { parsed, descriptor },
307        );
308
309        Ok(())
310    }
311
312    fn add_imported_file(&mut self, protobuf_path: &str) -> io::Result<()> {
313        for include_dir in self.includes {
314            let fs_path = Path::new(include_dir).join(protobuf_path);
315            if fs_path.exists() {
316                return self.add_file(protobuf_path, &fs_path);
317            }
318        }
319
320        let embedded = match protobuf_path {
321            "rustproto.proto" => Some(RUSTPROTO_PROTO),
322            "google/protobuf/any.proto" => Some(ANY_PROTO),
323            "google/protobuf/api.proto" => Some(API_PROTO),
324            "google/protobuf/descriptor.proto" => Some(DESCRIPTOR_PROTO),
325            "google/protobuf/duration.proto" => Some(DURATION_PROTO),
326            "google/protobuf/empty.proto" => Some(EMPTY_PROTO),
327            "google/protobuf/field_mask.proto" => Some(FIELD_MASK_PROTO),
328            "google/protobuf/source_context.proto" => Some(SOURCE_CONTEXT_PROTO),
329            "google/protobuf/struct.proto" => Some(STRUCT_PROTO),
330            "google/protobuf/timestamp.proto" => Some(TIMESTAMP_PROTO),
331            "google/protobuf/type.proto" => Some(TYPE_PROTO),
332            "google/protobuf/wrappers.proto" => Some(WRAPPERS_PROTO),
333            _ => None,
334        };
335
336        match embedded {
337            Some(content) => {
338                self.add_file_content(protobuf_path, Path::new(protobuf_path), content)
339            }
340            None => Err(io::Error::new(
341                io::ErrorKind::Other,
342                format!(
343                    "protobuf path {:?} is not found in import path {:?}",
344                    protobuf_path, self.includes
345                ),
346            )),
347        }
348    }
349
350    fn strip_prefix<'b>(path: &'b Path, prefix: &Path) -> Result<&'b Path, StripPrefixError> {
351        // special handling of `.` to allow successful `strip_prefix("foo.proto", ".")
352        if prefix == Path::new(".") {
353            Ok(path)
354        } else {
355            path.strip_prefix(prefix)
356        }
357    }
358
359    fn add_fs_file(&mut self, fs_path: &Path) -> io::Result<String> {
360        let relative_path = self
361            .includes
362            .iter()
363            .filter_map(|include_dir| Self::strip_prefix(fs_path, include_dir).ok())
364            .next();
365
366        match relative_path {
367            Some(relative_path) => {
368                let protobuf_path = relative_path_to_protobuf_path(relative_path);
369                self.add_file(&protobuf_path, fs_path)?;
370                Ok(protobuf_path)
371            }
372            None => Err(io::Error::new(
373                io::ErrorKind::Other,
374                format!(
375                    "file {:?} must reside in include path {:?}",
376                    fs_path, self.includes
377                ),
378            )),
379        }
380    }
381}
382
383#[doc(hidden)]
384pub struct ParsedAndTypechecked {
385    pub relative_paths: Vec<String>,
386    pub file_descriptors: Vec<protobuf::descriptor::FileDescriptorProto>,
387}
388
389#[doc(hidden)]
390pub fn parse_and_typecheck(
391    includes: &[&Path],
392    input: &[&Path],
393) -> io::Result<ParsedAndTypechecked> {
394    let mut run = Run {
395        parsed_files: LinkedHashMap::new(),
396        includes: includes,
397    };
398
399    let mut relative_paths = Vec::new();
400
401    for input in input {
402        relative_paths.push(run.add_fs_file(&Path::new(input))?);
403    }
404
405    let file_descriptors: Vec<_> = run
406        .parsed_files
407        .into_iter()
408        .map(|(_, v)| v.descriptor)
409        .collect();
410
411    Ok(ParsedAndTypechecked {
412        relative_paths,
413        file_descriptors,
414    })
415}
416
417const RUSTPROTO_PROTO: &str = include_str!("proto/rustproto.proto");
418const ANY_PROTO: &str = include_str!("proto/google/protobuf/any.proto");
419const API_PROTO: &str = include_str!("proto/google/protobuf/api.proto");
420const DESCRIPTOR_PROTO: &str = include_str!("proto/google/protobuf/descriptor.proto");
421const DURATION_PROTO: &str = include_str!("proto/google/protobuf/duration.proto");
422const EMPTY_PROTO: &str = include_str!("proto/google/protobuf/empty.proto");
423const FIELD_MASK_PROTO: &str = include_str!("proto/google/protobuf/field_mask.proto");
424const SOURCE_CONTEXT_PROTO: &str = include_str!("proto/google/protobuf/source_context.proto");
425const STRUCT_PROTO: &str = include_str!("proto/google/protobuf/struct.proto");
426const TIMESTAMP_PROTO: &str = include_str!("proto/google/protobuf/timestamp.proto");
427const TYPE_PROTO: &str = include_str!("proto/google/protobuf/type.proto");
428const WRAPPERS_PROTO: &str = include_str!("proto/google/protobuf/wrappers.proto");
429
430/// Like `protoc --rust_out=...` but without requiring `protoc` or `protoc-gen-rust`
431/// commands in `$PATH`.
432#[deprecated(since = "2.14", note = "Use Codegen instead")]
433#[allow(deprecated)]
434pub fn run(args: Args) -> io::Result<()> {
435    let includes: Vec<&Path> = args.includes.iter().map(|p| Path::new(p)).collect();
436    let inputs: Vec<&Path> = args.input.iter().map(|p| Path::new(p)).collect();
437    let p = parse_and_typecheck(&includes, &inputs)?;
438
439    protobuf_codegen::gen_and_write(
440        &p.file_descriptors,
441        &p.relative_paths,
442        &Path::new(&args.out_dir),
443        &args.customize,
444    )
445}
446
447#[cfg(test)]
448mod test {
449    use super::*;
450
451    #[cfg(windows)]
452    #[test]
453    fn test_relative_path_to_protobuf_path_windows() {
454        assert_eq!(
455            "foo/bar.proto",
456            relative_path_to_protobuf_path(&Path::new("foo\\bar.proto"))
457        );
458    }
459
460    #[test]
461    fn test_relative_path_to_protobuf_path() {
462        assert_eq!(
463            "foo/bar.proto",
464            relative_path_to_protobuf_path(&Path::new("foo/bar.proto"))
465        );
466    }
467}
protobuf_codegen_pure/lib.rs

protobuf_codegen_pure/
lib.rs
