1use std::{collections::HashMap, path::PathBuf, sync::Arc};
2
3use ahash::AHashMap;
4use faststr::FastStr;
5use itertools::Itertools;
6use normpath::PathExt;
7use pilota::Bytes;
8use protobuf::{
9 Message as _,
10 descriptor::{
11 DescriptorProto, EnumDescriptorProto, EnumValueDescriptorProto, FieldDescriptorProto,
12 MethodDescriptorProto, ServiceDescriptorProto,
13 field_descriptor_proto::{Label, Type},
14 },
15};
16use rustc_hash::{FxHashMap, FxHashSet};
17
18use super::Parser;
19use crate::{
20 IdentName,
21 index::Idx,
22 ir::{
23 self, FieldKind, Item, Path, TyKind,
24 ext::{
25 self,
26 pb::{ExtendeeIndex, ExtendeeKind, FieldType},
27 },
28 },
29 symbol::{EnumRepr, FileId, Ident},
30 tags::{
31 PilotaName, RustType, RustWrapperArc, SerdeAttribute, Tags,
32 protobuf::{
33 ClientStreaming, Deprecated, OneOf, OptionalRepeated, ProstType, Repeated,
34 ServerStreaming,
35 },
36 },
37};
38
39#[derive(Default)]
40pub struct ProtobufParser {
41 inner: protobuf_parse::Parser,
42 include_dirs: Vec<PathBuf>,
43 input_files: FxHashSet<PathBuf>,
44}
45
46#[derive(PartialEq, Eq)]
47pub enum Syntax {
48 Proto2,
49 Proto3,
50}
51
52struct Lower {
53 next_file_id: FileId,
54 files: FxHashMap<String, FileId>,
55 cur_package: Option<String>,
56 cur_syntax: Syntax,
57}
58
59impl Default for Lower {
60 fn default() -> Self {
61 Self {
62 next_file_id: FileId::from_u32(0),
63 files: Default::default(),
64 cur_package: None,
65 cur_syntax: Syntax::Proto3,
66 }
67 }
68}
69
70#[derive(Clone, Debug, PartialEq, Eq, Hash, Copy)]
71pub enum WellKnownFileName {
72 Descriptor,
73 Any,
74 Api,
75 Duration,
76 Empty,
77 FieldMask,
78 SourceContext,
79 Struct,
80 Timestamp,
81 Type,
82 Wrappers,
83 NotWellKnown,
84}
85
86pub const DESCRIPTOR_PROTO_NAME: &str = "google/protobuf/descriptor.proto";
87pub const ANY_PROTO_NAME: &str = "google/protobuf/any.proto";
88pub const API_PROTO_NAME: &str = "google/protobuf/api.proto";
89pub const DURATION_PROTO_NAME: &str = "google/protobuf/duration.proto";
90pub const EMPTY_PROTO_NAME: &str = "google/protobuf/empty.proto";
91pub const FIELD_MASK_PROTO_NAME: &str = "google/protobuf/field_mask.proto";
92pub const SOURCE_CONTEXT_PROTO_NAME: &str = "google/protobuf/source_context.proto";
93pub const STRUCT_PROTO_NAME: &str = "google/protobuf/struct.proto";
94pub const TIMESTAMP_PROTO_NAME: &str = "google/protobuf/timestamp.proto";
95pub const TYPE_PROTO_NAME: &str = "google/protobuf/type.proto";
96pub const WRAPPERS_PROTO_NAME: &str = "google/protobuf/wrappers.proto";
97
98impl WellKnownFileName {
99 pub fn name(&self) -> &str {
100 match self {
101 WellKnownFileName::Descriptor => DESCRIPTOR_PROTO_NAME,
102 WellKnownFileName::Any => ANY_PROTO_NAME,
103 WellKnownFileName::Api => API_PROTO_NAME,
104 WellKnownFileName::Duration => DURATION_PROTO_NAME,
105 WellKnownFileName::Empty => EMPTY_PROTO_NAME,
106 WellKnownFileName::FieldMask => FIELD_MASK_PROTO_NAME,
107 WellKnownFileName::SourceContext => SOURCE_CONTEXT_PROTO_NAME,
108 WellKnownFileName::Struct => STRUCT_PROTO_NAME,
109 WellKnownFileName::Timestamp => TIMESTAMP_PROTO_NAME,
110 WellKnownFileName::Type => TYPE_PROTO_NAME,
111 WellKnownFileName::Wrappers => WRAPPERS_PROTO_NAME,
112 WellKnownFileName::NotWellKnown => "",
113 }
114 }
115
116 pub fn mod_name(&self) -> &'static str {
117 match self {
118 WellKnownFileName::Descriptor => "::pilota::pb::descriptor",
119 WellKnownFileName::Any => "::pilota::pb::well_known_types::any",
120 WellKnownFileName::Api => "::pilota::pb::well_known_types::api",
121 WellKnownFileName::Duration => "::pilota::pb::well_known_types::duration",
122 WellKnownFileName::Empty => "::pilota::pb::well_known_types::empty",
123 WellKnownFileName::FieldMask => "::pilota::pb::well_known_types::field_mask",
124 WellKnownFileName::SourceContext => "::pilota::pb::well_known_types::source_context",
125 WellKnownFileName::Struct => "::pilota::pb::well_known_types::struct_",
126 WellKnownFileName::Timestamp => "::pilota::pb::well_known_types::timestamp",
127 WellKnownFileName::Type => "::pilota::pb::well_known_types::type_",
128 WellKnownFileName::Wrappers => "::pilota::pb::well_known_types::wrappers",
129 WellKnownFileName::NotWellKnown => "",
130 }
131 }
132}
133
134impl From<&str> for WellKnownFileName {
135 fn from(s: &str) -> Self {
136 match s {
137 DESCRIPTOR_PROTO_NAME => WellKnownFileName::Descriptor,
138 ANY_PROTO_NAME => WellKnownFileName::Any,
139 API_PROTO_NAME => WellKnownFileName::Api,
140 DURATION_PROTO_NAME => WellKnownFileName::Duration,
141 EMPTY_PROTO_NAME => WellKnownFileName::Empty,
142 FIELD_MASK_PROTO_NAME => WellKnownFileName::FieldMask,
143 SOURCE_CONTEXT_PROTO_NAME => WellKnownFileName::SourceContext,
144 STRUCT_PROTO_NAME => WellKnownFileName::Struct,
145 TIMESTAMP_PROTO_NAME => WellKnownFileName::Timestamp,
146 TYPE_PROTO_NAME => WellKnownFileName::Type,
147 WRAPPERS_PROTO_NAME => WellKnownFileName::Wrappers,
148 _ => WellKnownFileName::NotWellKnown,
149 }
150 }
151}
152
153impl Lower {
154 fn lower_extendee(&self, s: &str) -> Option<ExtendeeKind> {
155 match s {
156 ".google.protobuf.FileOptions" => Some(ExtendeeKind::File),
157 ".google.protobuf.MessageOptions" => Some(ExtendeeKind::Message),
158 ".google.protobuf.FieldOptions" => Some(ExtendeeKind::Field),
159 ".google.protobuf.EnumOptions" => Some(ExtendeeKind::Enum),
160 ".google.protobuf.EnumValueOptions" => Some(ExtendeeKind::EnumValue),
161 ".google.protobuf.ServiceOptions" => Some(ExtendeeKind::Service),
162 ".google.protobuf.MethodOptions" => Some(ExtendeeKind::Method),
163 ".google.protobuf.OneofOptions" => Some(ExtendeeKind::Oneof),
164 _ => None,
165 }
166 }
167
168 fn lower_pb_field_type(
169 &self,
170 ty: Option<protobuf::EnumOrUnknown<protobuf::descriptor::field_descriptor_proto::Type>>,
171 ) -> Option<ext::pb::FieldType> {
172 use protobuf::descriptor::field_descriptor_proto::Type as T;
173 let ty = ty?;
174 Some(match ty.enum_value().unwrap() {
175 T::TYPE_BOOL => FieldType::Bool,
176 T::TYPE_INT32 => FieldType::Int32,
177 T::TYPE_INT64 => FieldType::Int64,
178 T::TYPE_UINT32 => FieldType::UInt32,
179 T::TYPE_UINT64 => FieldType::UInt64,
180 T::TYPE_FLOAT => FieldType::Float,
181 T::TYPE_DOUBLE => FieldType::Double,
182 T::TYPE_STRING => FieldType::String,
183 T::TYPE_BYTES => FieldType::Bytes,
184 T::TYPE_MESSAGE => FieldType::Message,
185 T::TYPE_ENUM => FieldType::Enum,
186 _ => return None,
187 })
188 }
189
190 fn lower_extension(
191 &mut self,
192 f: &protobuf::descriptor::FieldDescriptorProto,
193 nested_messages: &AHashMap<FastStr, &DescriptorProto>,
194 ) -> Option<Arc<ext::pb::Extendee>> {
195 let extendee_str = f.extendee();
196 if extendee_str.is_empty() {
197 return None;
198 }
199 let extendee = self.lower_extendee(extendee_str)?;
200 let field_ty = self.lower_pb_field_type(f.type_)?;
201 let item_ty = self.lower_ty(f.type_, f.type_name.as_deref(), nested_messages, false);
202 let extendee = Arc::new(ext::pb::Extendee {
203 name: FastStr::new(f.name()).into(),
204 index: ExtendeeIndex {
205 extendee_kind: extendee,
206 tag_id: f.number() as u32,
207 },
208 extendee_ty: ext::pb::ExtendeeType { field_ty, item_ty },
209 });
210 Some(extendee)
211 }
212
213 fn str2path(&self, s: &str) -> ir::Path {
214 if s.is_empty() {
215 return ir::Path::default();
216 }
217 ir::Path {
218 segments: Arc::from_iter(s.split('.').map(FastStr::new).map(Ident::from)),
219 }
220 }
221
222 fn lower_ty(
223 &self,
224 type_: Option<protobuf::EnumOrUnknown<protobuf::descriptor::field_descriptor_proto::Type>>,
225 type_name: Option<&str>,
226 nested_messages: &AHashMap<FastStr, &DescriptorProto>,
227 is_wrapper_arc: bool,
228 ) -> ir::Ty {
229 let mut tags = Tags::default();
230 if is_wrapper_arc {
231 tags.insert(RustWrapperArc(true));
232 }
233
234 if let Some(name) = type_name {
235 if let Some(msg) = nested_messages.get(name) {
236 if msg.options.has_map_entry() {
237 let key = &msg.field[0];
238 let value = &msg.field[1];
239 assert_eq!("key", key.name());
240 assert_eq!("value", value.name());
241 return ir::Ty {
242 kind: ir::TyKind::Map(
243 Arc::from(self.lower_ty(
244 key.type_,
245 key.type_name.as_deref(),
246 nested_messages,
247 false,
248 )),
249 Arc::from(self.lower_ty(
250 value.type_,
251 value.type_name.as_deref(),
252 nested_messages,
253 false,
254 )),
255 ),
256 tags: Arc::new(tags),
257 };
258 }
259 }
260
261 assert_eq!(".", &name[..1]);
262
263 return ir::Ty {
264 kind: ir::TyKind::Path(self.str2path(&name[1..])),
265 tags: Arc::new(tags),
266 };
267 }
268 let Some(ty) = type_ else { panic!() };
269
270 let kind = match ty.enum_value().unwrap() {
271 protobuf::descriptor::field_descriptor_proto::Type::TYPE_DOUBLE => ir::TyKind::F64,
272 protobuf::descriptor::field_descriptor_proto::Type::TYPE_FLOAT => ir::TyKind::F32,
273 protobuf::descriptor::field_descriptor_proto::Type::TYPE_INT64 => ir::TyKind::I64,
274 protobuf::descriptor::field_descriptor_proto::Type::TYPE_UINT64 => ir::TyKind::UInt64,
275 protobuf::descriptor::field_descriptor_proto::Type::TYPE_INT32 => ir::TyKind::I32,
276 protobuf::descriptor::field_descriptor_proto::Type::TYPE_FIXED64 => {
277 tags.insert(ProstType::Fixed64);
278 ir::TyKind::UInt64
279 }
280 protobuf::descriptor::field_descriptor_proto::Type::TYPE_FIXED32 => {
281 tags.insert(ProstType::Fixed32);
282 ir::TyKind::UInt32
283 }
284 protobuf::descriptor::field_descriptor_proto::Type::TYPE_BOOL => ir::TyKind::Bool,
285 protobuf::descriptor::field_descriptor_proto::Type::TYPE_STRING => ir::TyKind::String,
286 protobuf::descriptor::field_descriptor_proto::Type::TYPE_GROUP => todo!(),
287 protobuf::descriptor::field_descriptor_proto::Type::TYPE_BYTES => ir::TyKind::Bytes,
288 protobuf::descriptor::field_descriptor_proto::Type::TYPE_UINT32 => ir::TyKind::UInt32,
289 protobuf::descriptor::field_descriptor_proto::Type::TYPE_SFIXED32 => {
290 tags.insert(ProstType::SFixed32);
291 ir::TyKind::I32
292 }
293 protobuf::descriptor::field_descriptor_proto::Type::TYPE_SFIXED64 => {
294 tags.insert(ProstType::SFixed64);
295 ir::TyKind::I64
296 }
297 protobuf::descriptor::field_descriptor_proto::Type::TYPE_SINT32 => {
298 tags.insert(ProstType::SInt32);
299 ir::TyKind::I32
300 }
301 protobuf::descriptor::field_descriptor_proto::Type::TYPE_SINT64 => {
302 tags.insert(ProstType::SInt64);
303 ir::TyKind::I64
304 }
305
306 protobuf::descriptor::field_descriptor_proto::Type::TYPE_MESSAGE
307 | protobuf::descriptor::field_descriptor_proto::Type::TYPE_ENUM => unreachable!(),
308 };
309
310 ir::Ty {
311 kind,
312 tags: Arc::new(tags),
313 }
314 }
315
316 fn lower_enum(&self, e: &EnumDescriptorProto, parent: Option<&str>) -> ir::Item {
317 ir::Item {
318 related_items: Default::default(),
319 tags: Arc::new(self.extract_enum_tags(e)),
320 kind: ir::ItemKind::Enum(ir::Enum {
321 leading_comments: "".into(),
322 trailing_comments: "".into(),
323 name: FastStr::new(e.name()).into(),
324 variants: e
325 .value
326 .iter()
327 .map(|v| ir::EnumVariant {
328 leading_comments: "".into(),
329 trailing_comments: "".into(),
330 id: v.number,
331 name: FastStr::new(v.name()).into(),
332 discr: v.number.map(|v| v as i64),
333 tags: Arc::new(self.extract_enum_value_tags(v)),
334 fields: Default::default(),
335 item_exts: ext::ItemExts::Pb(ext::pb::ItemExts {
336 used_options: ext::pb::UsedOptions::from_pb_unknown_fields(
337 ExtendeeKind::EnumValue,
338 v.options.special_fields.unknown_fields(),
339 ),
340 parent: None,
341 }),
342 })
343 .collect_vec(),
344 repr: Some(EnumRepr::I32),
345 item_exts: ext::ItemExts::Pb(ext::pb::ItemExts {
346 used_options: ext::pb::UsedOptions::from_pb_unknown_fields(
347 ExtendeeKind::Enum,
348 e.options.special_fields.unknown_fields(),
349 ),
350 parent: parent.map(|p| Path {
351 segments: Arc::from([FastStr::new(p).into()]),
352 }),
353 }),
354 }),
355 }
356 }
357
358 fn lower_default_value(&self, default_value: &str) -> Option<ir::Literal> {
359 if self.cur_syntax == Syntax::Proto3 {
360 if !default_value.is_empty() {
361 println!(
362 "cargo:warning=proto3 doesn't support default value, so we ignore it: {}",
363 default_value
364 );
365 }
366 return None;
367 }
368 let default_value = default_value.trim();
369 if default_value.is_empty() {
370 return None;
371 }
372 match default_value {
373 "true" => Some(ir::Literal::Bool(true)),
374 "false" => Some(ir::Literal::Bool(false)),
375 _ => {
376 if let Ok(i) = default_value.parse::<i64>() {
377 Some(ir::Literal::Int(i))
379 } else if default_value.parse::<f64>().is_ok() {
380 Some(ir::Literal::Float(Arc::from(default_value)))
382 } else {
383 Some(ir::Literal::String(Arc::from(default_value)))
385 }
386 }
387 }
388 }
389
390 fn lower_message(
391 &mut self,
392 message: &DescriptorProto,
393 parent_messages: &mut Vec<String>,
394 parent: Option<&str>,
395 ) -> Vec<ir::Item> {
396 let fq_message_name = format!(
397 "{}{}.{}{}",
398 if self.cur_package.is_none() { "" } else { "." },
399 self.cur_package.as_deref().unwrap_or(""),
400 {
401 let mut s = String::new();
402 parent_messages.iter().for_each(|m| {
403 s.push_str(m);
404 s.push('.')
405 });
406 s
407 },
408 message.name()
409 );
410
411 let nested_messages = message
412 .nested_type
413 .iter()
414 .map(|m| (format!("{}.{}", fq_message_name, m.name()).into(), m))
415 .collect::<AHashMap<FastStr, _>>();
416
417 let mut fields = Vec::default();
418 let mut oneof_fields = FxHashMap::default();
419
420 message.field.iter().enumerate().for_each(|(idx, field)| {
421 if field.proto3_optional.unwrap_or(false) {
422 fields.push((idx, field))
423 } else if let Some(oneof_index) = field.oneof_index {
424 oneof_fields
425 .entry(oneof_index)
426 .or_insert_with(Vec::default)
427 .push((idx, field))
428 } else {
429 fields.push((idx, field))
430 }
431 });
432
433 let mut nested_items: Vec<_> = Default::default();
434
435 let mut extra_fields = Vec::default();
436
437 message.oneof_decl.iter().enumerate().for_each(|(idx, d)| {
438 if let Some(fields) = oneof_fields.remove(&(idx as i32)) {
439 nested_items.push(Arc::new(ir::Item {
440 related_items: Default::default(),
441 tags: Arc::new(crate::tags!(OneOf)),
442 kind: ir::ItemKind::Enum(ir::Enum {
443 leading_comments: "".into(),
444 trailing_comments: "".into(),
445 name: FastStr::new(d.name()).into(),
446 repr: None,
447 variants: fields
448 .iter()
449 .map(|(_, f)| ir::EnumVariant {
450 leading_comments: "".into(),
451 trailing_comments: "".into(),
452 discr: None,
453 id: f.number,
454 name: FastStr::new(f.name()).into(),
455 fields: vec![self.lower_ty(
456 f.type_,
457 f.type_name.as_deref(),
458 &nested_messages,
459 false,
460 )],
461 tags: Arc::new(self.extract_field_tags(f)),
462 item_exts: ext::ItemExts::Pb(ext::pb::ItemExts {
463 used_options: ext::pb::UsedOptions::from_pb_unknown_fields(
464 ExtendeeKind::Field,
465 f.options.special_fields.unknown_fields(),
466 ),
467 parent: None,
468 }),
469 })
470 .collect_vec(),
471 item_exts: ext::ItemExts::Pb(ext::pb::ItemExts {
472 used_options: ext::pb::UsedOptions::from_pb_unknown_fields(
473 ExtendeeKind::Oneof,
474 d.options.special_fields.unknown_fields(),
475 ),
476 parent: Some(Path {
477 segments: Arc::from([FastStr::new(message.name()).into()]),
478 }),
479 }),
480 }),
481 }));
482
483 extra_fields.push((
484 fields[0].0,
485 ir::Field {
486 leading_comments: "".into(),
487 trailing_comments: "".into(),
488 name: FastStr::new(d.name()).into(),
489 id: -1,
490 ty: ir::Ty {
491 kind: ir::TyKind::Path(Path {
492 segments: Arc::from([
493 FastStr::new(message.name()).into(),
494 FastStr::new(d.name()).into(),
495 ]),
496 }),
497 tags: Default::default(),
498 },
499 tags: Arc::new(crate::tags!(OneOf)),
500 kind: ir::FieldKind::Optional,
501 default: None,
502 item_exts: ext::ItemExts::Pb(ext::pb::ItemExts {
503 used_options: ext::pb::UsedOptions::from_pb_unknown_fields(
504 ExtendeeKind::Oneof,
505 d.options.special_fields.unknown_fields(),
506 ),
507 parent: None,
508 }),
509 },
510 ));
511 }
512 });
513
514 parent_messages.push(message.name().into());
515
516 nested_messages
517 .iter()
518 .filter(|(_, m)| !m.options.has_map_entry())
519 .for_each(|(_, m)| {
520 self.lower_message(m, parent_messages, Some(message.name()))
521 .into_iter()
522 .for_each(|item| nested_items.push(Arc::new(item)))
523 });
524
525 parent_messages.pop();
526
527 message.enum_type.iter().for_each(|e| {
528 let item = self.lower_enum(e, Some(message.name()));
529 nested_items.push(Arc::new(item));
530 });
531
532 let item = ir::Item {
533 related_items: Default::default(),
534 tags: Arc::new(self.extract_message_tags(message)),
535 kind: ir::ItemKind::Message(ir::Message {
536 leading_comments: "".into(),
537 trailing_comments: "".into(),
538 fields: fields
539 .iter()
540 .map(|(idx, f)| {
541 let mut ty =
542 self.lower_ty(f.type_, f.type_name.as_deref(), &nested_messages, false);
543
544 let is_map = matches!(ty.kind, TyKind::Map(_, _));
545 let repeated = !is_map && matches!(f.label(), Label::LABEL_REPEATED);
546
547 if repeated {
548 ty = ir::Ty {
549 kind: ir::TyKind::Vec(Arc::from(ty)),
550 tags: Default::default(),
551 }
552 }
553
554 let optional = (|| {
555 if is_map {
556 return false;
557 }
558
559 match self.cur_syntax {
560 Syntax::Proto3 => {
561 f.proto3_optional()
562 || (!repeated && matches!(f.type_(), Type::TYPE_MESSAGE))
563 }
564 Syntax::Proto2 => f.label() == Label::LABEL_OPTIONAL,
565 }
566 })();
567
568 let mut tags = self.extract_field_tags(f);
569 if repeated {
570 tags.insert(Repeated);
571 }
572
573 (
574 *idx,
575 ir::Field {
576 leading_comments: "".into(),
577 trailing_comments: "".into(),
578 default: {
579 let default = f.default_value();
580 if !default.is_empty() {
581 if matches!(f.type_(), Type::TYPE_ENUM) {
583 if let Some(type_name) = f.type_name.as_deref() {
584 let enum_path = self.str2path(&type_name[1..]);
586 let mut segs = enum_path
587 .segments
588 .iter()
589 .cloned()
590 .collect::<Vec<_>>();
591 segs.push(Ident::from(FastStr::new(default)));
592 Some(ir::Literal::Path(ir::Path {
593 segments: Arc::from(segs),
594 }))
595 } else {
596 println!("cargo:warning=default value is not an enum member: {}", default);
597 None
598 }
599 } else {
600 self.lower_default_value(default)
601 }
602 } else {
603 None
604 }
605 },
606 id: f.number(),
607 name: FastStr::new(f.name()).into(),
608 ty,
609 tags: Arc::new(tags),
610 kind: if optional {
611 FieldKind::Optional
612 } else {
613 FieldKind::Required
614 },
615 item_exts: ext::ItemExts::Pb(ext::pb::ItemExts {
616 used_options: ext::pb::UsedOptions::from_pb_unknown_fields(
617 ExtendeeKind::Field,
618 f.options.special_fields.unknown_fields(),
619 ),
620 parent: None,
621 }),
622 },
623 )
624 })
625 .chain(extra_fields)
626 .sorted_unstable_by_key(|(idx, _)| *idx)
627 .map(|(_, f)| f)
628 .collect(),
629 name: FastStr::new(message.name()).into(),
630 is_wrapper: false,
631 item_exts: ext::ItemExts::Pb(ext::pb::ItemExts {
632 used_options: ext::pb::UsedOptions::from_pb_unknown_fields(
633 ExtendeeKind::Message,
634 message.options.special_fields.unknown_fields(),
635 ),
636 parent: parent.map(|p| Path {
637 segments: Arc::from([FastStr::new(p).into()]),
638 }),
639 }),
640 }),
641 };
642
643 let extendees = message
645 .extension
646 .iter()
647 .filter_map(|e| self.lower_extension(e, &nested_messages))
648 .collect::<Vec<_>>();
649
650 if nested_items.is_empty() && extendees.is_empty() {
651 vec![item]
652 } else {
653 let name = item.name();
654 let mut tags = Tags::default();
655 tags.insert(PilotaName(name.0.mod_ident()));
656 if !extendees.is_empty() {
657 nested_items.push(Arc::new(ir::Item {
658 related_items: Default::default(),
659 tags: Arc::new(Tags::default()),
660 kind: ir::ItemKind::Const(ir::Const {
661 leading_comments: "".into(),
662 trailing_comments: "".into(),
663 name: FastStr::new(format!("__PILOTA_PB_EXT_{}", name)).into(),
664 ty: ir::Ty {
665 kind: ir::TyKind::String,
666 tags: Default::default(),
667 },
668 lit: ir::Literal::String(Arc::from("extensions")),
669 }),
670 }));
671 }
672 vec![
673 item,
674 Item {
675 related_items: Default::default(),
676 tags: Arc::from(tags),
677 kind: ir::ItemKind::Mod(ir::Mod {
678 name: Ident { sym: name },
679 items: nested_items,
680 extensions: ext::ModExts::Pb(ext::pb::ModExts {
681 extendees: ext::pb::Extendees(extendees),
682 }),
683 }),
684 },
685 ]
686 }
687 }
688
689 pub fn lower_service(&self, service: &ServiceDescriptorProto) -> ir::Item {
690 let service_tags = self.extract_service_tags(service);
691 let rust_wrapper_arc_all = service_tags.get::<RustWrapperArc>().is_some_and(|v| v.0);
692
693 ir::Item {
694 tags: Arc::new(service_tags),
695 related_items: Default::default(),
696 kind: ir::ItemKind::Service(ir::Service {
697 leading_comments: "".into(),
698 trailing_comments: "".into(),
699 name: FastStr::new(service.name()).into(),
700 methods: service
701 .method
702 .iter()
703 .map(|m| {
704 let mut tags = self.extract_method_tags(m);
705 if m.client_streaming() {
706 tags.insert(ClientStreaming);
707 }
708 if m.server_streaming() {
709 tags.insert(ServerStreaming);
710 }
711
712 let mut arg_tags = Tags::default();
713 if rust_wrapper_arc_all {
714 arg_tags.insert(RustWrapperArc(true));
715 }
716
717 ir::Method {
718 leading_comments: "".into(),
719 trailing_comments: "".into(),
720 name: FastStr::new(m.name()).into(),
721 tags: Arc::new(tags),
722 args: vec![ir::Arg {
723 name: "req".into(),
724 id: -1,
725 ty: self.lower_ty(
726 None,
727 m.input_type.as_deref(),
728 &Default::default(),
729 rust_wrapper_arc_all,
730 ),
731 tags: Arc::new(arg_tags),
732 attribute: FieldKind::Required,
733 }],
734 oneway: false,
735 ret: self.lower_ty(
736 None,
737 m.output_type.as_deref(),
738 &Default::default(),
739 rust_wrapper_arc_all,
740 ),
741 exceptions: None,
742 item_exts: ext::ItemExts::Pb(ext::pb::ItemExts {
743 used_options: ext::pb::UsedOptions::from_pb_unknown_fields(
744 ExtendeeKind::Method,
745 m.options.special_fields.unknown_fields(),
746 ),
747 parent: None,
748 }),
749 }
750 })
751 .collect_vec(),
752 extend: vec![],
753 item_exts: ext::ItemExts::Pb(ext::pb::ItemExts {
754 used_options: ext::pb::UsedOptions::from_pb_unknown_fields(
755 ExtendeeKind::Service,
756 service.options.special_fields.unknown_fields(),
757 ),
758 parent: None,
759 }),
760 }),
761 }
762 }
763
764 pub fn lower(
765 &mut self,
766 files: &[protobuf::descriptor::FileDescriptorProto],
767 ) -> Vec<Arc<ir::File>> {
768 let mut file_map = HashMap::with_capacity(files.len());
769 files.iter().for_each(|f| {
770 self.files
771 .insert(f.name().to_string(), self.next_file_id.inc_one());
772 file_map.insert(f.name(), f);
773 });
774
775 files
776 .iter()
777 .map(|f| {
778 self.cur_package.clone_from(&f.package);
779 self.cur_syntax = match f.syntax() {
780 "proto3" => Syntax::Proto3,
781 _ => Syntax::Proto2,
782 };
783
784 let file_id = *self.files.get(f.name()).unwrap();
785
786 let package = self.str2path(f.package());
787
788 let messages = f
789 .message_type
790 .iter()
791 .flat_map(|m| self.lower_message(m, &mut Vec::new(), None))
792 .collect_vec()
793 .into_iter();
794 let enums = f
795 .enum_type
796 .iter()
797 .map(|e| self.lower_enum(e, None))
798 .collect_vec()
799 .into_iter();
800 let services = f
801 .service
802 .iter()
803 .map(|s| self.lower_service(s))
804 .collect_vec()
805 .into_iter();
806
807 let descriptor_bytes = {
808 let bytes_vec = f
809 .write_to_bytes()
810 .expect("serialize FileDescriptorProto failed");
811 Bytes::from(bytes_vec)
812 };
813
814 let mut f = ir::File {
815 package,
816 uses: f
817 .dependency
818 .iter()
819 .map(|d| {
820 (
821 self.str2path(file_map.get(&**d).unwrap().package()),
822 *self.files.get(d).unwrap(),
823 )
824 })
825 .collect(),
826 id: file_id,
827 items: messages
828 .chain(enums)
829 .chain(services)
830 .map(Arc::from)
831 .collect::<Vec<_>>(),
832 descriptor: descriptor_bytes,
833 extensions: ext::FileExts::Pb(ext::pb::FileExts {
834 well_known_file_name: WellKnownFileName::from(f.name()),
835 extendees: ext::pb::Extendees(
836 f.extension
837 .iter()
838 .filter_map(|e| self.lower_extension(e, &Default::default()))
839 .collect::<Vec<_>>(),
840 ),
841 used_options: ext::pb::UsedOptions::from_pb_unknown_fields(
842 ExtendeeKind::File,
843 f.options.special_fields.unknown_fields(),
844 ),
845 }),
846 comments: FastStr::from("".to_string()),
847 };
848
849 if f.items.is_empty() && f.extensions.has_extendees() {
850 f.items.push(Arc::new(ir::Item {
851 related_items: Default::default(),
852 tags: Arc::new(Tags::default()),
853 kind: ir::ItemKind::Const(ir::Const {
854 leading_comments: "".into(),
855 trailing_comments: "".into(),
856 name: FastStr::new(format!("__PILOTA_PB_EXT_{}", file_id.as_u32()))
857 .into(),
858 ty: ir::Ty {
859 kind: ir::TyKind::String,
860 tags: Default::default(),
861 },
862 lit: ir::Literal::String(Arc::from("extensions")),
863 }),
864 }));
865 }
866
867 let f = Arc::from(f);
868
869 self.cur_package = None;
870
871 f
872 })
873 .collect::<Vec<_>>()
874 }
875
876 fn extract_service_tags(&self, service: &ServiceDescriptorProto) -> Tags {
877 let mut tags = Tags::default();
878 if service.options.is_some() {
879 let options = &service.options;
880
881 if options.deprecated() {
883 tags.insert(Deprecated(true));
884 }
885
886 if options.rust_wrapper_arc() {
888 tags.insert(RustWrapperArc(true));
889 }
890 }
891 tags
892 }
893
894 fn extract_message_tags(&self, message: &DescriptorProto) -> Tags {
895 let mut tags = Tags::default();
896 if message.options.is_some() {
897 let options = &message.options;
898
899 if options.deprecated() {
901 tags.insert(Deprecated(true));
902 }
903
904 if let Some(serde_attr) = options.serde_attribute() {
906 tags.insert(SerdeAttribute(serde_attr));
907 }
908 if let Some(name) = options.name() {
909 tags.insert(PilotaName(name));
910 }
911 }
912 tags
913 }
914
915 fn extract_enum_tags(&self, field: &EnumDescriptorProto) -> Tags {
916 let mut tags = Tags::default();
917 if field.options.is_some() {
918 let options = &field.options;
919
920 if options.deprecated() {
922 tags.insert(Deprecated(true));
923 }
924
925 if let Some(serde_attr) = options.serde_attribute() {
927 tags.insert(SerdeAttribute(serde_attr));
928 }
929 if let Some(name) = options.name() {
930 tags.insert(PilotaName(name));
931 }
932 }
933 tags
934 }
935
936 fn extract_enum_value_tags(&self, field: &EnumValueDescriptorProto) -> Tags {
937 let mut tags = Tags::default();
938 if field.options.is_some() {
939 let options = &field.options;
940
941 if options.deprecated() {
943 tags.insert(Deprecated(true));
944 }
945
946 if let Some(serde_attr) = options.serde_attribute() {
948 tags.insert(SerdeAttribute(serde_attr));
949 }
950 }
951 tags
952 }
953
954 fn extract_method_tags(&self, field: &MethodDescriptorProto) -> Tags {
955 let mut tags = Tags::default();
956 if field.options.is_some() {
957 let options = &field.options;
958
959 if options.deprecated() {
961 tags.insert(Deprecated(true));
962 }
963 }
964 tags
965 }
966
967 fn extract_field_tags(&self, field: &FieldDescriptorProto) -> Tags {
968 let mut tags = Tags::default();
969 if field.options.is_some() {
970 let options = &field.options;
971
972 if options.deprecated() {
974 tags.insert(Deprecated(true));
975 }
976
977 if options.rust_wrapper_arc() {
979 tags.insert(RustWrapperArc(true));
980 }
981 if let Some(serde_attr) = options.serde_attribute() {
982 tags.insert(SerdeAttribute(serde_attr));
983 }
984 if let Some(name) = options.name() {
985 tags.insert(PilotaName(name));
986 }
987 if let Some(rust_type) = options.rust_type() {
988 tags.insert(RustType(rust_type));
989 }
990 if options.optional_repeated() {
991 tags.insert(OptionalRepeated(true));
992 }
993 }
994 tags
995 }
996}
997
998impl Parser for ProtobufParser {
999 fn input<P: AsRef<std::path::Path>>(&mut self, path: P) {
1000 let p = path.as_ref();
1001 self.input_files.insert(
1002 p.normalize()
1003 .unwrap_or_else(|_| panic!("normalize path failed: {}", p.display()))
1004 .into_path_buf(),
1005 );
1006 self.inner.input(path);
1007 }
1008
1009 fn include_dirs(&mut self, dirs: Vec<std::path::PathBuf>) {
1010 self.include_dirs.extend(dirs.clone());
1011 self.inner.includes(dirs);
1012 }
1013
1014 fn parse(self) -> super::ParseResult {
1015 let descriptors = self.inner.parse_and_typecheck().unwrap().file_descriptors;
1016
1017 let mut input_file_ids = vec![];
1018
1019 let mut lower = Lower::default();
1020
1021 let files = lower.lower(&descriptors);
1022
1023 let mut file_ids = FxHashMap::default();
1024 let mut file_paths = FxHashMap::default();
1025 let mut file_names = FxHashMap::default();
1026 descriptors.iter().for_each(|f| {
1027 self.include_dirs.iter().for_each(|p| {
1028 let path = p.join(f.name());
1029 if path.exists() {
1030 println!("cargo:rerun-if-changed={}", path.display());
1031 let file_id = *lower.files.get(f.name()).unwrap();
1032 let file_path: Arc<PathBuf> =
1033 Arc::from(path.normalize().unwrap().into_path_buf());
1034 file_ids.insert(file_path.clone(), file_id);
1035 file_names.insert(
1036 file_id,
1037 FastStr::new(file_path.file_stem().unwrap().to_string_lossy()),
1038 );
1039 file_paths.insert(file_id, file_path);
1040
1041 if self
1042 .input_files
1043 .contains(path.normalize().unwrap().as_path())
1044 {
1045 input_file_ids.push(*lower.files.get(f.name()).unwrap());
1046 }
1047 }
1048 });
1049 });
1050
1051 super::ParseResult {
1052 files,
1053 input_files: input_file_ids,
1054 file_ids_map: file_ids,
1055 file_paths,
1056 file_names,
1057 }
1058 }
1059}
1060
1061pub trait PbOptionsValueExtractor<T> {
1063 fn extract(&self, value: protobuf::UnknownValueRef) -> T;
1064}
1065pub struct PbOptionsValueExtractorImpl {
1066 id: u32,
1067}
1068
1069impl PbOptionsValueExtractor<bool> for PbOptionsValueExtractorImpl {
1070 fn extract(&self, value: protobuf::UnknownValueRef) -> bool {
1071 match value {
1072 protobuf::UnknownValueRef::Varint(v) => v != 0,
1073 _ => panic!("invalid value for option: {}", self.id),
1074 }
1075 }
1076}
1077
1078impl PbOptionsValueExtractor<FastStr> for PbOptionsValueExtractorImpl {
1079 fn extract(&self, value: protobuf::UnknownValueRef) -> FastStr {
1080 match value {
1081 protobuf::UnknownValueRef::LengthDelimited(v) => match std::str::from_utf8(v) {
1082 Ok(s) => FastStr::new(s),
1083 Err(_) => panic!("invalid value for option: {}", self.id),
1084 },
1085 _ => panic!("invalid value for option: {}", self.id),
1086 }
1087 }
1088}
1089
1090macro_rules! define_pb_option {
1092 ($name:ident, $id:expr, $default:expr) => {
1094 paste::paste! {
1095 pub const [<$name:upper _ID>]: u32 = $id;
1096 pub const [<$name:upper _DEFAULT>]: bool = $default;
1097 }
1098 };
1099 ($name:ident, $id:expr) => {
1101 paste::paste! {
1102 pub const [<$name:upper _ID>]: u32 = $id;
1103 }
1104 };
1105}
1106
1107macro_rules! define_all_options_traits {
1109 (
1110 $(
1111 $trait_name:ident for $options_type:ty {
1112 $($method_defs:tt)*
1113 }
1114 )*
1115 ) => {
1116 $(
1117 define_all_options_traits!(@process_trait $trait_name, $options_type, $($method_defs)*);
1118 )*
1119 };
1120
1121 (@process_trait $trait_name:ident, $options_type:ty, $($method_defs:tt)*) => {
1123 pub trait $trait_name {
1125 define_all_options_traits!(@collect_trait_methods $($method_defs)*);
1126 }
1127
1128 impl $trait_name for $options_type {
1130 define_all_options_traits!(@collect_impl_methods $($method_defs)*);
1131 }
1132 };
1133
1134 (@collect_trait_methods) => {};
1136 (@collect_trait_methods ($method:ident, $field_id:expr, $default:expr) -> $ret_type:ty; $($rest:tt)*) => {
1137 fn $method(&self) -> $ret_type;
1138 define_all_options_traits!(@collect_trait_methods $($rest)*);
1139 };
1140 (@collect_trait_methods opt ($method_opt:ident, $field_id_opt:expr) -> $ret_type_opt:ty; $($rest:tt)*) => {
1141 fn $method_opt(&self) -> Option<$ret_type_opt>;
1142 define_all_options_traits!(@collect_trait_methods $($rest)*);
1143 };
1144
1145 (@collect_impl_methods) => {};
1147 (@collect_impl_methods ($method:ident, $field_id:expr, $default:expr) -> $ret_type:ty; $($rest:tt)*) => {
1148 fn $method(&self) -> $ret_type {
1149 let Some(v) = self.special_fields.unknown_fields().get($field_id) else {
1150 return $default;
1151 };
1152 let extractor = PbOptionsValueExtractorImpl { id: $field_id };
1153 <PbOptionsValueExtractorImpl as PbOptionsValueExtractor<$ret_type>>::extract(&extractor, v)
1154 }
1155 define_all_options_traits!(@collect_impl_methods $($rest)*);
1156 };
1157 (@collect_impl_methods opt ($method_opt:ident, $field_id_opt:expr) -> $ret_type_opt:ty; $($rest:tt)*) => {
1158 fn $method_opt(&self) -> Option<$ret_type_opt> {
1159 let v = self.special_fields.unknown_fields().get($field_id_opt)?;
1160 let extractor = PbOptionsValueExtractorImpl { id: $field_id_opt };
1161 Some(<PbOptionsValueExtractorImpl as PbOptionsValueExtractor<$ret_type_opt>>::extract(&extractor, v))
1162 }
1163 define_all_options_traits!(@collect_impl_methods $($rest)*);
1164 };
1165}
1166
1167pub struct PbOptions;
1169impl PbOptions {
1170 define_pb_option!(serde_attribute, 1215201);
1173 define_pb_option!(name, 1215202);
1174 define_pb_option!(rust_wrapper_arc, 1215203, false);
1175 define_pb_option!(rust_type, 1215204);
1176 define_pb_option!(optional_repeated, 1215205, false);
1177}
1178
1179define_all_options_traits! {
1181
1182 PilotaMessageOptions for protobuf::descriptor::MessageOptions {
1187 opt (serde_attribute, PbOptions::SERDE_ATTRIBUTE_ID) -> FastStr;
1188 opt (name, PbOptions::NAME_ID) -> FastStr;
1189 }
1190
1191 PilotaFieldOptions for protobuf::descriptor::FieldOptions {
1192 (rust_wrapper_arc, PbOptions::RUST_WRAPPER_ARC_ID, PbOptions::RUST_WRAPPER_ARC_DEFAULT) -> bool;
1193 opt (serde_attribute, PbOptions::SERDE_ATTRIBUTE_ID) -> FastStr;
1194 opt (name, PbOptions::NAME_ID) -> FastStr;
1195 opt (rust_type, PbOptions::RUST_TYPE_ID) -> FastStr;
1196 (optional_repeated, PbOptions::OPTIONAL_REPEATED_ID, PbOptions::OPTIONAL_REPEATED_DEFAULT) -> bool;
1197 }
1198
1199 PilotaEnumOptions for protobuf::descriptor::EnumOptions {
1200 opt (serde_attribute, PbOptions::SERDE_ATTRIBUTE_ID) -> FastStr;
1201 opt (name, PbOptions::NAME_ID) -> FastStr;
1202 }
1203
1204 PilotaEnumValueOptions for protobuf::descriptor::EnumValueOptions {
1205 opt (serde_attribute, PbOptions::SERDE_ATTRIBUTE_ID) -> FastStr;
1206 }
1207
1208 PilotaServiceOptions for protobuf::descriptor::ServiceOptions {
1209 (rust_wrapper_arc, PbOptions::RUST_WRAPPER_ARC_ID, PbOptions::RUST_WRAPPER_ARC_DEFAULT) -> bool;
1210 }
1211}
1212
1213#[cfg(test)]
1227mod tests {
1228 use protobuf::descriptor::{MessageOptions, field_descriptor_proto};
1229
1230 use super::*;
1231
1232 #[test]
1233 fn lower_message_converts_map_entry_to_ir_map() {
1234 let mut lower = Lower::default();
1235 lower.cur_package = Some("pkg".into());
1236 lower.cur_syntax = Syntax::Proto3;
1237
1238 let mut map_entry = DescriptorProto::new();
1239 map_entry.set_name("EntriesEntry".into());
1240
1241 let mut entry_options = MessageOptions::new();
1242 entry_options.set_map_entry(true);
1243 map_entry.options = protobuf::MessageField::some(entry_options);
1244
1245 let mut key_field = FieldDescriptorProto::new();
1246 key_field.set_name("key".into());
1247 key_field.set_number(1);
1248 key_field.set_label(field_descriptor_proto::Label::LABEL_OPTIONAL);
1249 key_field.set_type(field_descriptor_proto::Type::TYPE_STRING);
1250 map_entry.field.push(key_field);
1251
1252 let mut value_field = FieldDescriptorProto::new();
1253 value_field.set_name("value".into());
1254 value_field.set_number(2);
1255 value_field.set_label(field_descriptor_proto::Label::LABEL_OPTIONAL);
1256 value_field.set_type(field_descriptor_proto::Type::TYPE_INT32);
1257 map_entry.field.push(value_field);
1258
1259 let mut message = DescriptorProto::new();
1260 message.set_name("Outer".into());
1261 message.nested_type.push(map_entry);
1262
1263 let mut field = FieldDescriptorProto::new();
1264 field.set_name("entries".into());
1265 field.set_number(1);
1266 field.set_label(field_descriptor_proto::Label::LABEL_REPEATED);
1267 field.set_type(field_descriptor_proto::Type::TYPE_MESSAGE);
1268 field.set_type_name(".pkg.Outer.EntriesEntry".into());
1269 message.field.push(field);
1270
1271 let items = lower.lower_message(&message, &mut Vec::new(), None);
1272
1273 assert_eq!(items.len(), 1);
1274
1275 let ir::ItemKind::Message(ir::Message { fields, .. }) = &items[0].kind else {
1276 panic!("expected message item");
1277 };
1278
1279 assert_eq!(fields.len(), 1);
1280 let map_field = &fields[0];
1281 assert!(matches!(map_field.kind, FieldKind::Required));
1282
1283 let ir::TyKind::Map(key_ty, value_ty) = &map_field.ty.kind else {
1284 panic!("expected map type");
1285 };
1286
1287 match (&key_ty.kind, &value_ty.kind) {
1288 (ir::TyKind::String, ir::TyKind::I32) => {}
1289 other => panic!("unexpected key/value types: {:?}", other),
1290 }
1291 }
1292
1293 #[test]
1294 fn lower_message_marks_proto3_optional_scalar_as_optional() {
1295 let mut lower = Lower::default();
1296 lower.cur_package = Some("pkg".into());
1297 lower.cur_syntax = Syntax::Proto3;
1298
1299 let mut message = DescriptorProto::new();
1300 message.set_name("Foo".into());
1301
1302 let mut field = FieldDescriptorProto::new();
1303 field.set_name("value".into());
1304 field.set_number(1);
1305 field.set_label(field_descriptor_proto::Label::LABEL_OPTIONAL);
1306 field.set_type(field_descriptor_proto::Type::TYPE_INT32);
1307 field.set_proto3_optional(true);
1308 message.field.push(field);
1309
1310 let items = lower.lower_message(&message, &mut Vec::new(), None);
1311
1312 assert_eq!(items.len(), 1);
1313
1314 let ir::ItemKind::Message(ir::Message { fields, .. }) = &items[0].kind else {
1315 panic!("expected message item");
1316 };
1317
1318 assert_eq!(fields.len(), 1);
1319 let optional_field = &fields[0];
1320 assert!(matches!(optional_field.kind, FieldKind::Optional));
1321 assert!(matches!(optional_field.ty.kind, ir::TyKind::I32));
1322 }
1323}