1extern crate self as protobuf_tonic;
7
8use bytes::Buf;
9use bytes::Bytes;
10use pbrs::{ClearAndParse, Parse, Serialize};
11use prost::Message as ProstMessage;
12use std::error::Error;
13use std::fmt;
14use std::marker::PhantomData;
15use tonic::Status;
16use tonic::codec::{BufferSettings, Codec, DecodeBuf, Decoder, EncodeBuf, Encoder};
17
18const MAX_PROTOBUF_MESSAGE_BYTES: usize = 0x7fff_ffff;
20
21#[derive(Clone, Copy, Debug, Default)]
23pub struct ProtobufCodec<E, D> {
24 _e: PhantomData<fn() -> E>,
25 _d: PhantomData<fn() -> D>,
26}
27
28impl<E, D> ProtobufCodec<E, D> {
29 #[must_use]
31 pub fn new() -> Self {
32 Self {
33 _e: PhantomData,
34 _d: PhantomData,
35 }
36 }
37}
38
39impl<E, D> ProtobufCodec<E, D>
40where
41 E: Serialize + Send + 'static,
42 D: Parse + Default + ClearAndParse + Send + 'static,
43{
44 #[must_use]
46 pub fn raw_encoder(buffer_settings: BufferSettings) -> <Self as Codec>::Encoder {
47 ProtobufEncoder::new(buffer_settings)
48 }
49
50 #[must_use]
52 pub fn raw_decoder(buffer_settings: BufferSettings) -> <Self as Codec>::Decoder {
53 ProtobufDecoder::new(buffer_settings)
54 }
55}
56
57impl<E, D> Codec for ProtobufCodec<E, D>
58where
59 E: Serialize + Send + 'static,
60 D: Parse + Default + ClearAndParse + Send + 'static,
61{
62 type Encode = E;
63 type Decode = D;
64 type Encoder = ProtobufEncoder<E>;
65 type Decoder = ProtobufDecoder<D>;
66
67 fn encoder(&mut self) -> Self::Encoder {
68 ProtobufEncoder(PhantomData, BufferSettings::default())
69 }
70
71 fn decoder(&mut self) -> Self::Decoder {
72 ProtobufDecoder(PhantomData, BufferSettings::default())
73 }
74}
75
76#[inline]
77fn reserve_message_len<T: Serialize>(item: &T, dst: &mut EncodeBuf<'_>) {
78 let len = item.serialized_len();
79 if len <= MAX_PROTOBUF_MESSAGE_BYTES {
80 dst.reserve(len);
81 }
82}
83
84#[derive(Clone, Copy, Debug, Default)]
86pub struct ProtobufEncoder<T>(PhantomData<fn() -> T>, BufferSettings);
87
88impl<T> ProtobufEncoder<T> {
89 #[must_use]
91 pub fn new(buffer_settings: BufferSettings) -> Self {
92 Self(PhantomData, buffer_settings)
93 }
94}
95
96impl<T: Serialize> Encoder for ProtobufEncoder<T> {
97 type Item = T;
98 type Error = Status;
99
100 fn encode(&mut self, item: Self::Item, dst: &mut EncodeBuf<'_>) -> Result<(), Self::Error> {
101 reserve_message_len(&item, dst);
102 Serialize::encode(&item, dst).map_err(|e| Status::internal(e.to_string()))
103 }
104
105 fn buffer_settings(&self) -> BufferSettings {
106 self.1
107 }
108}
109
110#[derive(Clone, Copy, Debug, Default)]
112pub struct ProtobufDecoder<T>(PhantomData<fn() -> T>, BufferSettings);
113
114impl<T> ProtobufDecoder<T> {
115 #[must_use]
117 pub fn new(buffer_settings: BufferSettings) -> Self {
118 Self(PhantomData, buffer_settings)
119 }
120}
121
122impl<T: Parse + Default + ClearAndParse> Decoder for ProtobufDecoder<T> {
123 type Item = T;
124 type Error = Status;
125
126 fn decode(&mut self, src: &mut DecodeBuf<'_>) -> Result<Option<Self::Item>, Self::Error> {
127 let n = src.remaining();
130 let bytes = src.copy_to_bytes(n);
131 Parse::parse_bytes(bytes)
132 .map(Some)
133 .map_err(|e| Status::internal(e.to_string()))
134 }
135
136 fn buffer_settings(&self) -> BufferSettings {
137 self.1
138 }
139}
140
141pub mod hello;
142
143#[derive(Clone, Debug, Eq, PartialEq)]
150pub struct ConversionError(String);
151
152impl fmt::Display for ConversionError {
153 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
154 f.write_str(&self.0)
155 }
156}
157
158impl Error for ConversionError {}
159
160pub fn prost_to_pbrs<P, R>(message: &P) -> Result<R, ConversionError>
165where
166 P: ProstMessage,
167 R: Parse,
168{
169 let len = message.encoded_len();
170 if len > MAX_PROTOBUF_MESSAGE_BYTES {
171 return Err(ConversionError("encoded message exceeds 2 GiB".to_owned()));
172 }
173 let mut wire = Vec::with_capacity(len);
174 message
175 .encode(&mut wire)
176 .map_err(|error| ConversionError(error.to_string()))?;
177 R::parse_bytes(Bytes::from(wire)).map_err(|error| ConversionError(error.to_string()))
178}
179
180pub fn pbrs_to_prost<P, R>(message: &P) -> Result<R, ConversionError>
185where
186 P: Serialize,
187 R: ProstMessage + Default,
188{
189 let wire = message
190 .serialize()
191 .map_err(|error| ConversionError(error.to_string()))?;
192 R::decode(Bytes::from(wire)).map_err(|error| ConversionError(error.to_string()))
193}
194
195#[cfg(test)]
196#[allow(
197 clippy::expect_used,
198 clippy::unwrap_used,
199 clippy::panic,
200 reason = "unit tests assert encode reservation and conversion invariants directly"
201)]
202mod tests {
203 use super::*;
204 use bytes::BufMut;
205 use prost::encoding::{DecodeContext, WireType};
206 use std::future::poll_fn;
207 use std::pin::Pin;
208 use tokio::sync::mpsc::{self, UnboundedSender};
209 use tonic::codec::{CompressionEncoding, EncodeBody, Streaming};
210
211 #[derive(Clone, Debug, Default, Eq, PartialEq)]
212 struct ReserveObservation {
213 initial_chunk_len: usize,
214 reserved_chunk_len: usize,
215 written: usize,
216 reallocated_mid_write: bool,
217 }
218
219 struct ObservingWireOut<'a, 'b> {
220 dst: &'a mut EncodeBuf<'b>,
221 reserved_ptr_addr: usize,
222 written: usize,
223 reallocated_mid_write: bool,
224 }
225
226 impl pbrs::WireOut for ObservingWireOut<'_, '_> {
227 fn put_u8(&mut self, b: u8) {
228 let chunk = self.dst.chunk_mut();
229 if chunk.as_mut_ptr().addr() != self.reserved_ptr_addr + self.written
230 || chunk.len() == 0
231 {
232 self.reallocated_mid_write = true;
233 }
234 BufMut::put_u8(self.dst, b);
235 self.written += 1;
236 }
237
238 fn put_slice(&mut self, data: &[u8]) {
239 if data.is_empty() {
240 return;
241 }
242 let chunk = self.dst.chunk_mut();
243 if chunk.as_mut_ptr().addr() != self.reserved_ptr_addr + self.written
244 || chunk.len() < data.len()
245 {
246 self.reallocated_mid_write = true;
247 }
248 BufMut::put_slice(self.dst, data);
249 self.written += data.len();
250 }
251 }
252
253 struct ObservingEncoder<T> {
254 settings: BufferSettings,
255 observation: UnboundedSender<ReserveObservation>,
256 _pd: PhantomData<fn() -> T>,
257 }
258
259 impl<T: Serialize> Encoder for ObservingEncoder<T> {
260 type Item = T;
261 type Error = Status;
262
263 fn encode(&mut self, item: Self::Item, dst: &mut EncodeBuf<'_>) -> Result<(), Self::Error> {
264 let initial_chunk_len = dst.chunk_mut().len();
265 reserve_message_len(&item, dst);
266 let reserved_chunk_len = dst.chunk_mut().len();
267 let reserved_ptr_addr = dst.chunk_mut().as_mut_ptr().addr();
268 let mut sink = ObservingWireOut {
269 dst,
270 reserved_ptr_addr,
271 written: 0,
272 reallocated_mid_write: false,
273 };
274 Serialize::encode(&item, &mut sink).map_err(|e| Status::internal(e.to_string()))?;
275 self.observation
276 .send(ReserveObservation {
277 initial_chunk_len,
278 reserved_chunk_len,
279 written: sink.written,
280 reallocated_mid_write: sink.reallocated_mid_write,
281 })
282 .expect("send observation");
283 Ok(())
284 }
285
286 fn buffer_settings(&self) -> BufferSettings {
287 self.settings
288 }
289 }
290
291 async fn drain_body_frames<B>(mut body: B) -> Bytes
292 where
293 B: tonic::codegen::Body<Data = Bytes, Error = Status> + Unpin,
294 {
295 let mut out = bytes::BytesMut::new();
296 while let Some(frame) = poll_fn(|cx| Pin::new(&mut body).poll_frame(cx)).await {
297 let frame = frame.expect("encode frame");
298 if let Ok(data) = frame.into_data() {
299 out.extend_from_slice(&data);
300 }
301 }
302 out.freeze()
303 }
304
305 #[tokio::test]
306 async fn encoder_reserves_full_message_capacity_up_front_without_mid_write_realloc() {
307 let payload = "x".repeat(256 * 1024);
308 let mut msg = hello::HelloRequest::new();
309 msg.set_name(payload.clone());
310 let expected_len = msg.serialized_len();
311 assert!(expected_len > 256 * 1024);
312
313 let (obs_tx, mut obs_rx) = mpsc::unbounded_channel();
314 let encoder = ObservingEncoder::<hello::HelloRequest> {
315 settings: BufferSettings::default(),
316 observation: obs_tx,
317 _pd: PhantomData,
318 };
319 let body = EncodeBody::new_client(encoder, tokio_stream::iter([Ok(msg)]), None, None);
320 let framed = drain_body_frames(body).await;
321 assert_eq!(framed.len(), expected_len + 5);
322
323 let obs = obs_rx.recv().await.expect("recorded observation");
324 assert!(obs.initial_chunk_len < 8 * 1024);
326 assert!(obs.reserved_chunk_len >= expected_len);
329 assert_eq!(obs.written, expected_len);
330 assert!(!obs.reallocated_mid_write);
331
332 for encoding in [None, Some(CompressionEncoding::Gzip)] {
334 let mut req = hello::HelloRequest::new();
335 req.set_name(payload.clone());
336 let encode_body = EncodeBody::new_client(
337 ProtobufCodec::<hello::HelloRequest, hello::HelloRequest>::new().encoder(),
338 tokio_stream::iter([Ok(req)]),
339 encoding,
340 Some(1024 * 1024),
341 );
342 let mut stream = Streaming::new_request(
343 ProtobufCodec::<hello::HelloRequest, hello::HelloRequest>::new().decoder(),
344 encode_body,
345 encoding,
346 Some(1024 * 1024),
347 );
348 let decoded = stream
349 .message()
350 .await
351 .expect("decode status")
352 .expect("decoded message");
353 assert_eq!(decoded.name().to_str(), Ok(payload.as_str()));
354 assert!(stream.message().await.expect("eof").is_none());
355 }
356 }
357
358 #[tokio::test]
359 async fn custom_buffer_settings_propagate_through_raw_encoder() {
360 let mut msg = hello::HelloRequest::new();
361 msg.set_name("hello".to_owned());
362
363 let (obs_tx, mut obs_rx) = mpsc::unbounded_channel();
364 let raw_encoder = ProtobufCodec::<hello::HelloRequest, hello::HelloReply>::raw_encoder(
365 BufferSettings::new(64 * 1024, 128 * 1024),
366 );
367 let encoder = ObservingEncoder::<hello::HelloRequest> {
368 settings: raw_encoder.buffer_settings(),
369 observation: obs_tx,
370 _pd: PhantomData,
371 };
372 let body = EncodeBody::new_client(encoder, tokio_stream::iter([Ok(msg)]), None, None);
373 let _ = drain_body_frames(body).await;
374
375 let obs = obs_rx.recv().await.expect("recorded observation");
376 assert!(obs.initial_chunk_len >= 64 * 1024 - 5);
377 }
378
379 #[derive(Debug, Default)]
380 struct OversizedProstMessage;
381
382 impl ProstMessage for OversizedProstMessage {
383 fn encode_raw(&self, _buf: &mut impl BufMut)
384 where
385 Self: Sized,
386 {
387 panic!("encode_raw must not be called when encoded_len exceeds 2 GiB");
388 }
389
390 fn merge_field(
391 &mut self,
392 _tag: u32,
393 _wire_type: WireType,
394 _buf: &mut impl Buf,
395 _ctx: DecodeContext,
396 ) -> Result<(), prost::DecodeError>
397 where
398 Self: Sized,
399 {
400 Ok(())
401 }
402
403 fn encoded_len(&self) -> usize {
404 MAX_PROTOBUF_MESSAGE_BYTES + 1
405 }
406
407 fn clear(&mut self) {}
408 }
409
410 #[test]
411 fn prost_to_pbrs_rejects_over_2gib_length_without_allocating() {
412 let err =
413 prost_to_pbrs::<OversizedProstMessage, hello::HelloRequest>(&OversizedProstMessage)
414 .expect_err("over-2-GiB prost message must be rejected");
415 assert_eq!(err.to_string(), "encoded message exceeds 2 GiB");
416 }
417}