Skip to main content

protobuf_tonic/
lib.rs

1//! A tonic `Codec` for pbrs messages using `Parse` and `Serialize`.
2//!
3//! Keeps tonic transport and middleware while using pbrs-generated messages.
4//! Code that requires `prost::Message` needs an adapter.
5
6extern crate self as protobuf_tonic;
7
8use bytes::Buf;
9use bytes::Bytes;
10use pbrs::{ClearAndParse, Parse, Serialize};
11use prost::Message as ProstMessage;
12use std::error::Error;
13use std::fmt;
14use std::marker::PhantomData;
15use tonic::Status;
16use tonic::codec::{BufferSettings, Codec, DecodeBuf, Decoder, EncodeBuf, Encoder};
17
18/// Google Protobuf wire-format message size ceiling (`i32::MAX`, 2 GiB - 1).
19const MAX_PROTOBUF_MESSAGE_BYTES: usize = 0x7fff_ffff;
20
21/// tonic [`Codec`] using pbrs [`Serialize`] / [`Parse`] (not `prost::Message`).
22#[derive(Clone, Copy, Debug, Default)]
23pub struct ProtobufCodec<E, D> {
24    _e: PhantomData<fn() -> E>,
25    _d: PhantomData<fn() -> D>,
26}
27
28impl<E, D> ProtobufCodec<E, D> {
29    /// Construct a new [`ProtobufCodec`] with default Tonic buffer settings.
30    #[must_use]
31    pub fn new() -> Self {
32        Self {
33            _e: PhantomData,
34            _d: PhantomData,
35        }
36    }
37}
38
39impl<E, D> ProtobufCodec<E, D>
40where
41    E: Serialize + Send + 'static,
42    D: Parse + Default + ClearAndParse + Send + 'static,
43{
44    /// Build an encoder half configured with explicit [`BufferSettings`].
45    #[must_use]
46    pub fn raw_encoder(buffer_settings: BufferSettings) -> <Self as Codec>::Encoder {
47        ProtobufEncoder::new(buffer_settings)
48    }
49
50    /// Build a decoder half configured with explicit [`BufferSettings`].
51    #[must_use]
52    pub fn raw_decoder(buffer_settings: BufferSettings) -> <Self as Codec>::Decoder {
53        ProtobufDecoder::new(buffer_settings)
54    }
55}
56
57impl<E, D> Codec for ProtobufCodec<E, D>
58where
59    E: Serialize + Send + 'static,
60    D: Parse + Default + ClearAndParse + Send + 'static,
61{
62    type Encode = E;
63    type Decode = D;
64    type Encoder = ProtobufEncoder<E>;
65    type Decoder = ProtobufDecoder<D>;
66
67    fn encoder(&mut self) -> Self::Encoder {
68        ProtobufEncoder(PhantomData, BufferSettings::default())
69    }
70
71    fn decoder(&mut self) -> Self::Decoder {
72        ProtobufDecoder(PhantomData, BufferSettings::default())
73    }
74}
75
76#[inline]
77fn reserve_message_len<T: Serialize>(item: &T, dst: &mut EncodeBuf<'_>) {
78    let len = item.serialized_len();
79    if len <= MAX_PROTOBUF_MESSAGE_BYTES {
80        dst.reserve(len);
81    }
82}
83
84/// Encoder half of [`ProtobufCodec`].
85#[derive(Clone, Copy, Debug, Default)]
86pub struct ProtobufEncoder<T>(PhantomData<fn() -> T>, BufferSettings);
87
88impl<T> ProtobufEncoder<T> {
89    /// Construct an encoder with explicit Tonic [`BufferSettings`].
90    #[must_use]
91    pub fn new(buffer_settings: BufferSettings) -> Self {
92        Self(PhantomData, buffer_settings)
93    }
94}
95
96impl<T: Serialize> Encoder for ProtobufEncoder<T> {
97    type Item = T;
98    type Error = Status;
99
100    fn encode(&mut self, item: Self::Item, dst: &mut EncodeBuf<'_>) -> Result<(), Self::Error> {
101        reserve_message_len(&item, dst);
102        Serialize::encode(&item, dst).map_err(|e| Status::internal(e.to_string()))
103    }
104
105    fn buffer_settings(&self) -> BufferSettings {
106        self.1
107    }
108}
109
110/// Decoder half of [`ProtobufCodec`].
111#[derive(Clone, Copy, Debug, Default)]
112pub struct ProtobufDecoder<T>(PhantomData<fn() -> T>, BufferSettings);
113
114impl<T> ProtobufDecoder<T> {
115    /// Construct a decoder with explicit Tonic [`BufferSettings`].
116    #[must_use]
117    pub fn new(buffer_settings: BufferSettings) -> Self {
118        Self(PhantomData, buffer_settings)
119    }
120}
121
122impl<T: Parse + Default + ClearAndParse> Decoder for ProtobufDecoder<T> {
123    type Item = T;
124    type Error = Status;
125
126    fn decode(&mut self, src: &mut DecodeBuf<'_>) -> Result<Option<Self::Item>, Self::Error> {
127        // One copy into an owned buffer (tonic's), then shared parse with no
128        // pbrs-side copy (PK-09): 2 copies before, 1 after, either branch.
129        let n = src.remaining();
130        let bytes = src.copy_to_bytes(n);
131        Parse::parse_bytes(bytes)
132            .map(Some)
133            .map_err(|e| Status::internal(e.to_string()))
134    }
135
136    fn buffer_settings(&self) -> BufferSettings {
137        self.1
138    }
139}
140
141pub mod hello;
142
143/// Failure while converting between wire-compatible prost and pbrs messages.
144///
145/// Conversion deliberately goes through protobuf wire bytes. This keeps the
146/// API schema-agnostic. The encoded buffer is moved into the decoder: neither
147/// direction copies that buffer after encoding, and the prost-to-pbrs
148/// direction retains it as pbrs' shared backing storage.
149#[derive(Clone, Debug, Eq, PartialEq)]
150pub struct ConversionError(String);
151
152impl fmt::Display for ConversionError {
153    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
154        f.write_str(&self.0)
155    }
156}
157
158impl Error for ConversionError {}
159
160/// Convert a prost message to its wire-compatible pbrs representation.
161///
162/// This allocates exactly one wire buffer. [`Bytes::from`] takes ownership of
163/// prost's `Vec`, and pbrs parses from that shared buffer without copying it.
164pub fn prost_to_pbrs<P, R>(message: &P) -> Result<R, ConversionError>
165where
166    P: ProstMessage,
167    R: Parse,
168{
169    let len = message.encoded_len();
170    if len > MAX_PROTOBUF_MESSAGE_BYTES {
171        return Err(ConversionError("encoded message exceeds 2 GiB".to_owned()));
172    }
173    let mut wire = Vec::with_capacity(len);
174    message
175        .encode(&mut wire)
176        .map_err(|error| ConversionError(error.to_string()))?;
177    R::parse_bytes(Bytes::from(wire)).map_err(|error| ConversionError(error.to_string()))
178}
179
180/// Convert a pbrs message to its wire-compatible prost representation.
181///
182/// This allocates one wire buffer for pbrs serialization. Prost consumes the
183/// owned buffer directly through its [`bytes::Buf`] decoder interface.
184pub fn pbrs_to_prost<P, R>(message: &P) -> Result<R, ConversionError>
185where
186    P: Serialize,
187    R: ProstMessage + Default,
188{
189    let wire = message
190        .serialize()
191        .map_err(|error| ConversionError(error.to_string()))?;
192    R::decode(Bytes::from(wire)).map_err(|error| ConversionError(error.to_string()))
193}
194
195#[cfg(test)]
196#[allow(
197    clippy::expect_used,
198    clippy::unwrap_used,
199    clippy::panic,
200    reason = "unit tests assert encode reservation and conversion invariants directly"
201)]
202mod tests {
203    use super::*;
204    use bytes::BufMut;
205    use prost::encoding::{DecodeContext, WireType};
206    use std::future::poll_fn;
207    use std::pin::Pin;
208    use tokio::sync::mpsc::{self, UnboundedSender};
209    use tonic::codec::{CompressionEncoding, EncodeBody, Streaming};
210
211    #[derive(Clone, Debug, Default, Eq, PartialEq)]
212    struct ReserveObservation {
213        initial_chunk_len: usize,
214        reserved_chunk_len: usize,
215        written: usize,
216        reallocated_mid_write: bool,
217    }
218
219    struct ObservingWireOut<'a, 'b> {
220        dst: &'a mut EncodeBuf<'b>,
221        reserved_ptr_addr: usize,
222        written: usize,
223        reallocated_mid_write: bool,
224    }
225
226    impl pbrs::WireOut for ObservingWireOut<'_, '_> {
227        fn put_u8(&mut self, b: u8) {
228            let chunk = self.dst.chunk_mut();
229            if chunk.as_mut_ptr().addr() != self.reserved_ptr_addr + self.written
230                || chunk.len() == 0
231            {
232                self.reallocated_mid_write = true;
233            }
234            BufMut::put_u8(self.dst, b);
235            self.written += 1;
236        }
237
238        fn put_slice(&mut self, data: &[u8]) {
239            if data.is_empty() {
240                return;
241            }
242            let chunk = self.dst.chunk_mut();
243            if chunk.as_mut_ptr().addr() != self.reserved_ptr_addr + self.written
244                || chunk.len() < data.len()
245            {
246                self.reallocated_mid_write = true;
247            }
248            BufMut::put_slice(self.dst, data);
249            self.written += data.len();
250        }
251    }
252
253    struct ObservingEncoder<T> {
254        settings: BufferSettings,
255        observation: UnboundedSender<ReserveObservation>,
256        _pd: PhantomData<fn() -> T>,
257    }
258
259    impl<T: Serialize> Encoder for ObservingEncoder<T> {
260        type Item = T;
261        type Error = Status;
262
263        fn encode(&mut self, item: Self::Item, dst: &mut EncodeBuf<'_>) -> Result<(), Self::Error> {
264            let initial_chunk_len = dst.chunk_mut().len();
265            reserve_message_len(&item, dst);
266            let reserved_chunk_len = dst.chunk_mut().len();
267            let reserved_ptr_addr = dst.chunk_mut().as_mut_ptr().addr();
268            let mut sink = ObservingWireOut {
269                dst,
270                reserved_ptr_addr,
271                written: 0,
272                reallocated_mid_write: false,
273            };
274            Serialize::encode(&item, &mut sink).map_err(|e| Status::internal(e.to_string()))?;
275            self.observation
276                .send(ReserveObservation {
277                    initial_chunk_len,
278                    reserved_chunk_len,
279                    written: sink.written,
280                    reallocated_mid_write: sink.reallocated_mid_write,
281                })
282                .expect("send observation");
283            Ok(())
284        }
285
286        fn buffer_settings(&self) -> BufferSettings {
287            self.settings
288        }
289    }
290
291    async fn drain_body_frames<B>(mut body: B) -> Bytes
292    where
293        B: tonic::codegen::Body<Data = Bytes, Error = Status> + Unpin,
294    {
295        let mut out = bytes::BytesMut::new();
296        while let Some(frame) = poll_fn(|cx| Pin::new(&mut body).poll_frame(cx)).await {
297            let frame = frame.expect("encode frame");
298            if let Ok(data) = frame.into_data() {
299                out.extend_from_slice(&data);
300            }
301        }
302        out.freeze()
303    }
304
305    #[tokio::test]
306    async fn encoder_reserves_full_message_capacity_up_front_without_mid_write_realloc() {
307        let payload = "x".repeat(256 * 1024);
308        let mut msg = hello::HelloRequest::new();
309        msg.set_name(payload.clone());
310        let expected_len = msg.serialized_len();
311        assert!(expected_len > 256 * 1024);
312
313        let (obs_tx, mut obs_rx) = mpsc::unbounded_channel();
314        let encoder = ObservingEncoder::<hello::HelloRequest> {
315            settings: BufferSettings::default(),
316            observation: obs_tx,
317            _pd: PhantomData,
318        };
319        let body = EncodeBody::new_client(encoder, tokio_stream::iter([Ok(msg)]), None, None);
320        let framed = drain_body_frames(body).await;
321        assert_eq!(framed.len(), expected_len + 5);
322
323        let obs = obs_rx.recv().await.expect("recorded observation");
324        // Tonic's default per-RPC buffer is 8 KiB minus the 5-byte gRPC prefix.
325        assert!(obs.initial_chunk_len < 8 * 1024);
326        // Single upfront reserve guarantees contiguous capacity for the entire message,
327        // and field-by-field serialization never reallocates mid-write.
328        assert!(obs.reserved_chunk_len >= expected_len);
329        assert_eq!(obs.written, expected_len);
330        assert!(!obs.reallocated_mid_write);
331
332        // Verify full ProtobufEncoder + ProtobufDecoder round-trip (identity and gzip).
333        for encoding in [None, Some(CompressionEncoding::Gzip)] {
334            let mut req = hello::HelloRequest::new();
335            req.set_name(payload.clone());
336            let encode_body = EncodeBody::new_client(
337                ProtobufCodec::<hello::HelloRequest, hello::HelloRequest>::new().encoder(),
338                tokio_stream::iter([Ok(req)]),
339                encoding,
340                Some(1024 * 1024),
341            );
342            let mut stream = Streaming::new_request(
343                ProtobufCodec::<hello::HelloRequest, hello::HelloRequest>::new().decoder(),
344                encode_body,
345                encoding,
346                Some(1024 * 1024),
347            );
348            let decoded = stream
349                .message()
350                .await
351                .expect("decode status")
352                .expect("decoded message");
353            assert_eq!(decoded.name().to_str(), Ok(payload.as_str()));
354            assert!(stream.message().await.expect("eof").is_none());
355        }
356    }
357
358    #[tokio::test]
359    async fn custom_buffer_settings_propagate_through_raw_encoder() {
360        let mut msg = hello::HelloRequest::new();
361        msg.set_name("hello".to_owned());
362
363        let (obs_tx, mut obs_rx) = mpsc::unbounded_channel();
364        let raw_encoder = ProtobufCodec::<hello::HelloRequest, hello::HelloReply>::raw_encoder(
365            BufferSettings::new(64 * 1024, 128 * 1024),
366        );
367        let encoder = ObservingEncoder::<hello::HelloRequest> {
368            settings: raw_encoder.buffer_settings(),
369            observation: obs_tx,
370            _pd: PhantomData,
371        };
372        let body = EncodeBody::new_client(encoder, tokio_stream::iter([Ok(msg)]), None, None);
373        let _ = drain_body_frames(body).await;
374
375        let obs = obs_rx.recv().await.expect("recorded observation");
376        assert!(obs.initial_chunk_len >= 64 * 1024 - 5);
377    }
378
379    #[derive(Debug, Default)]
380    struct OversizedProstMessage;
381
382    impl ProstMessage for OversizedProstMessage {
383        fn encode_raw(&self, _buf: &mut impl BufMut)
384        where
385            Self: Sized,
386        {
387            panic!("encode_raw must not be called when encoded_len exceeds 2 GiB");
388        }
389
390        fn merge_field(
391            &mut self,
392            _tag: u32,
393            _wire_type: WireType,
394            _buf: &mut impl Buf,
395            _ctx: DecodeContext,
396        ) -> Result<(), prost::DecodeError>
397        where
398            Self: Sized,
399        {
400            Ok(())
401        }
402
403        fn encoded_len(&self) -> usize {
404            MAX_PROTOBUF_MESSAGE_BYTES + 1
405        }
406
407        fn clear(&mut self) {}
408    }
409
410    #[test]
411    fn prost_to_pbrs_rejects_over_2gib_length_without_allocating() {
412        let err =
413            prost_to_pbrs::<OversizedProstMessage, hello::HelloRequest>(&OversizedProstMessage)
414                .expect_err("over-2-GiB prost message must be rejected");
415        assert_eq!(err.to_string(), "encoded message exceeds 2 GiB");
416    }
417}