Skip to main content

buffa/
unknown_fields.rs

1// Modified by ReallyMe Technologies Limited.
2//! Unknown field preservation for round-trip fidelity.
3
4use crate::encode_sink::EncodeSink;
5use alloc::vec::Vec;
6
7/// A collection of unknown fields encountered during decoding.
8///
9/// When a message is decoded with a schema that doesn't include all fields
10/// present on the wire, the unknown fields are stored here so they can be
11/// re-encoded without data loss.
12#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
13pub struct UnknownFields {
14    fields: Vec<UnknownField>,
15}
16
17impl UnknownFields {
18    /// Creates an empty set of unknown fields.
19    ///
20    /// `const` so an empty set can be a `static` (e.g. the default
21    /// [`ReflectMessage::unknown_fields`] return value for implementations
22    /// that don't preserve unknown fields).
23    ///
24    /// [`ReflectMessage::unknown_fields`]: https://docs.rs/reallyme-buffa-descriptor/latest/buffa_descriptor/reflect/trait.ReflectMessage.html#method.unknown_fields
25    pub const fn new() -> Self {
26        Self { fields: Vec::new() }
27    }
28
29    /// Returns `true` if no unknown fields have been recorded.
30    pub fn is_empty(&self) -> bool {
31        self.fields.is_empty()
32    }
33
34    /// Returns the number of unknown fields recorded.
35    pub fn len(&self) -> usize {
36        self.fields.len()
37    }
38
39    /// Appends an unknown field.
40    pub fn push(&mut self, field: UnknownField) {
41        self.fields.push(field);
42    }
43
44    /// Returns an iterator over the recorded unknown fields.
45    pub fn iter(&self) -> core::slice::Iter<'_, UnknownField> {
46        self.fields.iter()
47    }
48
49    /// Removes all recorded unknown fields.
50    pub fn clear(&mut self) {
51        self.fields.clear();
52    }
53
54    /// Retain only the fields for which the predicate returns `true`.
55    ///
56    /// Used by [`ExtensionSet::set_extension`](crate::ExtensionSet::set_extension)
57    /// and [`clear_extension`](crate::ExtensionSet::clear_extension) to remove
58    /// prior occurrences at a given field number.
59    pub fn retain(&mut self, f: impl FnMut(&UnknownField) -> bool) {
60        self.fields.retain(f);
61    }
62
63    /// Compute the encoded size of all unknown fields.
64    pub fn encoded_len(&self) -> usize {
65        self.fields.iter().map(|f| f.encoded_len()).sum()
66    }
67
68    /// Re-encode all unknown fields to `buf` in their original wire format.
69    pub fn write_to(&self, buf: &mut impl EncodeSink) {
70        for field in &self.fields {
71            field.write_to(buf);
72        }
73    }
74
75    /// Decode a concatenation of wire-format fields into [`UnknownFields`].
76    ///
77    /// Reads tag/data pairs until `data` is exhausted. Each field is decoded
78    /// via [`decode_unknown_field`](crate::encoding::decode_unknown_field)
79    /// with the full [`RECURSION_LIMIT`](crate::message::RECURSION_LIMIT)
80    /// depth budget and a fresh
81    /// [`DEFAULT_UNKNOWN_FIELD_LIMIT`](crate::DEFAULT_UNKNOWN_FIELD_LIMIT)
82    /// unknown-field allowance.
83    ///
84    /// Used by [`GroupCodec`](crate::extension::codecs::GroupCodec) to turn a
85    /// message's encoded bytes back into the inner-field representation that
86    /// [`UnknownFieldData::Group`] wraps.
87    ///
88    /// # Errors
89    ///
90    /// Returns [`DecodeError`](crate::DecodeError) if `data` contains a
91    /// malformed tag, truncated field, or exceeds the recursion limit or
92    /// unknown-field limit.
93    pub fn decode_from_slice(mut data: &[u8]) -> Result<Self, crate::DecodeError> {
94        use crate::encoding::{decode_unknown_field, Tag};
95        use crate::message::{DecodeContext, DEFAULT_UNKNOWN_FIELD_LIMIT, RECURSION_LIMIT};
96        let limit = core::cell::Cell::new(DEFAULT_UNKNOWN_FIELD_LIMIT);
97        let ctx = DecodeContext::new(RECURSION_LIMIT, &limit);
98        let mut out = Self::new();
99        while !data.is_empty() {
100            let tag = Tag::decode(&mut data)?;
101            out.push(decode_unknown_field(tag, &mut data, ctx)?);
102        }
103        Ok(out)
104    }
105}
106
107impl<'a> IntoIterator for &'a UnknownFields {
108    type Item = &'a UnknownField;
109    type IntoIter = core::slice::Iter<'a, UnknownField>;
110
111    fn into_iter(self) -> Self::IntoIter {
112        self.fields.iter()
113    }
114}
115
116impl IntoIterator for UnknownFields {
117    type Item = UnknownField;
118    type IntoIter = alloc::vec::IntoIter<UnknownField>;
119
120    fn into_iter(self) -> Self::IntoIter {
121        self.fields.into_iter()
122    }
123}
124
125impl Extend<UnknownField> for UnknownFields {
126    /// Appends each field, in iteration order. A field whose number is already
127    /// present is kept as a further record, as [`push`](UnknownFields::push)
128    /// does.
129    fn extend<T: IntoIterator<Item = UnknownField>>(&mut self, iter: T) {
130        self.fields.extend(iter);
131    }
132}
133
134impl FromIterator<UnknownField> for UnknownFields {
135    /// Collects unknown fields into an [`UnknownFields`], keeping their order
136    /// and any repeated field numbers.
137    fn from_iter<T: IntoIterator<Item = UnknownField>>(iter: T) -> Self {
138        Self {
139            fields: iter.into_iter().collect(),
140        }
141    }
142}
143
144/// A single unknown field (field number + wire data).
145#[derive(Clone, Debug, PartialEq, Eq, Hash)]
146pub struct UnknownField {
147    pub number: u32,
148    pub data: UnknownFieldData,
149}
150
151impl UnknownField {
152    /// Compute the encoded size of this field (tag + data).
153    pub fn encoded_len(&self) -> usize {
154        let tag_len =
155            crate::encoding::varint_len(((self.number as u64) << 3) | self.data.wire_type_value());
156        tag_len + self.data.encoded_len(self.number)
157    }
158
159    /// Re-encode this field (tag + data) to `buf` in its original wire format.
160    pub fn write_to(&self, buf: &mut impl EncodeSink) {
161        use crate::encoding::encode_varint;
162        let tag_value = ((self.number as u64) << 3) | self.data.wire_type_value();
163        encode_varint(tag_value, buf);
164        match &self.data {
165            UnknownFieldData::Varint(v) => encode_varint(*v, buf),
166            UnknownFieldData::Fixed64(v) => buf.put_u64_le(*v),
167            UnknownFieldData::Fixed32(v) => buf.put_u32_le(*v),
168            UnknownFieldData::LengthDelimited(data) => {
169                encode_varint(data.len() as u64, buf);
170                buf.put_slice(data);
171            }
172            UnknownFieldData::Group(fields) => {
173                fields.write_to(buf);
174                // End-group tag (wire type 4, same field number).
175                encode_varint(((self.number as u64) << 3) | 4, buf);
176            }
177        }
178    }
179}
180
181/// The wire data for an unknown field.
182#[derive(Clone, Debug, PartialEq, Eq, Hash)]
183pub enum UnknownFieldData {
184    Varint(u64),
185    Fixed64(u64),
186    Fixed32(u32),
187    LengthDelimited(Vec<u8>),
188    Group(UnknownFields),
189}
190
191impl UnknownFieldData {
192    fn wire_type_value(&self) -> u64 {
193        match self {
194            Self::Varint(_) => 0,
195            Self::Fixed64(_) => 1,
196            Self::LengthDelimited(_) => 2,
197            Self::Group(_) => 3,
198            Self::Fixed32(_) => 5,
199        }
200    }
201
202    fn encoded_len(&self, field_number: u32) -> usize {
203        match self {
204            Self::Varint(v) => crate::encoding::varint_len(*v),
205            Self::Fixed64(_) => 8,
206            Self::Fixed32(_) => 4,
207            Self::LengthDelimited(data) => {
208                crate::encoding::varint_len(data.len() as u64) + data.len()
209            }
210            Self::Group(fields) => {
211                // Group content + end-group tag (wire type 4, same field number).
212                let end_tag_len = crate::encoding::varint_len((field_number as u64) << 3 | 4);
213                fields.encoded_len() + end_tag_len
214            }
215        }
216    }
217}
218
219#[cfg(feature = "arbitrary")]
220impl<'a> arbitrary::Arbitrary<'a> for UnknownFields {
221    fn arbitrary(_u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result<Self> {
222        Ok(UnknownFields::new())
223    }
224}
225
226#[cfg(feature = "zeroize")]
227impl zeroize::Zeroize for UnknownFields {
228    fn zeroize(&mut self) {
229        // Also wipes the discriminants, payload buffer headers and padding
230        // the dropped fields leave in the retained buffer.
231        crate::sensitive::zeroize_vec(&mut self.fields);
232    }
233}
234#[cfg(feature = "zeroize")]
235impl zeroize::Zeroize for UnknownField {
236    fn zeroize(&mut self) {
237        self.number.zeroize();
238        self.data.zeroize();
239    }
240}
241#[cfg(feature = "zeroize")]
242impl zeroize::Zeroize for UnknownFieldData {
243    fn zeroize(&mut self) {
244        match self {
245            Self::Varint(value) | Self::Fixed64(value) => value.zeroize(),
246            Self::Fixed32(value) => value.zeroize(),
247            Self::LengthDelimited(value) => value.zeroize(),
248            Self::Group(value) => value.zeroize(),
249        }
250    }
251}
252
253#[cfg(test)]
254mod tests {
255    use super::*;
256    use crate::encoding::MAX_FIELD_NUMBER;
257
258    /// A message that preserves unknown fields holds an `UnknownFields`, so it
259    /// can be a `HashSet` or `HashMap` key only if this type is `Eq + Hash`.
260    #[test]
261    fn unknown_fields_are_usable_in_a_hashed_key() {
262        let mut fields = UnknownFields::default();
263        fields.push(UnknownField {
264            number: 1,
265            data: UnknownFieldData::Group(UnknownFields::default()),
266        });
267        let mut set = std::collections::HashSet::new();
268        assert!(set.insert(fields.clone()));
269        assert!(!set.insert(fields));
270        assert!(set.insert(UnknownFields::default()));
271    }
272
273    fn varint_field(number: u32, value: u64) -> UnknownField {
274        UnknownField {
275            number,
276            data: UnknownFieldData::Varint(value),
277        }
278    }
279
280    fn fixed32_field(number: u32, value: u32) -> UnknownField {
281        UnknownField {
282            number,
283            data: UnknownFieldData::Fixed32(value),
284        }
285    }
286
287    fn fixed64_field(number: u32, value: u64) -> UnknownField {
288        UnknownField {
289            number,
290            data: UnknownFieldData::Fixed64(value),
291        }
292    }
293
294    fn ld_field(number: u32, data: Vec<u8>) -> UnknownField {
295        UnknownField {
296            number,
297            data: UnknownFieldData::LengthDelimited(data),
298        }
299    }
300
301    fn group_field(number: u32, fields: UnknownFields) -> UnknownField {
302        UnknownField {
303            number,
304            data: UnknownFieldData::Group(fields),
305        }
306    }
307
308    #[test]
309    fn test_empty_encoded_len() {
310        assert_eq!(UnknownFields::new().encoded_len(), 0);
311    }
312
313    #[test]
314    fn test_varint_encoded_len() {
315        // field 1, value 0: tag=(1<<3)|0=8 (1 byte) + varint(0) (1 byte) = 2
316        assert_eq!(varint_field(1, 0).encoded_len(), 2);
317        // field 1, value 128: tag (1 byte) + varint(128) (2 bytes) = 3
318        assert_eq!(varint_field(1, 128).encoded_len(), 3);
319    }
320
321    #[test]
322    fn test_fixed32_encoded_len() {
323        // field 1: tag=(1<<3)|5=13 (1 byte) + 4 bytes = 5
324        assert_eq!(fixed32_field(1, 0xDEAD_BEEF).encoded_len(), 5);
325    }
326
327    #[test]
328    fn test_fixed64_encoded_len() {
329        // field 1: tag=(1<<3)|1=9 (1 byte) + 8 bytes = 9
330        assert_eq!(fixed64_field(1, 0xDEAD_BEEF_CAFE_BABE).encoded_len(), 9);
331    }
332
333    #[test]
334    fn test_length_delimited_encoded_len() {
335        // field 1, 3-byte data: tag (1) + len_varint(3)=1 + 3 = 5
336        assert_eq!(ld_field(1, vec![0xAA, 0xBB, 0xCC]).encoded_len(), 5);
337        // field 1, empty: tag (1) + len_varint(0)=1 + 0 = 2
338        assert_eq!(ld_field(1, vec![]).encoded_len(), 2);
339        // field 1, 128-byte data: tag (1) + len_varint(128)=2 + 128 = 131
340        assert_eq!(ld_field(1, vec![0u8; 128]).encoded_len(), 131);
341    }
342
343    #[test]
344    fn test_group_encoded_len() {
345        // field 1, empty group: start_tag (1) + end_tag (1) = 2
346        assert_eq!(group_field(1, UnknownFields::new()).encoded_len(), 2);
347        // field 1, group containing one varint(0) field (2 bytes):
348        //   start_tag (1) + inner (2) + end_tag (1) = 4
349        let mut inner = UnknownFields::new();
350        inner.push(varint_field(1, 0));
351        assert_eq!(group_field(1, inner).encoded_len(), 4);
352    }
353
354    #[test]
355    fn test_large_field_number_two_byte_tag() {
356        // field 16: tag=(16<<3)|0=128, varint_len(128)=2; varint(0)=1 → total 3
357        assert_eq!(varint_field(16, 0).encoded_len(), 3);
358    }
359
360    #[test]
361    fn test_multiple_fields_sum() {
362        let mut fields = UnknownFields::new();
363        fields.push(varint_field(1, 0)); // tag(1) + varint(0)(1) = 2
364        fields.push(fixed32_field(2, 0)); // tag=(2<<3)|5=21(1) + 4 = 5
365        assert_eq!(fields.encoded_len(), 7);
366    }
367
368    #[test]
369    fn test_into_iter_by_ref() {
370        let mut fields = UnknownFields::new();
371        fields.push(varint_field(1, 0));
372        fields.push(varint_field(2, 0));
373        let numbers: Vec<u32> = (&fields).into_iter().map(|f| f.number).collect();
374        assert_eq!(numbers, vec![1, 2]);
375        // `for f in &fields` syntax works:
376        let mut count = 0;
377        for _ in &fields {
378            count += 1;
379        }
380        assert_eq!(count, 2);
381    }
382
383    #[test]
384    fn test_from_iterator_preserves_order_and_duplicates() {
385        let expected = vec![
386            varint_field(1, 0),
387            fixed64_field(2, u64::MAX),
388            fixed32_field(3, u32::MAX),
389            ld_field(4, vec![1, 2]),
390            group_field(5, UnknownFields::new()),
391            varint_field(1, 42),
392        ];
393
394        let fields: UnknownFields = expected.clone().into_iter().collect();
395        assert_eq!(fields.into_iter().collect::<Vec<_>>(), expected);
396
397        let empty: UnknownFields = core::iter::empty().collect();
398        assert!(empty.is_empty());
399    }
400
401    #[test]
402    fn test_extend_appends_fields() {
403        let mut fields = UnknownFields::new();
404        fields.push(varint_field(7, 1));
405        fields.extend([fixed32_field(2, 3), ld_field(2, vec![]), varint_field(7, 4)]);
406
407        assert_eq!(
408            fields.into_iter().collect::<Vec<_>>(),
409            vec![
410                varint_field(7, 1),
411                fixed32_field(2, 3),
412                ld_field(2, vec![]),
413                varint_field(7, 4),
414            ]
415        );
416    }
417
418    // ── encoded_len vs write_to consistency ──────────────────────────────
419    //
420    // Size/write divergence corrupts downstream encoding (compute_size
421    // feeds the length prefix of length-delimited messages). These tests
422    // build fields spanning the full variant matrix + boundary field
423    // numbers, and assert write_to produces exactly encoded_len bytes.
424
425    fn assert_len_matches_write(f: UnknownField) {
426        let mut fields = UnknownFields::new();
427        let claimed = f.encoded_len();
428        fields.push(f);
429        let mut buf = alloc::vec::Vec::new();
430        fields.write_to(&mut buf);
431        assert_eq!(
432            buf.len(),
433            claimed,
434            "encoded_len={claimed} but wrote {} bytes for {fields:?}",
435            buf.len()
436        );
437    }
438
439    #[test]
440    fn test_encoded_len_matches_write_varint() {
441        for &num in &[1, 15, 16, 2047, 2048, MAX_FIELD_NUMBER] {
442            for &val in &[0, 1, 127, 128, u32::MAX as u64, u64::MAX] {
443                assert_len_matches_write(varint_field(num, val));
444            }
445        }
446    }
447
448    #[test]
449    fn test_encoded_len_matches_write_fixed() {
450        for &num in &[1, 16, MAX_FIELD_NUMBER] {
451            assert_len_matches_write(fixed32_field(num, 0));
452            assert_len_matches_write(fixed32_field(num, u32::MAX));
453            assert_len_matches_write(fixed64_field(num, 0));
454            assert_len_matches_write(fixed64_field(num, u64::MAX));
455        }
456    }
457
458    #[test]
459    fn test_encoded_len_matches_write_length_delimited() {
460        for &num in &[1, 16, MAX_FIELD_NUMBER] {
461            assert_len_matches_write(ld_field(num, alloc::vec![]));
462            assert_len_matches_write(ld_field(num, alloc::vec![0xAB]));
463            assert_len_matches_write(ld_field(num, alloc::vec![0; 127]));
464            assert_len_matches_write(ld_field(num, alloc::vec![0; 128]));
465        }
466    }
467
468    #[test]
469    fn test_encoded_len_matches_write_group() {
470        for &num in &[1, 16, MAX_FIELD_NUMBER] {
471            // Empty group.
472            assert_len_matches_write(group_field(num, UnknownFields::new()));
473            // Group with mixed children.
474            let mut inner = UnknownFields::new();
475            inner.push(varint_field(2, 42));
476            inner.push(ld_field(3, alloc::vec![1, 2, 3]));
477            assert_len_matches_write(group_field(num, inner.clone()));
478            // Nested group.
479            let mut inner2 = UnknownFields::new();
480            inner2.push(group_field(4, inner));
481            assert_len_matches_write(group_field(num, inner2));
482        }
483    }
484
485    #[test]
486    fn test_encoded_len_matches_write_multi_field() {
487        // All variants in one UnknownFields.
488        let mut fields = UnknownFields::new();
489        fields.push(varint_field(1, 42));
490        fields.push(fixed32_field(2, 0xDEADBEEF));
491        fields.push(fixed64_field(3, u64::MAX));
492        fields.push(ld_field(4, alloc::vec![1, 2, 3, 4, 5]));
493        let mut nested = UnknownFields::new();
494        nested.push(varint_field(10, 7));
495        fields.push(group_field(5, nested));
496        let claimed = fields.encoded_len();
497        let mut buf = alloc::vec::Vec::new();
498        fields.write_to(&mut buf);
499        assert_eq!(buf.len(), claimed);
500    }
501
502    // ── decode_from_slice ────────────────────────────────────────────────
503
504    #[test]
505    fn test_decode_from_slice_empty() {
506        let out = UnknownFields::decode_from_slice(&[]).unwrap();
507        assert!(out.is_empty());
508    }
509
510    #[test]
511    fn test_decode_from_slice_roundtrip() {
512        // Build a multi-variant field set, write, decode back, compare.
513        let mut orig = UnknownFields::new();
514        orig.push(varint_field(1, 42));
515        orig.push(fixed32_field(2, 0xDEAD_BEEF));
516        orig.push(fixed64_field(3, u64::MAX));
517        orig.push(ld_field(4, vec![1, 2, 3]));
518        let mut nested = UnknownFields::new();
519        nested.push(varint_field(10, 7));
520        orig.push(group_field(5, nested));
521
522        let mut buf = Vec::new();
523        orig.write_to(&mut buf);
524        let decoded = UnknownFields::decode_from_slice(&buf).unwrap();
525        assert_eq!(decoded, orig);
526    }
527
528    #[test]
529    fn test_decode_from_slice_truncated() {
530        // Tag 0x08 (field 1, varint) with no value — truncated.
531        assert!(UnknownFields::decode_from_slice(&[0x08]).is_err());
532    }
533}