Skip to main content

buffa/
unknown_fields.rs

1// Modified by ReallyMe Technologies Limited.
2//! Unknown field preservation for round-trip fidelity.
3
4use crate::encode_sink::EncodeSink;
5use alloc::vec::Vec;
6
7/// A collection of unknown fields encountered during decoding.
8///
9/// When a message is decoded with a schema that doesn't include all fields
10/// present on the wire, the unknown fields are stored here so they can be
11/// re-encoded without data loss.
12#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
13pub struct UnknownFields {
14    fields: Vec<UnknownField>,
15}
16
17impl UnknownFields {
18    /// Creates an empty set of unknown fields.
19    ///
20    /// `const` so an empty set can be a `static` (e.g. the default
21    /// [`ReflectMessage::unknown_fields`] return value for implementations
22    /// that don't preserve unknown fields).
23    ///
24    /// [`ReflectMessage::unknown_fields`]: https://docs.rs/reallyme-buffa-descriptor/latest/buffa_descriptor/reflect/trait.ReflectMessage.html#method.unknown_fields
25    pub const fn new() -> Self {
26        Self { fields: Vec::new() }
27    }
28
29    /// Returns `true` if no unknown fields have been recorded.
30    pub fn is_empty(&self) -> bool {
31        self.fields.is_empty()
32    }
33
34    /// Returns the number of unknown fields recorded.
35    pub fn len(&self) -> usize {
36        self.fields.len()
37    }
38
39    /// Appends an unknown field.
40    pub fn push(&mut self, field: UnknownField) {
41        self.fields.push(field);
42    }
43
44    /// Returns an iterator over the recorded unknown fields.
45    pub fn iter(&self) -> core::slice::Iter<'_, UnknownField> {
46        self.fields.iter()
47    }
48
49    /// Removes all recorded unknown fields.
50    pub fn clear(&mut self) {
51        self.fields.clear();
52    }
53
54    /// Retain only the fields for which the predicate returns `true`.
55    ///
56    /// Used by [`ExtensionSet::set_extension`](crate::ExtensionSet::set_extension)
57    /// and [`clear_extension`](crate::ExtensionSet::clear_extension) to remove
58    /// prior occurrences at a given field number.
59    pub fn retain(&mut self, f: impl FnMut(&UnknownField) -> bool) {
60        self.fields.retain(f);
61    }
62
63    /// Compute the encoded size of all unknown fields.
64    pub fn encoded_len(&self) -> usize {
65        self.fields.iter().map(|f| f.encoded_len()).sum()
66    }
67
68    /// Re-encode all unknown fields to `buf` in their original wire format.
69    pub fn write_to(&self, buf: &mut impl EncodeSink) {
70        for field in &self.fields {
71            field.write_to(buf);
72        }
73    }
74
75    /// Decode a concatenation of wire-format fields into [`UnknownFields`].
76    ///
77    /// Reads tag/data pairs until `data` is exhausted. Each field is decoded
78    /// via [`decode_unknown_field`](crate::encoding::decode_unknown_field)
79    /// with the full [`RECURSION_LIMIT`](crate::message::RECURSION_LIMIT)
80    /// depth budget and a fresh
81    /// [`DEFAULT_UNKNOWN_FIELD_LIMIT`](crate::DEFAULT_UNKNOWN_FIELD_LIMIT)
82    /// unknown-field allowance.
83    ///
84    /// Used by [`GroupCodec`](crate::extension::codecs::GroupCodec) to turn a
85    /// message's encoded bytes back into the inner-field representation that
86    /// [`UnknownFieldData::Group`] wraps.
87    ///
88    /// # Errors
89    ///
90    /// Returns [`DecodeError`](crate::DecodeError) if `data` contains a
91    /// malformed tag, truncated field, or exceeds the recursion limit or
92    /// unknown-field limit.
93    pub fn decode_from_slice(mut data: &[u8]) -> Result<Self, crate::DecodeError> {
94        use crate::encoding::{decode_unknown_field, Tag};
95        use crate::message::{DecodeContext, DEFAULT_UNKNOWN_FIELD_LIMIT, RECURSION_LIMIT};
96        let limit = core::cell::Cell::new(DEFAULT_UNKNOWN_FIELD_LIMIT);
97        let ctx = DecodeContext::new(RECURSION_LIMIT, &limit);
98        let mut out = Self::new();
99        while !data.is_empty() {
100            let tag = Tag::decode(&mut data)?;
101            out.push(decode_unknown_field(tag, &mut data, ctx)?);
102        }
103        Ok(out)
104    }
105}
106
107impl<'a> IntoIterator for &'a UnknownFields {
108    type Item = &'a UnknownField;
109    type IntoIter = core::slice::Iter<'a, UnknownField>;
110
111    fn into_iter(self) -> Self::IntoIter {
112        self.fields.iter()
113    }
114}
115
116impl IntoIterator for UnknownFields {
117    type Item = UnknownField;
118    type IntoIter = alloc::vec::IntoIter<UnknownField>;
119
120    fn into_iter(self) -> Self::IntoIter {
121        self.fields.into_iter()
122    }
123}
124
125impl Extend<UnknownField> for UnknownFields {
126    /// Appends each field, in iteration order. A field whose number is already
127    /// present is kept as a further record, as [`push`](UnknownFields::push)
128    /// does.
129    fn extend<T: IntoIterator<Item = UnknownField>>(&mut self, iter: T) {
130        self.fields.extend(iter);
131    }
132}
133
134impl FromIterator<UnknownField> for UnknownFields {
135    /// Collects unknown fields into an [`UnknownFields`], keeping their order
136    /// and any repeated field numbers.
137    fn from_iter<T: IntoIterator<Item = UnknownField>>(iter: T) -> Self {
138        Self {
139            fields: iter.into_iter().collect(),
140        }
141    }
142}
143
144/// A single unknown field (field number + wire data).
145#[derive(Clone, Debug, PartialEq, Eq, Hash)]
146pub struct UnknownField {
147    pub number: u32,
148    pub data: UnknownFieldData,
149}
150
151impl UnknownField {
152    /// Compute the encoded size of this field (tag + data).
153    pub fn encoded_len(&self) -> usize {
154        let tag_len =
155            crate::encoding::varint_len(((self.number as u64) << 3) | self.data.wire_type_value());
156        tag_len + self.data.encoded_len(self.number)
157    }
158
159    /// Re-encode this field (tag + data) to `buf` in its original wire format.
160    pub fn write_to(&self, buf: &mut impl EncodeSink) {
161        use crate::encoding::encode_varint;
162        let tag_value = ((self.number as u64) << 3) | self.data.wire_type_value();
163        encode_varint(tag_value, buf);
164        match &self.data {
165            UnknownFieldData::Varint(v) => encode_varint(*v, buf),
166            UnknownFieldData::Fixed64(v) => buf.put_u64_le(*v),
167            UnknownFieldData::Fixed32(v) => buf.put_u32_le(*v),
168            UnknownFieldData::LengthDelimited(data) => {
169                encode_varint(data.len() as u64, buf);
170                buf.put_slice(data);
171            }
172            UnknownFieldData::Group(fields) => {
173                fields.write_to(buf);
174                // End-group tag (wire type 4, same field number).
175                encode_varint(((self.number as u64) << 3) | 4, buf);
176            }
177        }
178    }
179}
180
181/// The wire data for an unknown field.
182#[derive(Clone, Debug, PartialEq, Eq, Hash)]
183pub enum UnknownFieldData {
184    Varint(u64),
185    Fixed64(u64),
186    Fixed32(u32),
187    LengthDelimited(Vec<u8>),
188    Group(UnknownFields),
189}
190
191impl UnknownFieldData {
192    fn wire_type_value(&self) -> u64 {
193        match self {
194            Self::Varint(_) => 0,
195            Self::Fixed64(_) => 1,
196            Self::LengthDelimited(_) => 2,
197            Self::Group(_) => 3,
198            Self::Fixed32(_) => 5,
199        }
200    }
201
202    fn encoded_len(&self, field_number: u32) -> usize {
203        match self {
204            Self::Varint(v) => crate::encoding::varint_len(*v),
205            Self::Fixed64(_) => 8,
206            Self::Fixed32(_) => 4,
207            Self::LengthDelimited(data) => {
208                crate::encoding::varint_len(data.len() as u64) + data.len()
209            }
210            Self::Group(fields) => {
211                // Group content + end-group tag (wire type 4, same field number).
212                let end_tag_len = crate::encoding::varint_len((field_number as u64) << 3 | 4);
213                fields.encoded_len() + end_tag_len
214            }
215        }
216    }
217}
218
219#[cfg(feature = "arbitrary")]
220impl<'a> arbitrary::Arbitrary<'a> for UnknownFields {
221    fn arbitrary(_u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result<Self> {
222        Ok(UnknownFields::new())
223    }
224}
225
226#[cfg(feature = "zeroize")]
227impl zeroize::Zeroize for UnknownFields {
228    fn zeroize(&mut self) {
229        self.fields.zeroize();
230    }
231}
232#[cfg(feature = "zeroize")]
233impl zeroize::Zeroize for UnknownField {
234    fn zeroize(&mut self) {
235        self.number.zeroize();
236        self.data.zeroize();
237    }
238}
239#[cfg(feature = "zeroize")]
240impl zeroize::Zeroize for UnknownFieldData {
241    fn zeroize(&mut self) {
242        match self {
243            Self::Varint(value) | Self::Fixed64(value) => value.zeroize(),
244            Self::Fixed32(value) => value.zeroize(),
245            Self::LengthDelimited(value) => value.zeroize(),
246            Self::Group(value) => value.zeroize(),
247        }
248    }
249}
250
251#[cfg(test)]
252mod tests {
253    use super::*;
254    use crate::encoding::MAX_FIELD_NUMBER;
255
256    /// A message that preserves unknown fields holds an `UnknownFields`, so it
257    /// can be a `HashSet` or `HashMap` key only if this type is `Eq + Hash`.
258    #[test]
259    fn unknown_fields_are_usable_in_a_hashed_key() {
260        let mut fields = UnknownFields::default();
261        fields.push(UnknownField {
262            number: 1,
263            data: UnknownFieldData::Group(UnknownFields::default()),
264        });
265        let mut set = std::collections::HashSet::new();
266        assert!(set.insert(fields.clone()));
267        assert!(!set.insert(fields));
268        assert!(set.insert(UnknownFields::default()));
269    }
270
271    fn varint_field(number: u32, value: u64) -> UnknownField {
272        UnknownField {
273            number,
274            data: UnknownFieldData::Varint(value),
275        }
276    }
277
278    fn fixed32_field(number: u32, value: u32) -> UnknownField {
279        UnknownField {
280            number,
281            data: UnknownFieldData::Fixed32(value),
282        }
283    }
284
285    fn fixed64_field(number: u32, value: u64) -> UnknownField {
286        UnknownField {
287            number,
288            data: UnknownFieldData::Fixed64(value),
289        }
290    }
291
292    fn ld_field(number: u32, data: Vec<u8>) -> UnknownField {
293        UnknownField {
294            number,
295            data: UnknownFieldData::LengthDelimited(data),
296        }
297    }
298
299    fn group_field(number: u32, fields: UnknownFields) -> UnknownField {
300        UnknownField {
301            number,
302            data: UnknownFieldData::Group(fields),
303        }
304    }
305
306    #[test]
307    fn test_empty_encoded_len() {
308        assert_eq!(UnknownFields::new().encoded_len(), 0);
309    }
310
311    #[test]
312    fn test_varint_encoded_len() {
313        // field 1, value 0: tag=(1<<3)|0=8 (1 byte) + varint(0) (1 byte) = 2
314        assert_eq!(varint_field(1, 0).encoded_len(), 2);
315        // field 1, value 128: tag (1 byte) + varint(128) (2 bytes) = 3
316        assert_eq!(varint_field(1, 128).encoded_len(), 3);
317    }
318
319    #[test]
320    fn test_fixed32_encoded_len() {
321        // field 1: tag=(1<<3)|5=13 (1 byte) + 4 bytes = 5
322        assert_eq!(fixed32_field(1, 0xDEAD_BEEF).encoded_len(), 5);
323    }
324
325    #[test]
326    fn test_fixed64_encoded_len() {
327        // field 1: tag=(1<<3)|1=9 (1 byte) + 8 bytes = 9
328        assert_eq!(fixed64_field(1, 0xDEAD_BEEF_CAFE_BABE).encoded_len(), 9);
329    }
330
331    #[test]
332    fn test_length_delimited_encoded_len() {
333        // field 1, 3-byte data: tag (1) + len_varint(3)=1 + 3 = 5
334        assert_eq!(ld_field(1, vec![0xAA, 0xBB, 0xCC]).encoded_len(), 5);
335        // field 1, empty: tag (1) + len_varint(0)=1 + 0 = 2
336        assert_eq!(ld_field(1, vec![]).encoded_len(), 2);
337        // field 1, 128-byte data: tag (1) + len_varint(128)=2 + 128 = 131
338        assert_eq!(ld_field(1, vec![0u8; 128]).encoded_len(), 131);
339    }
340
341    #[test]
342    fn test_group_encoded_len() {
343        // field 1, empty group: start_tag (1) + end_tag (1) = 2
344        assert_eq!(group_field(1, UnknownFields::new()).encoded_len(), 2);
345        // field 1, group containing one varint(0) field (2 bytes):
346        //   start_tag (1) + inner (2) + end_tag (1) = 4
347        let mut inner = UnknownFields::new();
348        inner.push(varint_field(1, 0));
349        assert_eq!(group_field(1, inner).encoded_len(), 4);
350    }
351
352    #[test]
353    fn test_large_field_number_two_byte_tag() {
354        // field 16: tag=(16<<3)|0=128, varint_len(128)=2; varint(0)=1 → total 3
355        assert_eq!(varint_field(16, 0).encoded_len(), 3);
356    }
357
358    #[test]
359    fn test_multiple_fields_sum() {
360        let mut fields = UnknownFields::new();
361        fields.push(varint_field(1, 0)); // tag(1) + varint(0)(1) = 2
362        fields.push(fixed32_field(2, 0)); // tag=(2<<3)|5=21(1) + 4 = 5
363        assert_eq!(fields.encoded_len(), 7);
364    }
365
366    #[test]
367    fn test_into_iter_by_ref() {
368        let mut fields = UnknownFields::new();
369        fields.push(varint_field(1, 0));
370        fields.push(varint_field(2, 0));
371        let numbers: Vec<u32> = (&fields).into_iter().map(|f| f.number).collect();
372        assert_eq!(numbers, vec![1, 2]);
373        // `for f in &fields` syntax works:
374        let mut count = 0;
375        for _ in &fields {
376            count += 1;
377        }
378        assert_eq!(count, 2);
379    }
380
381    #[test]
382    fn test_from_iterator_preserves_order_and_duplicates() {
383        let expected = vec![
384            varint_field(1, 0),
385            fixed64_field(2, u64::MAX),
386            fixed32_field(3, u32::MAX),
387            ld_field(4, vec![1, 2]),
388            group_field(5, UnknownFields::new()),
389            varint_field(1, 42),
390        ];
391
392        let fields: UnknownFields = expected.clone().into_iter().collect();
393        assert_eq!(fields.into_iter().collect::<Vec<_>>(), expected);
394
395        let empty: UnknownFields = core::iter::empty().collect();
396        assert!(empty.is_empty());
397    }
398
399    #[test]
400    fn test_extend_appends_fields() {
401        let mut fields = UnknownFields::new();
402        fields.push(varint_field(7, 1));
403        fields.extend([fixed32_field(2, 3), ld_field(2, vec![]), varint_field(7, 4)]);
404
405        assert_eq!(
406            fields.into_iter().collect::<Vec<_>>(),
407            vec![
408                varint_field(7, 1),
409                fixed32_field(2, 3),
410                ld_field(2, vec![]),
411                varint_field(7, 4),
412            ]
413        );
414    }
415
416    // ── encoded_len vs write_to consistency ──────────────────────────────
417    //
418    // Size/write divergence corrupts downstream encoding (compute_size
419    // feeds the length prefix of length-delimited messages). These tests
420    // build fields spanning the full variant matrix + boundary field
421    // numbers, and assert write_to produces exactly encoded_len bytes.
422
423    fn assert_len_matches_write(f: UnknownField) {
424        let mut fields = UnknownFields::new();
425        let claimed = f.encoded_len();
426        fields.push(f);
427        let mut buf = alloc::vec::Vec::new();
428        fields.write_to(&mut buf);
429        assert_eq!(
430            buf.len(),
431            claimed,
432            "encoded_len={claimed} but wrote {} bytes for {fields:?}",
433            buf.len()
434        );
435    }
436
437    #[test]
438    fn test_encoded_len_matches_write_varint() {
439        for &num in &[1, 15, 16, 2047, 2048, MAX_FIELD_NUMBER] {
440            for &val in &[0, 1, 127, 128, u32::MAX as u64, u64::MAX] {
441                assert_len_matches_write(varint_field(num, val));
442            }
443        }
444    }
445
446    #[test]
447    fn test_encoded_len_matches_write_fixed() {
448        for &num in &[1, 16, MAX_FIELD_NUMBER] {
449            assert_len_matches_write(fixed32_field(num, 0));
450            assert_len_matches_write(fixed32_field(num, u32::MAX));
451            assert_len_matches_write(fixed64_field(num, 0));
452            assert_len_matches_write(fixed64_field(num, u64::MAX));
453        }
454    }
455
456    #[test]
457    fn test_encoded_len_matches_write_length_delimited() {
458        for &num in &[1, 16, MAX_FIELD_NUMBER] {
459            assert_len_matches_write(ld_field(num, alloc::vec![]));
460            assert_len_matches_write(ld_field(num, alloc::vec![0xAB]));
461            assert_len_matches_write(ld_field(num, alloc::vec![0; 127]));
462            assert_len_matches_write(ld_field(num, alloc::vec![0; 128]));
463        }
464    }
465
466    #[test]
467    fn test_encoded_len_matches_write_group() {
468        for &num in &[1, 16, MAX_FIELD_NUMBER] {
469            // Empty group.
470            assert_len_matches_write(group_field(num, UnknownFields::new()));
471            // Group with mixed children.
472            let mut inner = UnknownFields::new();
473            inner.push(varint_field(2, 42));
474            inner.push(ld_field(3, alloc::vec![1, 2, 3]));
475            assert_len_matches_write(group_field(num, inner.clone()));
476            // Nested group.
477            let mut inner2 = UnknownFields::new();
478            inner2.push(group_field(4, inner));
479            assert_len_matches_write(group_field(num, inner2));
480        }
481    }
482
483    #[test]
484    fn test_encoded_len_matches_write_multi_field() {
485        // All variants in one UnknownFields.
486        let mut fields = UnknownFields::new();
487        fields.push(varint_field(1, 42));
488        fields.push(fixed32_field(2, 0xDEADBEEF));
489        fields.push(fixed64_field(3, u64::MAX));
490        fields.push(ld_field(4, alloc::vec![1, 2, 3, 4, 5]));
491        let mut nested = UnknownFields::new();
492        nested.push(varint_field(10, 7));
493        fields.push(group_field(5, nested));
494        let claimed = fields.encoded_len();
495        let mut buf = alloc::vec::Vec::new();
496        fields.write_to(&mut buf);
497        assert_eq!(buf.len(), claimed);
498    }
499
500    // ── decode_from_slice ────────────────────────────────────────────────
501
502    #[test]
503    fn test_decode_from_slice_empty() {
504        let out = UnknownFields::decode_from_slice(&[]).unwrap();
505        assert!(out.is_empty());
506    }
507
508    #[test]
509    fn test_decode_from_slice_roundtrip() {
510        // Build a multi-variant field set, write, decode back, compare.
511        let mut orig = UnknownFields::new();
512        orig.push(varint_field(1, 42));
513        orig.push(fixed32_field(2, 0xDEAD_BEEF));
514        orig.push(fixed64_field(3, u64::MAX));
515        orig.push(ld_field(4, vec![1, 2, 3]));
516        let mut nested = UnknownFields::new();
517        nested.push(varint_field(10, 7));
518        orig.push(group_field(5, nested));
519
520        let mut buf = Vec::new();
521        orig.write_to(&mut buf);
522        let decoded = UnknownFields::decode_from_slice(&buf).unwrap();
523        assert_eq!(decoded, orig);
524    }
525
526    #[test]
527    fn test_decode_from_slice_truncated() {
528        // Tag 0x08 (field 1, varint) with no value — truncated.
529        assert!(UnknownFields::decode_from_slice(&[0x08]).is_err());
530    }
531}