serde_yaml_bw 2.5.7

YAML support for Serde with an emphasis on panic-free parsing (including malformed YAML).
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
use crate::budget::BudgetBreach;
use crate::error::{self, Error, ErrorImpl, Result};
use crate::libyaml::cstr::{self, CStr};
use crate::libyaml::error::{Error as LibyamlError, Mark};
use crate::libyaml::tag::Tag;
use crate::libyaml::util::Owned;
use std::borrow::Cow;
use std::fmt::{self, Debug};
use std::io::{self, Read};
use std::mem::MaybeUninit;
use std::panic::{self, AssertUnwindSafe};
use std::ptr::{addr_of_mut, NonNull};
use std::slice;
use unsafe_libyaml_norway as sys;

pub(crate) const MAX_ANCHOR_LEN: usize = 65_536; // Keep in sync with tests/test_error.rs

pub(crate) struct Parser<'input> {
    pin: Owned<ParserPinned<'input>>,
}

struct ParserPinned<'input> {
    sys: sys::yaml_parser_t,
    input: Option<Cow<'input, [u8]>>,
    reader: Option<Box<dyn Read + 'input>>,
    read_error: Option<Error>,
    max_input_bytes: Option<usize>,
    total_input_bytes: usize,
}

#[derive(Debug)]
pub(crate) enum Event<'input> {
    StreamStart,
    StreamEnd,
    DocumentStart,
    DocumentEnd { implicit: bool },
    Alias(Anchor),
    Scalar(Scalar<'input>),
    SequenceStart(SequenceStart),
    SequenceEnd,
    MappingStart(MappingStart),
    MappingEnd,
    /// Placeholder event for unknown or empty libyaml events
    Void,
}

#[derive(Clone)]
pub(crate) struct Scalar<'input> {
    pub anchor: Option<Anchor>,
    pub tag: Option<Tag>,
    pub value: Box<[u8]>,
    pub style: ScalarStyle,
    pub repr: Option<&'input [u8]>,
}

#[derive(Clone, Debug)]
pub(crate) struct SequenceStart {
    pub anchor: Option<Anchor>,
    pub tag: Option<Tag>,
}

#[derive(Clone, Debug)]
pub(crate) struct MappingStart {
    pub anchor: Option<Anchor>,
    pub tag: Option<Tag>,
}

#[derive(Clone, Ord, PartialOrd, Eq, PartialEq, Hash)]
pub(crate) struct Anchor(pub(crate) Box<[u8]>);

#[derive(Copy, Clone, PartialEq, Eq, Debug)]
pub(crate) enum ScalarStyle {
    Plain,
    SingleQuoted,
    DoubleQuoted,
    Literal,
    Folded,
}

impl<'input> Parser<'input> {
    pub fn new(input: Cow<'input, [u8]>) -> Result<Parser<'input>> {
        let owned = Owned::<ParserPinned>::new_uninit();
        // SAFETY: `owned.ptr` points to uninitialized memory allocated for a
        // `yaml_parser_t`. The libyaml initialization functions expect a valid
        // pointer and we immediately initialize all fields before using them.
        let pin = unsafe {
            let parser = addr_of_mut!((*owned.ptr).sys);
            if sys::yaml_parser_initialize(parser).fail {
                return Err(Error::from(LibyamlError::parse_error(parser)));
            }
            sys::yaml_parser_set_encoding(parser, sys::YAML_UTF8_ENCODING);
            sys::yaml_parser_set_input_string(parser, input.as_ptr(), input.len() as u64);
            addr_of_mut!((*owned.ptr).input).write(Some(input));
            addr_of_mut!((*owned.ptr).reader).write(None);
            addr_of_mut!((*owned.ptr).read_error).write(None);
            addr_of_mut!((*owned.ptr).max_input_bytes).write(None);
            addr_of_mut!((*owned.ptr).total_input_bytes).write(0);
            Owned::assume_init(owned)
        };
        Ok(Parser { pin })
    }

    pub fn from_reader<R>(reader: R, max_input_bytes: Option<usize>) -> Result<Parser<'input>>
    where
        R: Read + 'input,
    {
        // SAFETY: Called by libyaml with pointers originating from our
        // `ParserPinned` state. All pointers are valid for the provided size and
        // live at least for the duration of the call.
        unsafe fn read_handler(
            data: *mut std::os::raw::c_void,
            buffer: *mut u8,
            size: u64,
            size_read: *mut u64,
        ) -> i32 {
            // SAFETY: `data` was set to point to a `ParserPinned` in `from_reader`.
            // `buffer` refers to a writable output buffer of length `size`.
            unsafe {
                let pinned = &mut *(data as *mut ParserPinned);
                let reader = match pinned.reader.as_mut() {
                    Some(reader) => reader,
                    None => {
                        pinned.read_error = Some(error::new(ErrorImpl::Io(io::Error::new(
                            io::ErrorKind::Other,
                            "reader is not set",
                        ))));
                        *size_read = 0;
                        return 0;
                    }
                };
                let slice = std::slice::from_raw_parts_mut(buffer, size as usize);
                match panic::catch_unwind(AssertUnwindSafe(|| reader.read(slice))) {
                    Ok(Ok(len)) => {
                        let total = pinned.total_input_bytes.saturating_add(len);
                        if let Some(limit) = pinned.max_input_bytes {
                            if total > limit {
                                pinned.read_error = Some(error::new(ErrorImpl::BudgetExceeded(
                                    BudgetBreach::ScalarBytes {
                                        total_scalar_bytes: total,
                                    },
                                )));
                                *size_read = 0;
                                return 0;
                            }
                        }
                        pinned.total_input_bytes = total;
                        *size_read = len as u64;
                        1
                    }
                    Ok(Err(err)) => {
                        pinned.read_error = Some(error::new(ErrorImpl::Io(err)));
                        *size_read = 0;
                        0
                    }
                    Err(_) => {
                        pinned.read_error = Some(error::new(ErrorImpl::Io(io::Error::other(
                            "reader panicked",
                        ))));
                        *size_read = 0;
                        0
                    }
                }
            }
        }

        let owned = Owned::<ParserPinned>::new_uninit();
        // SAFETY: as in `new`, we allocate space for a parser and fully
        // initialize it before use. The read handler and reader pointers remain
        // valid for the lifetime of the parser.
        let pin = unsafe {
            let parser = addr_of_mut!((*owned.ptr).sys);
            if sys::yaml_parser_initialize(parser).fail {
                return Err(Error::from(LibyamlError::parse_error(parser)));
            }
            sys::yaml_parser_set_encoding(parser, sys::YAML_UTF8_ENCODING);
            addr_of_mut!((*owned.ptr).reader).write(Some(Box::new(reader)));
            addr_of_mut!((*owned.ptr).read_error).write(None);
            addr_of_mut!((*owned.ptr).max_input_bytes).write(max_input_bytes);
            addr_of_mut!((*owned.ptr).total_input_bytes).write(0);
            let data = owned.ptr;
            sys::yaml_parser_set_input(
                parser,
                read_handler as sys::yaml_read_handler_t,
                data.cast(),
            );
            if let Some(err) = (*data).read_error.take() {
                return Err(err);
            }
            addr_of_mut!((*owned.ptr).input).write(None);
            Owned::assume_init(owned)
        };
        Ok(Parser { pin })
    }

    pub fn next(&mut self) -> Result<(Event<'input>, Mark)> {
        let mut event = MaybeUninit::<sys::yaml_event_t>::uninit();
        // SAFETY: libyaml uses the provided parser pointer and event structure to
        // produce the next event. `self.pin` contains a valid parser and the
        // event is properly initialized before being passed to libyaml.
        unsafe {
            if let Some(err) = (*self.pin.ptr).read_error.take() {
                return Err(err);
            }
            let parser = addr_of_mut!((*self.pin.ptr).sys);
            if (&*parser).error != sys::YAML_NO_ERROR {
                return Err(Error::from(LibyamlError::parse_error(parser)));
            }
            let event = event.as_mut_ptr();
            if sys::yaml_parser_parse(parser, event).fail {
                if let Some(err) = (*self.pin.ptr).read_error.take() {
                    return Err(err);
                }
                return Err(Error::from(LibyamlError::parse_error(parser)));
            }
            let mark = Mark {
                sys: (*event).start_mark,
            };
            let ret = convert_event(&*event, &(*self.pin.ptr).input).map_err(error::new);
            sys::yaml_event_delete(event);
            ret.map(|event| (event, mark))
        }
    }
}

// SAFETY: The caller guarantees that `sys` points to a valid libyaml event of
// the corresponding type and that any string pointers inside remain alive for
// the duration of the conversion.
unsafe fn convert_event<'input>(
    sys: &sys::yaml_event_t,
    input: &Option<Cow<'input, [u8]>>,
) -> std::result::Result<Event<'input>, ErrorImpl> {
    match sys.type_ {
        sys::YAML_STREAM_START_EVENT => Ok(Event::StreamStart),
        sys::YAML_STREAM_END_EVENT => Ok(Event::StreamEnd),
        sys::YAML_DOCUMENT_START_EVENT => Ok(Event::DocumentStart),
        sys::YAML_DOCUMENT_END_EVENT => Ok(Event::DocumentEnd {
            implicit: unsafe { sys.data.document_end.implicit },
        }),
        sys::YAML_ALIAS_EVENT => match unsafe {
            // SAFETY: The event is an alias; the union field `alias` is valid and
            // the pointer comes from libyaml.
            optional_anchor(sys.data.alias.anchor)?
        } {
            Some(anchor) => Ok(Event::Alias(anchor)),
            None => Err(ErrorImpl::UnknownAnchor(
                Mark {
                    sys: sys.start_mark,
                },
                Anchor(Box::from(&b""[..])),
            )),
        },
        sys::YAML_SCALAR_EVENT => Ok(Event::Scalar(Scalar {
            // SAFETY: The event is known to be a scalar, so the union fields for
            // `scalar` are valid and the pointers are trusted from libyaml.
            anchor: unsafe { optional_anchor(sys.data.scalar.anchor) }?,
            tag: unsafe { optional_tag(sys.data.scalar.tag) }?,
            value: Box::from(unsafe {
                // SAFETY: `value` points to `length` bytes of scalar data.
                slice::from_raw_parts(sys.data.scalar.value, sys.data.scalar.length as usize)
            }),
            style: match unsafe { sys.data.scalar.style } {
                sys::YAML_PLAIN_SCALAR_STYLE => ScalarStyle::Plain,
                sys::YAML_SINGLE_QUOTED_SCALAR_STYLE => ScalarStyle::SingleQuoted,
                sys::YAML_DOUBLE_QUOTED_SCALAR_STYLE => ScalarStyle::DoubleQuoted,
                sys::YAML_LITERAL_SCALAR_STYLE => ScalarStyle::Literal,
                sys::YAML_FOLDED_SCALAR_STYLE => ScalarStyle::Folded,
                // Treat any unrecognized style as plain to avoid panicking
                sys::YAML_ANY_SCALAR_STYLE | _ => ScalarStyle::Plain,
            },
            repr: if let Some(Cow::Borrowed(input)) = input {
                let start = sys.start_mark.index as usize;
                let end = sys.end_mark.index as usize;
                if start <= end && end <= input.len() {
                    Some(&input[start..end])
                } else {
                    None
                }
            } else {
                None
            },
        })),
        sys::YAML_SEQUENCE_START_EVENT => Ok(Event::SequenceStart(SequenceStart {
            // SAFETY: Union fields for `sequence_start` are valid in this match
            // arm and pointers come from libyaml.
            anchor: unsafe { optional_anchor(sys.data.sequence_start.anchor) }?,
            tag: unsafe { optional_tag(sys.data.sequence_start.tag) }?,
        })),
        sys::YAML_SEQUENCE_END_EVENT => Ok(Event::SequenceEnd),
        sys::YAML_MAPPING_START_EVENT => Ok(Event::MappingStart(MappingStart {
            // SAFETY: Union fields for `mapping_start` are valid for this event
            // type and pointers originate from libyaml.
            anchor: unsafe { optional_anchor(sys.data.mapping_start.anchor) }?,
            tag: unsafe { optional_tag(sys.data.mapping_start.tag) }?,
        })),
        sys::YAML_MAPPING_END_EVENT => Ok(Event::MappingEnd),
        // Unknown or empty events should not cause a panic
        sys::YAML_NO_EVENT => Ok(Event::Void),
        _ => Ok(Event::Void),
    }
}

// SAFETY: `anchor` must be a valid pointer to a NUL-terminated string or null
// if no anchor is present.
unsafe fn optional_anchor(anchor: *const u8) -> std::result::Result<Option<Anchor>, ErrorImpl> {
    let ptr = match NonNull::new(anchor as *mut i8) {
        Some(p) => p,
        None => return Ok(None),
    };
    // SAFETY: `ptr` is non-null and points to a valid NUL-terminated string from
    // libyaml.
    let cstr = unsafe { CStr::from_ptr(ptr) };
    match cstr.to_bytes() {
        Ok(bytes) => {
            // Enforce a maximum anchor length to avoid excessive memory/CPU usage.
            if bytes.len() > MAX_ANCHOR_LEN {
                return Err(ErrorImpl::Message(
                    format!(
                        "anchor too long: length {} exceeds maximum {}",
                        bytes.len(),
                        MAX_ANCHOR_LEN
                    ),
                    None,
                ));
            }
            Ok(Some(Anchor(Box::from(bytes))))
        }
        Err(_) => Err(ErrorImpl::TagError),
    }
}

// SAFETY: `tag` must be a valid pointer to a NUL-terminated string or null if
// no tag is provided.
unsafe fn optional_tag(tag: *const u8) -> std::result::Result<Option<Tag>, ErrorImpl> {
    let ptr = match NonNull::new(tag as *mut i8) {
        Some(p) => p,
        None => return Ok(None),
    };
    // SAFETY: `ptr` is non-null and points to a valid NUL-terminated string.
    let cstr = unsafe { CStr::from_ptr(ptr) };
    cstr.to_bytes()
        .map(|bytes| Some(Tag(Box::from(bytes))))
        .map_err(|_| ErrorImpl::TagError)
}

impl Debug for Scalar<'_> {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        let Scalar {
            anchor,
            tag,
            value,
            style,
            repr: _,
        } = self;

        struct LossySlice<'a>(&'a [u8]);

        impl Debug for LossySlice<'_> {
            fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
                cstr::debug_lossy(self.0, formatter)
            }
        }

        formatter
            .debug_struct("Scalar")
            .field("anchor", anchor)
            .field("tag", tag)
            .field("value", &LossySlice(value))
            .field("style", style)
            .finish()
    }
}

impl Debug for Anchor {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        cstr::debug_lossy(&self.0, formatter)
    }
}

impl Drop for ParserPinned<'_> {
    fn drop(&mut self) {
        // SAFETY: `self.sys` was initialized by libyaml and must be freed using
        // `yaml_parser_delete` to avoid leaking resources.
        unsafe { sys::yaml_parser_delete(&raw mut self.sys) }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::borrow::Cow;
    use std::io::{self, Read};

    #[test]
    fn repeated_parse_errors_do_not_leak() {
        let yaml = ":";
        for _ in 0..100 {
            let mut parser = Parser::new(Cow::Borrowed(yaml.as_bytes())).unwrap();
            loop {
                match parser.next() {
                    Ok(_) => continue,
                    Err(_) => break,
                }
            }
        }
    }

    struct FailingReader;

    impl Read for FailingReader {
        fn read(&mut self, _buf: &mut [u8]) -> io::Result<usize> {
            Err(io::Error::new(io::ErrorKind::Other, "fail"))
        }
    }

    #[test]
    fn read_error_is_propagated() {
        let mut parser = Parser::from_reader(FailingReader, None).unwrap();
        let err = parser.next().unwrap_err();
        assert_eq!(err.to_string(), "fail");
    }

    struct PanickingReader;

    impl Read for PanickingReader {
        fn read(&mut self, _buf: &mut [u8]) -> io::Result<usize> {
            panic!("boom");
        }
    }

    #[test]
    fn reader_panic_is_propagated_as_io_error() {
        let mut parser = Parser::from_reader(PanickingReader, None).unwrap();
        let err = parser.next().unwrap_err();
        assert_eq!(err.to_string(), "reader panicked");
    }

    struct ChunkedReader {
        chunks: Vec<&'static [u8]>,
    }

    impl Read for ChunkedReader {
        fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
            let Some(chunk) = self.chunks.first().copied() else {
                return Ok(0);
            };
            let len = chunk.len().min(buf.len());
            buf[..len].copy_from_slice(&chunk[..len]);
            if len == chunk.len() {
                self.chunks.remove(0);
            } else {
                self.chunks[0] = &chunk[len..];
            }
            Ok(len)
        }
    }

    #[test]
    fn oversized_reader_input_is_reported_as_budget_exceeded() {
        let reader = ChunkedReader {
            chunks: vec![b"a", b"b"],
        };
        let mut parser = Parser::from_reader(reader, Some(1)).unwrap();
        let _ = parser.next().unwrap();
        let err = parser.next().unwrap_err();
        assert_eq!(
            err.to_string(),
            "YAML budget exceeded: ScalarBytes { total_scalar_bytes: 2 }"
        );
    }
}