granit_parser/input.rs
1//! Utilities to create a source of input to the parser.
2//!
3//! [`Input`] must be implemented for the parser to fetch input. Make sure your needs aren't
4//! covered by the [`BufferedInput`].
5
6use alloc::string::String;
7
8use crate::error::ErrorKind;
9
10pub(crate) mod buffered;
11pub(crate) mod str;
12
13#[allow(clippy::module_name_repetitions)]
14pub use buffered::{BufferedInput, FallibleBufferedInput};
15
16/// A trait for inputs that can provide borrowed slices with a specific lifetime.
17///
18/// This trait enables zero-copy (`Cow::Borrowed`) token values for inputs that keep a stable
19/// backing string. The key difference from [`Input::slice_bytes`] is that this method returns
20/// a slice with the input's original lifetime `'a`, not tied to `&self`.
21///
22/// For inputs that support zero-copy (like [`str::StrInput`]), this returns `Some(&'a str)`.
23/// For streaming inputs that don't have stable backing storage, this returns `None`.
24pub trait BorrowedInput<'a>: Input {
25 /// Return a borrowed slice of the underlying source between two byte offsets.
26 ///
27 /// Unlike [`Input::slice_bytes`], this returns a slice with the input's lifetime `'a`,
28 /// allowing the slice to outlive the borrow of `&self`.
29 ///
30 /// `start` and `end` are byte offsets as returned by [`Input::byte_offset`]. The interval is
31 /// half-open: `[start, end)`.
32 ///
33 /// Returns `None` if the input does not support zero-copy slicing.
34 ///
35 /// # Panics
36 /// Implementations may panic in debug builds if `start` is greater than `end` or `end` is past
37 /// the end of the underlying source.
38 #[track_caller]
39 #[must_use]
40 fn slice_borrowed(&self, start: usize, end: usize) -> Option<&'a str>;
41}
42
43pub use crate::char_traits::{
44 is_alpha, is_blank, is_blank_or_breakz, is_break, is_breakz, is_digit, is_flow, is_z,
45};
46
47/// Interface for a source of characters.
48///
49/// Hiding the input's implementation behind this trait allows input-specific optimizations, such
50/// as using `str` methods instead of manually transferring one `char` at a time to a buffer.
51/// Implementations with stable backing storage can also return borrowed `&str` slices and avoid
52/// allocating token values.
53///
54/// # Scalar scanning hooks
55///
56/// [`Self::fetch_block_scalar_line`] and [`Self::take_quoted_scalar_ascii_chunk`] allow bulk
57/// processing of block and quoted scalar content. Both provide defaults: custom inputs need
58/// only override them when they can offer a more efficient implementation.
59///
60/// Overrides must consume from the logical stream front, including any characters already
61/// buffered for lookahead, and keep [`Self::byte_offset`] accurate if it is supported. They
62/// must work without a preceding [`Self::lookahead`] call; callers refresh lookahead before
63/// inspecting subsequent characters. Source errors must remain available through
64/// [`Self::take_source_error`], and a terminal source error must prevent further source reads.
65/// The scanner, not the input hook, updates line/column markers and interprets YAML syntax.
66pub trait Input {
67 /// A hint to the input source that we will need to read `count` characters.
68 ///
69 /// If the input is exhausted, `\0` can be used to pad the last characters and later returned.
70 /// The characters must not be consumed, but may be placed in an internal buffer.
71 ///
72 /// This method may be a no-op if buffering yields no performance improvement.
73 ///
74 /// Implementers of [`Input`] must _not_ expose a lookahead window larger than
75 /// [`Input::bufmaxlen`]. They may retain a larger window requested by an earlier call; callers
76 /// should use [`Input::buflen`] to observe the currently available window.
77 fn lookahead(&mut self, count: usize);
78
79 /// Return the number of characters in the active lookahead window.
80 ///
81 /// This is the number of characters that the input promises can be read through [`peek`] and
82 /// [`peek_nth`] after prior [`lookahead`] calls. It is not necessarily the number of source
83 /// characters remaining: inputs may keep the window available after consuming characters and
84 /// may pad positions past EOF with `\0`.
85 ///
86 /// [`lookahead`]: Input::lookahead
87 /// [`peek`]: Input::peek
88 /// [`peek_nth`]: Input::peek_nth
89 #[must_use]
90 fn buflen(&self) -> usize;
91
92 /// Return the maximum number of characters this input can buffer for lookahead.
93 #[must_use]
94 fn bufmaxlen(&self) -> usize;
95
96 /// Return whether the active lookahead window is empty.
97 ///
98 /// This is equivalent to `self.buflen() == 0`. It does not mean the underlying source is
99 /// exhausted: after a previous [`lookahead`] call, an input may keep a non-empty lookahead
100 /// window available even after all source characters have been consumed, with positions past
101 /// EOF observed as `\0`.
102 ///
103 /// [`lookahead`]: Input::lookahead
104 #[inline]
105 #[must_use]
106 fn buf_is_empty(&self) -> bool {
107 self.buflen() == 0
108 }
109
110 /// Read the next character from the logical input stream and return it directly.
111 ///
112 /// If an implementation has already fetched characters for lookahead, this consumes the
113 /// buffered stream front before reading farther from the underlying source.
114 #[must_use]
115 fn raw_read_ch(&mut self) -> char;
116
117 /// Read a non-breakz character from the input stream and return it directly.
118 ///
119 /// If an implementation has already fetched characters for lookahead, this consumes from the
120 /// buffered stream front before reading farther from the underlying source.
121 ///
122 /// If the next character is a breakz, it is either not consumed or placed into the buffer (if
123 /// any).
124 #[must_use]
125 fn raw_read_non_breakz_ch(&mut self) -> Option<char>;
126
127 /// Consume the next character.
128 fn skip(&mut self);
129
130 /// Consume the next `count` characters.
131 fn skip_n(&mut self, count: usize);
132
133 /// Return the next character, without consuming it.
134 ///
135 /// Users of the [`Input`] must make sure that the character has been loaded through a prior
136 /// call to [`Input::lookahead`]. Implementors of [`Input`] may assume that a valid call to
137 /// [`Input::lookahead`] has been made beforehand.
138 ///
139 /// # Return
140 /// If the input source is not exhausted, returns the next character to be fed into the
141 /// scanner. Otherwise, returns `\0`.
142 #[must_use]
143 fn peek(&self) -> char;
144
145 /// Return the `n`-th character in the buffer, without consuming it.
146 ///
147 /// This function assumes that the `n`-th character in the input has already been fetched through
148 /// [`Input::lookahead`].
149 #[must_use]
150 fn peek_nth(&self, n: usize) -> char;
151
152 /// Return the current byte offset in the underlying source, if available.
153 ///
154 /// This is an *optional* capability that enables zero-copy (`Cow::Borrowed`) token values
155 /// for inputs that keep a stable backing string (notably [`str::StrInput`]).
156 ///
157 /// The returned value (when `Some`) is the number of bytes that have been consumed so far,
158 /// i.e. an offset into the original source string.
159 ///
160 /// # Correctness contract
161 /// Implementations returning `Some(_)` must satisfy all of the following:
162 ///
163 /// - The offset is a valid UTF-8 boundary in the underlying source.
164 /// - The offset is monotonically non-decreasing as characters are consumed.
165 /// - The underlying source is stable for the duration of parsing (no reallocation/mutation)
166 /// so that slices returned by [`Input::slice_bytes`] remain valid.
167 ///
168 /// Inputs that cannot provide stable slicing (e.g. stream/iterator inputs) must return
169 /// `None`.
170 #[inline]
171 #[must_use]
172 fn byte_offset(&self) -> Option<usize> {
173 None
174 }
175
176 /// Return a borrowed slice of the underlying source between two byte offsets.
177 ///
178 /// This is an *optional* capability used to produce `Cow::Borrowed` values without
179 /// allocating.
180 ///
181 /// `start` and `end` are byte offsets as returned by [`Input::byte_offset`]. The interval is
182 /// half-open: `[start, end)`.
183 ///
184 /// # Correctness contract
185 /// Implementations returning `Some(&str)` must ensure:
186 ///
187 /// - `start <= end`.
188 /// - Both offsets are valid UTF-8 boundaries.
189 /// - The returned `&str` is a view into the stable underlying source associated with this
190 /// input.
191 ///
192 /// Implementations that return `None` from [`Input::byte_offset`] must also return `None`
193 /// here.
194 ///
195 /// # Panics
196 /// Implementations may panic in debug builds if `start` is greater than `end` or `end` is past
197 /// the end of the underlying source.
198 #[track_caller]
199 #[inline]
200 #[must_use]
201 fn slice_bytes(&self, _start: usize, _end: usize) -> Option<&str> {
202 None
203 }
204
205 /// Return whether this input may contain a `#` character.
206 ///
207 /// This is a conservative performance hint. Inputs that cannot answer cheaply should return
208 /// `true`, which keeps full comment handling enabled.
209 #[inline]
210 #[must_use]
211 fn may_contain_comments(&self) -> bool {
212 true
213 }
214
215 /// Take a terminal error reported by the underlying source.
216 ///
217 /// Infallible inputs use the default implementation. Fallible streaming inputs latch their
218 /// first source error and return it here so the scanner can distinguish the failure from clean
219 /// end-of-input. Once an implementation reports an error, it must not read from its source
220 /// again.
221 ///
222 /// Source adapters should use an input-related [`ErrorKind`] such as
223 /// [`ErrorKind::InputIo`], [`ErrorKind::InputDecoding`], or
224 /// [`ErrorKind::InputByteLimitExceeded`].
225 #[inline]
226 fn take_source_error(&mut self) -> Option<ErrorKind> {
227 None
228 }
229
230 /// Look for the next character and return it.
231 ///
232 /// The character is not consumed.
233 /// Equivalent to calling [`Input::lookahead`] and [`Input::peek`].
234 #[inline]
235 #[must_use]
236 fn look_ch(&mut self) -> char {
237 self.lookahead(1);
238 self.peek()
239 }
240
241 /// Return whether the next character in the input source is equal to `c`.
242 ///
243 /// This function assumes that the next character in the input has already been fetched through
244 /// [`Input::lookahead`].
245 #[inline]
246 #[must_use]
247 fn next_char_is(&self, c: char) -> bool {
248 self.peek() == c
249 }
250
251 /// Return whether the `n`-th character in the input source is equal to `c`.
252 ///
253 /// This function assumes that the `n`-th character in the input has already been fetched through
254 /// [`Input::lookahead`].
255 #[inline]
256 #[must_use]
257 fn nth_char_is(&self, n: usize, c: char) -> bool {
258 self.peek_nth(n) == c
259 }
260
261 /// Return whether the next 2 characters in the input source match the given characters.
262 ///
263 /// This function assumes that the next 2 characters in the input have already been fetched
264 /// through [`Input::lookahead`].
265 ///
266 /// # Panics
267 /// Panics if the active lookahead window contains fewer than 2 characters.
268 #[track_caller]
269 #[inline]
270 #[must_use]
271 fn next_2_are(&self, c1: char, c2: char) -> bool {
272 assert!(self.buflen() >= 2);
273 self.peek() == c1 && self.peek_nth(1) == c2
274 }
275
276 /// Return whether the next 3 characters in the input source match the given characters.
277 ///
278 /// This function assumes that the next 3 characters in the input have already been fetched
279 /// through [`Input::lookahead`].
280 ///
281 /// # Panics
282 /// Panics if the active lookahead window contains fewer than 3 characters.
283 #[track_caller]
284 #[inline]
285 #[must_use]
286 fn next_3_are(&self, c1: char, c2: char, c3: char) -> bool {
287 assert!(self.buflen() >= 3);
288 self.peek() == c1 && self.peek_nth(1) == c2 && self.peek_nth(2) == c3
289 }
290
291 /// Check whether the next characters correspond to a document indicator.
292 ///
293 /// This function assumes that the next 4 characters in the input have already been fetched
294 /// through [`Input::lookahead`].
295 ///
296 /// # Panics
297 /// Panics if the active lookahead window contains fewer than 4 characters.
298 #[track_caller]
299 #[inline]
300 #[must_use]
301 fn next_is_document_indicator(&self) -> bool {
302 assert!(self.buflen() >= 4);
303 is_blank_or_breakz(self.peek_nth(3))
304 && (self.next_3_are('.', '.', '.') || self.next_3_are('-', '-', '-'))
305 }
306
307 /// Check whether the next characters correspond to a start of document.
308 ///
309 /// This function assumes that the next 4 characters in the input have already been fetched
310 /// through [`Input::lookahead`].
311 ///
312 /// # Panics
313 /// Panics if the active lookahead window contains fewer than 4 characters.
314 #[track_caller]
315 #[inline]
316 #[must_use]
317 fn next_is_document_start(&self) -> bool {
318 assert!(self.buflen() >= 4);
319 self.next_3_are('-', '-', '-') && is_blank_or_breakz(self.peek_nth(3))
320 }
321
322 /// Check whether the next characters correspond to an end of document.
323 ///
324 /// This function assumes that the next 4 characters in the input have already been fetched
325 /// through [`Input::lookahead`].
326 ///
327 /// # Panics
328 /// Panics if the active lookahead window contains fewer than 4 characters.
329 #[track_caller]
330 #[inline]
331 #[must_use]
332 fn next_is_document_end(&self) -> bool {
333 assert!(self.buflen() >= 4);
334 self.next_3_are('.', '.', '.') && is_blank_or_breakz(self.peek_nth(3))
335 }
336
337 /// Skip YAML whitespace up to the end of the current line.
338 ///
339 /// Inline comments are consumed only after at least one preceding YAML whitespace character.
340 ///
341 /// # Return
342 /// Return a tuple with the number of characters that were consumed and the result of skipping
343 /// whitespace. The number of characters returned can be used to advance the index and column,
344 /// since no end-of-line character will be consumed.
345 /// See [`SkipTabs`] for more details on the success variant.
346 ///
347 /// # Errors
348 /// Returns [`ErrorKind::CommentNotSeparated`] if a comment is encountered without preceding
349 /// whitespace. In that event, the first tuple element contains the number of characters
350 /// consumed prior to reaching the `#`.
351 ///
352 /// # Panics
353 /// Panics if `skip_tabs` is [`SkipTabs::Result`], which is an output-only variant.
354 #[track_caller]
355 fn skip_ws_to_eol(&mut self, skip_tabs: SkipTabs) -> (usize, Result<SkipTabs, ErrorKind>) {
356 assert!(!matches!(skip_tabs, SkipTabs::Result(..)));
357
358 let mut encountered_tab = false;
359 let mut has_yaml_ws = false;
360 let mut chars_consumed = 0;
361 loop {
362 match self.look_ch() {
363 ' ' => {
364 has_yaml_ws = true;
365 self.skip();
366 }
367 '\t' if skip_tabs != SkipTabs::No => {
368 encountered_tab = true;
369 self.skip();
370 }
371 // YAML comments must be preceded by whitespace.
372 '#' if !encountered_tab && !has_yaml_ws => {
373 return (chars_consumed, Err(ErrorKind::CommentNotSeparated));
374 }
375 '#' => {
376 self.skip(); // Skip over '#'
377 while !is_breakz(self.look_ch()) {
378 self.skip();
379 chars_consumed += 1;
380 }
381 }
382 _ => break,
383 }
384 chars_consumed += 1;
385 }
386
387 (
388 chars_consumed,
389 Ok(SkipTabs::Result(encountered_tab, has_yaml_ws)),
390 )
391 }
392
393 /// Skip YAML blank characters, stopping before comments, line breaks, or other content.
394 ///
395 /// This is the comment-aware counterpart to [`Input::skip_ws_to_eol`]: it preserves a
396 /// following `#` for the scanner to tokenize while still letting input implementations batch
397 /// the common run of spaces and tabs.
398 ///
399 /// # Return
400 /// Returns the number of consumed characters and a [`SkipTabs::Result`] describing whether
401 /// tabs and valid YAML whitespace (` `) were encountered.
402 ///
403 /// # Panics
404 /// Panics if `skip_tabs` is [`SkipTabs::Result`], which is an output-only variant.
405 #[track_caller]
406 fn skip_ws_to_eol_blanks(&mut self, skip_tabs: SkipTabs) -> (usize, SkipTabs) {
407 assert!(!matches!(skip_tabs, SkipTabs::Result(..)));
408
409 let mut encountered_tab = false;
410 let mut has_yaml_ws = false;
411 let mut chars_consumed = 0;
412
413 loop {
414 match self.look_ch() {
415 ' ' => {
416 has_yaml_ws = true;
417 chars_consumed += 1;
418 self.skip();
419 }
420 '\t' if skip_tabs != SkipTabs::No => {
421 encountered_tab = true;
422 chars_consumed += 1;
423 self.skip();
424 }
425 _ => break,
426 }
427 }
428
429 (
430 chars_consumed,
431 SkipTabs::Result(encountered_tab, has_yaml_ws),
432 )
433 }
434
435 /// Check whether the next characters may be part of a plain scalar.
436 ///
437 /// This function assumes we are not given a blankz character.
438 #[allow(clippy::inline_always)]
439 #[inline(always)]
440 #[must_use]
441 fn next_can_be_plain_scalar(&self, in_flow: bool) -> bool {
442 let nc = self.peek_nth(1);
443 match self.peek() {
444 // indicators can end a plain scalar, see 7.3.3. Plain Style
445 ':' if is_blank_or_breakz(nc) || (in_flow && is_flow(nc)) => false,
446 c if in_flow && is_flow(c) => false,
447 _ => true,
448 }
449 }
450
451 /// Check whether the next character is [a blank] or [a break].
452 ///
453 /// The character must have previously been fetched through [`lookahead`]
454 ///
455 /// # Return
456 /// Returns true if the character is [a blank] or [a break], false otherwise.
457 ///
458 /// [`lookahead`]: Input::lookahead
459 /// [a blank]: is_blank
460 /// [a break]: is_break
461 #[inline]
462 #[must_use]
463 fn next_is_blank_or_break(&self) -> bool {
464 is_blank(self.peek()) || is_break(self.peek())
465 }
466
467 /// Check whether the next character is [a blank] or [a breakz].
468 ///
469 /// The character must have previously been fetched through [`lookahead`]
470 ///
471 /// # Return
472 /// Returns true if the character is [a blank] or [a break], false otherwise.
473 ///
474 /// [`lookahead`]: Input::lookahead
475 /// [a blank]: is_blank
476 /// [a breakz]: is_breakz
477 #[inline]
478 #[must_use]
479 fn next_is_blank_or_breakz(&self) -> bool {
480 is_blank(self.peek()) || is_breakz(self.peek())
481 }
482
483 /// Check whether the next character is [a blank].
484 ///
485 /// The character must have previously been fetched through [`lookahead`]
486 ///
487 /// # Return
488 /// Returns true if the character is [a blank], false otherwise.
489 ///
490 /// [`lookahead`]: Input::lookahead
491 /// [a blank]: is_blank
492 #[inline]
493 #[must_use]
494 fn next_is_blank(&self) -> bool {
495 is_blank(self.peek())
496 }
497
498 /// Check whether the next character is [a break].
499 ///
500 /// The character must have previously been fetched through [`lookahead`]
501 ///
502 /// # Return
503 /// Returns true if the character is [a break], false otherwise.
504 ///
505 /// [`lookahead`]: Input::lookahead
506 /// [a break]: is_break
507 #[inline]
508 #[must_use]
509 fn next_is_break(&self) -> bool {
510 is_break(self.peek())
511 }
512
513 /// Check whether the next character is [a breakz].
514 ///
515 /// The character must have previously been fetched through [`lookahead`]
516 ///
517 /// # Return
518 /// Returns true if the character is [a breakz], false otherwise.
519 ///
520 /// [`lookahead`]: Input::lookahead
521 /// [a breakz]: is_breakz
522 #[inline]
523 #[must_use]
524 fn next_is_breakz(&self) -> bool {
525 is_breakz(self.peek())
526 }
527
528 /// Check whether the input is at its physical end.
529 ///
530 /// The default implementation infers end-of-input from the `\0` sentinel returned by
531 /// [`Self::peek`]. Inputs that can distinguish a literal NUL from end-of-input should override
532 /// this method.
533 /// The next position must have previously been fetched through [`Self::lookahead`].
534 ///
535 /// # Return
536 /// Returns true if the input is exhausted, false otherwise.
537 #[inline]
538 #[must_use]
539 fn next_is_z(&self) -> bool {
540 is_z(self.peek())
541 }
542
543 /// Check whether the next character is [a flow].
544 ///
545 /// The character must have previously been fetched through [`lookahead`]
546 ///
547 /// # Return
548 /// Returns true if the character is [a flow], false otherwise.
549 ///
550 /// [`lookahead`]: Input::lookahead
551 /// [a flow]: is_flow
552 #[inline]
553 #[must_use]
554 fn next_is_flow(&self) -> bool {
555 is_flow(self.peek())
556 }
557
558 /// Check whether the next character is [a digit].
559 ///
560 /// The character must have previously been fetched through [`lookahead`]
561 ///
562 /// # Return
563 /// Returns true if the character is [a digit], false otherwise.
564 ///
565 /// [`lookahead`]: Input::lookahead
566 /// [a digit]: is_digit
567 #[inline]
568 #[must_use]
569 fn next_is_digit(&self) -> bool {
570 is_digit(self.peek())
571 }
572
573 /// Check whether the next character is [a letter].
574 ///
575 /// The character must have previously been fetched through [`lookahead`]
576 ///
577 /// # Return
578 /// Returns true if the character is [a letter], false otherwise.
579 ///
580 /// [`lookahead`]: Input::lookahead
581 /// [a letter]: is_alpha
582 #[inline]
583 #[must_use]
584 fn next_is_alpha(&self) -> bool {
585 is_alpha(self.peek())
586 }
587
588 /// Skip printable characters until a [breakz] or non-printable character is found.
589 ///
590 /// The stopping character is not consumed.
591 ///
592 /// # Return
593 /// Return the number of characters that were consumed. The number of characters returned can
594 /// be used to advance the index and column, since no end-of-line character will be consumed.
595 ///
596 /// [breakz]: is_breakz
597 #[inline]
598 fn skip_while_non_breakz(&mut self) -> usize {
599 let mut count = 0;
600 while {
601 let c = self.look_ch();
602 !is_breakz(c) && crate::char_traits::is_printable(c)
603 } {
604 count += 1;
605 self.skip();
606 }
607 count
608 }
609
610 /// Skip characters from the input while [blanks] are found.
611 ///
612 /// The characters are consumed from the input.
613 ///
614 /// # Return
615 /// Return the number of characters that were consumed. The number of characters returned can
616 /// be used to advance the index and column, since no end-of-line character will be consumed.
617 ///
618 /// [blanks]: is_blank
619 fn skip_while_blank(&mut self) -> usize {
620 let mut n_bytes = 0;
621 while is_blank(self.look_ch()) {
622 n_bytes += self.peek().len_utf8();
623 self.skip();
624 }
625 n_bytes
626 }
627
628 /// Fetch characters from the input while we encounter letters and store them in `out`.
629 ///
630 /// The characters are consumed from the input.
631 ///
632 /// # Return
633 /// Return the number of characters that were consumed. The number of characters returned can
634 /// be used to advance the index and column, since no end-of-line character will be consumed.
635 fn fetch_while_is_alpha(&mut self, out: &mut String) -> usize {
636 let mut n_bytes = 0;
637 while is_alpha(self.look_ch()) {
638 let c = self.peek();
639 n_bytes += c.len_utf8();
640 out.push(c);
641 self.skip();
642 }
643 n_bytes
644 }
645
646 /// Fetch characters as long as they satisfy `is_yaml_non_space(c)`.
647 ///
648 /// The characters are consumed from the input.
649 ///
650 /// # Return
651 /// Return the number of characters that were consumed. The number of characters returned can
652 /// be used to advance the index and column, since no end-of-line character will be consumed.
653 fn fetch_while_is_yaml_non_space(&mut self, out: &mut String) -> usize {
654 let mut chars_consumed = 0;
655 loop {
656 let c = self.look_ch();
657 if !crate::char_traits::is_yaml_non_space(c) || is_z(c) {
658 break;
659 }
660 let c = self.peek();
661 out.push(c);
662 self.skip();
663 chars_consumed += 1;
664 }
665 chars_consumed
666 }
667
668 /// Append a block scalar's content line to `out`, stopping before CR, LF, NUL, or EOF.
669 ///
670 /// The caller positions the input after the line's indentation. This method consumes the
671 /// entire remaining content line and appends it without clearing existing contents of `out`.
672 /// The stopping character is not consumed; in particular, both characters of CRLF remain
673 /// unconsumed, even if already buffered for lookahead.
674 ///
675 /// This copies content verbatim, including tabs and any non-printable characters other than
676 /// NUL. Unicode characters such as NEL (`U+0085`), line separator (`U+2028`), and paragraph
677 /// separator (`U+2029`) are content, not line terminators here. The scanner remains responsible
678 /// for validation, indentation, folding, and chomping; overrides must not discard or replace
679 /// invalid content.
680 ///
681 /// The default uses [`Self::raw_read_non_breakz_ch`], including any buffered lookahead.
682 /// Inputs with contiguous storage, such as [`str::StrInput`], can override this to append a
683 /// source slice in one operation. No prior lookahead is required; callers refresh lookahead
684 /// before inspecting the next character. If a source error interrupts the line, append and
685 /// count only the characters consumed before it. Keep the error available through
686 /// [`Self::take_source_error`]; the return value alone does not distinguish a source failure
687 /// from a normal line ending.
688 ///
689 /// # Returns
690 ///
691 /// The number of consumed Unicode scalar values (`char`s), **not UTF-8 bytes**, for advancing
692 /// the character index and column. A return value of zero means no content was appended or
693 /// consumed, for example when already at a line terminator or EOF.
694 ///
695 /// # Examples
696 ///
697 /// ```
698 /// use granit_parser::{Input, StrInput};
699 ///
700 /// let mut input = StrInput::new("é🦀\r\nnext");
701 /// let mut output = String::from("prefix:");
702 /// assert_eq!(input.fetch_block_scalar_line(&mut output), 2);
703 /// assert_eq!(output, "prefix:é🦀");
704 /// assert_eq!(input.byte_offset(), Some("é🦀".len()));
705 /// input.lookahead(2);
706 /// assert_eq!(input.peek(), '\r');
707 /// assert_eq!(input.peek_nth(1), '\n');
708 /// ```
709 fn fetch_block_scalar_line(&mut self, out: &mut String) -> usize {
710 // Raw reads consume the logical stream front even when lookahead is still buffered.
711 let mut chars_consumed = 0;
712 while let Some(character) = self.raw_read_non_breakz_ch() {
713 out.push(character);
714 chars_consumed += 1;
715 }
716 chars_consumed
717 }
718
719 /// Consume and return an ordinary ASCII run inside a quoted scalar, if supported.
720 ///
721 /// The caller has already consumed the opening quote. The boolean selects the quote style:
722 /// `true` for single quotes, `false` for double quotes. This optional optimization batches
723 /// characters needing no YAML escape or folding handling. The default returns an empty slice
724 /// without consuming input, leaving character-by-character scanning in place.
725 ///
726 /// # Override contract
727 ///
728 /// A non-empty result must contain exactly the consumed source prefix, with no decoding or
729 /// substitution. Only bytes in `0x21..=0x7e` may be consumed, excluding the matching quote
730 /// (`'` for single quotes, otherwise `"`) and, in double-quoted scalars, backslashes.
731 /// Backslashes and double quotes are ordinary content in single-quoted scalars; single quotes
732 /// are ordinary content in double-quoted scalars. Whitespace, non-ASCII text, control
733 /// characters, escapes, and closing or doubled matching quotes are left for the scanner.
734 /// The returned byte length is also the number of consumed characters, and any supported
735 /// [`Self::byte_offset`] must advance by that length.
736 ///
737 /// The run need not be maximal. Returning an empty slice must consume nothing, even if an
738 /// eligible run is present; it does **not** indicate EOF or the end of the scalar. No prior
739 /// lookahead is required, and already-buffered characters must not be skipped. Callers refresh
740 /// lookahead before inspecting the next character. The returned slice is tied to the borrow
741 /// of `self`, not to the original source lifetime used by [`BorrowedInput::slice_borrowed`].
742 ///
743 /// # Examples
744 ///
745 /// ```
746 /// use granit_parser::{BufferedInput, Input, StrInput};
747 ///
748 /// // Remaining content after a double-quoted scalar's opening quote.
749 /// let mut input = StrInput::new(r#"name\n""#);
750 /// assert_eq!(input.take_quoted_scalar_ascii_chunk(false), "name");
751 /// input.lookahead(1);
752 /// assert_eq!(input.peek(), '\\'); // The escape is left for the scanner.
753 ///
754 /// // Streaming inputs may retain the default and consume nothing.
755 /// let mut stream = BufferedInput::new("name".chars());
756 /// assert_eq!(stream.take_quoted_scalar_ascii_chunk(false), "");
757 /// stream.lookahead(1);
758 /// assert_eq!(stream.peek(), 'n');
759 /// ```
760 #[inline]
761 // Overrides return slices borrowed from the input, unlike this no-op default.
762 #[allow(clippy::unnecessary_literal_bound)]
763 fn take_quoted_scalar_ascii_chunk(&mut self, _single: bool) -> &str {
764 // Inputs with contiguous storage, such as StrInput, override this to return a whole run.
765 ""
766 }
767
768 /// Fetch a chunk of plain scalar characters.
769 ///
770 /// This optimization method allows the input to batch process characters.
771 /// Returns (stopped, `chars_consumed`).
772 /// stopped is true if the chunk ended because of a non-plain-scalar character.
773 fn fetch_plain_scalar_chunk(
774 &mut self,
775 out: &mut String,
776 count: usize,
777 flow_level_gt_0: bool,
778 ) -> (bool, usize) {
779 let mut chars_consumed = 0;
780 for _ in 0..count {
781 self.lookahead(1);
782 if self.next_is_blank_or_breakz() || !self.next_can_be_plain_scalar(flow_level_gt_0) {
783 return (true, chars_consumed);
784 }
785 out.push(self.peek());
786 self.skip();
787 chars_consumed += 1;
788 }
789 (false, chars_consumed)
790 }
791
792 /// Consume a chunk of plain-scalar characters without materializing them.
793 ///
794 /// Stable inputs use this while the scanner retains the source as a borrowed slice and only
795 /// promotes it to an owned buffer if YAML folding changes the scalar contents.
796 fn skip_plain_scalar_chunk(&mut self, count: usize, flow_level_gt_0: bool) -> (bool, usize) {
797 let mut chars_consumed = 0;
798 for _ in 0..count {
799 self.lookahead(1);
800 if self.next_is_blank_or_breakz() || !self.next_can_be_plain_scalar(flow_level_gt_0) {
801 return (true, chars_consumed);
802 }
803 self.skip();
804 chars_consumed += 1;
805 }
806 (false, chars_consumed)
807 }
808}
809
810/// Behavior to adopt regarding treating tabs as whitespace.
811///
812/// Although tab is valid YAML whitespace, it does not always behave the same as a space.
813#[derive(Copy, Clone, Eq, PartialEq)]
814pub enum SkipTabs {
815 /// Skip all tabs as whitespace.
816 Yes,
817 /// Don't skip any tab. Return from the function when encountering one.
818 No,
819 /// Return value from the function.
820 Result(
821 /// Whether tabs were encountered.
822 bool,
823 /// Whether at least one valid YAML whitespace character has been encountered.
824 bool,
825 ),
826}
827
828impl SkipTabs {
829 /// Whether tabs were found while skipping whitespace.
830 ///
831 /// This function must be called after a call to `skip_ws_to_eol`.
832 #[must_use]
833 pub fn found_tabs(self) -> bool {
834 matches!(self, SkipTabs::Result(true, _))
835 }
836
837 /// Whether a valid YAML whitespace has been found in skipped-over content.
838 ///
839 /// This function must be called after a call to `skip_ws_to_eol`.
840 #[must_use]
841 pub fn has_valid_yaml_ws(self) -> bool {
842 matches!(self, SkipTabs::Result(_, true))
843 }
844}
845
846#[cfg(test)]
847mod tests {
848 use super::{Input, SkipTabs};
849 use crate::error::ErrorKind;
850
851 struct MinimalInput;
852
853 impl Input for MinimalInput {
854 fn lookahead(&mut self, _count: usize) {}
855
856 fn buflen(&self) -> usize {
857 0
858 }
859
860 fn bufmaxlen(&self) -> usize {
861 0
862 }
863
864 fn raw_read_ch(&mut self) -> char {
865 '\0'
866 }
867
868 fn raw_read_non_breakz_ch(&mut self) -> Option<char> {
869 None
870 }
871
872 fn skip(&mut self) {}
873
874 fn skip_n(&mut self, _count: usize) {}
875
876 fn peek(&self) -> char {
877 '\0'
878 }
879
880 fn peek_nth(&self, _n: usize) -> char {
881 '\0'
882 }
883 }
884
885 #[test]
886 fn default_slice_bytes_returns_none() {
887 let mut input = MinimalInput;
888
889 input.lookahead(4);
890 assert_eq!(input.buflen(), 0);
891 assert_eq!(input.bufmaxlen(), 0);
892 assert_eq!(input.raw_read_ch(), '\0');
893 assert_eq!(input.raw_read_non_breakz_ch(), None);
894 input.skip();
895 input.skip_n(2);
896 assert_eq!(input.peek(), '\0');
897 assert_eq!(input.peek_nth(1), '\0');
898 assert_eq!(input.byte_offset(), None);
899 assert_eq!(input.slice_bytes(0, 0), None);
900 }
901
902 #[test]
903 fn default_skip_ws_to_eol_rejects_unseparated_comment() {
904 let mut input = super::buffered::BufferedInput::new("#comment\n".chars());
905
906 let (consumed, result) = input.skip_ws_to_eol(SkipTabs::Yes);
907
908 assert_eq!(consumed, 0);
909 assert_eq!(result.err(), Some(ErrorKind::CommentNotSeparated));
910 assert_eq!(input.peek(), '#');
911 }
912}