1use super::error::{ParseError, ParseErrorKind};
17
18#[derive(Debug, Clone, Copy, PartialEq, Eq)]
23pub enum TokenKind {
24 Eof,
26 Name,
30 Scalar,
34 MessageOpen,
36 MessageClose,
38 ListOpen,
40 ListClose,
42}
43
44#[derive(Debug, Clone, Copy, PartialEq, Eq)]
46pub enum NameKind {
47 Ident,
49 TypeName,
52 FieldNumber,
54}
55
56#[derive(Debug, Clone, Copy, PartialEq, Eq)]
58pub enum ScalarKind {
59 Number,
61 String,
63 Literal,
66}
67
68#[derive(Debug, Clone, Copy)]
70pub struct Token<'a> {
71 pub kind: TokenKind,
73 pub raw: &'a str,
77 pub pos: usize,
80 pub name_kind: NameKind,
82 pub scalar_kind: ScalarKind,
84 pub has_separator: bool,
88}
89
90#[derive(Clone, Copy, PartialEq, Eq)]
92enum OpenKind {
93 Top,
95 Message(u8),
98 List { scalars: bool },
101}
102
103const MESSAGE_LIST: u8 = b'(';
107
108pub(super) const COLON_BEFORE_SCALAR: &str = "':' before scalar value";
113
114#[derive(Clone, Copy, PartialEq, Eq)]
118enum LastKind {
119 Bof,
121 Name(bool),
122 Scalar,
123 MessageOpen,
124 MessageClose,
125 ListOpen,
126 ListClose,
127 Comma,
128 Semicolon,
129}
130
131pub struct Tokenizer<'a> {
137 input: &'a str,
138 cursor: usize,
139 last_kind: LastKind,
140 open_stack: [u8; crate::message::RECURSION_LIMIT as usize],
144 open_depth: usize,
145 peeked: Option<(Token<'a>, LastKind, usize)>,
147}
148
149impl<'a> Tokenizer<'a> {
150 pub fn new(input: &'a str) -> Self {
156 let cursor = if input.starts_with('\u{FEFF}') { 3 } else { 0 };
158 Tokenizer {
159 input,
160 cursor,
161 last_kind: LastKind::Bof,
162 open_stack: [0u8; crate::message::RECURSION_LIMIT as usize],
163 open_depth: 0,
164 peeked: None,
165 }
166 }
167
168 pub fn peek(&mut self) -> Result<Token<'a>, ParseError> {
176 if let Some((tok, _, _)) = self.peeked {
177 return Ok(tok);
178 }
179 let save_cursor = self.cursor;
192 let save_depth = self.open_depth;
193 let save_last = self.last_kind;
194 let tok = self.parse_next()?;
195 let new_last = self.last_kind;
196 let new_cursor = self.cursor;
197 self.cursor = save_cursor;
199 self.open_depth = save_depth;
200 self.last_kind = save_last;
201 self.peeked = Some((tok, new_last, new_cursor));
202 Ok(tok)
203 }
204
205 pub fn read(&mut self) -> Result<Token<'a>, ParseError> {
211 if let Some((tok, last, cur)) = self.peeked.take() {
212 self.last_kind = last;
217 self.cursor = cur;
218 match tok.kind {
219 TokenKind::MessageOpen | TokenKind::ListOpen => self.open_depth += 1,
220 TokenKind::MessageClose | TokenKind::ListClose => self.open_depth -= 1,
221 _ => {}
222 }
223 return Ok(tok);
224 }
225 self.parse_next()
226 }
227
228 pub fn line_col(&self, pos: usize) -> (u32, u32) {
237 let mut pos = pos.min(self.input.len());
238 while !self.input.is_char_boundary(pos) {
239 pos -= 1;
240 }
241 let before = &self.input[..pos];
242 let line = before.bytes().filter(|&b| b == b'\n').count() as u32 + 1;
243 let line_start = before.rfind('\n').map(|i| i + 1).unwrap_or(0);
244 let col = before[line_start..].chars().count() as u32 + 1;
245 (line, col)
246 }
247
248 fn err(&self, pos: usize, kind: ParseErrorKind) -> ParseError {
251 let (line, col) = self.line_col(pos);
252 ParseError::new(line, col, kind)
253 }
254
255 pub(super) fn in_list_without_separator(&self) -> bool {
261 self.current_open() == (OpenKind::List { scalars: false })
262 }
263
264 pub(super) fn err_here(&self, kind: ParseErrorKind) -> ParseError {
265 self.err(self.cursor, kind)
266 }
267
268 #[inline]
269 fn rest(&self) -> &'a [u8] {
270 &self.input.as_bytes()[self.cursor..]
271 }
272
273 fn current_open(&self) -> OpenKind {
275 if self.open_depth == 0 {
276 return OpenKind::Top;
277 }
278 match self.open_stack[self.open_depth - 1] {
279 b'{' => OpenKind::Message(b'}'),
280 b'<' => OpenKind::Message(b'>'),
281 b'[' => OpenKind::List { scalars: true },
282 MESSAGE_LIST => OpenKind::List { scalars: false },
283 _ => unreachable!("open_stack holds only {{, <, [ and MESSAGE_LIST"),
284 }
285 }
286
287 fn push_open(&mut self, ch: u8) -> Result<(), ParseError> {
288 if self.open_depth >= self.open_stack.len() {
289 return Err(self.err_here(ParseErrorKind::RecursionLimitExceeded));
290 }
291 self.open_stack[self.open_depth] = ch;
292 self.open_depth += 1;
293 Ok(())
294 }
295
296 fn pop_open(&mut self) {
297 debug_assert!(self.open_depth > 0);
298 self.open_depth -= 1;
299 }
300
301 fn consume(&mut self, n: usize) {
304 self.cursor += n;
305 loop {
306 match self.rest().first() {
307 Some(&c) if is_textproto_ws(c) => self.cursor += 1,
308 Some(b'#') => {
309 let rest = self.rest();
311 match rest.iter().position(|&b| b == b'\n') {
312 Some(i) => self.cursor += i + 1,
313 None => self.cursor = self.input.len(),
314 }
315 }
316 _ => break,
317 }
318 }
319 }
320
321 fn try_consume_char(&mut self, c: u8) -> bool {
323 if self.rest().first() == Some(&c) {
324 self.consume(1);
325 true
326 } else {
327 false
328 }
329 }
330
331 fn parse_next(&mut self) -> Result<Token<'a>, ParseError> {
336 loop {
338 self.consume(0);
339 let at_eof = self.rest().is_empty();
340 let open = self.current_open();
341
342 match self.last_kind {
343 LastKind::Bof => {
344 if at_eof {
346 return Ok(self.emit_eof());
347 }
348 return self.parse_field_name();
349 }
350
351 LastKind::Name(has_separator) => {
352 if at_eof {
354 return Err(self.err_here(ParseErrorKind::UnexpectedEof));
355 }
356 let ch = self.rest()[0];
357 match ch {
358 b'{' | b'<' => {
359 self.push_open(ch)?;
360 return Ok(self.emit(TokenKind::MessageOpen, 1));
361 }
362 b'[' => {
363 self.push_open(if has_separator { ch } else { MESSAGE_LIST })?;
364 return Ok(self.emit(TokenKind::ListOpen, 1));
365 }
366 _ if !has_separator => {
367 return Err(self.err_here(ParseErrorKind::UnexpectedToken {
368 expected: COLON_BEFORE_SCALAR,
369 }));
370 }
371 _ => return self.parse_scalar(),
372 }
373 }
374
375 LastKind::Scalar | LastKind::MessageClose | LastKind::ListClose => {
376 match open {
379 OpenKind::Top => {
380 if at_eof {
381 return Ok(self.emit_eof());
382 }
383 match self.rest()[0] {
384 b',' => {
385 self.consume(1);
386 self.last_kind = LastKind::Comma;
387 continue;
388 }
389 b';' => {
390 self.consume(1);
391 self.last_kind = LastKind::Semicolon;
392 continue;
393 }
394 _ => return self.parse_field_name(),
395 }
396 }
397 OpenKind::Message(close) => {
398 if at_eof {
399 return Err(self.err_here(ParseErrorKind::UnexpectedEof));
400 }
401 let ch = self.rest()[0];
402 if ch == close {
403 self.pop_open();
404 return Ok(self.emit(TokenKind::MessageClose, 1));
405 }
406 if ch == other_close(close) {
407 return Err(self.err_here(ParseErrorKind::DelimiterMismatch));
408 }
409 match ch {
410 b',' => {
411 self.consume(1);
412 self.last_kind = LastKind::Comma;
413 continue;
414 }
415 b';' => {
416 self.consume(1);
417 self.last_kind = LastKind::Semicolon;
418 continue;
419 }
420 _ => return self.parse_field_name(),
421 }
422 }
423 OpenKind::List { .. } => {
424 if at_eof {
425 return Err(self.err_here(ParseErrorKind::UnexpectedEof));
426 }
427 let ch = self.rest()[0];
428 match ch {
429 b']' => {
430 self.pop_open();
431 return Ok(self.emit(TokenKind::ListClose, 1));
432 }
433 b',' => {
434 self.consume(1);
435 self.last_kind = LastKind::Comma;
436 continue;
437 }
438 _ => {
439 return Err(self.err_here(ParseErrorKind::UnexpectedToken {
440 expected: "',' or ']'",
441 }));
442 }
443 }
444 }
445 }
446 }
447
448 LastKind::MessageOpen => {
449 if at_eof {
451 return Err(self.err_here(ParseErrorKind::UnexpectedEof));
452 }
453 let OpenKind::Message(close) = open else {
454 unreachable!("MessageOpen always pushes a Message frame")
455 };
456 let ch = self.rest()[0];
457 if ch == close {
458 self.pop_open();
459 return Ok(self.emit(TokenKind::MessageClose, 1));
460 }
461 if ch == other_close(close) {
462 return Err(self.err_here(ParseErrorKind::DelimiterMismatch));
463 }
464 return self.parse_field_name();
465 }
466
467 LastKind::ListOpen => {
468 if at_eof {
470 return Err(self.err_here(ParseErrorKind::UnexpectedEof));
471 }
472 let OpenKind::List { scalars } = open else {
473 unreachable!("ListOpen always pushes a List frame")
474 };
475 let ch = self.rest()[0];
476 match ch {
477 b']' => {
478 self.pop_open();
479 return Ok(self.emit(TokenKind::ListClose, 1));
480 }
481 b'{' | b'<' => {
482 self.push_open(ch)?;
483 return Ok(self.emit(TokenKind::MessageOpen, 1));
484 }
485 _ => return self.parse_list_scalar(scalars),
486 }
487 }
488
489 LastKind::Comma | LastKind::Semicolon => {
490 match open {
492 OpenKind::Top => {
493 if at_eof {
494 return Ok(self.emit_eof());
495 }
496 return self.parse_field_name();
497 }
498 OpenKind::Message(close) => {
499 if at_eof {
500 return Err(self.err_here(ParseErrorKind::UnexpectedEof));
501 }
502 let ch = self.rest()[0];
503 if ch == close {
504 self.pop_open();
505 return Ok(self.emit(TokenKind::MessageClose, 1));
506 }
507 if ch == other_close(close) {
508 return Err(self.err_here(ParseErrorKind::DelimiterMismatch));
509 }
510 return self.parse_field_name();
511 }
512 OpenKind::List { scalars } => {
513 if at_eof {
517 return Err(self.err_here(ParseErrorKind::UnexpectedEof));
518 }
519 let ch = self.rest()[0];
520 match ch {
521 b'{' | b'<' => {
522 self.push_open(ch)?;
523 return Ok(self.emit(TokenKind::MessageOpen, 1));
524 }
525 _ => return self.parse_list_scalar(scalars),
526 }
527 }
528 }
529 }
530 }
531 }
532 }
533
534 fn parse_list_scalar(&mut self, scalars: bool) -> Result<Token<'a>, ParseError> {
538 if scalars {
539 return self.parse_scalar();
540 }
541 let missing_colon = self.err_here(ParseErrorKind::UnexpectedToken {
542 expected: COLON_BEFORE_SCALAR,
543 });
544 let (cursor, last_kind) = (self.cursor, self.last_kind);
547 self.parse_scalar()?;
548 self.cursor = cursor;
549 self.last_kind = last_kind;
550 Err(missing_colon)
551 }
552
553 fn emit(&mut self, kind: TokenKind, len: usize) -> Token<'a> {
554 let pos = self.cursor;
555 let raw = &self.input[pos..pos + len];
556 self.consume(len);
557 self.last_kind = match kind {
558 TokenKind::Name => LastKind::Name(false),
559 TokenKind::Scalar => LastKind::Scalar,
560 TokenKind::MessageOpen => LastKind::MessageOpen,
561 TokenKind::MessageClose => LastKind::MessageClose,
562 TokenKind::ListOpen => LastKind::ListOpen,
563 TokenKind::ListClose => LastKind::ListClose,
564 TokenKind::Eof => LastKind::Bof, };
566 Token {
567 kind,
568 raw,
569 pos,
570 name_kind: NameKind::Ident,
571 scalar_kind: ScalarKind::Number,
572 has_separator: false,
573 }
574 }
575
576 fn emit_eof(&mut self) -> Token<'a> {
577 Token {
578 kind: TokenKind::Eof,
579 raw: &self.input[self.input.len()..],
580 pos: self.input.len(),
581 name_kind: NameKind::Ident,
582 scalar_kind: ScalarKind::Number,
583 has_separator: false,
584 }
585 }
586
587 fn parse_field_name(&mut self) -> Result<Token<'a>, ParseError> {
591 let start = self.cursor;
592 let rest = self.rest();
593
594 if rest[0] == b'[' {
596 let mut i = 1;
599 while i < rest.len() {
600 match rest[i] {
601 b']' => break,
602 b'#' => {
603 while i < rest.len() && rest[i] != b'\n' {
604 i += 1;
605 }
606 }
607 _ => i += 1,
608 }
609 }
610 if i >= rest.len() {
611 return Err(self.err(start, ParseErrorKind::UnexpectedEof));
612 }
613 let len = i + 1; let raw = &self.input[start..start + len];
615 self.consume(len);
616 let has_separator = self.try_consume_char(b':');
617 self.last_kind = LastKind::Name(has_separator);
618 return Ok(Token {
619 kind: TokenKind::Name,
620 raw,
621 pos: start,
622 name_kind: NameKind::TypeName,
623 scalar_kind: ScalarKind::Number,
624 has_separator,
625 });
626 }
627
628 let ilen = parse_ident(rest, false);
630 if ilen > 0 {
631 let raw = &self.input[start..start + ilen];
632 self.consume(ilen);
633 let has_separator = self.try_consume_char(b':');
634 self.last_kind = LastKind::Name(has_separator);
635 return Ok(Token {
636 kind: TokenKind::Name,
637 raw,
638 pos: start,
639 name_kind: NameKind::Ident,
640 scalar_kind: ScalarKind::Number,
641 has_separator,
642 });
643 }
644
645 if let Some(num) = lex_number(rest) {
647 if !num.neg && num.kind == NumKind::Dec {
648 let s = &self.input[start..start + num.len];
651 if s.parse::<i32>().is_ok() {
652 let raw = s;
653 self.consume(num.len);
654 let has_separator = self.try_consume_char(b':');
655 self.last_kind = LastKind::Name(has_separator);
656 return Ok(Token {
657 kind: TokenKind::Name,
658 raw,
659 pos: start,
660 name_kind: NameKind::FieldNumber,
661 scalar_kind: ScalarKind::Number,
662 has_separator,
663 });
664 }
665 }
666 }
667
668 Err(self.err(
669 start,
670 ParseErrorKind::UnexpectedToken {
671 expected: "field name",
672 },
673 ))
674 }
675
676 fn parse_scalar(&mut self) -> Result<Token<'a>, ParseError> {
678 let start = self.cursor;
679 let rest = self.rest();
680 let first = rest[0];
681
682 if first == b'"' || first == b'\'' {
684 let len = lex_string_run(rest).ok_or_else(|| {
685 self.err(start, ParseErrorKind::InvalidString("unterminated string"))
686 })?;
687 let raw = &self.input[start..start + len];
688 self.consume(len);
689 self.last_kind = LastKind::Scalar;
690 return Ok(Token {
691 kind: TokenKind::Scalar,
692 raw,
693 pos: start,
694 name_kind: NameKind::Ident,
695 scalar_kind: ScalarKind::String,
696 has_separator: false,
697 });
698 }
699
700 let ilen = parse_ident(rest, true);
702 if ilen > 0 {
703 let raw = &self.input[start..start + ilen];
704 self.consume(ilen);
705 self.last_kind = LastKind::Scalar;
706 return Ok(Token {
707 kind: TokenKind::Scalar,
708 raw,
709 pos: start,
710 name_kind: NameKind::Ident,
711 scalar_kind: ScalarKind::Literal,
712 has_separator: false,
713 });
714 }
715
716 if let Some(num) = lex_number(rest) {
718 let raw = &self.input[start..start + num.len];
719 self.consume(num.len);
720 self.last_kind = LastKind::Scalar;
721 return Ok(Token {
722 kind: TokenKind::Scalar,
723 raw,
724 pos: start,
725 name_kind: NameKind::Ident,
726 scalar_kind: ScalarKind::Number,
727 has_separator: false,
728 });
729 }
730
731 Err(self.err(
732 start,
733 ParseErrorKind::UnexpectedToken {
734 expected: "scalar value",
735 },
736 ))
737 }
738}
739
740#[inline]
745fn other_close(c: u8) -> u8 {
746 match c {
747 b'}' => b'>',
748 b'>' => b'}',
749 _ => unreachable!(),
750 }
751}
752
753#[inline]
756fn is_delim(c: u8) -> bool {
757 !(c == b'-' || c == b'+' || c == b'.' || c == b'_' || c.is_ascii_alphanumeric())
758}
759
760#[inline]
766pub(super) const fn is_textproto_ws(c: u8) -> bool {
767 matches!(c, b' ' | b'\t' | b'\r' | b'\n' | b'\x0B' | b'\x0C')
768}
769
770fn parse_ident(s: &[u8], allow_neg: bool) -> usize {
775 let mut i = 0;
776 if allow_neg && s.first() == Some(&b'-') {
777 i = 1;
778 let after = consume_ws(&s[1..]);
784 i += s.len() - 1 - after.len();
785 if s.get(i).is_none() {
786 return 0;
787 }
788 }
789 match s.get(i) {
790 Some(b'_') | Some(b'a'..=b'z') | Some(b'A'..=b'Z') => i += 1,
791 _ => return 0,
792 }
793 while let Some(&c) = s.get(i) {
794 if c == b'_' || c.is_ascii_alphanumeric() {
795 i += 1;
796 } else {
797 break;
798 }
799 }
800 if let Some(&c) = s.get(i) {
803 if !is_delim(c) {
804 return 0;
805 }
806 }
807 i
808}
809
810#[derive(Clone, Copy, PartialEq, Eq, Debug)]
812pub(super) enum NumKind {
813 Dec,
814 Hex,
815 Oct,
816 Float,
817}
818
819#[derive(Clone, Copy, Debug)]
820pub(super) struct LexNumber {
821 pub kind: NumKind,
822 pub neg: bool,
823 pub len: usize,
825 pub sep: usize,
828}
829
830pub(super) fn lex_number(input: &[u8]) -> Option<LexNumber> {
837 let mut s = input;
838 let mut len = 0;
839 let mut neg = false;
840 let mut sep = 0;
841 let mut kind = NumKind::Dec;
842
843 if s.is_empty() {
844 return None;
845 }
846
847 if s[0] == b'-' {
848 neg = true;
849 s = &s[1..];
850 len += 1;
851 let before = s.len();
853 s = consume_ws(s);
854 sep = before - s.len();
855 len += sep;
856 if s.is_empty() {
857 return None;
858 }
859 }
860
861 match s[0] {
862 b'0' => {
863 if s.len() > 1 {
864 match s[1] {
865 b'x' | b'X' => {
866 kind = NumKind::Hex;
867 let mut n = 2;
868 while s.get(n).is_some_and(|c| c.is_ascii_hexdigit()) {
869 n += 1;
870 }
871 if n == 2 {
872 return None; }
874 len += n;
875 s = &s[n..];
876 return finish_number(s, kind, neg, len, sep);
877 }
878 b'0'..=b'7' => {
879 kind = NumKind::Oct;
880 let mut n = 2;
881 while s.get(n).is_some_and(|&c| (b'0'..=b'7').contains(&c)) {
882 n += 1;
883 }
884 len += n;
885 s = &s[n..];
886 return finish_number(s, kind, neg, len, sep);
887 }
888 _ => {}
889 }
890 }
891 s = &s[1..];
892 len += 1;
893 }
894 b'1'..=b'9' => {
895 let mut n = 1;
896 while s.get(n).is_some_and(u8::is_ascii_digit) {
897 n += 1;
898 }
899 s = &s[n..];
900 len += n;
901 }
902 b'.' => {
903 kind = NumKind::Float;
905 }
906 _ => return None,
907 }
908
909 if s.first() == Some(&b'.') {
911 let mut n = 1;
912 let had_digits = kind != NumKind::Float;
915 while s.get(n).is_some_and(u8::is_ascii_digit) {
916 n += 1;
917 }
918 if !had_digits && n == 1 {
919 return None; }
921 s = &s[n..];
922 len += n;
923 kind = NumKind::Float;
924 }
925
926 if s.len() >= 2 && matches!(s[0], b'e' | b'E') {
928 kind = NumKind::Float;
929 let mut n = 1;
930 if matches!(s[1], b'+' | b'-') {
931 n = 2;
932 if s.len() <= 2 {
933 return None;
934 }
935 }
936 let start = n;
937 while s.get(n).is_some_and(u8::is_ascii_digit) {
938 n += 1;
939 }
940 if n == start {
941 return None; }
943 s = &s[n..];
944 len += n;
945 }
946
947 if matches!(s.first(), Some(b'f' | b'F')) {
949 kind = NumKind::Float;
950 s = &s[1..];
951 len += 1;
952 }
953
954 finish_number(s, kind, neg, len, sep)
955}
956
957#[inline]
958fn finish_number(
959 rest: &[u8],
960 kind: NumKind,
961 neg: bool,
962 len: usize,
963 sep: usize,
964) -> Option<LexNumber> {
965 if let Some(&c) = rest.first() {
967 if !is_delim(c) {
968 return None;
969 }
970 }
971 Some(LexNumber {
972 kind,
973 neg,
974 len,
975 sep,
976 })
977}
978
979pub(super) fn number_for_parse<'a>(raw: &'a str, num: &LexNumber) -> alloc::borrow::Cow<'a, str> {
983 let bytes = raw.as_bytes();
984 let mut end = bytes.len();
985 if num.kind == NumKind::Float && matches!(bytes.last(), Some(b'f' | b'F')) {
986 end -= 1;
987 }
988 if num.neg && num.sep > 0 {
989 let mut s = alloc::string::String::with_capacity(end - num.sep);
991 s.push('-');
992 s.push_str(&raw[1 + num.sep..end]);
993 alloc::borrow::Cow::Owned(s)
994 } else {
995 alloc::borrow::Cow::Borrowed(&raw[..end])
996 }
997}
998
999pub(super) fn consume_ws(mut s: &[u8]) -> &[u8] {
1001 loop {
1002 match s.first() {
1003 Some(&c) if is_textproto_ws(c) => s = &s[1..],
1004 Some(b'#') => match s.iter().position(|&b| b == b'\n') {
1005 Some(i) => s = &s[i + 1..],
1006 None => return &[],
1007 },
1008 _ => return s,
1009 }
1010 }
1011}
1012
1013pub(super) fn normalize_bracket_name(name: &str) -> Option<alloc::borrow::Cow<'_, str>> {
1021 let is_trivia_start = |b: u8| is_textproto_ws(b) || b == b'#';
1022 let inner = name.strip_prefix('[')?.strip_suffix(']')?;
1023 if !inner.bytes().any(is_trivia_start) {
1024 return Some(alloc::borrow::Cow::Borrowed(inner));
1025 }
1026
1027 let mut out = alloc::string::String::with_capacity(inner.len());
1028 let mut rest = inner;
1029 while !rest.is_empty() {
1030 rest = &rest[rest.len() - consume_ws(rest.as_bytes()).len()..];
1033 let end = rest.bytes().position(is_trivia_start).unwrap_or(rest.len());
1034 out.push_str(&rest[..end]);
1035 rest = &rest[end..];
1036 }
1037 Some(alloc::borrow::Cow::Owned(out))
1038}
1039
1040fn lex_string_run(s: &[u8]) -> Option<usize> {
1046 let mut i = 0;
1047 loop {
1048 let quote = *s.get(i)?;
1050 debug_assert!(quote == b'"' || quote == b'\'');
1051 i += 1;
1052 loop {
1053 match s.get(i)? {
1054 &c if c == quote => {
1055 i += 1;
1056 break;
1057 }
1058 b'\n' | 0 => return None,
1059 b'\\' => {
1060 i += 2;
1063 if i > s.len() {
1064 return None;
1065 }
1066 }
1067 _ => i += 1,
1068 }
1069 }
1070 let rest = consume_ws(&s[i..]);
1073 let j = s.len() - rest.len();
1074 if matches!(s.get(j), Some(b'"') | Some(b'\'')) {
1075 i = j;
1076 continue;
1077 }
1078 return Some(i);
1079 }
1080}
1081
1082#[cfg(test)]
1083mod tests {
1084 use super::*;
1085 use alloc::vec::Vec;
1086
1087 fn drain(input: &str) -> Result<Vec<(TokenKind, &str)>, ParseError> {
1089 let mut t = Tokenizer::new(input);
1090 let mut out = Vec::new();
1091 loop {
1092 let tok = t.read()?;
1093 if tok.kind == TokenKind::Eof {
1094 return Ok(out);
1095 }
1096 out.push((tok.kind, tok.raw));
1097 }
1098 }
1099
1100 #[test]
1103 fn tokenize_simple_field() {
1104 let toks = drain("foo: 42").unwrap();
1105 assert_eq!(toks, [(TokenKind::Name, "foo"), (TokenKind::Scalar, "42")]);
1106 }
1107
1108 #[test]
1109 fn tokenize_nested_message() {
1110 let toks = drain("child { a: 1 b: 2 }").unwrap();
1111 assert_eq!(
1112 toks,
1113 [
1114 (TokenKind::Name, "child"),
1115 (TokenKind::MessageOpen, "{"),
1116 (TokenKind::Name, "a"),
1117 (TokenKind::Scalar, "1"),
1118 (TokenKind::Name, "b"),
1119 (TokenKind::Scalar, "2"),
1120 (TokenKind::MessageClose, "}"),
1121 ]
1122 );
1123 }
1124
1125 #[test]
1126 fn tokenize_angle_delimiters() {
1127 let toks = drain("m < x: 1 >").unwrap();
1128 assert_eq!(toks[1], (TokenKind::MessageOpen, "<"));
1129 assert_eq!(toks[4], (TokenKind::MessageClose, ">"));
1130 }
1131
1132 #[test]
1133 fn tokenize_list() {
1134 let toks = drain("r: [1, 2, 3]").unwrap();
1135 assert_eq!(
1136 toks,
1137 [
1138 (TokenKind::Name, "r"),
1139 (TokenKind::ListOpen, "["),
1140 (TokenKind::Scalar, "1"),
1141 (TokenKind::Scalar, "2"),
1142 (TokenKind::Scalar, "3"),
1143 (TokenKind::ListClose, "]"),
1144 ]
1145 );
1146 }
1147
1148 #[test]
1149 fn tokenize_empty_list() {
1150 let toks = drain("r: []").unwrap();
1151 assert_eq!(
1152 toks,
1153 [
1154 (TokenKind::Name, "r"),
1155 (TokenKind::ListOpen, "["),
1156 (TokenKind::ListClose, "]"),
1157 ]
1158 );
1159 }
1160
1161 #[test]
1162 fn tokenize_message_list() {
1163 let toks = drain("r: [{a: 1}, {a: 2}]").unwrap();
1165 assert_eq!(toks[1], (TokenKind::ListOpen, "["));
1166 assert_eq!(toks[2], (TokenKind::MessageOpen, "{"));
1167 assert_eq!(toks[5], (TokenKind::MessageClose, "}"));
1168 assert_eq!(toks[6], (TokenKind::MessageOpen, "{"));
1169 assert_eq!(toks[9], (TokenKind::MessageClose, "}"));
1170 assert_eq!(toks[10], (TokenKind::ListClose, "]"));
1171 }
1172
1173 #[test]
1174 fn separators_consumed() {
1175 let a = drain("a: 1 b: 2").unwrap();
1177 let b = drain("a: 1, b: 2").unwrap();
1178 let c = drain("a: 1; b: 2").unwrap();
1179 assert_eq!(a, b);
1180 assert_eq!(a, c);
1181 }
1182
1183 #[test]
1184 fn comments_skipped() {
1185 let toks = drain("# leading\nfoo: 1 # trailing\nbar: 2").unwrap();
1186 assert_eq!(
1187 toks,
1188 [
1189 (TokenKind::Name, "foo"),
1190 (TokenKind::Scalar, "1"),
1191 (TokenKind::Name, "bar"),
1192 (TokenKind::Scalar, "2"),
1193 ]
1194 );
1195 }
1196
1197 #[test]
1198 fn empty_input() {
1199 assert_eq!(drain("").unwrap(), []);
1200 assert_eq!(drain(" \n\t ").unwrap(), []);
1201 assert_eq!(drain("# just a comment").unwrap(), []);
1202 }
1203
1204 #[test]
1207 fn name_kinds() {
1208 #[rustfmt::skip]
1209 let cases: &[(&str, NameKind, &str, bool)] = &[
1210 ("foo: 1", NameKind::Ident, "foo", true),
1211 ("foo_bar: 1", NameKind::Ident, "foo_bar", true),
1212 ("_private: 1", NameKind::Ident, "_private", true),
1213 ("msg {}", NameKind::Ident, "msg", false), ("[pkg.ext]: 1", NameKind::TypeName, "[pkg.ext]", true),
1215 ("[a.b.c/d.e]: 1", NameKind::TypeName, "[a.b.c/d.e]", true),
1216 ("[type.googleapis.com/Foo] {}", NameKind::TypeName, "[type.googleapis.com/Foo]", false),
1217 ("42: 1", NameKind::FieldNumber, "42", true),
1218 ];
1219 for &(input, want_kind, want_raw, want_sep) in cases {
1220 let mut t = Tokenizer::new(input);
1221 let tok = t.read().unwrap();
1222 assert_eq!(tok.kind, TokenKind::Name, "input: {input}");
1223 assert_eq!(tok.name_kind, want_kind, "input: {input}");
1224 assert_eq!(tok.raw, want_raw, "input: {input}");
1225 assert_eq!(tok.has_separator, want_sep, "input: {input}");
1226 }
1227 }
1228
1229 #[test]
1230 fn bracket_name_comment_may_contain_close_bracket() {
1231 let input = "[pkg # ] and é are comment text\n .ext]: 1";
1232 let mut t = Tokenizer::new(input);
1233 let tok = t.read().unwrap();
1234 assert_eq!(tok.kind, TokenKind::Name);
1235 assert_eq!(tok.name_kind, NameKind::TypeName);
1236 assert_eq!(tok.raw, "[pkg # ] and é are comment text\n .ext]");
1237 }
1238
1239 #[test]
1240 fn normalize_bracket_name_trivia() {
1241 assert!(matches!(
1242 normalize_bracket_name("[pkg.ext]"),
1243 Some(alloc::borrow::Cow::Borrowed("pkg.ext"))
1244 ));
1245
1246 #[rustfmt::skip]
1247 let cases = [
1248 ("[ pkg . ext ]", "pkg.ext"),
1249 ("[pkg# comment\n.ext]", "pkg.ext"),
1250 ("[pkg # ] and é are comment text\n . ext]", "pkg.ext"),
1251 ("[type.googleapis.com / pkg . Msg]", "type.googleapis.com/pkg.Msg"),
1252 ("[type.googleapis.com/# comment\npkg.Msg]", "type.googleapis.com/pkg.Msg"),
1253 ("[pkg.e xt]", "pkg.ext"),
1255 ("[pkg.e# comment\nxt]", "pkg.ext"),
1256 ("[ pkg.é ]", "pkg.é"),
1258 ("[pkg.ext # comment]", "pkg.ext"),
1260 ("[ # only a comment\n ]", ""),
1261 ];
1262 for (input, want) in cases {
1263 assert_eq!(
1264 normalize_bracket_name(input).as_deref(),
1265 Some(want),
1266 "input: {input:?}"
1267 );
1268 }
1269
1270 for unbracketed in ["pkg.ext", "[pkg.ext", "pkg.ext]", ""] {
1271 assert_eq!(normalize_bracket_name(unbracketed), None);
1272 }
1273 }
1274
1275 #[test]
1276 fn bracket_name_closed_only_inside_a_comment_is_unterminated() {
1277 for input in ["[pkg # ]", "[pkg # ]\n .ext"] {
1278 let err = Tokenizer::new(input).read().unwrap_err();
1279 assert_eq!(err.kind, ParseErrorKind::UnexpectedEof, "input: {input:?}");
1280 }
1281 }
1282
1283 #[test]
1286 fn scalar_kinds() {
1287 #[rustfmt::skip]
1288 let cases: &[(&str, ScalarKind, &str)] = &[
1289 ("f: 42", ScalarKind::Number, "42"),
1290 ("f: -7", ScalarKind::Number, "-7"),
1291 ("f: 0x1F", ScalarKind::Number, "0x1F"),
1292 ("f: 0777", ScalarKind::Number, "0777"),
1293 ("f: 1.5", ScalarKind::Number, "1.5"),
1294 ("f: 1.5e-3", ScalarKind::Number, "1.5e-3"),
1295 ("f: .5", ScalarKind::Number, ".5"),
1296 ("f: 1f", ScalarKind::Number, "1f"),
1297 ("f: 1.5F", ScalarKind::Number, "1.5F"),
1298 (r#"f: "hello""#, ScalarKind::String, r#""hello""#),
1299 (r#"f: 'world'"#, ScalarKind::String, r#"'world'"#),
1300 (r#"f: "a" "b""#, ScalarKind::String, r#""a" "b""#), ("f: true", ScalarKind::Literal, "true"),
1302 ("f: False", ScalarKind::Literal, "False"),
1303 ("f: FOO_BAR", ScalarKind::Literal, "FOO_BAR"), ("f: inf", ScalarKind::Literal, "inf"),
1305 ("f: -inf", ScalarKind::Literal, "-inf"), ("f: nan", ScalarKind::Literal, "nan"),
1307 ];
1308 for &(input, want_kind, want_raw) in cases {
1309 let mut t = Tokenizer::new(input);
1310 t.read().unwrap(); let tok = t.read().unwrap();
1312 assert_eq!(tok.kind, TokenKind::Scalar, "input: {input}");
1313 assert_eq!(tok.scalar_kind, want_kind, "input: {input}");
1314 assert_eq!(tok.raw, want_raw, "input: {input}");
1315 }
1316 }
1317
1318 #[test]
1319 fn string_escape_not_closing() {
1320 let mut t = Tokenizer::new(r#"f: "say \"hi\"""#);
1322 t.read().unwrap();
1323 let tok = t.read().unwrap();
1324 assert_eq!(tok.scalar_kind, ScalarKind::String);
1325 assert_eq!(tok.raw, r#""say \"hi\"""#);
1326 }
1327
1328 #[test]
1329 fn adjacent_strings_no_whitespace() {
1330 let mut t = Tokenizer::new(r#"f: "a"'b'"c""#);
1331 t.read().unwrap();
1332 let tok = t.read().unwrap();
1333 assert_eq!(tok.raw, r#""a"'b'"c""#);
1334 }
1335
1336 #[test]
1337 fn adjacent_strings_with_comment() {
1338 let input = "f: \"a\" # between\n 'b'";
1339 let mut t = Tokenizer::new(input);
1340 t.read().unwrap();
1341 let tok = t.read().unwrap();
1342 assert_eq!(tok.raw, "\"a\" # between\n 'b'");
1343 }
1344
1345 #[test]
1346 fn trailing_comment_is_not_part_of_string_token() {
1347 let mut t = Tokenizer::new("f: \"a\" # \"not a literal\"\ng: 1");
1348 t.read().unwrap();
1349 assert_eq!(t.read().unwrap().raw, "\"a\"");
1350 assert_eq!(t.read().unwrap().raw, "g");
1351 }
1352
1353 #[test]
1354 fn unterminated_literal_after_comment_is_an_error() {
1355 let mut t = Tokenizer::new("f: \"a\" # c\n \"b");
1356 t.read().unwrap();
1357 let err = t.read().unwrap_err();
1358 assert!(matches!(err.kind, ParseErrorKind::InvalidString(_)));
1359 }
1360
1361 #[test]
1364 fn lex_number_table() {
1365 #[rustfmt::skip]
1367 let cases: &[(&str, Option<(NumKind, usize)>)] = &[
1368 ("0", Some((NumKind::Dec, 1))),
1369 ("42", Some((NumKind::Dec, 2))),
1370 ("-7", Some((NumKind::Dec, 2))),
1371 ("- 7", Some((NumKind::Dec, 3))), ("0x1F", Some((NumKind::Hex, 4))),
1373 ("0XFF", Some((NumKind::Hex, 4))),
1374 ("-0x1", Some((NumKind::Hex, 4))),
1375 ("0777", Some((NumKind::Oct, 4))),
1376 ("1.5", Some((NumKind::Float, 3))),
1377 (".5", Some((NumKind::Float, 2))),
1378 ("1.", Some((NumKind::Float, 2))),
1379 ("1e3", Some((NumKind::Float, 3))),
1380 ("1.5e-3", Some((NumKind::Float, 6))),
1381 ("1.5E+3", Some((NumKind::Float, 6))),
1382 ("1f", Some((NumKind::Float, 2))),
1383 ("1.5F", Some((NumKind::Float, 4))),
1384 ("", None),
1386 ("abc", None),
1387 ("0x", None), (".", None), ("1e", None), ("1e+", None), ("-", None), ("0x1g", None), ("42abc", None), ];
1395 for &(input, want) in cases {
1396 let got = lex_number(input.as_bytes()).map(|n| (n.kind, n.len));
1397 assert_eq!(got, want, "input: {input:?}");
1398 }
1399 }
1400
1401 #[test]
1402 fn number_for_parse_strips_suffix() {
1403 let n = lex_number(b"1.5f").unwrap();
1404 assert_eq!(number_for_parse("1.5f", &n), "1.5");
1405 }
1406
1407 #[test]
1408 fn number_for_parse_strips_separator() {
1409 let n = lex_number(b"- 42").unwrap();
1410 assert_eq!(n.sep, 1);
1411 assert_eq!(number_for_parse("- 42", &n), "-42");
1412 }
1413
1414 #[test]
1415 fn number_for_parse_neg_hex_with_sep() {
1416 let n = lex_number(b"- 0xFF").unwrap();
1418 assert_eq!(n.kind, NumKind::Hex);
1419 assert!(n.neg);
1420 assert_eq!(n.sep, 1);
1421 assert_eq!(number_for_parse("- 0xFF", &n), "-0xFF");
1422 }
1423
1424 #[test]
1427 fn direct_scalar_requires_colon() {
1428 for name in ["field", "[pkg.ext]", "42"] {
1429 for value in [
1430 "7", "- 7", "0x1F", ".5", "true", "ACTIVE", "-inf", "\"text\"", "'bytes'",
1431 ] {
1432 let input = alloc::format!("{name} {value}");
1433 let mut t = Tokenizer::new(&input);
1434 assert!(!t.read().unwrap().has_separator);
1435 let err = t.read().unwrap_err();
1436 assert_eq!(
1437 err.kind,
1438 ParseErrorKind::UnexpectedToken {
1439 expected: COLON_BEFORE_SCALAR,
1440 },
1441 "input: {input:?}"
1442 );
1443 assert_eq!((err.line, err.col), (1, name.len() as u32 + 2));
1444
1445 let valid = alloc::format!("{name}: {value}");
1446 assert!(drain(&valid).is_ok(), "input: {valid:?}");
1447 }
1448 }
1449 for input in ["field\"text\"", "field # comment\n 7", "m { field 7 }"] {
1450 assert!(drain(input).is_err(), "input: {input:?}");
1451 }
1452 }
1453
1454 #[test]
1455 fn scalar_list_requires_colon() {
1456 for (input, col) in [
1458 ("f [1]", 4),
1459 ("f [1, 2]", 4),
1460 ("f [\"a\"]", 4),
1461 ("f [ACTIVE]", 4),
1462 ("f [{}, 1]", 8),
1463 ("[pkg.ext] [true]", 12),
1464 ("m { f [1] }", 8),
1465 ("m [{ f [1] }]", 9),
1469 ("m [{ f: [1] }, 2]", 16),
1470 ("a: [1] b [1]", 11),
1471 ] {
1472 let err = drain(input).unwrap_err();
1473 assert_eq!(
1474 err.kind,
1475 ParseErrorKind::UnexpectedToken {
1476 expected: COLON_BEFORE_SCALAR,
1477 },
1478 "input: {input:?}"
1479 );
1480 assert_eq!((err.line, err.col), (1, col), "input: {input:?}");
1481 }
1482 for input in [
1485 "f: [1]",
1486 "f: [1, 2]",
1487 "f: [\"a\"]",
1488 "f: [ACTIVE]",
1489 "f: [{}, 1]",
1490 "[pkg.ext]: [true]",
1491 "m { f: [1] }",
1492 "m [{ f: [1] }, { g: 2 }]",
1493 "m: [{ f [{}] }, 1]",
1494 "a [{}] b: [1]",
1495 "f # comment\n : [1]",
1496 ] {
1497 assert!(drain(input).is_ok(), "input: {input:?}");
1498 }
1499 let err = drain("f # comment\n [1]").unwrap_err();
1501 assert_eq!((err.line, err.col), (2, 3));
1502 }
1503
1504 #[test]
1505 fn malformed_element_of_colonless_list_keeps_its_own_error() {
1506 let with_colon = drain("m: [{},]").unwrap_err();
1508 let without = drain("m [{},]").unwrap_err();
1509 assert_eq!(without.kind, with_colon.kind);
1510 assert_ne!(
1511 without.kind,
1512 ParseErrorKind::UnexpectedToken {
1513 expected: COLON_BEFORE_SCALAR,
1514 }
1515 );
1516 }
1517
1518 #[test]
1519 fn peeked_colonless_list_open_still_rejects_scalar() {
1520 let mut t = Tokenizer::new("f [1]");
1521 t.read().unwrap();
1522 assert_eq!(t.peek().unwrap().kind, TokenKind::ListOpen);
1523 assert_eq!(t.read().unwrap().kind, TokenKind::ListOpen);
1524 let expected = ParseErrorKind::UnexpectedToken {
1525 expected: COLON_BEFORE_SCALAR,
1526 };
1527 assert_eq!(t.peek().unwrap_err().kind, expected);
1528 assert_eq!(t.read().unwrap_err().kind, expected);
1529 }
1530
1531 #[test]
1532 fn peek_preserves_field_separator() {
1533 for input in ["field: 7", "field 7"] {
1534 let mut t = Tokenizer::new(input);
1535 assert_eq!(t.peek().unwrap().has_separator, input.contains(':'));
1536 assert_eq!(t.peek().unwrap().has_separator, input.contains(':'));
1537 t.read().unwrap();
1538 if input.contains(':') {
1539 assert_eq!(t.peek().unwrap().raw, "7");
1540 assert_eq!(t.read().unwrap().raw, "7");
1541 } else {
1542 assert!(t.peek().is_err());
1543 assert!(t.read().is_err());
1544 }
1545 }
1546 }
1547
1548 #[test]
1549 fn message_values_allow_optional_colon() {
1550 for input in [
1551 "m {}",
1552 "m <i: 7>",
1553 "m: {}",
1554 "m: <i: 7>",
1555 "m [{} , <i: 7>]",
1556 "m: [{} , <i: 7>]",
1557 "m []",
1558 "m: []",
1559 "[pkg.ext] {}",
1560 "[type.googleapis.com/pkg.Msg] {}",
1561 ] {
1562 assert!(drain(input).is_ok(), "input: {input:?}");
1563 }
1564 }
1565
1566 #[test]
1567 fn delimiter_mismatch() {
1568 let err = drain("m { a: 1 >").unwrap_err();
1570 assert_eq!(err.kind, ParseErrorKind::DelimiterMismatch);
1571 }
1572
1573 #[test]
1574 fn delimiter_mismatch_angle() {
1575 let err = drain("m < a: 1 }").unwrap_err();
1576 assert_eq!(err.kind, ParseErrorKind::DelimiterMismatch);
1577 }
1578
1579 #[test]
1580 fn unexpected_eof_in_message() {
1581 let err = drain("m { a: 1").unwrap_err();
1582 assert_eq!(err.kind, ParseErrorKind::UnexpectedEof);
1583 }
1584
1585 #[test]
1586 fn unexpected_eof_in_list() {
1587 let err = drain("r: [1, 2").unwrap_err();
1588 assert_eq!(err.kind, ParseErrorKind::UnexpectedEof);
1589 }
1590
1591 #[test]
1592 fn unterminated_string() {
1593 let err = drain(r#"f: "oops"#).unwrap_err();
1594 assert!(matches!(err.kind, ParseErrorKind::InvalidString(_)));
1595 }
1596
1597 #[test]
1598 fn list_missing_comma() {
1599 let err = drain("r: [1 2]").unwrap_err();
1601 assert!(matches!(
1602 err.kind,
1603 ParseErrorKind::UnexpectedToken {
1604 expected: "',' or ']'"
1605 }
1606 ));
1607 }
1608
1609 #[test]
1610 fn recursion_limit() {
1611 let depth = crate::message::RECURSION_LIMIT as usize + 1;
1613 let mut s = alloc::string::String::new();
1614 for _ in 0..depth {
1615 s.push_str("m { ");
1616 }
1617 let err = drain(&s).unwrap_err();
1618 assert_eq!(err.kind, ParseErrorKind::RecursionLimitExceeded);
1619 }
1620
1621 #[test]
1624 fn peek_does_not_advance() {
1625 let mut t = Tokenizer::new("a: 1 b: 2");
1626 let p1 = t.peek().unwrap();
1627 let p2 = t.peek().unwrap();
1628 assert_eq!(p1.raw, "a");
1629 assert_eq!(p2.raw, "a");
1630 let r = t.read().unwrap();
1631 assert_eq!(r.raw, "a");
1632 let next = t.read().unwrap();
1633 assert_eq!(next.raw, "1");
1634 }
1635
1636 #[test]
1637 fn peek_then_read_preserves_nesting() {
1638 let mut t = Tokenizer::new("m { a: 1 }");
1640 t.read().unwrap(); let p = t.peek().unwrap();
1642 assert_eq!(p.kind, TokenKind::MessageOpen);
1643 t.read().unwrap(); assert_eq!(t.read().unwrap().raw, "a");
1646 assert_eq!(t.read().unwrap().raw, "1");
1647 assert_eq!(t.read().unwrap().kind, TokenKind::MessageClose);
1648 }
1649
1650 #[test]
1653 fn line_col_table() {
1654 let input = "ab\ncde\nfg";
1655 let t = Tokenizer::new(input);
1656 #[rustfmt::skip]
1657 let cases: &[(usize, (u32, u32))] = &[
1658 (0, (1, 1)), (1, (1, 2)), (2, (1, 3)), (3, (2, 1)), (5, (2, 3)), (7, (3, 1)), (999, (3, 3)), ];
1666 for &(pos, want) in cases {
1667 assert_eq!(t.line_col(pos), want, "pos: {pos}");
1668 }
1669 }
1670
1671 #[test]
1672 fn line_col_unicode() {
1673 let t = Tokenizer::new("éx");
1675 assert_eq!(t.line_col(0), (1, 1));
1676 assert_eq!(t.line_col(1), (1, 1));
1677 assert_eq!(t.line_col(2), (1, 2)); }
1679
1680 #[test]
1681 fn line_col_unicode_interior_offsets_and_eof() {
1682 let t = Tokenizer::new("😀");
1683 for pos in 1..4 {
1684 assert_eq!(t.line_col(pos), (1, 1), "pos: {pos}");
1685 }
1686 assert_eq!(t.line_col(4), (1, 2));
1687 assert_eq!(t.line_col(usize::MAX), (1, 2));
1688
1689 let t = Tokenizer::new("a€b");
1691 assert_eq!(t.line_col(2), (1, 2));
1692 assert_eq!(t.line_col(3), (1, 2));
1693 assert_eq!(t.line_col(4), (1, 3));
1694 }
1695
1696 #[test]
1697 fn line_col_multiline_unicode_interior_offset() {
1698 let t = Tokenizer::new("é\nx😀y");
1699 let offset_inside_emoji = 6;
1700 assert_eq!(t.line_col(offset_inside_emoji), (2, 2));
1701 }
1702
1703 #[test]
1704 fn error_has_correct_position() {
1705 let err = drain("a: 1\nb: 2\nm { x: 1 >").unwrap_err();
1706 assert_eq!(err.line, 3);
1707 assert_eq!(err.col, 10);
1709 }
1710
1711 #[test]
1712 fn bom_is_skipped() {
1713 let toks = drain("\u{FEFF}a: 1").unwrap();
1715 assert_eq!(toks, &[(TokenKind::Name, "a"), (TokenKind::Scalar, "1")]);
1716 }
1717
1718 #[test]
1719 fn bom_only_is_empty() {
1720 let toks = drain("\u{FEFF}").unwrap();
1721 assert!(toks.is_empty());
1722 }
1723
1724 #[test]
1725 fn vertical_tab_and_form_feed_are_whitespace() {
1726 let toks = drain("a:\x0B1\x0Cb: 2").unwrap();
1728 assert_eq!(
1729 toks,
1730 &[
1731 (TokenKind::Name, "a"),
1732 (TokenKind::Scalar, "1"),
1733 (TokenKind::Name, "b"),
1734 (TokenKind::Scalar, "2"),
1735 ]
1736 );
1737 }
1738
1739 #[test]
1740 fn signed_literal_with_whitespace() {
1741 let toks = drain("f: - inf").unwrap();
1744 assert_eq!(
1745 toks,
1746 &[(TokenKind::Name, "f"), (TokenKind::Scalar, "- inf")]
1747 );
1748 }
1749}