yaml_rt_core/
lexer.rs

1use crate::{Diagnostic, DiagnosticKind, Source, Span, YamlError};
2
3/// Lexical token preserving its exact source span.
4#[derive(Debug, Clone, PartialEq, Eq)]
5pub struct Token {
6    /// Token classification.
7    pub kind: TokenKind,
8    /// Original source span for this token.
9    pub span: Span,
10}
11
12/// Token kinds emitted by the lossless lexer.
13#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14pub enum TokenKind {
15    /// UTF-8 byte-order mark at the start of a stream.
16    Bom,
17    /// Spaces, tabs, or other separation characters that are not line breaks.
18    Whitespace,
19    /// A YAML line break, preserving the original spelling through the span.
20    Newline,
21    /// A comment from `#` through the byte before the line break.
22    Comment,
23    /// `---` document start marker.
24    DocumentStart,
25    /// `...` document end marker.
26    DocumentEnd,
27    /// `:` mapping value indicator.
28    Colon,
29    /// `-` sequence entry indicator or dash token.
30    Dash,
31    /// `[` flow sequence start.
32    FlowSequenceStart,
33    /// `]` flow sequence end.
34    FlowSequenceEnd,
35    /// `{` flow mapping start.
36    FlowMappingStart,
37    /// `}` flow mapping end.
38    FlowMappingEnd,
39    /// `,` flow separator.
40    Comma,
41    /// `?` explicit mapping key indicator.
42    Question,
43    /// A double-quoted scalar, including its quotes.
44    DoubleQuotedScalar,
45    /// A single-quoted scalar, including its quotes.
46    SingleQuotedScalar,
47    /// An unquoted scalar chunk.
48    PlainScalar,
49}
50
51/// Lexes YAML source into lossless tokens.
52///
53/// # Errors
54///
55/// Returns an error when the source contains malformed quoted scalars or other
56/// token-level syntax that the lexer can diagnose.
57pub fn lex(source: &Source) -> Result<Vec<Token>, YamlError> {
58    Lexer::new(source).lex()
59}
60
61/// Reconstructs source text from token spans.
62#[must_use]
63pub fn tokens_to_string(tokens: &[Token], source: &Source) -> String {
64    let mut output = String::new();
65    for token in tokens {
66        output.push_str(source.slice(token.span));
67    }
68    output
69}
70
71struct Lexer<'source> {
72    source: &'source Source,
73    text: &'source str,
74    position: usize,
75    tokens: Vec<Token>,
76}
77
78impl<'source> Lexer<'source> {
79    fn new(source: &'source Source) -> Self {
80        Self {
81            source,
82            text: source.as_str(),
83            position: 0,
84            tokens: Vec::with_capacity(source.len() / 4 + 1),
85        }
86    }
87
88    fn lex(mut self) -> Result<Vec<Token>, YamlError> {
89        while self.position < self.text.len() {
90            let start = self.position;
91
92            if self.consume_bom() {
93                self.push(TokenKind::Bom, start);
94            } else if self.consume_line_break() {
95                self.push(TokenKind::Newline, start);
96            } else if self.consume_horizontal_whitespace() {
97                self.push(TokenKind::Whitespace, start);
98            } else if self.consume_comment() {
99                self.push(TokenKind::Comment, start);
100            } else if self.consume_document_marker("---") {
101                self.push(TokenKind::DocumentStart, start);
102            } else if self.consume_document_marker("...") {
103                self.push(TokenKind::DocumentEnd, start);
104            } else if self.can_start_quoted_scalar() && self.consume_double_quoted_scalar()? {
105                self.push(TokenKind::DoubleQuotedScalar, start);
106            } else if self.can_start_quoted_scalar() && self.consume_single_quoted_scalar()? {
107                self.push(TokenKind::SingleQuotedScalar, start);
108            } else if self.consume_single_byte_indicator() {
109                let kind = match self.text.as_bytes()[start] {
110                    b':' => TokenKind::Colon,
111                    b'-' => TokenKind::Dash,
112                    b'[' => TokenKind::FlowSequenceStart,
113                    b']' => TokenKind::FlowSequenceEnd,
114                    b'{' => TokenKind::FlowMappingStart,
115                    b'}' => TokenKind::FlowMappingEnd,
116                    b',' => TokenKind::Comma,
117                    b'?' => TokenKind::Question,
118                    _ => unreachable!("consume_single_byte_indicator only consumes indicators"),
119                };
120                self.push(kind, start);
121            } else {
122                self.consume_plain_scalar();
123                self.push(TokenKind::PlainScalar, start);
124            }
125        }
126
127        Ok(self.tokens)
128    }
129
130    fn push(&mut self, kind: TokenKind, start: usize) {
131        self.tokens.push(Token {
132            kind,
133            span: Span::from_usize(start, self.position),
134        });
135    }
136
137    fn consume_bom(&mut self) -> bool {
138        if self.position == 0 && self.text[self.position..].starts_with('\u{FEFF}') {
139            self.position += '\u{FEFF}'.len_utf8();
140            true
141        } else {
142            false
143        }
144    }
145
146    fn consume_line_break(&mut self) -> bool {
147        let remaining = &self.text[self.position..];
148        if remaining.starts_with("\r\n") {
149            self.position += 2;
150            true
151        } else if remaining.starts_with('\n') || remaining.starts_with('\r') {
152            self.position += 1;
153            true
154        } else {
155            false
156        }
157    }
158
159    fn consume_horizontal_whitespace(&mut self) -> bool {
160        let start = self.position;
161        while let Some(character) = self.current_char() {
162            if character == ' ' || character == '\t' {
163                self.position += character.len_utf8();
164            } else {
165                break;
166            }
167        }
168
169        self.position != start
170    }
171
172    fn consume_comment(&mut self) -> bool {
173        if !self.text[self.position..].starts_with('#') {
174            return false;
175        }
176
177        self.position += 1;
178        while let Some(character) = self.current_char() {
179            if character == '\n' || character == '\r' {
180                break;
181            }
182            self.position += character.len_utf8();
183        }
184
185        true
186    }
187
188    fn consume_document_marker(&mut self, marker: &str) -> bool {
189        if !self.is_line_start() || !self.text[self.position..].starts_with(marker) {
190            return false;
191        }
192
193        let end = self.position + marker.len();
194        let followed_by_boundary = self.text[end..]
195            .chars()
196            .next()
197            .is_none_or(|character| matches!(character, ' ' | '\t' | '\r' | '\n'));
198
199        if followed_by_boundary {
200            self.position = end;
201            true
202        } else {
203            false
204        }
205    }
206
207    fn consume_double_quoted_scalar(&mut self) -> Result<bool, YamlError> {
208        if !self.text[self.position..].starts_with('"') {
209            return Ok(false);
210        }
211
212        let start = self.position;
213        self.position += 1;
214        let mut escaped = false;
215
216        while let Some(character) = self.current_char() {
217            self.position += character.len_utf8();
218            if escaped {
219                escaped = false;
220            } else if character == '\\' {
221                escaped = true;
222            } else if character == '"' {
223                return Ok(true);
224            }
225        }
226
227        Err(self.unterminated_scalar_error(start, "double-quoted scalar", '"'))
228    }
229
230    fn consume_single_quoted_scalar(&mut self) -> Result<bool, YamlError> {
231        if !self.text[self.position..].starts_with('\'') {
232            return Ok(false);
233        }
234
235        let start = self.position;
236        self.position += 1;
237
238        while let Some(character) = self.current_char() {
239            self.position += character.len_utf8();
240            if character == '\'' {
241                if self.text[self.position..].starts_with('\'') {
242                    self.position += 1;
243                } else {
244                    return Ok(true);
245                }
246            }
247        }
248
249        Err(self.unterminated_scalar_error(start, "single-quoted scalar", '\''))
250    }
251
252    fn consume_single_byte_indicator(&mut self) -> bool {
253        if matches!(
254            self.text.as_bytes()[self.position],
255            b':' | b'-' | b'[' | b']' | b'{' | b'}' | b',' | b'?'
256        ) {
257            self.position += 1;
258            true
259        } else {
260            false
261        }
262    }
263
264    fn consume_plain_scalar(&mut self) {
265        while let Some(character) = self.current_char() {
266            if matches!(
267                character,
268                ' ' | '\t' | '\r' | '\n' | '#' | ':' | '-' | '[' | ']' | '{' | '}' | ',' | '?'
269            ) {
270                break;
271            }
272            self.position += character.len_utf8();
273        }
274
275        if self.position == 0
276            || self.position
277                == self
278                    .tokens
279                    .last()
280                    .map_or(0, |token| token.span.end as usize)
281        {
282            self.position += self.current_char().map_or(0, char::len_utf8);
283        }
284    }
285
286    fn can_start_quoted_scalar(&self) -> bool {
287        self.tokens.last().is_none_or(|token| {
288            matches!(
289                token.kind,
290                TokenKind::Bom
291                    | TokenKind::Whitespace
292                    | TokenKind::Newline
293                    | TokenKind::Colon
294                    | TokenKind::Dash
295                    | TokenKind::Question
296                    | TokenKind::FlowSequenceStart
297                    | TokenKind::FlowMappingStart
298                    | TokenKind::Comma
299            )
300        })
301    }
302
303    fn current_char(&self) -> Option<char> {
304        self.text[self.position..].chars().next()
305    }
306
307    fn is_line_start(&self) -> bool {
308        self.position == 0
309            || matches!(
310                self.text.as_bytes().get(self.position.wrapping_sub(1)),
311                Some(b'\n' | b'\r')
312            )
313    }
314
315    fn unterminated_scalar_error(
316        &self,
317        start: usize,
318        scalar_name: &'static str,
319        terminator: char,
320    ) -> YamlError {
321        YamlError::new(
322            Diagnostic::new(
323                DiagnosticKind::Lexer,
324                format!("unterminated {scalar_name}"),
325                Span::from_usize(start, self.source.len()),
326            )
327            .with_expected(format!("closing {terminator}")),
328        )
329    }
330}
yaml_rt_core/lexer.rs

yaml_rt_core/
lexer.rs
