harper_core/parsers/
plain_english.rs

1use super::Parser;
2use crate::lexing::{FoundToken, lex_token};
3use crate::{Span, Token};
4
5/// A parser that will attempt to lex as many tokens as possible,
6/// without discrimination and until the end of input.
7#[derive(Clone, Copy)]
8pub struct PlainEnglish;
9
10impl Parser for PlainEnglish {
11    fn parse(&self, source: &[char]) -> Vec<Token> {
12        let mut cursor = 0;
13
14        // Lex tokens
15        let mut tokens = Vec::new();
16
17        loop {
18            if cursor >= source.len() {
19                return tokens;
20            }
21
22            if let Some(FoundToken { token, next_index }) = lex_token(&source[cursor..]) {
23                tokens.push(Token {
24                    span: Span::new(cursor, cursor + next_index),
25                    kind: token,
26                });
27                cursor += next_index;
28            } else {
29                panic!()
30            }
31        }
32    }
33}