backyard_lexer/
lib.rs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
pub mod token;
mod internal;
mod lexer;
pub mod error;

use bumpalo::Bump;
use error::LexError;
use internal::inline::InlineToken;
use lexer::{ ControlSnapshot, Lexer };
use token::Token;

pub fn lex(input: &str) -> Result<Vec<Token>, LexError> {
  let arena = Bump::new();
  let result = arena_lex(&arena, input)?;
  Ok(result.to_vec())
}

pub fn lex_eval(input: &str) -> Result<Vec<Token>, LexError> {
  let arena = Bump::new();
  let result = arena_lex_eval(&arena, input)?;
  Ok(result.to_vec())
}

pub fn arena_lex<'a>(
  arena: &'a Bump,
  input: &str
) -> Result<bumpalo::collections::Vec<'a, Token>, LexError> {
  let mut lexer = Lexer::new(arena, input);
  InlineToken::lex(&mut lexer, &(ControlSnapshot { line: 1, column: 0, offset: 0 }))?;
  lexer.start()?;
  Ok(lexer.tokens)
}

pub fn arena_lex_eval<'a>(
  arena: &'a Bump,
  input: &str
) -> Result<bumpalo::collections::Vec<'a, Token>, LexError> {
  let mut lexer = Lexer::new(arena, input);
  lexer.start()?;
  Ok(lexer.tokens)
}