1pub mod token;
2mod internal;
3mod lexer;
4pub mod error;
5
6use bstr::BString;
7use bumpalo::Bump;
8use error::LexError;
9use lexer::Lexer;
10use token::Token;
11
12pub fn lex_bytes<'a>(
13 arena: &'a Bump,
14 input: Vec<u8>
15) -> Result<bumpalo::collections::Vec<'a, Token>, LexError> {
16 let mut lexer = Lexer::new(arena, BString::new(input));
17 lexer.start(true)?;
18 Ok(lexer.tokens)
19}
20
21pub fn lex_eval_bytes<'a>(
22 arena: &'a Bump,
23 input: Vec<u8>
24) -> Result<bumpalo::collections::Vec<'a, Token>, LexError> {
25 let mut lexer = Lexer::new(arena, BString::new(input));
26 lexer.start(false)?;
27 Ok(lexer.tokens)
28}
29
30pub fn lex<'a>(
31 arena: &'a Bump,
32 input: &str
33) -> Result<bumpalo::collections::Vec<'a, Token>, LexError> {
34 lex_bytes(arena, input.as_bytes().to_vec())
35}
36
37pub fn lex_eval<'a>(
38 arena: &'a Bump,
39 input: &str
40) -> Result<bumpalo::collections::Vec<'a, Token>, LexError> {
41 lex_eval_bytes(arena, input.as_bytes().to_vec())
42}