mod cursor;
mod error;
mod scanner;
mod span;
mod token;
pub use error::{LexerError, LexerErrorKind, LexerResult};
pub use scanner::Lexer;
pub use span::{
macro_ident_rename, macro_ident_renames_in, register_macro_ident_rename, source_text_for_id,
BytePos, Position, SourceFile, SourceId, Span,
};
pub use token::{
is_integer_only_suffix, is_valid_float_suffix, is_valid_int_suffix, validate_numeric_suffix,
Delimiter, DocComment, DocCommentKind, DocComments, IntBase, InterpolatedPart, Keyword,
LiteralKind, NumericSuffixKind, NumericSuffixValidation, Token, TokenKind, FLOAT_SUFFIXES,
INTEGER_SUFFIXES, NUMERIC_SUFFIXES,
};
pub fn tokenize(source: &str) -> LexerResult<Vec<Token>> {
let file = SourceFile::anonymous(source);
let mut lexer = Lexer::new(&file);
lexer.tokenize()
}
pub fn tokenize_file(filename: &str, source: &str) -> LexerResult<Vec<Token>> {
let file = SourceFile::new(filename, source);
let mut lexer = Lexer::new(&file);
lexer.tokenize()
}
pub fn tokenize_with_docs(source: &str) -> LexerResult<(Vec<Token>, DocComments)> {
let file = SourceFile::anonymous(source);
let mut lexer = Lexer::new(&file);
lexer.tokenize_with_docs()
}
pub fn unescape_str(s: &str) -> String {
let mut result = String::with_capacity(s.len());
let mut chars = s.chars().peekable();
while let Some(c) = chars.next() {
if c != '\\' {
result.push(c);
continue;
}
match chars.next() {
Some('n') => result.push('\n'),
Some('r') => result.push('\r'),
Some('t') => result.push('\t'),
Some('\\') => result.push('\\'),
Some('\'') => result.push('\''),
Some('"') => result.push('"'),
Some('0') => result.push('\0'),
Some('x') => {
let hi = chars.next().unwrap_or('0');
let lo = chars.next().unwrap_or('0');
let val = u8::from_str_radix(&format!("{hi}{lo}"), 16).unwrap_or(0);
result.push(val as char);
}
Some('u') => {
chars.next(); let mut hex = String::new();
while let Some(&d) = chars.peek() {
chars.next();
if d == '}' {
break;
}
hex.push(d);
}
if let Some(ch) = u32::from_str_radix(&hex, 16).ok().and_then(char::from_u32) {
result.push(ch);
}
}
Some(other) => result.push(other),
None => {}
}
}
result
}
pub fn tokenize_file_with_docs(
filename: &str,
source: &str,
) -> LexerResult<(Vec<Token>, DocComments)> {
let file = SourceFile::new(filename, source);
let mut lexer = Lexer::new(&file);
lexer.tokenize_with_docs()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_basic_tokenization() {
let tokens = tokenize("let x = 42").unwrap();
assert!(tokens.len() >= 4); }
}