mod basic_multi_token_tests;
mod escape_newline_tests;
mod generative_multi_token_tests;
mod single_token_tests;
use crate::lexer::Token;
fn build_string_from_tokens(tokens: &mut Vec<Token<'static>>) -> String {
let mut byte_increment = 0;
let mut out = String::new();
for tok in tokens.iter_mut() {
tok.start_byte += byte_increment;
tok.end_byte += byte_increment;
byte_increment += tok.text.len();
out.push_str(&tok.text.clone());
}
out
}