use crate::SyntaxKind;
use crate::lexer::{self, LexFlag};
use rowan::{TextRange, TextSize};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum LexError {
UnexpectedCharacter,
BadString,
UnterminatedComment,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Token {
pub kind: SyntaxKind,
pub range: TextRange,
pub error: Option<LexError>,
}
#[must_use]
pub fn tokenize_spec(src: &str) -> Vec<Token> {
lexer::lex_spec(src)
.into_iter()
.map(|t| Token {
kind: t.kind,
range: TextRange::new(TextSize::from(t.start), TextSize::from(t.end)),
error: match t.flag {
LexFlag::Ok => None,
LexFlag::BadChar => Some(LexError::UnexpectedCharacter),
LexFlag::BadString => Some(LexError::BadString),
LexFlag::UnterminatedComment => Some(LexError::UnterminatedComment),
},
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn spec_state_from_offset_zero() {
let src = "INIT x";
let t: Vec<_> = tokenize_spec(src)
.into_iter()
.filter(|t| !t.kind.is_trivia())
.collect();
assert_eq!(t.len(), 2);
assert_eq!(&src[t[0].range], "INIT");
}
#[test]
fn errors_are_flagged() {
let kinds = |s: &str| {
tokenize_spec(s)
.into_iter()
.filter_map(|t| t.error)
.collect::<Vec<_>>()
};
assert_eq!(kinds(";"), [LexError::UnexpectedCharacter]);
assert_eq!(kinds("\"a\\q\""), [LexError::BadString]);
assert_eq!(kinds("(* a (* b *)"), [LexError::UnterminatedComment]);
assert_eq!(kinds("(* a (* b *) *) \\* c"), []);
}
#[test]
fn lossless() {
let src = "CONSTANT S = {a, \"b\"} (* c *)\r\n\\* d";
let joined: String = tokenize_spec(src).iter().map(|t| &src[t.range]).collect();
assert_eq!(joined, src);
}
}