dive_syntax/tokens.rs
1//! Raw tokens of SANY's `SPEC` lexical state.
2//!
3//! Other TLA+ formats reuse SANY's token manager. TLC's model configuration
4//! reader (`tlc2.tool.impl.ModelConfig`) creates a `TLAplusParserTokenManager`
5//! directly in the `SPEC` state, so a `.cfg` file is tokenized exactly like
6//! the body of a module. [`tokenize_spec`] exposes that lexer.
7//!
8//! # Example
9//!
10//! ```
11//! use dive_syntax::SyntaxKind;
12//! use dive_syntax::tokens::tokenize_spec;
13//!
14//! let src = "INIT Init \\* comment\nCONSTANT N <- 3";
15//! let words: Vec<&str> = tokenize_spec(src)
16//! .iter()
17//! .filter(|t| !t.kind.is_trivia())
18//! .map(|t| &src[t.range])
19//! .collect();
20//! assert_eq!(words, ["INIT", "Init", "CONSTANT", "N", "<-", "3"]);
21//! assert_eq!(tokenize_spec("\"open").last().map(|t| t.kind), Some(SyntaxKind::LEX_ERROR));
22//! ```
23
24use crate::SyntaxKind;
25use crate::lexer::{self, LexFlag};
26use rowan::{TextRange, TextSize};
27
28/// Why a token is a lexical error. SANY's token manager would throw a
29/// `TokenMgrError` when it reaches such a token.
30#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
31#[non_exhaustive]
32pub enum LexError {
33 /// A character no token can start with.
34 UnexpectedCharacter,
35 /// A string literal with a bad escape, or no closing quote on its line.
36 BadString,
37 /// A block comment still open at end of input.
38 UnterminatedComment,
39}
40
41/// One token, trivia included. Concatenating the ranges of all tokens
42/// reproduces the input.
43#[derive(Debug, Clone, Copy, PartialEq, Eq)]
44pub struct Token {
45 /// The token kind. Lexical errors have kind [`SyntaxKind::LEX_ERROR`],
46 /// except unterminated block comments, which keep
47 /// [`SyntaxKind::BLOCK_COMMENT`] and carry [`Token::error`].
48 pub kind: SyntaxKind,
49 /// The token's bytes in the input.
50 pub range: TextRange,
51 /// Set when SANY's token manager would fail on this token.
52 pub error: Option<LexError>,
53}
54
55/// Tokenizes `src` from the start in SANY's `SPEC` lexical state.
56///
57/// Inputs larger than 4 GiB cannot be represented by [`TextRange`]; callers
58/// must refuse them first. Offsets past `u32::MAX` are clamped.
59#[must_use]
60pub fn tokenize_spec(src: &str) -> Vec<Token> {
61 lexer::lex_spec(src)
62 .into_iter()
63 .map(|t| Token {
64 kind: t.kind,
65 range: TextRange::new(TextSize::from(t.start), TextSize::from(t.end)),
66 error: match t.flag {
67 LexFlag::Ok => None,
68 LexFlag::BadChar => Some(LexError::UnexpectedCharacter),
69 LexFlag::BadString => Some(LexError::BadString),
70 LexFlag::UnterminatedComment => Some(LexError::UnterminatedComment),
71 },
72 })
73 .collect()
74}
75
76#[cfg(test)]
77mod tests {
78 use super::*;
79
80 #[test]
81 fn spec_state_from_offset_zero() {
82 // No module header is needed: `INIT` is lexed as a token at once.
83 let src = "INIT x";
84 let t: Vec<_> = tokenize_spec(src)
85 .into_iter()
86 .filter(|t| !t.kind.is_trivia())
87 .collect();
88 assert_eq!(t.len(), 2);
89 assert_eq!(&src[t[0].range], "INIT");
90 }
91
92 #[test]
93 fn errors_are_flagged() {
94 let kinds = |s: &str| {
95 tokenize_spec(s)
96 .into_iter()
97 .filter_map(|t| t.error)
98 .collect::<Vec<_>>()
99 };
100 assert_eq!(kinds(";"), [LexError::UnexpectedCharacter]);
101 assert_eq!(kinds("\"a\\q\""), [LexError::BadString]);
102 assert_eq!(kinds("(* a (* b *)"), [LexError::UnterminatedComment]);
103 assert_eq!(kinds("(* a (* b *) *) \\* c"), []);
104 }
105
106 #[test]
107 fn lossless() {
108 let src = "CONSTANT S = {a, \"b\"} (* c *)\r\n\\* d";
109 let joined: String = tokenize_spec(src).iter().map(|t| &src[t.range]).collect();
110 assert_eq!(joined, src);
111 }
112}