sct_ecl/
lexer.rs

1//! The token layer of ECL: the grammar's lexer rules folded into the tokens
2//! the parser consumes (`ECL.g4`, the lexer rules and `ws`, `comment`,
3//! `term`, and the quoted forms).
4//!
5//! The grammar is written character by character; the tokens here are the
6//! maximal runs a parser rule never splits: a pipe-delimited term, a quoted
7//! string, an alternate identifier, a run of digits, a word, and each
8//! operator. Whitespace and comments are skipped, and each token keeps its
9//! byte span so the parser can require the grammar's mandatory whitespace
10//! (`mws`) and its adjacency (`sctid`, `numericvalue`, `cardinality`).
11
12use std::ops::Range;
13
14use logos::Logos;
15
16/// A token kind.
17#[derive(Logos, Debug, Clone, Copy, PartialEq, Eq, Hash)]
18#[logos(skip r"[ \t\r\n]+")]
19#[logos(skip r"/\*([^*]|\*[^/])*\*/")]
20pub enum Kind {
21    /// `|term|` (`eclconceptreference`, `altidentifier`).
22    #[regex(r"\|[^|]*\|")]
23    Term,
24    /// A quoted string (`typedsearchterm`, `timevalue`, the quoted
25    /// `altidentifier`, and the string concrete value).
26    #[regex(r#""([^"\\]|\\.)*""#)]
27    String,
28    /// `scheme#code` (`altidentifier` without quotes).
29    #[regex(r"[A-Za-z][A-Za-z0-9\-]*#[A-Za-z0-9\-._]+")]
30    AltIdentifier,
31    /// A run of digits (`sctid`, `integervalue`, `nonnegativeintegervalue`).
32    #[regex(r"[0-9]+")]
33    Integer,
34    /// A word: the keywords, `refsetfieldname`, `languagecode`,
35    /// `dialectalias`, and the reverse flag `R`.
36    #[regex(r"[A-Za-z][A-Za-z0-9_\-]*")]
37    Identifier,
38    /// `<<!`
39    #[token("<<!")]
40    ChildOrSelfOf,
41    /// `<<`
42    #[token("<<")]
43    DescendantOrSelfOf,
44    /// `<!`
45    #[token("<!")]
46    ChildOf,
47    /// `<=`
48    #[token("<=")]
49    LessOrEqual,
50    /// `<`
51    #[token("<")]
52    LessThan,
53    /// `>>!`
54    #[token(">>!")]
55    ParentOrSelfOf,
56    /// `>>`
57    #[token(">>")]
58    AncestorOrSelfOf,
59    /// `>!`
60    #[token(">!")]
61    ParentOf,
62    /// `>=`
63    #[token(">=")]
64    GreaterOrEqual,
65    /// `>`
66    #[token(">")]
67    GreaterThan,
68    /// `!!>`
69    #[token("!!>")]
70    Top,
71    /// `!!<`
72    #[token("!!<")]
73    Bottom,
74    /// `!=`
75    #[token("!=")]
76    NotEqual,
77    /// `=`
78    #[token("=")]
79    Equal,
80    /// `(`
81    #[token("(")]
82    LeftParen,
83    /// `)`
84    #[token(")")]
85    RightParen,
86    /// `{{`
87    #[token("{{")]
88    DoubleLeftBrace,
89    /// `}}`
90    #[token("}}")]
91    DoubleRightBrace,
92    /// `{`
93    #[token("{")]
94    LeftBrace,
95    /// `}`
96    #[token("}")]
97    RightBrace,
98    /// `[`
99    #[token("[")]
100    LeftBracket,
101    /// `]`
102    #[token("]")]
103    RightBracket,
104    /// `:`
105    #[token(":")]
106    Colon,
107    /// `,`
108    #[token(",")]
109    Comma,
110    /// `^`
111    #[token("^")]
112    Caret,
113    /// `..`
114    #[token("..")]
115    To,
116    /// `.`
117    #[token(".")]
118    Period,
119    /// `*`
120    #[token("*")]
121    Asterisk,
122    /// `#`
123    #[token("#")]
124    Hash,
125    /// `+`
126    #[token("+")]
127    Plus,
128    /// `-`
129    #[token("-")]
130    Dash,
131}
132
133impl Kind {
134    /// The token class as an error message names it.
135    #[must_use]
136    pub const fn describe(self) -> &'static str {
137        match self {
138            Self::Term => "a term between pipes",
139            Self::String => "a quoted string",
140            Self::AltIdentifier => "an alternate identifier",
141            Self::Integer => "a number",
142            Self::Identifier => "a word",
143            Self::ChildOrSelfOf => "'<<!'",
144            Self::DescendantOrSelfOf => "'<<'",
145            Self::ChildOf => "'<!'",
146            Self::LessOrEqual => "'<='",
147            Self::LessThan => "'<'",
148            Self::ParentOrSelfOf => "'>>!'",
149            Self::AncestorOrSelfOf => "'>>'",
150            Self::ParentOf => "'>!'",
151            Self::GreaterOrEqual => "'>='",
152            Self::GreaterThan => "'>'",
153            Self::Top => "'!!>'",
154            Self::Bottom => "'!!<'",
155            Self::NotEqual => "'!='",
156            Self::Equal => "'='",
157            Self::LeftParen => "'('",
158            Self::RightParen => "')'",
159            Self::DoubleLeftBrace => "'{{'",
160            Self::DoubleRightBrace => "'}}'",
161            Self::LeftBrace => "'{'",
162            Self::RightBrace => "'}'",
163            Self::LeftBracket => "'['",
164            Self::RightBracket => "']'",
165            Self::Colon => "':'",
166            Self::Comma => "','",
167            Self::Caret => "'^'",
168            Self::To => "'..'",
169            Self::Period => "'.'",
170            Self::Asterisk => "'*'",
171            Self::Hash => "'#'",
172            Self::Plus => "'+'",
173            Self::Dash => "'-'",
174        }
175    }
176}
177
178/// A token with its source text and byte span.
179#[derive(Debug, Clone, PartialEq, Eq)]
180pub struct Token<'s> {
181    /// The kind.
182    pub kind: Kind,
183    /// The source text.
184    pub text: &'s str,
185    /// The byte span in the input.
186    pub span: Range<usize>,
187}
188
189impl PartialEq<Kind> for Token<'_> {
190    fn eq(&self, other: &Kind) -> bool {
191        self.kind == *other
192    }
193}
194
195/// A character no token starts with.
196#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
197#[error("unexpected character {found:?} at byte {offset}")]
198pub struct LexError {
199    /// The byte offset of the character.
200    pub offset: usize,
201    /// The character.
202    pub found: char,
203}
204
205/// Splits `input` into tokens, skipping whitespace and comments.
206///
207/// # Errors
208///
209/// Returns [`LexError`] at the first character no token starts with (an
210/// unterminated term, string, or comment among them).
211pub fn lex(input: &str) -> Result<Vec<Token<'_>>, LexError> {
212    let mut lexer = Kind::lexer(input);
213    let mut tokens = Vec::new();
214    while let Some(kind) = lexer.next() {
215        let span = lexer.span();
216        let kind = kind.map_err(|()| LexError {
217            offset: span.start,
218            found: lexer.slice().chars().next().unwrap_or('\u{0}'),
219        })?;
220        tokens.push(Token {
221            kind,
222            text: lexer.slice(),
223            span,
224        });
225    }
226    Ok(tokens)
227}
228
229#[cfg(test)]
230mod tests {
231    use super::{Kind, lex};
232
233    fn kinds(input: &str) -> Vec<Kind> {
234        lex(input)
235            .expect("lexes")
236            .into_iter()
237            .map(|t| t.kind)
238            .collect()
239    }
240
241    #[test]
242    fn operators_take_the_longest_match_and_comments_are_skipped() {
243        assert_eq!(
244            kinds("<<! /* c */ 123 |a b| {{ D term = \"x\" }} [1..*] LOINC#54-6 !!>"),
245            [
246                Kind::ChildOrSelfOf,
247                Kind::Integer,
248                Kind::Term,
249                Kind::DoubleLeftBrace,
250                Kind::Identifier,
251                Kind::Identifier,
252                Kind::Equal,
253                Kind::String,
254                Kind::DoubleRightBrace,
255                Kind::LeftBracket,
256                Kind::Integer,
257                Kind::To,
258                Kind::Asterisk,
259                Kind::RightBracket,
260                Kind::AltIdentifier,
261                Kind::Top,
262            ]
263        );
264        assert_eq!(
265            kinds("#-5.5"),
266            [
267                Kind::Hash,
268                Kind::Dash,
269                Kind::Integer,
270                Kind::Period,
271                Kind::Integer
272            ]
273        );
274        let error = lex("< 123 |unterminated").expect_err("refused");
275        assert_eq!(error.offset, 6);
276        assert_eq!(error.found, '|');
277        assert_eq!(lex("a /* open").expect_err("refused").offset, 2);
278    }
279}
sct_ecl/lexer.rs

sct_ecl/
lexer.rs
