sct_ecl/
lexer.rs

1// SPDX-FileCopyrightText: Cadasto B.V.
2// SPDX-License-Identifier: BUSL-1.1
3
4//! The token layer of ECL: the grammar's lexer rules folded into the tokens
5//! the parser consumes (`ECL.g4`, the lexer rules and `ws`, `comment`,
6//! `term`, and the quoted forms).
7//!
8//! The grammar is written character by character; the tokens here are the
9//! maximal runs a parser rule never splits: a pipe-delimited term, a quoted
10//! string, an alternate identifier, a run of digits, a word, and each
11//! operator. Whitespace and comments are skipped, and each token keeps its
12//! byte span so the parser can require the grammar's mandatory whitespace
13//! (`mws`) and its adjacency (`sctid`, `numericvalue`, `cardinality`).
14
15use std::ops::Range;
16
17use logos::Logos;
18
19/// A token kind.
20#[derive(Logos, Debug, Clone, Copy, PartialEq, Eq, Hash)]
21#[logos(skip r"[ \t\r\n]+")]
22#[logos(skip r"/\*([^*]|\*[^/])*\*/")]
23pub enum Kind {
24    /// `|term|` (`eclconceptreference`, `altidentifier`).
25    #[regex(r"\|[^|]*\|")]
26    Term,
27    /// A quoted string (`typedsearchterm`, `timevalue`, the quoted
28    /// `altidentifier`, and the string concrete value).
29    #[regex(r#""([^"\\]|\\.)*""#)]
30    String,
31    /// `scheme#code` (`altidentifier` without quotes).
32    #[regex(r"[A-Za-z][A-Za-z0-9\-]*#[A-Za-z0-9\-._]+")]
33    AltIdentifier,
34    /// A run of digits (`sctid`, `integervalue`, `nonnegativeintegervalue`).
35    #[regex(r"[0-9]+")]
36    Integer,
37    /// A word: the keywords, `refsetfieldname`, `languagecode`,
38    /// `dialectalias`, and the reverse flag `R`.
39    #[regex(r"[A-Za-z][A-Za-z0-9_\-]*")]
40    Identifier,
41    /// `<<!`
42    #[token("<<!")]
43    ChildOrSelfOf,
44    /// `<<`
45    #[token("<<")]
46    DescendantOrSelfOf,
47    /// `<!`
48    #[token("<!")]
49    ChildOf,
50    /// `<=`
51    #[token("<=")]
52    LessOrEqual,
53    /// `<`
54    #[token("<")]
55    LessThan,
56    /// `>>!`
57    #[token(">>!")]
58    ParentOrSelfOf,
59    /// `>>`
60    #[token(">>")]
61    AncestorOrSelfOf,
62    /// `>!`
63    #[token(">!")]
64    ParentOf,
65    /// `>=`
66    #[token(">=")]
67    GreaterOrEqual,
68    /// `>`
69    #[token(">")]
70    GreaterThan,
71    /// `!!>`
72    #[token("!!>")]
73    Top,
74    /// `!!<`
75    #[token("!!<")]
76    Bottom,
77    /// `!=`
78    #[token("!=")]
79    NotEqual,
80    /// `=`
81    #[token("=")]
82    Equal,
83    /// `(`
84    #[token("(")]
85    LeftParen,
86    /// `)`
87    #[token(")")]
88    RightParen,
89    /// `{{`
90    #[token("{{")]
91    DoubleLeftBrace,
92    /// `}}`
93    #[token("}}")]
94    DoubleRightBrace,
95    /// `{`
96    #[token("{")]
97    LeftBrace,
98    /// `}`
99    #[token("}")]
100    RightBrace,
101    /// `[`
102    #[token("[")]
103    LeftBracket,
104    /// `]`
105    #[token("]")]
106    RightBracket,
107    /// `:`
108    #[token(":")]
109    Colon,
110    /// `,`
111    #[token(",")]
112    Comma,
113    /// `^`
114    #[token("^")]
115    Caret,
116    /// `..`
117    #[token("..")]
118    To,
119    /// `.`
120    #[token(".")]
121    Period,
122    /// `*`
123    #[token("*")]
124    Asterisk,
125    /// `#`
126    #[token("#")]
127    Hash,
128    /// `+`
129    #[token("+")]
130    Plus,
131    /// `-`
132    #[token("-")]
133    Dash,
134}
135
136impl Kind {
137    /// The token class as an error message names it.
138    #[must_use]
139    pub const fn describe(self) -> &'static str {
140        match self {
141            Self::Term => "a term between pipes",
142            Self::String => "a quoted string",
143            Self::AltIdentifier => "an alternate identifier",
144            Self::Integer => "a number",
145            Self::Identifier => "a word",
146            Self::ChildOrSelfOf => "'<<!'",
147            Self::DescendantOrSelfOf => "'<<'",
148            Self::ChildOf => "'<!'",
149            Self::LessOrEqual => "'<='",
150            Self::LessThan => "'<'",
151            Self::ParentOrSelfOf => "'>>!'",
152            Self::AncestorOrSelfOf => "'>>'",
153            Self::ParentOf => "'>!'",
154            Self::GreaterOrEqual => "'>='",
155            Self::GreaterThan => "'>'",
156            Self::Top => "'!!>'",
157            Self::Bottom => "'!!<'",
158            Self::NotEqual => "'!='",
159            Self::Equal => "'='",
160            Self::LeftParen => "'('",
161            Self::RightParen => "')'",
162            Self::DoubleLeftBrace => "'{{'",
163            Self::DoubleRightBrace => "'}}'",
164            Self::LeftBrace => "'{'",
165            Self::RightBrace => "'}'",
166            Self::LeftBracket => "'['",
167            Self::RightBracket => "']'",
168            Self::Colon => "':'",
169            Self::Comma => "','",
170            Self::Caret => "'^'",
171            Self::To => "'..'",
172            Self::Period => "'.'",
173            Self::Asterisk => "'*'",
174            Self::Hash => "'#'",
175            Self::Plus => "'+'",
176            Self::Dash => "'-'",
177        }
178    }
179}
180
181/// A token with its source text and byte span.
182#[derive(Debug, Clone, PartialEq, Eq)]
183pub struct Token<'s> {
184    /// The kind.
185    pub kind: Kind,
186    /// The source text.
187    pub text: &'s str,
188    /// The byte span in the input.
189    pub span: Range<usize>,
190}
191
192impl PartialEq<Kind> for Token<'_> {
193    fn eq(&self, other: &Kind) -> bool {
194        self.kind == *other
195    }
196}
197
198/// A character no token starts with.
199#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
200#[error("unexpected character {found:?} at byte {offset}")]
201pub struct LexError {
202    /// The byte offset of the character.
203    pub offset: usize,
204    /// The character.
205    pub found: char,
206}
207
208/// Splits `input` into tokens, skipping whitespace and comments.
209///
210/// # Errors
211///
212/// Returns [`LexError`] at the first character no token starts with (an
213/// unterminated term, string, or comment among them).
214pub fn lex(input: &str) -> Result<Vec<Token<'_>>, LexError> {
215    let mut lexer = Kind::lexer(input);
216    let mut tokens = Vec::new();
217    while let Some(kind) = lexer.next() {
218        let span = lexer.span();
219        let kind = kind.map_err(|()| LexError {
220            offset: span.start,
221            found: lexer.slice().chars().next().unwrap_or('\u{0}'),
222        })?;
223        tokens.push(Token {
224            kind,
225            text: lexer.slice(),
226            span,
227        });
228    }
229    Ok(tokens)
230}
231
232#[cfg(test)]
233mod tests {
234    use super::{Kind, lex};
235
236    fn kinds(input: &str) -> Vec<Kind> {
237        lex(input)
238            .expect("lexes")
239            .into_iter()
240            .map(|t| t.kind)
241            .collect()
242    }
243
244    #[test]
245    fn operators_take_the_longest_match_and_comments_are_skipped() {
246        assert_eq!(
247            kinds("<<! /* c */ 123 |a b| {{ D term = \"x\" }} [1..*] LOINC#54-6 !!>"),
248            [
249                Kind::ChildOrSelfOf,
250                Kind::Integer,
251                Kind::Term,
252                Kind::DoubleLeftBrace,
253                Kind::Identifier,
254                Kind::Identifier,
255                Kind::Equal,
256                Kind::String,
257                Kind::DoubleRightBrace,
258                Kind::LeftBracket,
259                Kind::Integer,
260                Kind::To,
261                Kind::Asterisk,
262                Kind::RightBracket,
263                Kind::AltIdentifier,
264                Kind::Top,
265            ]
266        );
267        assert_eq!(
268            kinds("#-5.5"),
269            [
270                Kind::Hash,
271                Kind::Dash,
272                Kind::Integer,
273                Kind::Period,
274                Kind::Integer
275            ]
276        );
277        let error = lex("< 123 |unterminated").expect_err("refused");
278        assert_eq!(error.offset, 6);
279        assert_eq!(error.found, '|');
280        assert_eq!(lex("a /* open").expect_err("refused").offset, 2);
281    }
282}
sct_ecl/lexer.rs

sct_ecl/
lexer.rs
