jia_parse/jia_lang/
lexer.rs

1//! Tokenizer for the Jia modelling language.
2//!
3//! Converts raw `.jia` source text into a flat vector of [`Token`]s, each carrying
4//! a [`TokenKind`] and a [`Span`] for error reporting.
5//!
6//! Token conventions:
7//! - `//` → line comment (skipped entirely)
8//! - Two-char operators: `<=`, `>=`, `==`, `!=`, `..`
9//! - Identifiers are case-sensitive (types are capitalized, user names lowercase)
10//! - `@model` is tokenized as `At` + `Ident("model")`
11//! - Numbers: integer `42` or float `3.14`
12
13use crate::error::{ParseError, Span};
14
15/// The kind of a lexed CP token.
16#[derive(Debug, Clone, PartialEq)]
17pub enum TokenKind {
18    /// `(`
19    LParen,
20    /// `)`
21    RParen,
22    /// `{`
23    LBrace,
24    /// `}`
25    RBrace,
26    /// `[`
27    LBracket,
28    /// `]`
29    RBracket,
30    /// `,`
31    Comma,
32    /// `:`
33    Colon,
34    /// `..`
35    DotDot,
36    /// `+`
37    Plus,
38    /// `-`
39    Minus,
40    /// `*`
41    Star,
42    /// `/`
43    Slash,
44    /// `@`
45    At,
46    /// `<=`
47    Le,
48    /// `>=`
49    Ge,
50    /// `<`
51    Lt,
52    /// `>`
53    Gt,
54    /// `==`
55    EqEq,
56    /// `!=`
57    Ne,
58    /// `=`
59    Eq,
60    /// An integer literal.
61    Number(i64),
62    /// A floating-point literal.
63    Float(f64),
64    /// An identifier or keyword.
65    Ident(String),
66}
67
68/// A single token produced by the CP lexer.
69#[derive(Debug, Clone, PartialEq)]
70pub struct Token {
71    /// What kind of token this is.
72    pub kind: TokenKind,
73    /// Where in the source text this token appears.
74    pub span: Span,
75}
76
77/// Tokenize a CP source string into a flat vector of [`Token`]s.
78///
79/// Line comments (starting with `//`) are stripped. Whitespace is consumed
80/// between tokens. Returns an error for any character that doesn't belong
81/// to a valid token.
82pub fn tokenize(input: &str) -> Result<Vec<Token>, ParseError> {
83    let mut tokens = Vec::new();
84    let bytes = input.as_bytes();
85    let mut pos = 0;
86    let mut line = 1usize;
87    let mut col = 1usize;
88
89    while pos < bytes.len() {
90        let b = bytes[pos];
91
92        // Skip whitespace
93        if b.is_ascii_whitespace() {
94            if b == b'\n' {
95                line += 1;
96                col = 1;
97            } else {
98                col += 1;
99            }
100            pos += 1;
101            continue;
102        }
103
104        // Line comments (must check before `/` as single-char operator)
105        if b == b'/' && pos + 1 < bytes.len() && bytes[pos + 1] == b'/' {
106            // Skip to end of line
107            while pos < bytes.len() && bytes[pos] != b'\n' {
108                pos += 1;
109            }
110            continue;
111        }
112
113        let span = Span::new(pos, line, col);
114
115        // Two-char operators first
116        if pos + 1 < bytes.len() {
117            let next = bytes[pos + 1];
118            let two_char = match (b, next) {
119                (b'<', b'=') => Some(TokenKind::Le),
120                (b'>', b'=') => Some(TokenKind::Ge),
121                (b'=', b'=') => Some(TokenKind::EqEq),
122                (b'!', b'=') => Some(TokenKind::Ne),
123                (b'.', b'.') => Some(TokenKind::DotDot),
124                _ => None,
125            };
126            if let Some(kind) = two_char {
127                tokens.push(Token { kind, span });
128                pos += 2;
129                col += 2;
130                continue;
131            }
132        }
133
134        // Single-char tokens
135        let single = match b {
136            b'(' => Some(TokenKind::LParen),
137            b')' => Some(TokenKind::RParen),
138            b'{' => Some(TokenKind::LBrace),
139            b'}' => Some(TokenKind::RBrace),
140            b'[' => Some(TokenKind::LBracket),
141            b']' => Some(TokenKind::RBracket),
142            b',' => Some(TokenKind::Comma),
143            b':' => Some(TokenKind::Colon),
144            b'+' => Some(TokenKind::Plus),
145            b'-' => Some(TokenKind::Minus),
146            b'*' => Some(TokenKind::Star),
147            b'/' => Some(TokenKind::Slash),
148            b'@' => Some(TokenKind::At),
149            b'<' => Some(TokenKind::Lt),
150            b'>' => Some(TokenKind::Gt),
151            b'=' => Some(TokenKind::Eq),
152            _ => None,
153        };
154        if let Some(kind) = single {
155            tokens.push(Token { kind, span });
156            pos += 1;
157            col += 1;
158            continue;
159        }
160
161        // Numbers (integer or float)
162        if b.is_ascii_digit() {
163            let start = pos;
164            while pos < bytes.len() && bytes[pos].is_ascii_digit() {
165                pos += 1;
166            }
167            // Check for decimal point: must be `.` followed by a digit (not `..` range)
168            let is_float = pos < bytes.len()
169                && bytes[pos] == b'.'
170                && pos + 1 < bytes.len()
171                && bytes[pos + 1].is_ascii_digit();
172            if is_float {
173                pos += 1; // consume '.'
174                while pos < bytes.len() && bytes[pos].is_ascii_digit() {
175                    pos += 1;
176                }
177                let text = &input[start..pos];
178                let value: f64 = match text.parse() {
179                    Ok(value) => value,
180                    Err(_) => {
181                        return Err(ParseError::new(
182                            format!("invalid float literal: {text}"),
183                            span,
184                        ));
185                    }
186                };
187                tokens.push(Token {
188                    kind: TokenKind::Float(value),
189                    span,
190                });
191            } else {
192                let text = &input[start..pos];
193                let value: i64 = text.parse().map_err(|_| {
194                    ParseError::new(format!("invalid number literal: {text}"), span)
195                })?;
196                tokens.push(Token {
197                    kind: TokenKind::Number(value),
198                    span,
199                });
200            }
201            col += pos - start;
202            continue;
203        }
204
205        // Identifiers (start with letter or underscore)
206        if b.is_ascii_alphabetic() || b == b'_' {
207            let start = pos;
208            while pos < bytes.len() && (bytes[pos].is_ascii_alphanumeric() || bytes[pos] == b'_') {
209                pos += 1;
210            }
211            let text = input[start..pos].to_string();
212            tokens.push(Token {
213                kind: TokenKind::Ident(text),
214                span,
215            });
216            col += pos - start;
217            continue;
218        }
219
220        return Err(ParseError::new(
221            format!("unexpected character: '{}'", b as char),
222            span,
223        ));
224    }
225
226    Ok(tokens)
227}
228
229#[cfg(test)]
230mod tests {
231    use super::*;
232
233    #[test]
234    fn test_basic_tokens() {
235        let tokens = tokenize("model foo").unwrap();
236        assert_eq!(tokens.len(), 2);
237        assert_eq!(tokens[0].kind, TokenKind::Ident("model".to_string()));
238        assert_eq!(tokens[1].kind, TokenKind::Ident("foo".to_string()));
239    }
240
241    #[test]
242    fn test_operators() {
243        let tokens = tokenize("<= >= == != < > = ..").unwrap();
244        let kinds: Vec<_> = tokens.iter().map(|t| &t.kind).collect();
245        assert_eq!(
246            kinds,
247            vec![
248                &TokenKind::Le,
249                &TokenKind::Ge,
250                &TokenKind::EqEq,
251                &TokenKind::Ne,
252                &TokenKind::Lt,
253                &TokenKind::Gt,
254                &TokenKind::Eq,
255                &TokenKind::DotDot,
256            ]
257        );
258    }
259
260    #[test]
261    fn test_punctuation() {
262        let tokens = tokenize("( ) { } [ ] , :").unwrap();
263        assert_eq!(tokens.len(), 8);
264        assert_eq!(tokens[0].kind, TokenKind::LParen);
265        assert_eq!(tokens[1].kind, TokenKind::RParen);
266        assert_eq!(tokens[2].kind, TokenKind::LBrace);
267        assert_eq!(tokens[3].kind, TokenKind::RBrace);
268        assert_eq!(tokens[4].kind, TokenKind::LBracket);
269        assert_eq!(tokens[5].kind, TokenKind::RBracket);
270        assert_eq!(tokens[6].kind, TokenKind::Comma);
271        assert_eq!(tokens[7].kind, TokenKind::Colon);
272    }
273
274    #[test]
275    fn test_numbers() {
276        let tokens = tokenize("42 0 100").unwrap();
277        assert_eq!(tokens[0].kind, TokenKind::Number(42));
278        assert_eq!(tokens[1].kind, TokenKind::Number(0));
279        assert_eq!(tokens[2].kind, TokenKind::Number(100));
280    }
281
282    #[test]
283    fn test_line_comment() {
284        let tokens = tokenize("model foo // this is a comment\nvariables").unwrap();
285        assert_eq!(tokens.len(), 3);
286        assert_eq!(tokens[0].kind, TokenKind::Ident("model".to_string()));
287        assert_eq!(tokens[1].kind, TokenKind::Ident("foo".to_string()));
288        assert_eq!(tokens[2].kind, TokenKind::Ident("variables".to_string()));
289    }
290
291    #[test]
292    fn test_span_tracking() {
293        let tokens = tokenize("a\nb").unwrap();
294        assert_eq!(tokens[0].span, Span::new(0, 1, 1));
295        assert_eq!(tokens[1].span, Span::new(2, 2, 1));
296    }
297
298    #[test]
299    fn test_arithmetic_operators() {
300        let tokens = tokenize("+ - *").unwrap();
301        assert_eq!(tokens[0].kind, TokenKind::Plus);
302        assert_eq!(tokens[1].kind, TokenKind::Minus);
303        assert_eq!(tokens[2].kind, TokenKind::Star);
304    }
305
306    #[test]
307    fn test_unexpected_char() {
308        let err = tokenize("model #foo").unwrap_err();
309        assert!(err.message.contains("unexpected character"));
310    }
311
312    #[test]
313    fn test_float_literals() {
314        let tokens = tokenize("3.25 0.5 100.0").unwrap();
315        assert_eq!(tokens[0].kind, TokenKind::Float(3.25));
316        assert_eq!(tokens[1].kind, TokenKind::Float(0.5));
317        assert_eq!(tokens[2].kind, TokenKind::Float(100.0));
318    }
319
320    #[test]
321    fn test_integer_followed_by_dotdot_not_float() {
322        // "0..10" should be Number(0), DotDot, Number(10) — not Float
323        let tokens = tokenize("0..10").unwrap();
324        assert_eq!(tokens[0].kind, TokenKind::Number(0));
325        assert_eq!(tokens[1].kind, TokenKind::DotDot);
326        assert_eq!(tokens[2].kind, TokenKind::Number(10));
327    }
328
329    #[test]
330    fn test_slash_operator() {
331        let tokens = tokenize("x / 2").unwrap();
332        assert_eq!(tokens[1].kind, TokenKind::Slash);
333    }
334
335    #[test]
336    fn test_slash_vs_comment() {
337        // "/" alone is Slash; "//" is a comment
338        let tokens = tokenize("x / y // comment").unwrap();
339        assert_eq!(tokens.len(), 3);
340        assert_eq!(tokens[1].kind, TokenKind::Slash);
341    }
342
343    #[test]
344    fn test_at_token() {
345        let tokens = tokenize("@model lp").unwrap();
346        assert_eq!(tokens[0].kind, TokenKind::At);
347        assert_eq!(tokens[1].kind, TokenKind::Ident("model".to_string()));
348        assert_eq!(tokens[2].kind, TokenKind::Ident("lp".to_string()));
349    }
350
351    #[test]
352    fn test_integer_overflow_reports_error() {
353        let err = tokenize("999999999999999999999999999999999999").unwrap_err();
354        assert!(err.message.contains("invalid number literal"));
355    }
356}
jia_parse/jia_lang/lexer.rs

jia_parse/jia_lang/
lexer.rs
