datex_core/ast/
lexer.rs

1use std::{
2    fmt::{self, Display},
3    ops::Range,
4};
5
6use logos::{Lexer, Logos};
7
8pub type SourceId = usize;
9
10#[derive(Debug, Clone, PartialEq, Hash)]
11pub struct Loc {
12    pub source: SourceId,
13    pub span: core::ops::Range<usize>,
14}
15use strum::IntoEnumIterator;
16
17use crate::values::core_values::{
18    decimal::typed_decimal::DecimalTypeVariant,
19    integer::typed_integer::IntegerTypeVariant,
20};
21
22impl Loc {
23    pub fn new(source: SourceId, span: core::ops::Range<usize>) -> Self {
24        Self { source, span }
25    }
26}
27fn extract_line_doc(lex: &mut Lexer<Token>) -> String {
28    lex.slice()[3..].to_owned()
29}
30#[derive(Logos, Debug, Clone, PartialEq, Eq)]
31#[logos(error = Range<usize>)]
32// single line comments
33#[logos(skip r"//[^\n]*")]
34// multiline comments
35#[logos(skip r"/\*[^*]*\*+(?:[^/*][^*]*\*+)*/")]
36// #[logos(skip r"[ \n\t\r\f]+")]
37#[rustfmt::skip]
38pub enum Token {
39    #[regex(r"///[^\n]*", extract_line_doc)]
40    LineDoc(String),
41
42    // Operators & Separators
43    #[token("(")] LeftParen,
44    #[token(")")] RightParen,
45    #[token("[")] LeftBracket,
46    #[token("]")] RightBracket,
47    #[token("{")] LeftCurly,
48    #[token("}")] RightCurly,
49    #[token("<")] LeftAngle,
50    #[token(">")] RightAngle,
51
52    #[token("%")] Percent,
53    #[token("+")] Plus,
54    #[token("-")] Minus,
55    #[token("*")] Star,
56    #[token("/")] Slash,
57    #[token(":")] Colon,
58    #[token("::")] DoubleColon,
59    #[token(":::")] TripleColon,
60    #[token(";")] Semicolon,
61    #[token(".")] Dot,
62    #[token(",")] Comma,
63    #[token("=")] Assign,
64
65    #[token("++")] Increment,
66    #[token("--")] Decrement,
67    #[token("&&")] Conjunction,
68    #[token("||")] Disjunction,
69    #[token("+=")] AddAssign,
70    #[token("-=")] SubAssign,
71    #[token("*=")] MulAssign,
72    #[token("/=")] DivAssign,
73    #[token("%=")] ModAssign,
74
75    #[token("->")] Arrow,
76    #[token("=>")] FatArrow,
77    #[token("..")] Range,
78    #[token("..=")] RangeInclusive,
79    #[token("...")] Spread,
80    #[token("@")] At,
81    #[token("&")] Ampersand,
82    #[token("|")] Pipe,
83    #[token("!")] Exclamation,
84    #[token("`")] Backtick,
85
86    #[token("<=")] LessEqual,
87    // #[token(">=")] GreaterEqual, // can not use because of generic overlap type X<test>= 4;
88    #[token("!=")] NotStructuralEqual,
89    #[token("!==")] NotEqual,
90    #[token("==")] StructuralEqual,
91    #[token("===")] Equal,
92    #[token("is")] Is,
93    #[token("matches")] Matches,
94
95    // Keywords
96    #[token("true")] True,
97    #[token("false")] False,
98    #[token("null")] Null,
99
100    #[token("?")] Placeholder,
101    #[token("const")] Const,
102    #[token("var")] Variable,
103    #[token("mut")] Mutable,
104    #[token("final")] Final,
105    #[token("function")] Function,
106    #[token("if")] If,
107    #[token("else")] Else,
108
109    // pointer address (e.g. $1234ab, exactly 3, 5 or 26 bytes)
110    #[regex(r"\$(?:[0-9a-fA-F]{6}|[0-9a-fA-F]{10}|[0-9a-fA-F]{52})", allocated_string)] PointerAddress(String),
111
112    // decimal literals (infinity, nan)
113    // FIXME #366 remove +- from lexing and let it be handled by unary operators as for other numbers
114    #[regex(r"[+-]?[Ii]nfinity", allocated_string)] Infinity(String),
115    #[regex(r"[+-]?(?:nan|NaN)")] Nan,
116
117    // Value literals
118    // decimal
119    // ### Supported formats:
120    // - Standard decimals:
121    //   - `123.456`
122    //   - `0.001`
123    //   - `.789`
124    //   - `123.`
125    //   - `3.e10`
126    //   - `534.e-124`
127    // - Decimals with exponent:
128    //   - `1.23e10`
129    //   - `4.56E-3`
130    //   - `789e+2`
131    //   - `42e0`
132    // - Integer with exponent (no decimal point):
133    //   - `123e5`
134    //   - `42E-1`
135    // - Special values:
136    //   - `NaN`, `nan`
137    //   - `Infinity`, `infinity`
138    // - Optional leading sign is supported for all formats:
139    //   - `-123.45`, `+123.45`
140    //   - `-Infinity`, `+Infinity`
141    //   - `-3.e10`, `+3.e10`
142    #[regex(r"(((0|[1-9])(\d|_)*)?\.(\d|_)+(?:[eE][+-]?(\d|_)+)?|((0|[1-9])(\d|_)*)\.|((0|[1-9])(\d|_)*)[eE][+-]?(\d|_)+)(?:f32|f64)?", parse_typed_literal::<DecimalTypeVariant>)] 
143    DecimalLiteral(DecimalLiteral),
144    // integer
145    // ### Supported formats:
146    // - Hexadecimal integers:
147    //     - `0x1A2B3C4D5E6F`
148    //     - `0X1A2B3C4D5E6F`
149    // - Octal integers:
150    //     - `0o755`
151    //     - `0O755`
152    // - Binary integers:
153    //     - `0b101010`
154    //     - `0B101010`
155    // - Decimal integers:
156    //     - `123456789`
157    //     - `-123456789`
158    // - Integers with underscores:
159    //     - `1_234_567`
160    //     - `-1_234_567`
161    // - Decimal integers with leading zeros:
162    // - `0123`
163    // - `-0123`
164    #[regex(r"(0|[1-9])(\d|_)*(?:u8|u16|u32|u64|u128|i8|i16|i32|i64|i128|big)?", parse_typed_literal::<IntegerTypeVariant>)] 
165    DecimalIntegerLiteral(IntegerLiteral),
166    // binary integer
167    #[regex(r"0[bB][01_]+(?:u8|u16|u32|u64|u128|i8|i16|i32|i64|i128|big)?", parse_typed_literal::<IntegerTypeVariant>)] 
168    BinaryIntegerLiteral(IntegerLiteral),
169    // octal integer
170    #[regex(r"0[oO][0-7_]+(?:u8|u16|u32|u64|u128|i8|i16|i32|i64|i128|big)?", parse_typed_literal::<IntegerTypeVariant>)] 
171    OctalIntegerLiteral(IntegerLiteral),
172    // hexadecimal integer
173    #[regex(r"0[xX][0-9a-fA-F_]+(?:u8|u16|u32|u64|u128|i8|i16|i32|i64|i128|big)?", parse_typed_literal::<IntegerTypeVariant>)] 
174    HexadecimalIntegerLiteral(IntegerLiteral),
175
176    // fraction (e.g. 1/2)
177    #[regex(r"\d+/\d+", allocated_string)] 
178    FractionLiteral(String),
179
180    #[regex(r#"[a-z0-9]*("(?:\\.|[^\\"])*"|'(?:\\.|[^\\'])*')"#, allocated_string)] StringLiteral(String),
181
182
183    #[regex(r"@[+@]?[a-zA-Z0-9_-]+", allocated_string)] Endpoint(String),
184
185    // identifiers
186    #[regex(r"[_\p{L}][_\p{L}\p{N}]*", allocated_string)] Identifier(String),
187
188    // number slots (starting with #, followed by digits)
189    #[regex(r"#\d+", allocated_string)] Slot(String),
190
191    // named slots (starting with #, followed by A-Z or a-z)
192    #[regex(r"#[_a-zA-Z]+", allocated_string)] NamedSlot(String),
193
194    #[regex(r"[ \t\n\f]")]
195    Whitespace,
196
197
198    Error
199}
200
201impl Token {
202    pub fn as_string(&self) -> String {
203        let literal_token = match self {
204            Token::LeftParen => Some("("),
205            Token::RightParen => Some(")"),
206            Token::LeftBracket => Some("["),
207            Token::RightBracket => Some("]"),
208            Token::LeftCurly => Some("{"),
209            Token::RightCurly => Some("}"),
210            Token::LeftAngle => Some("<"),
211            Token::RightAngle => Some(">"),
212            Token::Percent => Some("%"),
213            Token::Plus => Some("+"),
214            Token::Minus => Some("-"),
215            Token::Slash => Some("/"),
216            Token::Colon => Some(":"),
217            Token::DoubleColon => Some("::"),
218            Token::TripleColon => Some(":::"),
219            Token::Semicolon => Some(";"),
220            Token::Dot => Some("."),
221            Token::Comma => Some(","),
222            Token::Assign => Some("="),
223            Token::Increment => Some("++"),
224            Token::Decrement => Some("--"),
225            Token::Conjunction => Some("&&"),
226            Token::Disjunction => Some("||"),
227            Token::AddAssign => Some("+="),
228            Token::SubAssign => Some("-="),
229            Token::MulAssign => Some("*="),
230            Token::DivAssign => Some("/="),
231            Token::ModAssign => Some("%="),
232            Token::Arrow => Some("->"),
233            Token::FatArrow => Some("=>"),
234            Token::Range => Some(".."),
235            Token::RangeInclusive => Some("..="),
236            Token::Spread => Some("..."),
237            Token::At => Some("@"),
238            Token::Ampersand => Some("&"),
239            Token::Pipe => Some("|"),
240            Token::Backtick => Some("`"),
241            Token::LessEqual => Some("<="),
242            // Token::GreaterEqual => Some(">="),
243            Token::NotStructuralEqual => Some("!="),
244            Token::NotEqual => Some("!=="),
245            Token::StructuralEqual => Some("=="),
246            Token::Equal => Some("==="),
247            Token::Is => Some("is"),
248            Token::True => Some("true"),
249            Token::False => Some("false"),
250            Token::Null => Some("null"),
251            Token::Placeholder => Some("?"),
252            Token::Const => Some("const"),
253            Token::Variable => Some("var"),
254            Token::Mutable => Some("mut"),
255            Token::Final => Some("final"),
256            Token::Function => Some("function"),
257            Token::Whitespace => Some(" "),
258            Token::Error => Some("error"),
259            Token::Infinity(_) => Some("infinity"),
260            Token::Nan => Some("nan"),
261            Token::Star => Some("*"),
262            Token::Exclamation => Some("!"),
263            _ => None,
264        };
265        if let Some(token) = literal_token {
266            return format!("'{}'", token);
267        }
268
269        let identifier_token = match self {
270            Token::LineDoc(_) => "line doc",
271            Token::DecimalLiteral(_) => "decimal literal",
272            Token::DecimalIntegerLiteral(_) => "decimal integer literal",
273            Token::BinaryIntegerLiteral(_) => "binary integer literal",
274            Token::OctalIntegerLiteral(_) => "octal integer literal",
275            Token::HexadecimalIntegerLiteral(_) => {
276                "hexadecimal integer literal"
277            }
278            Token::FractionLiteral(_) => "fraction literal",
279            Token::StringLiteral(_) => "string literal",
280            Token::Endpoint(_) => "endpoint",
281            Token::Slot(_) => "slot",
282            Token::NamedSlot(_) => "named slot",
283            Token::Error => "error",
284            Token::Identifier(s) => s,
285            Token::Matches => "matches",
286            Token::If => "if",
287            Token::Else => "else",
288            e => todo!("#367 Unhandled token in as_string: {:?}", e),
289        };
290
291        identifier_token.to_string()
292    }
293}
294
295pub type IntegerLiteral = TypedLiteral<IntegerTypeVariant>;
296pub type DecimalLiteral = TypedLiteral<DecimalTypeVariant>;
297
298#[derive(Debug, Clone, PartialEq, Eq)]
299pub struct TypedLiteral<T> {
300    pub value: String,
301    pub variant: Option<T>,
302}
303
304impl Display for TypedLiteral<IntegerTypeVariant> {
305    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
306        if let Some(variant) = &self.variant {
307            write!(f, "{}{}", self.value, variant.as_ref())
308        } else {
309            write!(f, "{}", self.value)
310        }
311    }
312}
313
314trait TypeSuffix: IntoEnumIterator + Copy + AsRef<str> {}
315impl<T> TypeSuffix for T where T: IntoEnumIterator + Copy + AsRef<str> {}
316
317fn parse_typed_literal<T: TypeSuffix>(
318    lex: &mut Lexer<Token>,
319) -> TypedLiteral<T> {
320    let mut variant = None;
321    let mut number_part = lex.slice();
322    for suffix in T::iter() {
323        let suffix_str = suffix.as_ref();
324        if number_part.ends_with(suffix_str) {
325            variant = Some(suffix);
326            number_part = &number_part[..number_part.len() - suffix_str.len()];
327            break;
328        }
329    }
330    TypedLiteral {
331        value: number_part.to_string(),
332        variant,
333    }
334}
335
336impl fmt::Display for Token {
337    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
338        write!(f, "{self:?}")
339    }
340}
341
342#[inline(always)]
343fn allocated_string(lex: &mut Lexer<Token>) -> String {
344    lex.slice().to_owned()
345}
346
347#[cfg(test)]
348mod tests {
349    use super::*;
350    use logos::Logos;
351
352    #[test]
353    fn integer() {
354        let mut lexer = Token::lexer("42");
355        assert_eq!(
356            lexer.next().unwrap(),
357            Ok(Token::DecimalIntegerLiteral(IntegerLiteral {
358                value: "42".to_string(),
359                variant: None
360            }))
361        );
362    }
363
364    #[test]
365    fn integer_type() {
366        let mut lexer = Token::lexer("42u8");
367        let res = lexer.next().unwrap();
368        if let Ok(Token::DecimalIntegerLiteral(literal)) = res {
369            assert_eq!(literal.value, "42");
370            assert_eq!(literal.variant, Some(IntegerTypeVariant::U8));
371            assert_eq!(format!("{}", literal), "42u8".to_string());
372        } else {
373            panic!("Expected DecimalIntegerLiteral with variant U8");
374        }
375
376        let mut lexer = Token::lexer("42");
377        let res = lexer.next().unwrap();
378        if let Ok(Token::DecimalIntegerLiteral(literal)) = res {
379            assert_eq!(literal.value, "42");
380            assert_eq!(literal.variant, None);
381            assert_eq!(format!("{}", literal), "42".to_string());
382        } else {
383            panic!("Expected DecimalIntegerLiteral with no variant");
384        }
385    }
386
387    #[test]
388    fn integer_with_type() {
389        let mut lexer = Token::lexer("42u8");
390        assert_eq!(
391            lexer.next().unwrap(),
392            Ok(Token::DecimalIntegerLiteral(IntegerLiteral {
393                value: "42".to_string(),
394                variant: Some(IntegerTypeVariant::U8)
395            }))
396        );
397
398        let mut lexer = Token::lexer("42i32");
399        assert_eq!(
400            lexer.next().unwrap(),
401            Ok(Token::DecimalIntegerLiteral(IntegerLiteral {
402                value: "42".to_string(),
403                variant: Some(IntegerTypeVariant::I32)
404            }))
405        );
406
407        let mut lexer = Token::lexer("42big");
408        assert_eq!(
409            lexer.next().unwrap(),
410            Ok(Token::DecimalIntegerLiteral(IntegerLiteral {
411                value: "42".to_string(),
412                variant: Some(IntegerTypeVariant::Big)
413            }))
414        );
415    }
416
417    #[test]
418    fn decimal() {
419        let mut lexer = Token::lexer("3.14");
420        assert_eq!(
421            lexer.next().unwrap(),
422            Ok(Token::DecimalLiteral(TypedLiteral::<DecimalTypeVariant> {
423                value: "3.14".to_string(),
424                variant: None
425            }))
426        );
427    }
428
429    #[test]
430    fn decimal_with_type() {
431        let mut lexer = Token::lexer("3.14f32");
432        assert_eq!(
433            lexer.next().unwrap(),
434            Ok(Token::DecimalLiteral(TypedLiteral::<DecimalTypeVariant> {
435                value: "3.14".to_string(),
436                variant: Some(DecimalTypeVariant::F32)
437            }))
438        );
439
440        let mut lexer = Token::lexer("3.14f64");
441        assert_eq!(
442            lexer.next().unwrap(),
443            Ok(Token::DecimalLiteral(TypedLiteral::<DecimalTypeVariant> {
444                value: "3.14".to_string(),
445                variant: Some(DecimalTypeVariant::F64)
446            }))
447        );
448    }
449
450    #[test]
451    fn infinity() {
452        let mut lexer = Token::lexer("Infinity");
453        assert_eq!(
454            lexer.next().unwrap(),
455            Ok(Token::Infinity("Infinity".to_string()))
456        );
457
458        let mut lexer = Token::lexer("infinity");
459        assert_eq!(
460            lexer.next().unwrap(),
461            Ok(Token::Infinity("infinity".to_string()))
462        );
463
464        // FIXME #368, this is not ideal, should be handled by unary minus
465        let mut lexer = Token::lexer("-Infinity");
466        assert_eq!(
467            lexer.next().unwrap(),
468            Ok(Token::Infinity("-Infinity".to_string()))
469        );
470
471        let mut lexer = Token::lexer("+Infinity");
472        assert_eq!(
473            lexer.next().unwrap(),
474            Ok(Token::Infinity("+Infinity".to_string()))
475        );
476    }
477
478    #[test]
479    fn nan() {
480        let mut lexer = Token::lexer("NaN");
481        assert_eq!(lexer.next().unwrap(), Ok(Token::Nan));
482
483        let mut lexer = Token::lexer("nan");
484        assert_eq!(lexer.next().unwrap(), Ok(Token::Nan));
485
486        let mut lexer = Token::lexer("-NaN");
487        assert_eq!(lexer.next().unwrap(), Ok(Token::Nan));
488
489        let mut lexer = Token::lexer("+NaN");
490        assert_eq!(lexer.next().unwrap(), Ok(Token::Nan));
491    }
492
493    #[test]
494    fn fraction() {
495        let mut lexer = Token::lexer("1/2");
496        assert_eq!(
497            lexer.next().unwrap(),
498            Ok(Token::FractionLiteral("1/2".to_string()))
499        );
500
501        let mut lexer = Token::lexer("3/4");
502        assert_eq!(
503            lexer.next().unwrap(),
504            Ok(Token::FractionLiteral("3/4".to_string()))
505        );
506
507        let mut lexer = Token::lexer("5111/6");
508        assert_eq!(
509            lexer.next().unwrap(),
510            Ok(Token::FractionLiteral("5111/6".to_string()))
511        );
512    }
513
514    #[test]
515    fn hexadecimal_integer() {
516        let mut lexer = Token::lexer("0x1A3F");
517        assert_eq!(
518            lexer.next().unwrap(),
519            Ok(Token::HexadecimalIntegerLiteral(IntegerLiteral {
520                value: "0x1A3F".to_string(),
521                variant: None
522            }))
523        );
524
525        let mut lexer = Token::lexer("0XABC");
526        assert_eq!(
527            lexer.next().unwrap(),
528            Ok(Token::HexadecimalIntegerLiteral(IntegerLiteral {
529                value: "0XABC".to_string(),
530                variant: None
531            }))
532        );
533    }
534
535    #[test]
536    fn binary_integer() {
537        let mut lexer = Token::lexer("0b1010");
538        assert_eq!(
539            lexer.next().unwrap(),
540            Ok(Token::BinaryIntegerLiteral(IntegerLiteral {
541                value: "0b1010".to_string(),
542                variant: None
543            }))
544        );
545
546        let mut lexer = Token::lexer("0B1101");
547        assert_eq!(
548            lexer.next().unwrap(),
549            Ok(Token::BinaryIntegerLiteral(IntegerLiteral {
550                value: "0B1101".to_string(),
551                variant: None
552            }))
553        );
554    }
555
556    #[test]
557    fn octal_integer() {
558        let mut lexer = Token::lexer("0o755");
559        assert_eq!(
560            lexer.next().unwrap(),
561            Ok(Token::OctalIntegerLiteral(IntegerLiteral {
562                value: "0o755".to_string(),
563                variant: None
564            }))
565        );
566
567        let mut lexer = Token::lexer("0O644");
568        assert_eq!(
569            lexer.next().unwrap(),
570            Ok(Token::OctalIntegerLiteral(IntegerLiteral {
571                value: "0O644".to_string(),
572                variant: None
573            }))
574        );
575    }
576
577    #[test]
578    fn integers_with_underscores() {
579        let mut lexer = Token::lexer("1_000");
580        assert_eq!(
581            lexer.next().unwrap(),
582            Ok(Token::DecimalIntegerLiteral(IntegerLiteral {
583                value: "1_000".to_string(),
584                variant: None
585            }))
586        );
587
588        let mut lexer = Token::lexer("0xFF_FF_FF");
589        assert_eq!(
590            lexer.next().unwrap(),
591            Ok(Token::HexadecimalIntegerLiteral(IntegerLiteral {
592                value: "0xFF_FF_FF".to_string(),
593                variant: None
594            }))
595        );
596
597        let mut lexer = Token::lexer("0b1010_1010");
598        assert_eq!(
599            lexer.next().unwrap(),
600            Ok(Token::BinaryIntegerLiteral(IntegerLiteral {
601                value: "0b1010_1010".to_string(),
602                variant: None
603            }))
604        );
605    }
606
607    #[test]
608    fn decimals_with_underscores() {
609        let mut lexer = Token::lexer("1_000.123_456");
610        assert_eq!(
611            lexer.next().unwrap(),
612            Ok(Token::DecimalLiteral(TypedLiteral::<DecimalTypeVariant> {
613                value: "1_000.123_456".to_string(),
614                variant: None
615            }))
616        );
617
618        let mut lexer = Token::lexer("0.123_456e2");
619        assert_eq!(
620            lexer.next().unwrap(),
621            Ok(Token::DecimalLiteral(TypedLiteral::<DecimalTypeVariant> {
622                value: "0.123_456e2".to_string(),
623                variant: None
624            }))
625        );
626
627        let mut lexer = Token::lexer("1.234_567e-8");
628        assert_eq!(
629            lexer.next().unwrap(),
630            Ok(Token::DecimalLiteral(TypedLiteral::<DecimalTypeVariant> {
631                value: "1.234_567e-8".to_string(),
632                variant: None
633            }))
634        );
635    }
636
637    #[test]
638    fn add() {
639        let mut lexer = Token::lexer("1 + 2");
640        assert_eq!(
641            lexer.next().unwrap(),
642            Ok(Token::DecimalIntegerLiteral(IntegerLiteral {
643                value: "1".to_string(),
644                variant: None
645            }))
646        );
647        assert_eq!(lexer.next().unwrap(), Ok(Token::Whitespace));
648        assert_eq!(lexer.next().unwrap(), Ok(Token::Plus));
649        assert_eq!(lexer.next().unwrap(), Ok(Token::Whitespace));
650        assert_eq!(
651            lexer.next().unwrap(),
652            Ok(Token::DecimalIntegerLiteral(IntegerLiteral {
653                value: "2".to_string(),
654                variant: None
655            }))
656        );
657        assert_eq!(lexer.next(), None);
658    }
659
660    #[test]
661    fn invalid_fraction() {
662        let mut lexer = Token::lexer("42.4/3");
663        assert_eq!(
664            lexer.next().unwrap(),
665            Ok(Token::DecimalLiteral(TypedLiteral::<DecimalTypeVariant> {
666                value: "42.4".to_string(),
667                variant: None
668            }))
669        );
670        assert_eq!(lexer.next().unwrap(), Ok(Token::Slash));
671        assert_eq!(
672            lexer.next().unwrap(),
673            Ok(Token::DecimalIntegerLiteral(IntegerLiteral {
674                value: "3".to_string(),
675                variant: None
676            }))
677        );
678    }
679
680    #[test]
681    fn equality() {
682        let mut lexer = Token::lexer("a == b");
683        assert_eq!(
684            lexer.next().unwrap(),
685            Ok(Token::Identifier("a".to_string()))
686        );
687        assert_eq!(lexer.next().unwrap(), Ok(Token::Whitespace));
688        assert_eq!(lexer.next().unwrap(), Ok(Token::StructuralEqual));
689        assert_eq!(lexer.next().unwrap(), Ok(Token::Whitespace));
690        assert_eq!(
691            lexer.next().unwrap(),
692            Ok(Token::Identifier("b".to_string()))
693        );
694        assert_eq!(lexer.next(), None);
695    }
696
697    #[test]
698    fn is_operator() {
699        let mut lexer = Token::lexer("a is b");
700        assert_eq!(
701            lexer.next().unwrap(),
702            Ok(Token::Identifier("a".to_string()))
703        );
704        assert_eq!(lexer.next().unwrap(), Ok(Token::Whitespace));
705        assert_eq!(lexer.next().unwrap(), Ok(Token::Is));
706        assert_eq!(lexer.next().unwrap(), Ok(Token::Whitespace));
707        assert_eq!(
708            lexer.next().unwrap(),
709            Ok(Token::Identifier("b".to_string()))
710        );
711        assert_eq!(lexer.next(), None);
712    }
713
714    #[test]
715    fn matches_operator() {
716        let mut lexer = Token::lexer("a matches b");
717        assert_eq!(
718            lexer.next().unwrap(),
719            Ok(Token::Identifier("a".to_string()))
720        );
721        assert_eq!(lexer.next().unwrap(), Ok(Token::Whitespace));
722        assert_eq!(lexer.next().unwrap(), Ok(Token::Matches));
723        assert_eq!(lexer.next().unwrap(), Ok(Token::Whitespace));
724        assert_eq!(
725            lexer.next().unwrap(),
726            Ok(Token::Identifier("b".to_string()))
727        );
728        assert_eq!(lexer.next(), None);
729    }
730
731    #[test]
732    fn line_doc() {
733        let mut lexer = Token::lexer("/// This is a line doc\n42");
734        assert_eq!(
735            lexer.next().unwrap(),
736            Ok(Token::LineDoc(" This is a line doc".to_string()))
737        );
738        assert_eq!(lexer.next().unwrap(), Ok(Token::Whitespace));
739        assert_eq!(
740            lexer.next().unwrap(),
741            Ok(Token::DecimalIntegerLiteral(IntegerLiteral {
742                value: "42".to_string(),
743                variant: None
744            }))
745        );
746        assert_eq!(lexer.next(), None);
747    }
748
749    #[test]
750    fn divide() {
751        let mut lexer = Token::lexer("8 /2");
752        assert_eq!(
753            lexer.next().unwrap(),
754            Ok(Token::DecimalIntegerLiteral(IntegerLiteral {
755                value: "8".to_string(),
756                variant: None
757            }))
758        );
759        assert_eq!(lexer.next().unwrap(), Ok(Token::Whitespace));
760        assert_eq!(lexer.next().unwrap(), Ok(Token::Slash));
761        assert_eq!(
762            lexer.next().unwrap(),
763            Ok(Token::DecimalIntegerLiteral(IntegerLiteral {
764                value: "2".to_string(),
765                variant: None
766            }))
767        );
768        assert_eq!(lexer.next(), None);
769    }
770}
datex_core/ast/lexer.rs

datex_core/ast/
lexer.rs
