1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39
use crate::Token; use crate::TokenType; use logos::Logos; use std::convert::TryFrom; use std::ops::Range; use text_size::{TextRange, TextSize}; pub struct Lexer<'l> { inner: logos::Lexer<'l, TokenType<'l>>, } impl<'l> Lexer<'l> { pub fn new(source: &'l str) -> Self { Self { inner: TokenType::lexer(source), } } } impl<'l> Iterator for Lexer<'l> { type Item = Token<'l>; fn next(&mut self) -> Option<Self::Item> { let kind = self.inner.next()?; let slice = self.inner.slice(); let range = { let Range { start, end } = self.inner.span(); TextRange::new( TextSize::try_from(start).unwrap(), TextSize::try_from(end).unwrap(), ) }; Some(Self::Item { kind, slice, range }) } }