1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53
use crate::Token; use crate::TokenType; use crate::Extras; use logos::{Logos}; use std::convert::TryFrom; use std::ops::Range; use text_size::{TextRange, TextSize}; #[derive(Clone)] pub struct Lexer<'l> { inner: logos::Lexer<'l, TokenType>, } impl<'l> Lexer<'l> { pub fn new(source: &'l str) -> Self { Self { inner: TokenType::lexer_with_extras(source, Extras { line: 1 }), } } pub fn extras(&self) -> Extras { self.inner.extras.clone() } pub fn peek(&mut self) -> Option<Token> { let mut clone = self.clone(); clone.next() } } impl<'l> Iterator for Lexer<'l> { type Item = Token<'l>; fn next(&mut self) -> Option<Self::Item> { let kind = self.inner.next()?; let slice = self.inner.slice(); let range = { let Range { start, end } = self.inner.span(); TextRange::new( TextSize::try_from(start).unwrap(), TextSize::try_from(end).unwrap(), ) }; Some(Self::Item { kind, slice, range }) } }