[][src]Struct logos::Lexer

pub struct Lexer<'source, Token: Logos<'source>> {
    pub extras: Token::Extras,
    // some fields omitted

Lexer is the main struct of the crate that allows you to read through a Source and produce tokens for enums implementing the Logos trait.


extras: Token::Extras

Extras associated with the Token.


impl<'source, Token: Logos<'source>> Lexer<'source, Token>[src]

pub fn new(source: &'source Token::Source) -> Self[src]

Create a new Lexer.

Due to type inference, it might be more ergonomic to construct it by calling Token::lexer on any Token with derived Logos.

pub fn source(&self) -> &'source Token::Source[src]

Source from which this Lexer is reading tokens.

pub fn spanned(self) -> SpannedIter<'source, Token>[src]

Wrap the Lexer in an Iterator that produces tuples of (Token, Span).


use logos::Logos;

#[derive(Logos, Debug, PartialEq)]
enum Example {
    #[regex(r"[ \n\t\f]+", logos::skip)]

    #[regex("-?[0-9]+", |lex| lex.slice().parse())]

    #[regex("-?[0-9]+\\.[0-9]+", |lex| lex.slice().parse())]

let tokens: Vec<_> = Example::lexer("42 3.14 -5 f").spanned().collect();

        (Example::Integer(42), 0..2),
        (Example::Float(3.14), 3..7),
        (Example::Integer(-5), 8..10),
        (Example::Error, 11..12), // 'f' is not a recognized token

pub fn span(&self) -> Span[src]

Get the range for the current token in Source.

pub fn slice(&self) -> &'source <Token::Source as Source>::Slice[src]

Get a string slice of the current token.

pub fn remainder(&self) -> &'source <Token::Source as Source>::Slice[src]

Get a slice of remaining source, starting at the end of current token.

pub fn morph<Token2>(self) -> Lexer<'source, Token2> where
    Token2: Logos<'source, Source = Token::Source>,
    Token::Extras: Into<Token2::Extras>, 

Turn this lexer into a lexer for a new token type.

The new lexer continues to point at the same span as the current lexer, and the current token becomes the error token of the new token type. If you want to start reading from the new lexer immediately, consider using Lexer::advance_as instead.

pub fn bump(&mut self, n: usize)[src]

Bumps the end of currently lexed token by n bytes.


Panics if adding n to current offset would place the Lexer beyond the last byte, or in the middle of an UTF-8 code point (does not apply when lexing raw &[u8]).

Trait Implementations

impl<'source, Token> Clone for Lexer<'source, Token> where
    Token: Logos<'source> + Clone,
    Token::Extras: Clone

impl<'source, Token> Iterator for Lexer<'source, Token> where
    Token: Logos<'source>, 

type Item = Token

The type of the elements being iterated over.

Auto Trait Implementations

impl<'source, Token> RefUnwindSafe for Lexer<'source, Token> where
    Token: RefUnwindSafe,
    <Token as Logos<'source>>::Extras: RefUnwindSafe,
    <Token as Logos<'source>>::Source: RefUnwindSafe

impl<'source, Token> Send for Lexer<'source, Token> where
    Token: Send,
    <Token as Logos<'source>>::Extras: Send,
    <Token as Logos<'source>>::Source: Sync

impl<'source, Token> Sync for Lexer<'source, Token> where
    Token: Sync,
    <Token as Logos<'source>>::Extras: Sync,
    <Token as Logos<'source>>::Source: Sync

impl<'source, Token> Unpin for Lexer<'source, Token> where
    Token: Unpin,
    <Token as Logos<'source>>::Extras: Unpin

impl<'source, Token> UnwindSafe for Lexer<'source, Token> where
    Token: UnwindSafe,
    <Token as Logos<'source>>::Extras: UnwindSafe,
    <Token as Logos<'source>>::Source: RefUnwindSafe

Blanket Implementations

impl<T> Any for T where
    T: 'static + ?Sized

impl<T> Borrow<T> for T where
    T: ?Sized

impl<T> BorrowMut<T> for T where
    T: ?Sized

impl<T> From<T> for T[src]

impl<T, U> Into<U> for T where
    U: From<T>, 

impl<I> IntoIterator for I where
    I: Iterator

type Item = <I as Iterator>::Item

The type of the elements being iterated over.

type IntoIter = I

Which kind of iterator are we turning this into?

impl<T> ToOwned for T where
    T: Clone

type Owned = T

The resulting type after obtaining ownership.

impl<T, U> TryFrom<U> for T where
    U: Into<T>, 

type Error = Infallible

The type returned in the event of a conversion error.

impl<T, U> TryInto<U> for T where
    U: TryFrom<T>, 

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.