Trait logos::Logos

pub trait Logos<'source>: Sized {
    type Extras;
    type Source: Source + ?Sized + 'source;
    type Error: Default + Clone + PartialEq + Debug + 'source;

    // Required method
    fn lex(lexer: &mut Lexer<'source, Self>);

    // Provided methods
    fn lexer(source: &'source Self::Source) -> Lexer<'source, Self> 
       where Self::Extras: Default { ... }
    fn lexer_with_extras(
        source: &'source Self::Source,
        extras: Self::Extras
    ) -> Lexer<'source, Self>  { ... }
}

Trait implemented for an enum representing all tokens. You should never have to implement it manually, use the #[derive(Logos)] attribute on your enum.

Required Associated Types

type Extras

Associated type Extras for the particular lexer. This can be set using #[logos(extras = MyExtras)] and accessed inside callbacks.

type Source: Source + ?Sized + 'source

Source type this token can be lexed from. This will default to str, unless one of the defined patterns explicitly uses non-unicode byte values or byte slices, in which case that implementation will use [u8].

type Error: Default + Clone + PartialEq + Debug + 'source

Error type returned by the lexer. This can be set using #[logos(error = MyError)]. Defaults to () if not set.

Required Methods

fn lex(lexer: &mut Lexer<'source, Self>)

The heart of Logos. Called by the Lexer. The implementation for this function is generated by the logos-derive crate.

Provided Methods

fn lexer(source: &'source Self::Source) -> Lexer<'source, Self>

where Self::Extras: Default,

Create a new instance of a Lexer that will produce tokens implementing this Logos.

Examples found in repository

examples/extras.rs (line 65)

fn main() {
    let src = fs::read_to_string(env::args().nth(1).expect("Expected file argument"))
        .expect("Failed to read file");

    let mut lex = Token::lexer(src.as_str());

    while let Some(token) = lex.next() {
        match token {
            Ok(Token::Word((line, column))) => {
                println!("Word '{}' found at ({}, {})", lex.slice(), line, column);
            }
            _ => (),
        }
    }
}

examples/custom_error.rs (line 45)

fn main() {
    // 256 overflows u8, since u8's max value is 255.
    // 'é' is not a valid ascii letter.
    let mut lex = Token::lexer("Hello 256 Jérome");

    assert_eq!(lex.next(), Some(Ok(Token::Word)));
    assert_eq!(lex.slice(), "Hello");

    assert_eq!(
        lex.next(),
        Some(Err(LexingError::InvalidInteger(
            "overflow error".to_owned()
        )))
    );
    assert_eq!(lex.slice(), "256");

    assert_eq!(lex.next(), Some(Ok(Token::Word)));
    assert_eq!(lex.slice(), "J");

    assert_eq!(lex.next(), Some(Err(LexingError::NonAsciiCharacter)));
    assert_eq!(lex.slice(), "é");

    assert_eq!(lex.next(), Some(Ok(Token::Word)));
    assert_eq!(lex.slice(), "rome");

    assert_eq!(lex.next(), None);
}

examples/json.rs (line 208)

fn main() {
    let filename = env::args().nth(1).expect("Expected file argument");
    let src = fs::read_to_string(&filename).expect("Failed to read file");

    let mut lexer = Token::lexer(src.as_str());

    match parse_value(&mut lexer) {
        Ok(value) => println!("{:#?}", value),
        Err((msg, span)) => {
            use ariadne::{ColorGenerator, Label, Report, ReportKind, Source};

            let mut colors = ColorGenerator::new();

            let a = colors.next();

            Report::build(ReportKind::Error, &filename, 12)
                .with_message(format!("Invalid JSON"))
                .with_label(
                    Label::new((&filename, span))
                        .with_message(msg)
                        .with_color(a),
                )
                .finish()
                .eprint((&filename, Source::from(src)))
                .unwrap();
        }
    }
}

examples/brainfuck.rs (line 89)

pub fn execute(code: &str) {
    let operations: Vec<_> = Op::lexer(code).filter_map(|op| op.ok()).collect();
    let mut data = [0u8; 30_000]; // Minimum recommended size
    let mut pointer: usize = 0;
    let len = operations.len();

    // We pre-process matching jump commands, and we create
    // a mapping between them.
    let mut queue = Vec::new();
    let mut pairs = HashMap::new();
    let mut pairs_reverse = HashMap::new();

    for (i, op) in operations.iter().enumerate() {
        match op {
            Op::CondJumpForward => queue.push(i),
            Op::CondJumpBackward => {
                if let Some(start) = queue.pop() {
                    pairs.insert(start, i);
                    pairs_reverse.insert(i, start);
                } else {
                    panic!(
                        "Unexpected conditional backward jump at position {}, does not match any '['",
                        i
                    );
                }
            }
            _ => (),
        }
    }

    if !queue.is_empty() {
        panic!("Unmatched conditional forward jump at positions {:?}, expecting a closing ']' for each of them", queue);
    }

    /* ANCHOR: fsm */
    let mut i: usize = 0;
    // True program execution.
    loop {
        match operations[i] {
            Op::IncPointer => pointer += 1,
            Op::DecPointer => pointer -= 1,
            Op::IncData => data[pointer] = data[pointer].wrapping_add(1),
            Op::DecData => data[pointer] = data[pointer].wrapping_sub(1),
            Op::OutData => print_byte(data[pointer]),
            Op::InpData => data[pointer] = read_byte(),
            Op::CondJumpForward => {
                if data[pointer] == 0 {
                    // Skip until matching end.
                    i = *pairs.get(&i).unwrap();
                }
            }
            Op::CondJumpBackward => {
                if data[pointer] != 0 {
                    // Go back to matching start.
                    i = *pairs_reverse.get(&i).unwrap();
                }
            }
        }
        i += 1;

        if i >= len {
            break;
        }
    }
    /* ANCHOR_END: fsm */
}

fn lexer_with_extras( source: &'source Self::Source, extras: Self::Extras ) -> Lexer<'source, Self>

Create a new instance of a Lexer with the provided Extras that will produce tokens implementing this Logos.

Object Safety

This trait is not object safe.

Implementors