Function lex 

pub fn lex<'a, 'o>(source: &'a str, options: &'o Options) -> Lexer<'a, 'o> 

Lex source under options, yielding a token stream that tiles the input. source must be at most u32::MAX bytes (Span stores u32 offsets).

Unlike the tree, the token stream keeps trivia (whitespace, comments) — the layer a formatter or parinfer backend wants:

use lispexp::{lex, Options, TokenKind};

let comments = lex("(a ; note\n b)", &Options::scheme())
    .filter(|t| matches!(t.kind, TokenKind::LineComment))
    .count();
assert_eq!(comments, 1);

Examples found in repository

examples/lex_tokens.rs (line 15)

fn main() {
    let source = "(define x ; the answer\n  42)";

    for token in lex(source, &Options::scheme()) {
        let text = token.span.text(source);
        // Show newlines/spaces legibly.
        let shown = text.replace('\n', "\\n");
        println!(
            "{:>2}..{:<2} {:<16} {:?}",
            token.span.start,
            token.span.end,
            format!("{:?}", token.kind),
            shown,
        );
    }
}