lex_tokens/
lex_tokens.rs

1//! Print the token stream, including the trivia (whitespace, comments) the
2//! datum tree drops — the layer a formatter or parinfer backend works from.
3//!
4//! Run with:
5//!
6//! ```sh
7//! cargo run --example lex_tokens
8//! ```
9
10use lispexp::{lex, Options};
11
12fn main() {
13    let source = "(define x ; the answer\n  42)";
14
15    for token in lex(source, &Options::scheme()) {
16        let text = token.span.text(source);
17        // Show newlines/spaces legibly.
18        let shown = text.replace('\n', "\\n");
19        println!(
20            "{:>2}..{:<2} {:<16} {:?}",
21            token.span.start,
22            token.span.end,
23            format!("{:?}", token.kind),
24            shown,
25        );
26    }
27}
lex_tokens/lex_tokens.rs

lex_tokens/
lex_tokens.rs
