perl_parser_core/tokens/
mod.rs

1//! Token stream and trivia utilities for parser workflows.
2//!
3//! This module hosts the parser-facing token infrastructure that depends
4//! on `perl-error` and `perl-ast-v2` (cannot live in `perl-lexer` without
5//! creating a dependency cycle):
6//!
7//! - [`token_stream`] — buffered [`TokenStream`](token_stream::TokenStream)
8//!   over the raw lexer with 3-token lookahead, trivia skipping, and
9//!   statement-boundary mode resets.
10//! - [`trivia`] — whitespace/comment/POD classification with
11//!   [`Trivia`](trivia::Trivia) and [`TriviaLexer`](trivia::TriviaLexer).
12//! - [`trivia_parser`] — trivia-preserving parser context and `format_with_trivia`.
13//!
14//! AST-agnostic utilities (token position wrapping, `__DATA__`/`__END__`
15//! marker scanning) live in [`perl_lexer::tokenizer`] and are re-exported
16//! here for discoverability.
17
18/// Buffered token stream over the raw lexer (with trivia skipping).
19pub mod token_stream;
20/// Token wrapper utilities for preserving original lexemes.
21pub use perl_lexer::tokenizer::token_wrapper;
22/// Trivia tokens (whitespace/comments/POD) used for formatting and diagnostics.
23pub mod trivia;
24/// Trivia-preserving parser helpers for formatting context.
25pub mod trivia_parser;
perl_parser_core/tokens/mod.rs

perl_parser_core/tokens/
mod.rs
