perl_lexer/tokenizer/
mod.rs

1//! Token utilities bridging raw lexer output to parser consumption.
2//!
3//! This module consolidates the AST-agnostic slice of the former
4//! `perl-tokenizer` crate that has no dependency on `perl-error` or
5//! `perl-ast-v2`:
6//!
7//! - [`token_wrapper`] — position-tracked wrappers over lexer tokens.
8//! - [`util`] — [`__DATA__`/`__END__`](util) marker utilities.
9//!
10//! The buffered [`TokenStream`](perl_parser_core::tokens::token_stream::TokenStream)
11//! lives in `perl-parser-core` because it uses `perl-error`'s `ParseError`
12//! type, which would create a dependency cycle if it lived here. Trivia
13//! preservation (comments/whitespace → AST) also lives in `perl-parser-core`,
14//! since it depends on `perl-ast-v2`.
15
16pub mod token_wrapper;
17pub mod util;
18
19pub use token_wrapper::{PositionTracker, TokenWithPosition};
perl_lexer/tokenizer/mod.rs

perl_lexer/tokenizer/
mod.rs
