perl_tokenizer/lib.rs
1//! Token stream and trivia utilities for the parser.
2//!
3//! Wraps the raw token output of `perl-lexer` into a position-aware
4//! [`TokenStream`] and preserves whitespace/comment trivia via the
5//! [`TriviaPreservingParser`]. Used by the parser and formatting provider
6//! to maintain lossless round-trip fidelity.
7
8#![deny(unsafe_code)]
9#![cfg_attr(test, allow(clippy::panic, clippy::unwrap_used, clippy::expect_used))]
10#![warn(rust_2018_idioms)]
11
12pub mod token_stream;
13pub mod token_wrapper;
14pub mod trivia;
15pub mod trivia_parser;
16pub mod util;
17
18pub use perl_token::{Token, TokenKind};
19pub use token_stream::TokenStream;
20pub use token_wrapper::TokenWithPosition;
21pub use trivia::{Trivia, TriviaToken};
22pub use trivia_parser::{TriviaParserContext, TriviaPreservingParser};