perl_parser_core/tokens/mod.rs
1//! Token stream and trivia utilities for parser workflows.
2//!
3//! This module re-exports tokenizer helpers used during the Parse → Index → Analyze
4//! stages to power LSP features such as diagnostics, completion, and navigation.
5//!
6//! # Examples
7//!
8//! ```rust
9//! use perl_parser_core::tokens::token_stream::TokenStream;
10//!
11//! let mut stream = TokenStream::new("my $x = 1;");
12//! let _ = stream.peek();
13//! ```
14
15/// Token stream adapters used during the Parse stage for LSP workflows.
16pub mod token_stream;
17/// Token wrapper utilities for preserving original lexemes and trivia.
18pub use perl_tokenizer::token_wrapper;
19/// Trivia tokens (whitespace/comments) used for formatting and diagnostics.
20pub use perl_tokenizer::trivia;
21/// Trivia parser helpers for preserving formatting context.
22pub use perl_tokenizer::trivia_parser;