lex_core/lex/token/normalization.rs
1//! Token normalization utilities
2//!
3//! This module provides functions to normalize various token formats into
4//! standard `Vec<(Token, Range<usize>)>` representations for AST building.
5
6pub(crate) mod normalize;
7pub mod utilities;
8
9// Re-export commonly used functions from normalize
10pub(crate) use normalize::{normalize_line_token, normalize_line_tokens};
11
12// Re-export utilities (individual items to avoid unused warning)