oak_rust/lexer/
mod.rs

1pub use self::token_type::{RustToken, RustTokenType};
2use crate::language::RustLanguage;
3use oak_core::{Lexer, LexerCache, LexerState, lexer::LexOutput, source::Source};
4
5mod lex;
6mod token_type;
7
8/// A lexer for the Rust programming language.
9///
10/// The `RustLexer` is responsible for tokenizing Rust source code into a sequence of tokens
11/// that can be used by the parser. It handles all Rust syntax including modern features like
12/// raw strings, byte strings, lifetimes, and all standard Rust keywords.
13///
14/// # Examples
15///
16/// Basic usage:
17///
18/// ```rust,ignore
19/// use oak_core::{Lexer, LexerState, SourceText};
20/// use oak_rust::{RustLanguage, RustLexer};
21///
22/// let language = RustLanguage::default();
23/// let lexer = RustLexer::new(&language);
24/// let source = SourceText::new("fn main() { println!(\"Hello, world!\"); }");
25/// let output = lexer.lex(source, 0);
26///
27/// // The output contains tokens for the entire source code
28/// assert!(!output.tokens.is_empty());
29/// ```
30///
31/// Tokenizing different Rust constructs:
32///
33/// ```rust,ignore
34/// use oak_core::{Lexer, LexerState, SourceText};
35/// use oak_rust::{RustLanguage, RustLexer};
36///
37/// let language = RustLanguage::default();
38/// let lexer = RustLexer::new(&language);
39///
40/// // Tokenize a function with various Rust features
41/// let source = SourceText::new(
42///     r#"
43/// fn calculate<'a>(x: &'a i32, y: i32) -> i32 {
44///     let result = x + y;
45///     println!("Result: {}", result);
46///     result
47/// }
48/// "#,
49/// );
50/// let output = lexer.lex(source, 0);
51///
52/// // Verify that tokens were generated
53/// assert!(output.tokens.len() > 10);
54/// ```
55#[derive(Clone)]
56pub struct RustLexer<'config> {
57    _config: &'config RustLanguage,
58}
59
60type State<'a, S> = LexerState<'a, S, RustLanguage>;
61
62impl<'config> Lexer<RustLanguage> for RustLexer<'config> {
63    fn lex<'a, S: Source + ?Sized>(&self, source: &'a S, _edits: &[oak_core::TextEdit], cache: &'a mut impl LexerCache<RustLanguage>) -> LexOutput<RustLanguage> {
64        let mut state = State::new_with_cache(source, 0, cache);
65        let result = self.run(&mut state);
66        if result.is_ok() {
67            state.add_eof();
68        }
69        state.finish_with_cache(result, cache)
70    }
71}
72
73impl<'config> RustLexer<'config> {
74    /// Creates a new `RustLexer` with the given language configuration.
75    ///
76    /// # Parameters
77    ///
78    /// * `config` - A reference to the `RustLanguage` configuration that controls
79    ///   language-specific parsing behavior.
80    ///
81    /// # Examples
82    ///
83    /// ```
84    /// # use oak_rust::{RustLexer, RustLanguage};
85    ///
86    /// let language = RustLanguage::default();
87    /// let lexer = RustLexer::new(&language);
88    /// ```
89    pub fn new(config: &'config RustLanguage) -> Self {
90        Self { _config: config }
91    }
92
93    /// Internal method to run the lexer on the given state.
94    /// This delegates to the implementation in the `lex` module.
95    pub(crate) fn run<'s, S: Source + ?Sized>(&self, state: &mut LexerState<'s, S, RustLanguage>) -> Result<(), oak_core::OakError> {
96        lex::run(self, state)
97    }
98}
oak_rust/lexer/mod.rs

oak_rust/lexer/
mod.rs
