oak_rust/lexer/
mod.rs

1#![doc = include_str!("readme.md")]
2/// Rust token type definitions.
3pub mod token_type;
4pub use self::token_type::{RustToken, RustTokenType};
5use crate::language::RustLanguage;
6use oak_core::{Lexer, LexerCache, LexerState, Source, lexer::LexOutput};
7
8mod lex;
9
10/// A lexer for the Rust programming language.
11///
12/// The `RustLexer` is responsible for tokenizing Rust source code into a sequence of tokens
13/// that can be used by the parser. It handles all Rust syntax including modern features like
14/// raw strings, byte strings, lifetimes, and all standard Rust keywords.
15///
16/// # Examples
17///
18/// Basic usage:
19///
20/// ```rust,ignore
21/// use oak_core::{Lexer, LexerState, SourceText};
22/// use oak_rust::{RustLanguage, RustLexer};
23///
24/// let language = RustLanguage::default();
25/// let lexer = RustLexer::new(&language);
26/// let source = SourceText::new("fn main() { println!(\"Hello, world!\") }");
27/// let mut cache = oak_core::parser::session::ParseSession::<RustLanguage>::default();
28/// let output = lexer.lex(&source, &[], &mut cache);
29///
30/// // The output contains tokens for the entire source code
31/// assert!(output.result.is_ok());
32/// ```
33///
34/// Tokenizing different Rust constructs:
35///
36/// ```rust,ignore
37/// use oak_core::{Lexer, LexerState, SourceText};
38/// use oak_rust::{RustLanguage, RustLexer};
39///
40/// let language = RustLanguage::default();
41/// let lexer = RustLexer::new(&language);
42///
43/// // Tokenize a function with various Rust features
44/// let source = SourceText::new(
45///     r#"
46/// fn calculate<'a>(x: &'a i32, y: i32) -> i32 {
47///     let result = x + y;
48///     println!("Result: {}", result);
49///     result
50/// }
51/// "#,
52/// );
53/// let mut cache = oak_core::parser::session::ParseSession::<RustLanguage>::default();
54/// let output = lexer.lex(&source, &[], &mut cache);
55///
56/// // Verify that tokens were generated
57/// assert!(output.result.is_ok());
58/// ```
59#[derive(Clone)]
60pub struct RustLexer<'config> {
61    config: &'config RustLanguage,
62}
63
64pub(crate) type State<'a, S> = LexerState<'a, S, RustLanguage>;
65
66impl<'config> Lexer<RustLanguage> for RustLexer<'config> {
67    fn lex<'a, S: Source + ?Sized>(&self, source: &'a S, _edits: &[oak_core::TextEdit], cache: &'a mut impl LexerCache<RustLanguage>) -> LexOutput<RustLanguage> {
68        let mut state = State::new_with_cache(source, 0, cache);
69        let result = self.run(&mut state);
70        if result.is_ok() {
71            state.add_eof()
72        }
73        state.finish_with_cache(result, cache)
74    }
75}
76
77impl<'config> RustLexer<'config> {
78    /// Creates a new `RustLexer` with the given language configuration.
79    ///
80    /// # Parameters
81    ///
82    /// * `config` - A `RustLanguage` configuration that controls
83    ///   language-specific parsing behavior.
84    ///
85    /// # Examples
86    ///
87    /// ```
88    /// # use oak_rust::{RustLexer, RustLanguage};
89    ///
90    /// let language = RustLanguage::default();
91    /// let lexer = RustLexer::new(&language);
92    /// ```
93    pub fn new(config: &'config RustLanguage) -> Self {
94        Self { config }
95    }
96
97    /// Internal method to run the lexer on the given state.
98    /// This delegates to the implementation in the `lex` module.
99    pub(crate) fn run<'s, S: Source + ?Sized>(&self, state: &mut LexerState<'s, S, RustLanguage>) -> Result<(), oak_core::OakError> {
100        lex::run(self, state)
101    }
102}
oak_rust/lexer/mod.rs

oak_rust/lexer/
mod.rs
