oak_valkyrie/lexer/
mod.rs

1pub use self::keywords::ValkyrieKeywords;
2use crate::language::ValkyrieLanguage;
3use oak_core::{Lexer, LexerCache, LexerState, lexer::LexOutput, source::Source};
4mod keywords;
5mod lex;
6
7/// The lexer for the Valkyrie programming language.
8#[derive(Clone, Debug)]
9pub struct ValkyrieLexer<'config> {
10    _config: &'config ValkyrieLanguage,
11}
12
13impl<'config> Lexer<ValkyrieLanguage> for ValkyrieLexer<'config> {
14    fn lex<'a, S: Source + ?Sized>(&self, source: &S, _edits: &[oak_core::TextEdit], cache: &'a mut impl LexerCache<ValkyrieLanguage>) -> LexOutput<ValkyrieLanguage> {
15        let mut state = LexerState::new_with_cache(source, 0, cache);
16        let result = self.run(&mut state);
17        if result.is_ok() {
18            state.add_eof();
19        }
20        state.finish_with_cache(result, cache)
21    }
22}
23
24impl<'config> ValkyrieLexer<'config> {
25    /// Create a new lexer with the given configuration.
26    pub fn new(config: &'config ValkyrieLanguage) -> Self {
27        Self { _config: config }
28    }
29
30    /// Tokenize the given source code.
31    pub fn tokenize<S: Source + ?Sized>(&self, source: &S) -> impl Iterator<Item = oak_core::lexer::Token<crate::kind::ValkyrieSyntaxKind>> {
32        let mut cache = oak_core::parser::session::ParseSession::<ValkyrieLanguage>::default();
33        let output = self.lex(source, &[], &mut cache);
34        output.result.unwrap_or_else(|_| oak_core::Arc::from_iter(Vec::new())).to_vec().into_iter()
35    }
36}
oak_valkyrie/lexer/mod.rs

oak_valkyrie/lexer/
mod.rs
