dpscript/ir/tokenizer/
mod.rs

1mod inner;
2mod token;
3
4pub use token::*;
5
6use crate::{Cursor, Result, Spanned};
7
8pub type IRStringCursor = Cursor<String, String>;
9
10#[derive(Debug, Clone)]
11pub struct IRTokenizer {
12    pub tokens: Vec<Spanned<IRToken>>,
13    pub cursor: IRStringCursor,
14}
15
16impl IRTokenizer {
17    pub fn new(data: impl AsRef<str>) -> Self {
18        Self {
19            tokens: Vec::new(),
20            cursor: IRStringCursor::new_from_code(data),
21        }
22    }
23
24    pub fn tokenize(&mut self) -> Result<&mut Self> {
25        while let Some(ch) = self.cursor.next() {
26            self.tokenize_inner(ch)?;
27        }
28
29        Ok(self)
30    }
31
32    pub fn tokens(&self) -> Vec<Spanned<IRToken>> {
33        self.tokens.clone()
34    }
35}

dpscript/ir/tokenizer/mod.rs

dpscript/ir/tokenizer/
mod.rs