ass_core/tokenizer/tokens/
position.rs

1//! Source position tracking for streaming tokenization.
2//!
3//! Defines the [`TokenPosition`] cursor that tracks byte offset, line, and
4//! column while advancing through source text one character at a time.
5
6/// Token stream position for streaming tokenization
7#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8pub struct TokenPosition {
9    /// Byte offset in source
10    pub offset: usize,
11
12    /// Line number (1-based)
13    pub line: usize,
14
15    /// Column number (1-based)
16    pub column: usize,
17}
18
19impl TokenPosition {
20    /// Create new position
21    #[must_use]
22    pub const fn new(offset: usize, line: usize, column: usize) -> Self {
23        Self {
24            offset,
25            line,
26            column,
27        }
28    }
29
30    /// Create position at start of input
31    #[must_use]
32    pub const fn start() -> Self {
33        Self::new(0, 1, 1)
34    }
35
36    /// Advance position by one character
37    #[must_use]
38    pub const fn advance(mut self, ch: char) -> Self {
39        self.offset += ch.len_utf8();
40        if ch == '\n' {
41            self.line += 1;
42            self.column = 1;
43        } else {
44            self.column += 1;
45        }
46        self
47    }
48
49    /// Advance position by string length
50    #[must_use]
51    pub fn advance_by_str(mut self, s: &str) -> Self {
52        for ch in s.chars() {
53            self = self.advance(ch);
54        }
55        self
56    }
57}
58
59impl Default for TokenPosition {
60    fn default() -> Self {
61        Self::start()
62    }
63}
ass_core/tokenizer/tokens/position.rs

ass_core/tokenizer/tokens/
position.rs
