stack_compiler/
lossless.rs

1//! Lossless lexical source model for formatters and editor tooling.
2
3use crate::diagnostic::{SourcePosition, Span};
4use crate::lexer;
5
6/// A syntactically valid Stack document represented as authored lexemes.
7#[derive(Debug, Clone, PartialEq, Eq)]
8pub struct Document {
9    tokens: Vec<Token>,
10}
11
12impl Document {
13    pub(crate) fn from_lexer_tokens(source: &str, tokens: Vec<lexer::Token>) -> Self {
14        let mut lossless_tokens = Vec::new();
15        let mut cursor = SourcePosition::start();
16
17        for token in tokens {
18            append_trivia(source, cursor, token.span.start, &mut lossless_tokens);
19
20            let span = token.span;
21            let text = source[span.start.byte_offset..span.end.byte_offset].to_owned();
22            let kind = match token.kind {
23                lexer::TokenKind::Bare(_) => TokenKind::Bare,
24                lexer::TokenKind::String(value) => TokenKind::String(value),
25                lexer::TokenKind::LeftBrace => TokenKind::LeftBrace,
26                lexer::TokenKind::RightBrace => TokenKind::RightBrace,
27                lexer::TokenKind::LeftBracket => TokenKind::LeftBracket,
28                lexer::TokenKind::RightBracket => TokenKind::RightBracket,
29                lexer::TokenKind::Comma => TokenKind::Comma,
30                lexer::TokenKind::Dot => TokenKind::Dot,
31                lexer::TokenKind::ForwardArrow => TokenKind::ForwardArrow,
32                lexer::TokenKind::BidirectionalArrow => TokenKind::BidirectionalArrow,
33                lexer::TokenKind::Association => TokenKind::Association,
34                lexer::TokenKind::End => TokenKind::End,
35            };
36            lossless_tokens.push(Token { kind, text, span });
37            cursor = span.end;
38        }
39
40        Self {
41            tokens: lossless_tokens,
42        }
43    }
44
45    /// Returns every authored token and trivia segment in source order.
46    pub fn tokens(&self) -> &[Token] {
47        &self.tokens
48    }
49
50    /// Reconstructs the original UTF-8 source byte-for-byte.
51    pub fn reconstruct(&self) -> String {
52        let mut source = String::new();
53        for token in &self.tokens {
54            source.push_str(&token.text);
55        }
56        source
57    }
58}
59
60/// One authored lexeme or trivia segment with its exact source text and span.
61#[derive(Debug, Clone, PartialEq, Eq)]
62pub struct Token {
63    /// Lexical category. String tokens also expose their decoded value.
64    pub kind: TokenKind,
65    /// Exact authored text, including escapes and original line endings.
66    pub text: String,
67    /// End-exclusive span in the original source.
68    pub span: Span,
69}
70
71/// Lexical category in a lossless Stack document.
72#[derive(Debug, Clone, PartialEq, Eq)]
73pub enum TokenKind {
74    /// One or more spaces, tabs, LF line endings, or CRLF line endings.
75    Whitespace,
76    /// A `//` comment without its following line ending.
77    LineComment,
78    /// An identifier, contextual keyword, integer, or unknown bare token.
79    Bare,
80    /// A source string and its decoded value.
81    String(String),
82    /// `{`.
83    LeftBrace,
84    /// `}`.
85    RightBrace,
86    /// `[`.
87    LeftBracket,
88    /// `]`.
89    RightBracket,
90    /// `,`.
91    Comma,
92    /// `.`.
93    Dot,
94    /// `->`.
95    ForwardArrow,
96    /// `<->`.
97    BidirectionalArrow,
98    /// `--`.
99    Association,
100    /// The zero-width end of the source.
101    End,
102}
103
104fn append_trivia(
105    source: &str,
106    mut position: SourcePosition,
107    end: SourcePosition,
108    tokens: &mut Vec<Token>,
109) {
110    while position.byte_offset < end.byte_offset {
111        let start = position;
112        let is_comment = source[position.byte_offset..end.byte_offset].starts_with("//");
113
114        if is_comment {
115            while position.byte_offset < end.byte_offset
116                && next_character(source, position.byte_offset)
117                    .is_some_and(|character| !matches!(character, '\n' | '\r'))
118            {
119                advance_position(source, &mut position);
120            }
121        } else {
122            while position.byte_offset < end.byte_offset
123                && !source[position.byte_offset..end.byte_offset].starts_with("//")
124            {
125                advance_position(source, &mut position);
126            }
127        }
128
129        let text = source[start.byte_offset..position.byte_offset].to_owned();
130        tokens.push(Token {
131            kind: if is_comment {
132                TokenKind::LineComment
133            } else {
134                TokenKind::Whitespace
135            },
136            text,
137            span: Span {
138                start,
139                end: position,
140            },
141        });
142    }
143}
144
145fn next_character(source: &str, offset: usize) -> Option<char> {
146    source[offset..].chars().next()
147}
148
149fn advance_position(source: &str, position: &mut SourcePosition) {
150    let Some(character) = next_character(source, position.byte_offset) else {
151        return;
152    };
153
154    if character == '\r' && source[position.byte_offset..].starts_with("\r\n") {
155        position.byte_offset += 2;
156        position.line += 1;
157        position.column = 1;
158    } else {
159        position.byte_offset += character.len_utf8();
160        if matches!(character, '\n' | '\r') {
161            position.line += 1;
162            position.column = 1;
163        } else {
164            position.column += 1;
165        }
166    }
167}
168
169#[cfg(test)]
170mod tests {
171    use super::{Document, TokenKind};
172    use crate::lexer::tokenize;
173
174    #[test]
175    fn preserves_every_lexeme_and_decoded_string_value() {
176        let source = " \t// lead\r\nword \"\\u0041\" {}[],. a->b a<->b a--b // tail";
177        let result = tokenize(source);
178        assert!(result.is_ok(), "{result:?}");
179        let tokens: Vec<_> = result.into_iter().flatten().collect();
180        let document = Document::from_lexer_tokens(source, tokens);
181
182        assert_eq!(document.reconstruct(), source);
183        assert_eq!(document.tokens[0].kind, TokenKind::Whitespace);
184        assert_eq!(document.tokens[0].text, " \t");
185        assert_eq!(document.tokens[1].kind, TokenKind::LineComment);
186        assert_eq!(document.tokens[1].text, "// lead");
187        assert_eq!(document.tokens[1].span.start.line, 1);
188        assert_eq!(document.tokens[2].text, "\r\n");
189        assert_eq!(document.tokens[2].span.end.line, 2);
190        assert!(document.tokens.iter().any(
191            |token| matches!(&token.kind, TokenKind::String(value) if value == "A")
192                && token.text == "\"\\u0041\""
193        ));
194
195        for expected in [
196            TokenKind::LeftBrace,
197            TokenKind::RightBrace,
198            TokenKind::LeftBracket,
199            TokenKind::RightBracket,
200            TokenKind::Comma,
201            TokenKind::Dot,
202            TokenKind::ForwardArrow,
203            TokenKind::BidirectionalArrow,
204            TokenKind::Association,
205            TokenKind::End,
206        ] {
207            assert!(document.tokens.iter().any(|token| token.kind == expected));
208        }
209
210        let mut byte_offset = 0;
211        for token in document.tokens() {
212            assert_eq!(token.span.start.byte_offset, byte_offset);
213            byte_offset = token.span.end.byte_offset;
214        }
215        assert_eq!(byte_offset, source.len());
216    }
217
218    #[test]
219    fn defensive_position_advance_stops_at_end_of_source() {
220        let mut position = crate::diagnostic::SourcePosition {
221            byte_offset: 0,
222            line: 1,
223            column: 1,
224        };
225        super::advance_position("", &mut position);
226        assert_eq!(position.byte_offset, 0);
227    }
228}
stack_compiler/lossless.rs

stack_compiler/
lossless.rs
