Skip to main content

scon/
source.rs

1use crate::ast::Span;
2
3#[derive(Clone, Debug, Eq, PartialEq)]
4pub struct SourcePosition {
5    pub line: usize,
6    pub character: usize,
7    pub byte: usize,
8}
9
10#[derive(Clone, Debug, Eq, PartialEq)]
11pub struct SourceRange {
12    pub start: SourcePosition,
13    pub end: SourcePosition,
14}
15
16#[derive(Clone, Debug, Eq, PartialEq)]
17pub struct Utf16Position {
18    pub line: usize,
19    pub character: usize,
20}
21
22#[derive(Clone, Debug)]
23pub struct LineIndex {
24    line_starts: Vec<usize>,
25}
26
27impl LineIndex {
28    pub fn new(source: &str) -> Self {
29        let mut line_starts = vec![0];
30        for (index, byte) in source.bytes().enumerate() {
31            if byte == b'\n' {
32                line_starts.push(index + 1);
33            }
34        }
35        Self { line_starts }
36    }
37
38    pub fn source_position(&self, source: &str, byte: usize) -> SourcePosition {
39        let byte = byte.min(source.len());
40        let line = self.line_for_byte(byte);
41        let line_start = self.line_starts[line];
42        SourcePosition {
43            line,
44            character: source[line_start..byte].chars().count(),
45            byte,
46        }
47    }
48
49    pub fn byte_for_line_character(
50        &self,
51        source: &str,
52        line: usize,
53        character: usize,
54    ) -> Option<usize> {
55        let line_start = *self.line_starts.get(line)?;
56        let line_end = self.line_end(source, line);
57        let mut chars_seen = 0usize;
58        for (offset, _) in source[line_start..line_end].char_indices() {
59            if chars_seen == character {
60                return Some(line_start + offset);
61            }
62            chars_seen += 1;
63        }
64        (chars_seen == character).then_some(line_end)
65    }
66
67    pub fn utf16_position(&self, source: &str, byte: usize) -> Utf16Position {
68        let byte = byte.min(source.len());
69        let line = self.line_for_byte(byte);
70        let line_start = self.line_starts[line];
71        Utf16Position {
72            line,
73            character: source[line_start..byte].chars().map(char::len_utf16).sum(),
74        }
75    }
76
77    pub fn byte_for_utf16_position(
78        &self,
79        source: &str,
80        line: usize,
81        character: usize,
82    ) -> Option<usize> {
83        let line_start = *self.line_starts.get(line)?;
84        let line_end = self.line_end(source, line);
85        let mut units_seen = 0usize;
86        for (offset, ch) in source[line_start..line_end].char_indices() {
87            if units_seen == character {
88                return Some(line_start + offset);
89            }
90            units_seen += ch.len_utf16();
91            if units_seen > character {
92                return None;
93            }
94        }
95        (units_seen == character).then_some(line_end)
96    }
97
98    pub fn source_range(&self, source: &str, span: Span) -> SourceRange {
99        SourceRange {
100            start: self.source_position(source, span.start_byte),
101            end: self.source_position(source, span.end_byte),
102        }
103    }
104
105    fn line_for_byte(&self, byte: usize) -> usize {
106        match self.line_starts.binary_search(&byte) {
107            Ok(line) => line,
108            Err(next) => next.saturating_sub(1),
109        }
110    }
111
112    fn line_end(&self, source: &str, line: usize) -> usize {
113        self.line_starts
114            .get(line + 1)
115            .map(|next_start| next_start.saturating_sub(1))
116            .unwrap_or(source.len())
117    }
118}
119
120#[derive(Clone, Copy, Debug, Eq, PartialEq)]
121pub enum CommentKind {
122    Hash,
123    SlashSlash,
124}
125
126#[derive(Clone, Copy, Debug, Eq, PartialEq)]
127pub enum TokenKind {
128    Identifier,
129    String,
130    Number,
131    Keyword,
132    Punctuation,
133    Comment(CommentKind),
134    Whitespace,
135    Newline,
136    Unknown,
137}
138
139#[derive(Clone, Debug, Eq, PartialEq)]
140pub struct Token {
141    pub kind: TokenKind,
142    pub span: Span,
143}
144
145#[derive(Clone, Debug, Eq, PartialEq)]
146pub struct Comment {
147    pub kind: CommentKind,
148    pub text: String,
149    pub range: SourceRange,
150}
151
152pub fn collect_tokens(source: &str) -> Vec<Token> {
153    let bytes = source.as_bytes();
154    let mut tokens = Vec::new();
155    let mut pos = 0usize;
156    while pos < bytes.len() {
157        let start = pos;
158        match bytes[pos] {
159            b' ' | b'\t' => {
160                pos += 1;
161                while matches!(bytes.get(pos), Some(b' ' | b'\t')) {
162                    pos += 1;
163                }
164                tokens.push(token(TokenKind::Whitespace, start, pos));
165            }
166            b'\n' => {
167                pos += 1;
168                tokens.push(token(TokenKind::Newline, start, pos));
169            }
170            b'\r' if bytes.get(pos + 1) == Some(&b'\n') => {
171                pos += 2;
172                tokens.push(token(TokenKind::Newline, start, pos));
173            }
174            b'#' => {
175                pos = consume_comment(bytes, pos);
176                tokens.push(token(TokenKind::Comment(CommentKind::Hash), start, pos));
177            }
178            b'/' if bytes.get(pos + 1) == Some(&b'/') => {
179                pos = consume_comment(bytes, pos);
180                tokens.push(token(
181                    TokenKind::Comment(CommentKind::SlashSlash),
182                    start,
183                    pos,
184                ));
185            }
186            b'"' => {
187                pos = consume_string(bytes, pos);
188                tokens.push(token(TokenKind::String, start, pos));
189            }
190            b'-' | b'0'..=b'9' => {
191                pos += 1;
192                while matches!(
193                    bytes.get(pos),
194                    Some(b'0'..=b'9' | b'.' | b'e' | b'E' | b'+' | b'-')
195                ) {
196                    pos += 1;
197                }
198                tokens.push(token(TokenKind::Number, start, pos));
199            }
200            ch if is_ident_start(ch) => {
201                pos += 1;
202                while bytes.get(pos).is_some_and(|ch| is_ident_continue(*ch)) {
203                    pos += 1;
204                }
205                let text = &source[start..pos];
206                let kind = if matches!(text, "include" | "true" | "false" | "null") {
207                    TokenKind::Keyword
208                } else {
209                    TokenKind::Identifier
210                };
211                tokens.push(token(kind, start, pos));
212            }
213            b'{' | b'}' | b'[' | b']' | b'=' | b',' | b'.' | b'$' => {
214                pos += 1;
215                tokens.push(token(TokenKind::Punctuation, start, pos));
216            }
217            _ => {
218                pos += source[pos..]
219                    .chars()
220                    .next()
221                    .map(char::len_utf8)
222                    .unwrap_or(1);
223                tokens.push(token(TokenKind::Unknown, start, pos));
224            }
225        }
226    }
227    tokens
228}
229
230pub fn comments_from_tokens(
231    source: &str,
232    line_index: &LineIndex,
233    tokens: &[Token],
234) -> Vec<Comment> {
235    tokens
236        .iter()
237        .filter_map(|token| {
238            let TokenKind::Comment(kind) = token.kind else {
239                return None;
240            };
241            Some(Comment {
242                kind,
243                text: source[token.span.start_byte..token.span.end_byte].to_string(),
244                range: line_index.source_range(source, token.span),
245            })
246        })
247        .collect()
248}
249
250fn token(kind: TokenKind, start: usize, end: usize) -> Token {
251    Token {
252        kind,
253        span: Span::new(start, end),
254    }
255}
256
257fn consume_comment(bytes: &[u8], mut pos: usize) -> usize {
258    while bytes
259        .get(pos)
260        .is_some_and(|ch| !matches!(ch, b'\n' | b'\r'))
261    {
262        pos += 1;
263    }
264    pos
265}
266
267fn consume_string(bytes: &[u8], mut pos: usize) -> usize {
268    pos += 1;
269    let mut escaped = false;
270    while let Some(byte) = bytes.get(pos).copied() {
271        pos += 1;
272        if escaped {
273            escaped = false;
274            continue;
275        }
276        match byte {
277            b'\\' => escaped = true,
278            b'"' => break,
279            b'\n' | b'\r' => break,
280            _ => {}
281        }
282    }
283    pos
284}
285
286fn is_ident_start(ch: u8) -> bool {
287    ch.is_ascii_alphabetic() || ch == b'_'
288}
289
290fn is_ident_continue(ch: u8) -> bool {
291    ch.is_ascii_alphanumeric() || ch == b'_' || ch == b'-'
292}