1use crate::ast::Span;
2
3#[derive(Clone, Debug, Eq, PartialEq)]
4pub struct SourcePosition {
5 pub line: usize,
6 pub character: usize,
7 pub byte: usize,
8}
9
10#[derive(Clone, Debug, Eq, PartialEq)]
11pub struct SourceRange {
12 pub start: SourcePosition,
13 pub end: SourcePosition,
14}
15
16#[derive(Clone, Debug, Eq, PartialEq)]
17pub struct Utf16Position {
18 pub line: usize,
19 pub character: usize,
20}
21
22#[derive(Clone, Debug)]
23pub struct LineIndex {
24 line_starts: Vec<usize>,
25}
26
27impl LineIndex {
28 pub fn new(source: &str) -> Self {
29 let mut line_starts = vec![0];
30 for (index, byte) in source.bytes().enumerate() {
31 if byte == b'\n' {
32 line_starts.push(index + 1);
33 }
34 }
35 Self { line_starts }
36 }
37
38 pub fn source_position(&self, source: &str, byte: usize) -> SourcePosition {
39 let byte = byte.min(source.len());
40 let line = self.line_for_byte(byte);
41 let line_start = self.line_starts[line];
42 SourcePosition {
43 line,
44 character: source[line_start..byte].chars().count(),
45 byte,
46 }
47 }
48
49 pub fn byte_for_line_character(
50 &self,
51 source: &str,
52 line: usize,
53 character: usize,
54 ) -> Option<usize> {
55 let line_start = *self.line_starts.get(line)?;
56 let line_end = self.line_end(source, line);
57 let mut chars_seen = 0usize;
58 for (offset, _) in source[line_start..line_end].char_indices() {
59 if chars_seen == character {
60 return Some(line_start + offset);
61 }
62 chars_seen += 1;
63 }
64 (chars_seen == character).then_some(line_end)
65 }
66
67 pub fn utf16_position(&self, source: &str, byte: usize) -> Utf16Position {
68 let byte = byte.min(source.len());
69 let line = self.line_for_byte(byte);
70 let line_start = self.line_starts[line];
71 Utf16Position {
72 line,
73 character: source[line_start..byte].chars().map(char::len_utf16).sum(),
74 }
75 }
76
77 pub fn byte_for_utf16_position(
78 &self,
79 source: &str,
80 line: usize,
81 character: usize,
82 ) -> Option<usize> {
83 let line_start = *self.line_starts.get(line)?;
84 let line_end = self.line_end(source, line);
85 let mut units_seen = 0usize;
86 for (offset, ch) in source[line_start..line_end].char_indices() {
87 if units_seen == character {
88 return Some(line_start + offset);
89 }
90 units_seen += ch.len_utf16();
91 if units_seen > character {
92 return None;
93 }
94 }
95 (units_seen == character).then_some(line_end)
96 }
97
98 pub fn source_range(&self, source: &str, span: Span) -> SourceRange {
99 SourceRange {
100 start: self.source_position(source, span.start_byte),
101 end: self.source_position(source, span.end_byte),
102 }
103 }
104
105 fn line_for_byte(&self, byte: usize) -> usize {
106 match self.line_starts.binary_search(&byte) {
107 Ok(line) => line,
108 Err(next) => next.saturating_sub(1),
109 }
110 }
111
112 fn line_end(&self, source: &str, line: usize) -> usize {
113 self.line_starts
114 .get(line + 1)
115 .map(|next_start| next_start.saturating_sub(1))
116 .unwrap_or(source.len())
117 }
118}
119
120#[derive(Clone, Copy, Debug, Eq, PartialEq)]
121pub enum CommentKind {
122 Hash,
123 SlashSlash,
124}
125
126#[derive(Clone, Copy, Debug, Eq, PartialEq)]
127pub enum TokenKind {
128 Identifier,
129 String,
130 Number,
131 Keyword,
132 Punctuation,
133 Comment(CommentKind),
134 Whitespace,
135 Newline,
136 Unknown,
137}
138
139#[derive(Clone, Debug, Eq, PartialEq)]
140pub struct Token {
141 pub kind: TokenKind,
142 pub span: Span,
143}
144
145#[derive(Clone, Debug, Eq, PartialEq)]
146pub struct Comment {
147 pub kind: CommentKind,
148 pub text: String,
149 pub range: SourceRange,
150}
151
152pub fn collect_tokens(source: &str) -> Vec<Token> {
153 let bytes = source.as_bytes();
154 let mut tokens = Vec::new();
155 let mut pos = 0usize;
156 while pos < bytes.len() {
157 let start = pos;
158 match bytes[pos] {
159 b' ' | b'\t' => {
160 pos += 1;
161 while matches!(bytes.get(pos), Some(b' ' | b'\t')) {
162 pos += 1;
163 }
164 tokens.push(token(TokenKind::Whitespace, start, pos));
165 }
166 b'\n' => {
167 pos += 1;
168 tokens.push(token(TokenKind::Newline, start, pos));
169 }
170 b'\r' if bytes.get(pos + 1) == Some(&b'\n') => {
171 pos += 2;
172 tokens.push(token(TokenKind::Newline, start, pos));
173 }
174 b'#' => {
175 pos = consume_comment(bytes, pos);
176 tokens.push(token(TokenKind::Comment(CommentKind::Hash), start, pos));
177 }
178 b'/' if bytes.get(pos + 1) == Some(&b'/') => {
179 pos = consume_comment(bytes, pos);
180 tokens.push(token(
181 TokenKind::Comment(CommentKind::SlashSlash),
182 start,
183 pos,
184 ));
185 }
186 b'"' => {
187 pos = consume_string(bytes, pos);
188 tokens.push(token(TokenKind::String, start, pos));
189 }
190 b'-' | b'0'..=b'9' => {
191 pos += 1;
192 while matches!(
193 bytes.get(pos),
194 Some(b'0'..=b'9' | b'.' | b'e' | b'E' | b'+' | b'-')
195 ) {
196 pos += 1;
197 }
198 tokens.push(token(TokenKind::Number, start, pos));
199 }
200 ch if is_ident_start(ch) => {
201 pos += 1;
202 while bytes.get(pos).is_some_and(|ch| is_ident_continue(*ch)) {
203 pos += 1;
204 }
205 let text = &source[start..pos];
206 let kind = if matches!(text, "include" | "true" | "false" | "null") {
207 TokenKind::Keyword
208 } else {
209 TokenKind::Identifier
210 };
211 tokens.push(token(kind, start, pos));
212 }
213 b'{' | b'}' | b'[' | b']' | b'=' | b',' | b'.' | b'$' => {
214 pos += 1;
215 tokens.push(token(TokenKind::Punctuation, start, pos));
216 }
217 _ => {
218 pos += source[pos..]
219 .chars()
220 .next()
221 .map(char::len_utf8)
222 .unwrap_or(1);
223 tokens.push(token(TokenKind::Unknown, start, pos));
224 }
225 }
226 }
227 tokens
228}
229
230pub fn comments_from_tokens(
231 source: &str,
232 line_index: &LineIndex,
233 tokens: &[Token],
234) -> Vec<Comment> {
235 tokens
236 .iter()
237 .filter_map(|token| {
238 let TokenKind::Comment(kind) = token.kind else {
239 return None;
240 };
241 Some(Comment {
242 kind,
243 text: source[token.span.start_byte..token.span.end_byte].to_string(),
244 range: line_index.source_range(source, token.span),
245 })
246 })
247 .collect()
248}
249
250fn token(kind: TokenKind, start: usize, end: usize) -> Token {
251 Token {
252 kind,
253 span: Span::new(start, end),
254 }
255}
256
257fn consume_comment(bytes: &[u8], mut pos: usize) -> usize {
258 while bytes
259 .get(pos)
260 .is_some_and(|ch| !matches!(ch, b'\n' | b'\r'))
261 {
262 pos += 1;
263 }
264 pos
265}
266
267fn consume_string(bytes: &[u8], mut pos: usize) -> usize {
268 pos += 1;
269 let mut escaped = false;
270 while let Some(byte) = bytes.get(pos).copied() {
271 pos += 1;
272 if escaped {
273 escaped = false;
274 continue;
275 }
276 match byte {
277 b'\\' => escaped = true,
278 b'"' => break,
279 b'\n' | b'\r' => break,
280 _ => {}
281 }
282 }
283 pos
284}
285
286fn is_ident_start(ch: u8) -> bool {
287 ch.is_ascii_alphabetic() || ch == b'_'
288}
289
290fn is_ident_continue(ch: u8) -> bool {
291 ch.is_ascii_alphanumeric() || ch == b'_' || ch == b'-'
292}