1use daml_parser::ast::{Module, Span as ParserSpan};
9use daml_parser::layout::resolve_layout;
10use daml_parser::lexer::{lex_with_trivia, LexError, Token, Trivia};
11use daml_parser::parse::parse_module;
12use std::sync::OnceLock;
13
14pub use text_size::{TextRange, TextSize};
15
16#[derive(Debug, Clone, Copy, PartialEq, Eq)]
17pub struct LineCol {
18 pub line: usize,
19 pub column: usize,
20}
21
22#[derive(Debug, Clone, PartialEq, Eq)]
23pub struct Diagnostic {
24 pub range: TextRange,
25 pub line: usize,
26 pub column: usize,
27 pub end_column: Option<usize>,
28 pub message: String,
29 pub category: &'static str,
30}
31
32#[derive(Debug, Clone, PartialEq, Eq)]
33pub struct LineIndex {
34 source_len: usize,
35 line_start_bytes: Vec<usize>,
36 utf16_offset_by_byte: Vec<usize>,
37}
38
39impl LineIndex {
40 #[must_use]
41 pub fn new(source: &str) -> Self {
42 let mut line_start_bytes = vec![0];
43 for (idx, byte) in source.bytes().enumerate() {
44 if byte == b'\n' {
45 line_start_bytes.push(idx + 1);
46 }
47 }
48
49 let mut utf16_offset_by_byte = vec![0; source.len() + 1];
50 let mut utf16 = 0usize;
51 let mut prev = 0usize;
52 for (idx, ch) in source.char_indices() {
53 for slot in utf16_offset_by_byte.iter_mut().take(idx).skip(prev) {
54 *slot = utf16;
55 }
56 let char_end = idx + ch.len_utf8();
57 for slot in utf16_offset_by_byte.iter_mut().take(char_end).skip(idx) {
58 *slot = utf16;
59 }
60 utf16 += ch.len_utf16();
61 prev = char_end;
62 }
63 for slot in utf16_offset_by_byte
64 .iter_mut()
65 .take(source.len() + 1)
66 .skip(prev)
67 {
68 *slot = utf16;
69 }
70
71 Self {
72 source_len: source.len(),
73 line_start_bytes,
74 utf16_offset_by_byte,
75 }
76 }
77
78 #[must_use]
79 pub fn line_col(&self, offset: TextSize) -> LineCol {
80 let byte = usize::from(offset).min(self.source_len);
81 let line_idx = match self.line_start_bytes.binary_search(&byte) {
82 Ok(idx) => idx,
83 Err(idx) => idx.saturating_sub(1),
84 };
85 LineCol {
86 line: line_idx + 1,
87 column: byte - self.line_start_bytes[line_idx] + 1,
88 }
89 }
90
91 #[must_use]
92 pub fn char_line_col(&self, source: &str, offset: TextSize) -> LineCol {
93 let byte = usize::from(offset).min(self.source_len);
94 let line_idx = match self.line_start_bytes.binary_search(&byte) {
95 Ok(idx) => idx,
96 Err(idx) => idx.saturating_sub(1),
97 };
98 let line_start = self.line_start_bytes[line_idx];
99 LineCol {
100 line: line_idx + 1,
101 column: source[line_start..byte].chars().count() + 1,
102 }
103 }
104
105 #[must_use]
106 pub fn utf16_col(&self, line: usize, byte_col: usize) -> usize {
107 let line_start = self
108 .line_start_bytes
109 .get(line.saturating_sub(1))
110 .copied()
111 .unwrap_or(self.source_len);
112 let byte = line_start
113 .saturating_add(byte_col.saturating_sub(1))
114 .min(self.source_len);
115 self.utf16_offset_by_byte[byte] - self.utf16_offset_by_byte[line_start]
116 }
117
118 #[must_use]
119 pub fn utf16_range(&self, range: TextRange) -> (usize, usize) {
120 let start = usize::from(range.start()).min(self.source_len);
121 let end = usize::from(range.end()).min(self.source_len).max(start);
122 (
123 self.utf16_offset_by_byte[start],
124 self.utf16_offset_by_byte[end],
125 )
126 }
127}
128
129#[derive(Debug)]
130pub struct SourceTokens {
131 tokens: Vec<Token>,
132 trivia: Vec<Trivia>,
133 lex_errors: Vec<LexError>,
134 laid_out_tokens: OnceLock<Vec<Token>>,
135}
136
137impl SourceTokens {
138 #[must_use]
139 pub fn lex(source: &str) -> Self {
140 let lexed = lex_with_trivia(source);
141 Self {
142 tokens: lexed.tokens,
143 trivia: lexed.trivia,
144 lex_errors: lexed.errors,
145 laid_out_tokens: OnceLock::new(),
146 }
147 }
148
149 #[must_use]
150 pub fn tokens(&self) -> &[Token] {
151 &self.tokens
152 }
153
154 #[must_use]
155 pub fn trivia(&self) -> &[Trivia] {
156 &self.trivia
157 }
158
159 #[must_use]
160 pub fn lex_errors(&self) -> &[LexError] {
161 &self.lex_errors
162 }
163
164 #[must_use]
165 pub fn laid_out_tokens(&self) -> &[Token] {
166 self.laid_out_tokens
167 .get_or_init(|| resolve_layout(self.tokens.clone()))
168 }
169}
170
171#[derive(Debug)]
172pub struct SourceFile {
173 source: String,
174 module: Module,
175 diagnostics: Vec<Diagnostic>,
176 line_index: LineIndex,
177 tokens: OnceLock<SourceTokens>,
178}
179
180impl SourceFile {
181 #[must_use]
182 pub fn parse(source: &str) -> Self {
183 let parsed = parse_module(source);
184 let line_index = LineIndex::new(source);
185 let diagnostics = parsed
186 .diagnostics
187 .into_iter()
188 .map(|diagnostic| {
189 let range = parser_span_to_text_range(source, diagnostic.span);
190 let start = range.start();
191 let end_column = source
192 .get(usize::from(range.start())..usize::from(range.end()))
193 .filter(|s| !s.is_empty() && !s.contains('\n'))
194 .map(|s| diagnostic.pos.column + s.chars().count());
195 Diagnostic {
196 range,
197 line: line_index.char_line_col(source, start).line,
198 column: diagnostic.pos.column,
199 end_column,
200 message: diagnostic.message,
201 category: diagnostic.category.as_str(),
202 }
203 })
204 .collect();
205
206 Self {
207 source: source.to_string(),
208 module: parsed.module,
209 diagnostics,
210 line_index,
211 tokens: OnceLock::new(),
212 }
213 }
214
215 #[must_use]
216 pub fn source(&self) -> &str {
217 &self.source
218 }
219
220 #[must_use]
221 pub const fn module(&self) -> &Module {
222 &self.module
223 }
224
225 #[must_use]
226 pub fn diagnostics(&self) -> &[Diagnostic] {
227 &self.diagnostics
228 }
229
230 #[must_use]
231 pub const fn line_index(&self) -> &LineIndex {
232 &self.line_index
233 }
234
235 #[must_use]
236 pub fn tokens(&self) -> &[Token] {
237 self.source_tokens().tokens()
238 }
239
240 #[must_use]
241 pub fn trivia(&self) -> &[Trivia] {
242 self.source_tokens().trivia()
243 }
244
245 #[must_use]
246 pub fn laid_out_tokens(&self) -> &[Token] {
247 self.source_tokens().laid_out_tokens()
248 }
249
250 #[must_use]
251 pub fn parser_span_to_text_range(&self, span: ParserSpan) -> TextRange {
252 parser_span_to_text_range(&self.source, span)
253 }
254
255 fn source_tokens(&self) -> &SourceTokens {
256 self.tokens.get_or_init(|| SourceTokens::lex(&self.source))
257 }
258}
259
260#[must_use]
261pub fn parser_span_to_text_range(source: &str, span: ParserSpan) -> TextRange {
262 let start = span.start.min(source.len());
263 let end = span.end.min(source.len()).max(start);
264 TextRange::new(
265 TextSize::try_from(start).unwrap(),
266 TextSize::try_from(end).unwrap(),
267 )
268}
269
270#[cfg(test)]
271mod tests {
272 use super::*;
273 use daml_parser::ast_span::render_from_ast;
274 use daml_parser::lexer::render_lossless;
275
276 #[test]
277 fn maps_empty_source_to_first_line() {
278 let index = LineIndex::new("");
279
280 assert_eq!(index.line_col(0.into()), LineCol { line: 1, column: 1 });
281 assert_eq!(index.utf16_range(TextRange::empty(0.into())), (0, 0));
282 }
283
284 #[test]
285 fn maps_ascii_byte_lines() {
286 let source = "module M where\nfoo = 1\n";
287 let index = LineIndex::new(source);
288
289 assert_eq!(index.line_col(15.into()), LineCol { line: 2, column: 1 });
290 assert_eq!(index.utf16_col(2, 4), 3);
291 }
292
293 #[test]
294 fn maps_utf8_and_utf16_offsets() {
295 let source = "a😀b\nz";
296 let index = LineIndex::new(source);
297
298 assert_eq!(
299 index.utf16_range(TextRange::new(0.into(), 6.into())),
300 (0, 4)
301 );
302 assert_eq!(index.utf16_col(1, 6), 3);
303 assert_eq!(
304 index.char_line_col(source, 5.into()),
305 LineCol { line: 1, column: 3 }
306 );
307 }
308
309 #[test]
310 fn preserves_trailing_newline_line_start() {
311 let index = LineIndex::new("a\n");
312
313 assert_eq!(index.line_col(2.into()), LineCol { line: 2, column: 1 });
314 }
315
316 #[test]
317 fn treats_crlf_as_bytes_without_normalization() {
318 let index = LineIndex::new("a\r\nb");
319
320 assert_eq!(index.line_col(3.into()), LineCol { line: 2, column: 1 });
321 }
322
323 #[test]
324 fn clamps_ranges_to_source_end() {
325 let index = LineIndex::new("abc");
326 let range = TextRange::new(1.into(), 99.into());
327
328 assert_eq!(index.utf16_range(range), (1, 3));
329 }
330
331 #[test]
332 fn source_file_exposes_parser_pipeline_facts() {
333 let source = "module M where\nfoo : Int\nfoo = 1\n";
334 let file = SourceFile::parse(source);
335
336 assert_eq!(file.source(), source);
337 assert_eq!(file.module().name, "M");
338 assert!(file.diagnostics().is_empty());
339 assert!(!file.tokens().is_empty());
340 assert!(!file.laid_out_tokens().is_empty());
341 assert_eq!(
342 render_lossless(source, file.tokens(), file.trivia()).as_deref(),
343 Ok(source)
344 );
345 assert_eq!(
346 render_from_ast(source, file.module(), file.trivia()).as_deref(),
347 Ok(source)
348 );
349 }
350
351 #[test]
352 fn source_tokens_exposes_lex_only_pipeline_facts() {
353 let source = "module M where\nfoo : Int\nfoo = 1\n";
354 let tokens = SourceTokens::lex(source);
355
356 assert!(tokens.lex_errors().is_empty());
357 assert!(!tokens.tokens().is_empty());
358 assert!(!tokens.laid_out_tokens().is_empty());
359 assert_eq!(
360 render_lossless(source, tokens.tokens(), tokens.trivia()).as_deref(),
361 Ok(source)
362 );
363 }
364
365 #[test]
366 fn malformed_source_keeps_source_file_and_diagnostics() {
367 let file = SourceFile::parse("module M where\nfoo = \"unterminated\nbar = 1\n");
368
369 assert_eq!(file.module().name, "M");
370 assert!(file
371 .diagnostics()
372 .iter()
373 .any(|diagnostic| diagnostic.category == "lexical-error"));
374 }
375
376 #[test]
377 fn converts_parser_spans_to_text_ranges() {
378 let file = SourceFile::parse("module M where\nfoo = 1\n");
379 let range = file.parser_span_to_text_range(ParserSpan::new(0, 99));
380
381 assert_eq!(
382 range,
383 TextRange::new(0.into(), file.source().len().try_into().unwrap())
384 );
385 }
386}