1use daml_parser::ast::{Module, Span as ParserSpan};
9use daml_parser::layout::resolve_layout;
10use daml_parser::lexer::{lex_with_trivia, LexError, Token, Trivia};
11use daml_parser::parse::parse_module;
12use std::sync::OnceLock;
13
14pub use text_size::{TextRange, TextSize};
15
16#[derive(Debug, Clone, Copy, PartialEq, Eq)]
17pub struct LineCol {
18 pub line: usize,
19 pub column: usize,
20}
21
22#[derive(Debug, Clone, PartialEq, Eq)]
23pub struct Diagnostic {
24 pub range: TextRange,
25 pub line: usize,
26 pub column: usize,
27 pub end_column: Option<usize>,
28 pub message: String,
29 pub category: &'static str,
30}
31
32#[derive(Debug, Clone, PartialEq, Eq)]
33pub struct LineIndex {
34 source_len: usize,
35 line_start_bytes: Vec<usize>,
36 utf16_offset_by_byte: Vec<usize>,
37}
38
39impl LineIndex {
40 #[must_use]
41 pub fn new(source: &str) -> Self {
42 let mut line_start_bytes = vec![0];
43 for (idx, byte) in source.bytes().enumerate() {
44 if byte == b'\n' {
45 line_start_bytes.push(idx + 1);
46 }
47 }
48
49 let mut utf16_offset_by_byte = vec![0; source.len() + 1];
50 let mut utf16 = 0usize;
51 let mut prev = 0usize;
52 for (idx, ch) in source.char_indices() {
53 for slot in utf16_offset_by_byte.iter_mut().take(idx).skip(prev) {
54 *slot = utf16;
55 }
56 let char_end = idx + ch.len_utf8();
57 for slot in utf16_offset_by_byte.iter_mut().take(char_end).skip(idx) {
58 *slot = utf16;
59 }
60 utf16 += ch.len_utf16();
61 prev = char_end;
62 }
63 for slot in utf16_offset_by_byte
64 .iter_mut()
65 .take(source.len() + 1)
66 .skip(prev)
67 {
68 *slot = utf16;
69 }
70
71 Self {
72 source_len: source.len(),
73 line_start_bytes,
74 utf16_offset_by_byte,
75 }
76 }
77
78 #[must_use]
79 pub fn line_col(&self, offset: TextSize) -> LineCol {
80 let byte = usize::from(offset).min(self.source_len);
81 let line_idx = match self.line_start_bytes.binary_search(&byte) {
82 Ok(idx) => idx,
83 Err(idx) => idx.saturating_sub(1),
84 };
85 LineCol {
86 line: line_idx + 1,
87 column: byte - self.line_start_bytes[line_idx] + 1,
88 }
89 }
90
91 #[must_use]
92 pub fn char_line_col(&self, source: &str, offset: TextSize) -> LineCol {
93 let byte = usize::from(offset).min(self.source_len);
94 let line_idx = match self.line_start_bytes.binary_search(&byte) {
95 Ok(idx) => idx,
96 Err(idx) => idx.saturating_sub(1),
97 };
98 let line_start = self.line_start_bytes[line_idx];
99 LineCol {
100 line: line_idx + 1,
101 column: source[line_start..byte].chars().count() + 1,
102 }
103 }
104
105 #[must_use]
106 pub fn utf16_col(&self, line: usize, byte_col: usize) -> usize {
107 let line_start = self
108 .line_start_bytes
109 .get(line.saturating_sub(1))
110 .copied()
111 .unwrap_or(self.source_len);
112 let byte = line_start
113 .saturating_add(byte_col.saturating_sub(1))
114 .min(self.source_len);
115 self.utf16_offset_by_byte[byte] - self.utf16_offset_by_byte[line_start]
116 }
117
118 #[must_use]
119 pub fn utf16_range(&self, range: TextRange) -> (usize, usize) {
120 let start = usize::from(range.start()).min(self.source_len);
121 let end = usize::from(range.end()).min(self.source_len).max(start);
122 (
123 self.utf16_offset_by_byte[start],
124 self.utf16_offset_by_byte[end],
125 )
126 }
127}
128
129#[derive(Debug)]
130pub struct SourceTokens {
131 tokens: Vec<Token>,
132 trivia: Vec<Trivia>,
133 lex_errors: Vec<LexError>,
134 laid_out_tokens: OnceLock<Vec<Token>>,
135}
136
137impl SourceTokens {
138 #[must_use]
139 pub fn lex(source: &str) -> Self {
140 let (tokens, trivia, lex_errors) = lex_with_trivia(source);
141 Self {
142 tokens,
143 trivia,
144 lex_errors,
145 laid_out_tokens: OnceLock::new(),
146 }
147 }
148
149 #[must_use]
150 pub fn tokens(&self) -> &[Token] {
151 &self.tokens
152 }
153
154 #[must_use]
155 pub fn trivia(&self) -> &[Trivia] {
156 &self.trivia
157 }
158
159 #[must_use]
160 pub fn lex_errors(&self) -> &[LexError] {
161 &self.lex_errors
162 }
163
164 #[must_use]
165 pub fn laid_out_tokens(&self) -> &[Token] {
166 self.laid_out_tokens
167 .get_or_init(|| resolve_layout(self.tokens.clone()))
168 }
169}
170
171#[derive(Debug)]
172pub struct SourceFile {
173 source: String,
174 module: Module,
175 diagnostics: Vec<Diagnostic>,
176 line_index: LineIndex,
177 tokens: OnceLock<SourceTokens>,
178}
179
180impl SourceFile {
181 #[must_use]
182 pub fn parse(source: &str) -> Self {
183 let (module, parse_diagnostics) = parse_module(source);
184 let line_index = LineIndex::new(source);
185 let diagnostics = parse_diagnostics
186 .into_iter()
187 .map(|diagnostic| {
188 let range = parser_span_to_text_range(source, diagnostic.span);
189 let start = range.start();
190 let end_column = source
191 .get(usize::from(range.start())..usize::from(range.end()))
192 .filter(|s| !s.is_empty() && !s.contains('\n'))
193 .map(|s| diagnostic.pos.column + s.chars().count());
194 Diagnostic {
195 range,
196 line: line_index.char_line_col(source, start).line,
197 column: diagnostic.pos.column,
198 end_column,
199 message: diagnostic.message,
200 category: diagnostic.category.as_str(),
201 }
202 })
203 .collect();
204
205 Self {
206 source: source.to_string(),
207 module,
208 diagnostics,
209 line_index,
210 tokens: OnceLock::new(),
211 }
212 }
213
214 #[must_use]
215 pub fn source(&self) -> &str {
216 &self.source
217 }
218
219 #[must_use]
220 pub const fn module(&self) -> &Module {
221 &self.module
222 }
223
224 #[must_use]
225 pub fn diagnostics(&self) -> &[Diagnostic] {
226 &self.diagnostics
227 }
228
229 #[must_use]
230 pub const fn line_index(&self) -> &LineIndex {
231 &self.line_index
232 }
233
234 #[must_use]
235 pub fn tokens(&self) -> &[Token] {
236 self.source_tokens().tokens()
237 }
238
239 #[must_use]
240 pub fn trivia(&self) -> &[Trivia] {
241 self.source_tokens().trivia()
242 }
243
244 #[must_use]
245 pub fn laid_out_tokens(&self) -> &[Token] {
246 self.source_tokens().laid_out_tokens()
247 }
248
249 #[must_use]
250 pub fn parser_span_to_text_range(&self, span: ParserSpan) -> TextRange {
251 parser_span_to_text_range(&self.source, span)
252 }
253
254 fn source_tokens(&self) -> &SourceTokens {
255 self.tokens.get_or_init(|| SourceTokens::lex(&self.source))
256 }
257}
258
259#[must_use]
260pub fn parser_span_to_text_range(source: &str, span: ParserSpan) -> TextRange {
261 let start = span.start.min(source.len());
262 let end = span.end.min(source.len()).max(start);
263 TextRange::new(
264 TextSize::try_from(start).unwrap(),
265 TextSize::try_from(end).unwrap(),
266 )
267}
268
269#[cfg(test)]
270mod tests {
271 use super::*;
272 use daml_parser::ast_span::render_from_ast;
273 use daml_parser::lexer::render_lossless;
274
275 #[test]
276 fn maps_empty_source_to_first_line() {
277 let index = LineIndex::new("");
278
279 assert_eq!(index.line_col(0.into()), LineCol { line: 1, column: 1 });
280 assert_eq!(index.utf16_range(TextRange::empty(0.into())), (0, 0));
281 }
282
283 #[test]
284 fn maps_ascii_byte_lines() {
285 let source = "module M where\nfoo = 1\n";
286 let index = LineIndex::new(source);
287
288 assert_eq!(index.line_col(15.into()), LineCol { line: 2, column: 1 });
289 assert_eq!(index.utf16_col(2, 4), 3);
290 }
291
292 #[test]
293 fn maps_utf8_and_utf16_offsets() {
294 let source = "a😀b\nz";
295 let index = LineIndex::new(source);
296
297 assert_eq!(
298 index.utf16_range(TextRange::new(0.into(), 6.into())),
299 (0, 4)
300 );
301 assert_eq!(index.utf16_col(1, 6), 3);
302 assert_eq!(
303 index.char_line_col(source, 5.into()),
304 LineCol { line: 1, column: 3 }
305 );
306 }
307
308 #[test]
309 fn preserves_trailing_newline_line_start() {
310 let index = LineIndex::new("a\n");
311
312 assert_eq!(index.line_col(2.into()), LineCol { line: 2, column: 1 });
313 }
314
315 #[test]
316 fn treats_crlf_as_bytes_without_normalization() {
317 let index = LineIndex::new("a\r\nb");
318
319 assert_eq!(index.line_col(3.into()), LineCol { line: 2, column: 1 });
320 }
321
322 #[test]
323 fn clamps_ranges_to_source_end() {
324 let index = LineIndex::new("abc");
325 let range = TextRange::new(1.into(), 99.into());
326
327 assert_eq!(index.utf16_range(range), (1, 3));
328 }
329
330 #[test]
331 fn source_file_exposes_parser_pipeline_facts() {
332 let source = "module M where\nfoo : Int\nfoo = 1\n";
333 let file = SourceFile::parse(source);
334
335 assert_eq!(file.source(), source);
336 assert_eq!(file.module().name, "M");
337 assert!(file.diagnostics().is_empty());
338 assert!(!file.tokens().is_empty());
339 assert!(!file.laid_out_tokens().is_empty());
340 assert_eq!(
341 render_lossless(source, file.tokens(), file.trivia()).as_deref(),
342 Ok(source)
343 );
344 assert_eq!(
345 render_from_ast(source, file.module(), file.trivia()).as_deref(),
346 Ok(source)
347 );
348 }
349
350 #[test]
351 fn source_tokens_exposes_lex_only_pipeline_facts() {
352 let source = "module M where\nfoo : Int\nfoo = 1\n";
353 let tokens = SourceTokens::lex(source);
354
355 assert!(tokens.lex_errors().is_empty());
356 assert!(!tokens.tokens().is_empty());
357 assert!(!tokens.laid_out_tokens().is_empty());
358 assert_eq!(
359 render_lossless(source, tokens.tokens(), tokens.trivia()).as_deref(),
360 Ok(source)
361 );
362 }
363
364 #[test]
365 fn malformed_source_keeps_source_file_and_diagnostics() {
366 let file = SourceFile::parse("module M where\nfoo = \"unterminated\nbar = 1\n");
367
368 assert_eq!(file.module().name, "M");
369 assert!(file
370 .diagnostics()
371 .iter()
372 .any(|diagnostic| diagnostic.category == "lexical-error"));
373 }
374
375 #[test]
376 fn converts_parser_spans_to_text_ranges() {
377 let file = SourceFile::parse("module M where\nfoo = 1\n");
378 let range = file.parser_span_to_text_range(ParserSpan::new(0, 99));
379
380 assert_eq!(
381 range,
382 TextRange::new(0.into(), file.source().len().try_into().unwrap())
383 );
384 }
385}