Skip to main content

dsl_parse/
split.rs

1//! Split a SQL document on top-level semicolons.
2//!
3//! Why we slice ourselves rather than letting the parser do it: `sqlparser`
4//! aborts on the first syntax error, which would poison the rest of the
5//! file. By slicing first and feeding each piece in independently we
6//! contain failures to a single statement and keep editing other parts of
7//! the file fully analysed.
8//!
9//! Edge cases handled:
10//!   - Single-quoted strings: semicolons inside are part of the literal.
11//!   - Double-quoted identifiers: same.
12//!   - Postgres dollar-quoted blocks: `$tag$ ... $tag$` (any tag, including
13//!     the empty tag `$$ ... $$`).
14//!   - Backslash escapes inside quoted strings.
15//!
16//! Line comments (`--` to end-of-line) and block comments (`/* ... */`,
17//! including nested PG-style blocks) are stripped *for the purpose of
18//! finding statement boundaries*. A `;` inside a comment must not
19//! split the statement, otherwise the body fragment after `;` parses
20//! as broken SQL and produces a sql000 syntax error.
21
22use text_size::{TextRange, TextSize};
23
24/// Returns `(trimmed_chunk, range_in_source)` for each top-level statement.
25/// Whitespace-only chunks are dropped.
26pub fn split_statements(src: &str) -> Vec<(String, TextRange)> {
27  let mut out: Vec<(String, TextRange)> = Vec::new();
28  let bytes = src.as_bytes();
29  let mut start = 0usize;
30  let mut i = 0usize;
31  let mut in_single = false;
32  let mut in_double = false;
33  let mut dollar_tag: Option<String> = None;
34  let mut block_depth: u32 = 0;
35
36  while i < bytes.len() {
37    let c = bytes[i] as char;
38
39    // Block comment open / close (nestable, PG-flavoured).
40    if !in_single && !in_double && dollar_tag.is_none() {
41      if block_depth > 0 {
42        if i + 1 < bytes.len() && bytes[i] == b'*' && bytes[i + 1] == b'/' {
43          block_depth -= 1;
44          i += 2;
45          continue;
46        }
47        if i + 1 < bytes.len() && bytes[i] == b'/' && bytes[i + 1] == b'*' {
48          block_depth += 1;
49          i += 2;
50          continue;
51        }
52        i += 1;
53        continue;
54      }
55      if i + 1 < bytes.len() && bytes[i] == b'/' && bytes[i + 1] == b'*' {
56        block_depth = 1;
57        i += 2;
58        continue;
59      }
60      // Line comment: skip to end of line.
61      if i + 1 < bytes.len() && bytes[i] == b'-' && bytes[i + 1] == b'-' {
62        while i < bytes.len() && bytes[i] != b'\n' {
63          i += 1;
64        }
65        continue;
66      }
67    }
68
69    if let Some(tag) = &dollar_tag {
70      let closer = format!("${tag}$");
71      // Byte comparison, not `src[i..]`: inside a dollar-quoted body
72      // the cursor advances one byte at a time, so it lands mid-
73      // character on any multi-byte content and slicing the `str`
74      // panics with "byte index is not a char boundary".
75      if src.as_bytes()[i..].starts_with(closer.as_bytes()) {
76        i += closer.len();
77        dollar_tag = None;
78        continue;
79      }
80      i += 1;
81      continue;
82    }
83
84    if !in_single && !in_double && c == '$' {
85      // Try to read an opening dollar tag: $ident$ or $$.
86      let rest = &src[i + 1..];
87      if let Some(end) = rest.find('$') {
88        let tag = &rest[..end];
89        if tag.chars().all(|ch| ch.is_alphanumeric() || ch == '_') {
90          dollar_tag = Some(tag.to_string());
91          i += 1 + end + 1; // skip past $tag$
92          continue;
93        }
94      }
95    }
96
97    if !in_double && c == '\'' && (i == 0 || bytes[i - 1] != b'\\') {
98      in_single = !in_single;
99    } else if !in_single && c == '"' && (i == 0 || bytes[i - 1] != b'\\') {
100      in_double = !in_double;
101    } else if !in_single && !in_double && c == ';' {
102      push_chunk(src, start, i, &mut out);
103      start = i + 1;
104    }
105    i += 1;
106  }
107
108  push_chunk(src, start, src.len(), &mut out);
109  out
110}
111
112fn push_chunk(src: &str, start: usize, end: usize, out: &mut Vec<(String, TextRange)>) {
113  let raw = &src[start..end];
114  let chunk = raw.trim().to_string();
115  if chunk.is_empty() {
116    return;
117  }
118  // The raw slice spans `start..end` (semicolon-bounded), but the
119  // trimmed chunk excludes leading/trailing whitespace and the
120  // trailing semicolon. The range must follow the trimmed chunk so
121  // diagnostics and inlay hints anchor on the actual statement, not
122  // a preceding blank line, the previous statement's terminator, or
123  // a leading comment block (the codelens above-line anchor must
124  // sit above the SQL, not above any preceding doc-comment).
125  let trimmed_start = start + skip_leading_ws_and_comments(raw);
126  let trimmed_end = trimmed_start + (end - trimmed_start) - trailing_ws_bytes(&src[trimmed_start..end]);
127  // Also re-slice the chunk to match the new range so the stored
128  // chunk string excludes the leading comment block consistently.
129  let trimmed_chunk = src[trimmed_start..trimmed_end].trim().to_string();
130  if trimmed_chunk.is_empty() {
131    return;
132  }
133  // Some callers expect the chunk to end exactly where its range
134  // ends, so use the chunk's length for the right edge after the
135  // trim_end above strips trailing whitespace.
136  let final_end = trimmed_start + trimmed_chunk.len();
137  out.push((trimmed_chunk, TextRange::new(TextSize::from(trimmed_start as u32), TextSize::from(final_end as u32))));
138}
139
140/// Count leading bytes of `s` that are whitespace, `-- line comments`,
141/// or `/* block comments */`. Used to advance the statement range so
142/// the editor anchors codelens / inlay hints on the SQL itself rather
143/// than on a comment block that documents the statement.
144fn skip_leading_ws_and_comments(s: &str) -> usize {
145  let bytes = s.as_bytes();
146  let n = bytes.len();
147  let mut i = 0usize;
148  loop {
149    let before = i;
150    while i < n && bytes[i].is_ascii_whitespace() {
151      i += 1;
152    }
153    if i + 1 < n && bytes[i] == b'-' && bytes[i + 1] == b'-' {
154      while i < n && bytes[i] != b'\n' {
155        i += 1;
156      }
157      continue;
158    }
159    if i + 1 < n && bytes[i] == b'/' && bytes[i + 1] == b'*' {
160      let mut depth = 1u32;
161      i += 2;
162      while i + 1 < n && depth > 0 {
163        if bytes[i] == b'/' && bytes[i + 1] == b'*' {
164          depth += 1;
165          i += 2;
166        } else if bytes[i] == b'*' && bytes[i + 1] == b'/' {
167          depth -= 1;
168          i += 2;
169        } else {
170          i += 1;
171        }
172      }
173      continue;
174    }
175    if i == before {
176      return i;
177    }
178  }
179}
180
181fn trailing_ws_bytes(s: &str) -> usize {
182  s.len() - s.trim_end().len()
183}
184
185#[cfg(test)]
186mod tests_r4 {
187  use super::*;
188  #[test]
189  fn r4_stmt_range_skips_leading_line_comment() {
190    // CYCLE 4: stmt.range used to include the leading `-- comment\n`
191    // which made codelens anchor above the comment, not the SQL.
192    // Now skipped.
193    let src = "-- header comment\nSELECT 1;";
194    let chunks = split_statements(src);
195    assert_eq!(chunks.len(), 1);
196    let (chunk, range) = &chunks[0];
197    assert_eq!(chunk.trim(), "SELECT 1");
198    let s: u32 = range.start().into();
199    let select_pos = src.find("SELECT").unwrap() as u32;
200    assert_eq!(s, select_pos, "range should start at SELECT, not comment");
201  }
202
203  #[test]
204  fn r4_stmt_range_skips_block_comment() {
205    let src = "/* block doc */\nSELECT 1;";
206    let chunks = split_statements(src);
207    let (_, range) = &chunks[0];
208    let s: u32 = range.start().into();
209    let select_pos = src.find("SELECT").unwrap() as u32;
210    assert_eq!(s, select_pos);
211  }
212
213  #[test]
214  fn r4_stmt_range_skips_multi_comment_block() {
215    let src = "-- doc1\n-- doc2\n/* doc3 */\nSELECT * FROM users;";
216    let chunks = split_statements(src);
217    let (_, range) = &chunks[0];
218    let s: u32 = range.start().into();
219    let select_pos = src.find("SELECT").unwrap() as u32;
220    assert_eq!(s, select_pos);
221  }
222}