sentinel_core/normalize/
sql.rs

1//! Homemade SQL tokenizer/normalizer.
2//!
3//! Replaces numeric literals, string literals, and UUIDs with `?`
4//! placeholders. `PostgreSQL` positional parameters (`$1`, `$2`) are
5//! recognized as driver placeholders and emitted as `$?` with empty
6//! `params` (not extracted as literals). This keeps `params` empty for
7//! parameterized queries so the sanitizer-aware detection path can
8//! fire. Collapses `IN (?, ?, ?)` into `IN (?)`. Quoted identifiers are
9//! preserved verbatim: ANSI `"id"` and `MySQL` `` `id` ``.
10
11use regex::Regex;
12use std::borrow::Cow;
13use std::sync::LazyLock;
14
15static IN_LIST_RE: LazyLock<Regex> =
16    LazyLock::new(|| Regex::new(r"(?i)IN\s*\(\s*\?(?:\s*,\s*\?)*\s*\)").expect("static regex"));
17
18/// Result of SQL normalization.
19#[derive(Debug, Clone, PartialEq, Eq)]
20pub struct SqlNormalized {
21    pub template: String,
22    pub params: Vec<String>,
23}
24
25#[derive(Debug, Clone, Copy, PartialEq)]
26enum State {
27    Normal,
28    InString,
29    InNumber,
30    /// Inside a double-quoted identifier (e.g., `"MyTable"`). Preserved as-is.
31    InDoubleQuote,
32    /// Inside a `MySQL` backtick-quoted identifier (e.g. `` `col` ``). Preserved as-is.
33    InBacktick,
34    /// Inside a `PostgreSQL` dollar-quoted string (e.g., `$$body$$` or `$tag$body$tag$`).
35    InDollarQuote,
36}
37
38/// Mutable tokenizer state carried between steps.
39struct Tokenizer<'a> {
40    query: &'a str,
41    bytes: &'a [u8],
42    template: String,
43    params: Vec<String>,
44    i: usize,
45    state: State,
46    current_value: String,
47    seen_dot: bool,
48    has_in_list: bool,
49    normal_start: usize,
50    /// Start of the current string/dollar-quote literal content being accumulated.
51    value_start: usize,
52    /// The closing tag for dollar-quoted strings (e.g., `$$` or `$tag$`).
53    dollar_tag: Vec<u8>,
54}
55
56/// Maximum query length accepted for normalization (64 KB).
57/// Queries exceeding this are truncated to prevent unbounded memory usage.
58const MAX_QUERY_LEN: usize = 65_536;
59
60/// Normalize a SQL query by replacing literal values with `?` placeholders.
61#[must_use]
62pub fn normalize_sql(query: &str) -> SqlNormalized {
63    // Truncate at a char boundary to prevent unbounded allocation from adversarial input
64    let query = if query.len() > MAX_QUERY_LEN {
65        &query[..query.floor_char_boundary(MAX_QUERY_LEN)]
66    } else {
67        query
68    };
69    let mut t = Tokenizer {
70        query,
71        bytes: query.as_bytes(),
72        template: String::with_capacity(query.len()),
73        params: Vec::with_capacity(4),
74        i: 0,
75        state: State::Normal,
76        current_value: String::new(),
77        seen_dot: false,
78        has_in_list: false,
79        normal_start: 0,
80        value_start: 0,
81        dollar_tag: Vec::new(),
82    };
83
84    while t.i < t.bytes.len() {
85        match t.state {
86            State::Normal => step_normal(&mut t),
87            State::InString => step_in_string(&mut t),
88            State::InNumber => step_in_number(&mut t),
89            State::InDoubleQuote => step_in_double_quote(&mut t),
90            State::InBacktick => step_in_backtick(&mut t),
91            State::InDollarQuote => step_in_dollar_quote(&mut t),
92        }
93    }
94
95    flush_pending(&mut t);
96    collapse_in_lists(t.template, t.has_in_list, t.params)
97}
98
99/// Slice `query` by byte bounds, asserting in debug builds that both
100/// bounds fall on UTF-8 char boundaries.
101///
102/// The tokenizer only ever anchors slice bounds on ASCII delimiters
103/// (`'`, `"`, `$...$`, digits), and every ASCII byte is a char boundary,
104/// so this holds by construction today. The guard is a refactor net: a
105/// future slice taken at a non-ASCII-anchored position would panic on
106/// multi-byte input, invisible to a grep for `unwrap`/`panic`. Compiles
107/// out in release.
108fn checked_query_slice(query: &str, start: usize, end: usize) -> &str {
109    debug_assert!(
110        query.is_char_boundary(start) && query.is_char_boundary(end),
111        "SQL tokenizer query slice [{start}..{end}) is off a char boundary; \
112         bounds must anchor on ASCII delimiters"
113    );
114    &query[start..end]
115}
116
117fn step_normal(t: &mut Tokenizer<'_>) {
118    let b = t.bytes[t.i];
119    if b == b'\'' {
120        flush_normal_run(t);
121        t.state = State::InString;
122        t.current_value.clear();
123        t.value_start = t.i + 1; // points after the opening '
124    } else if b == b'"' {
125        // Double-quoted identifier: preserve as-is (don't replace literals inside)
126        t.state = State::InDoubleQuote;
127        t.i += 1;
128        return;
129    } else if b == b'`' {
130        // MySQL backtick-quoted identifier: preserve as-is.
131        t.state = State::InBacktick;
132        t.i += 1;
133        return;
134    } else if b == b'$' && is_dollar_param(t.i, t.bytes) {
135        // PostgreSQL positional parameter: $1, $2, etc. Preserve as
136        // `$?` in the template WITHOUT extracting the index as a
137        // param. The index is a placeholder, not a literal value.
138        // Without this, `$1` → params=["1"] which breaks
139        // `looks_sanitized` (params must be empty for sanitized
140        // queries to enter the sanitizer-aware classification path).
141        flush_normal_run(t);
142        let mut end = t.i + 1; // skip the `$`
143        while end < t.bytes.len() && t.bytes[end].is_ascii_digit() {
144            end += 1;
145        }
146        t.template.push_str("$?");
147        t.i = end;
148        t.normal_start = t.i;
149        return;
150    } else if b == b'$' && is_dollar_quote_start(t.i, t.bytes) {
151        // PostgreSQL dollar-quoted string: $$ or $tag$
152        let tag = extract_dollar_tag(t.i, t.bytes);
153        flush_normal_run(t);
154        let tag_len = tag.len();
155        t.dollar_tag = tag;
156        t.state = State::InDollarQuote;
157        t.current_value.clear();
158        t.i += tag_len;
159        t.value_start = t.i; // points after the opening tag
160        return;
161    } else if b.is_ascii_digit() && !is_identifier_byte_before(t.i, t.bytes) {
162        flush_normal_run(t);
163        t.state = State::InNumber;
164        t.seen_dot = false;
165        t.current_value.clear();
166        t.current_value.push(b as char);
167    } else if !t.has_in_list {
168        t.has_in_list = is_in_keyword(t.i, t.bytes);
169    }
170    t.i += 1;
171}
172
173fn step_in_string(t: &mut Tokenizer<'_>) {
174    let b = t.bytes[t.i];
175    if b == b'\'' {
176        if t.i + 1 < t.bytes.len() && t.bytes[t.i + 1] == b'\'' {
177            // Escaped quote '': flush accumulated slice, push a single quote, reset start
178            t.current_value
179                .push_str(checked_query_slice(t.query, t.value_start, t.i));
180            t.current_value.push('\'');
181            t.i += 2;
182            t.value_start = t.i;
183        } else {
184            // Closing quote: flush remaining content as a proper &str slice
185            t.current_value
186                .push_str(checked_query_slice(t.query, t.value_start, t.i));
187            t.params.push(std::mem::take(&mut t.current_value));
188            t.template.push('?');
189            t.state = State::Normal;
190            t.i += 1;
191            t.normal_start = t.i;
192        }
193    } else {
194        t.i += 1;
195    }
196}
197
198fn step_in_number(t: &mut Tokenizer<'_>) {
199    let b = t.bytes[t.i];
200    if b.is_ascii_digit() {
201        t.current_value.push(b as char);
202        t.i += 1;
203    } else if b == b'.' && !t.seen_dot {
204        t.seen_dot = true;
205        t.current_value.push('.');
206        t.i += 1;
207    } else {
208        t.params.push(std::mem::take(&mut t.current_value));
209        t.template.push('?');
210        t.state = State::Normal;
211        t.normal_start = t.i;
212    }
213}
214
215fn step_in_double_quote(t: &mut Tokenizer<'_>) {
216    if t.bytes[t.i] == b'"' {
217        t.state = State::Normal;
218    }
219    t.i += 1;
220}
221
222fn step_in_backtick(t: &mut Tokenizer<'_>) {
223    // Closes on the first backtick; doubled-backtick escapes (`` `a``b` ``)
224    // are not handled, matching the existing double-quote behavior.
225    if t.bytes[t.i] == b'`' {
226        t.state = State::Normal;
227    }
228    t.i += 1;
229}
230
231fn step_in_dollar_quote(t: &mut Tokenizer<'_>) {
232    // Look for the closing dollar tag at current position
233    let remaining = &t.bytes[t.i..];
234    if remaining.starts_with(&t.dollar_tag) {
235        // Found closing tag -- flush content as a proper &str slice
236        t.current_value
237            .push_str(checked_query_slice(t.query, t.value_start, t.i));
238        t.params.push(std::mem::take(&mut t.current_value));
239        t.template.push('?');
240        t.i += t.dollar_tag.len();
241        t.state = State::Normal;
242        t.normal_start = t.i;
243    } else {
244        t.i += 1;
245    }
246}
247
248/// Check if position `i` starts a `PostgreSQL` positional parameter
249/// (`$1`, `$2`, `$12`): `$` followed by at least one ASCII digit.
250/// The `$$` case (dollar-quoting) is excluded because `$` is not a
251/// digit. `PostgreSQL` forbids digit-starting dollar-quote tags
252/// (tags must follow identifier rules: letter or underscore first),
253/// so `$1$body$1$` is not valid and does not need a trailing-`$`
254/// guard. Must be checked BEFORE `is_dollar_quote_start` in
255/// `step_normal`.
256fn is_dollar_param(i: usize, bytes: &[u8]) -> bool {
257    debug_assert_eq!(bytes[i], b'$');
258    let next = i + 1;
259    next < bytes.len() && bytes[next].is_ascii_digit()
260}
261
262/// Check if position `i` starts a dollar-quote tag (`$$` or `$identifier$`).
263fn is_dollar_quote_start(i: usize, bytes: &[u8]) -> bool {
264    if i >= bytes.len() || bytes[i] != b'$' {
265        return false;
266    }
267    // $$ case
268    if i + 1 < bytes.len() && bytes[i + 1] == b'$' {
269        return true;
270    }
271    // $tag$ case: $ followed by identifier chars then $
272    let mut j = i + 1;
273    while j < bytes.len() && (bytes[j].is_ascii_alphanumeric() || bytes[j] == b'_') {
274        j += 1;
275    }
276    j > i + 1 && j < bytes.len() && bytes[j] == b'$'
277}
278
279/// Extract the dollar-quote tag starting at position `i` (e.g., `$$` or `$tag$`).
280///
281/// Allocates a small `Vec<u8>` per call. Dollar-quoted strings are rare
282/// in practice (PostgreSQL-specific), so this is not on the hot path.
283/// A stack-allocated array would avoid the heap, but adds complexity
284/// for negligible gain.
285fn extract_dollar_tag(i: usize, bytes: &[u8]) -> Vec<u8> {
286    // $$ case
287    if i + 1 < bytes.len() && bytes[i + 1] == b'$' {
288        return vec![b'$', b'$'];
289    }
290    // $tag$ case
291    let mut j = i + 1;
292    while j < bytes.len() && (bytes[j].is_ascii_alphanumeric() || bytes[j] == b'_') {
293        j += 1;
294    }
295    bytes[i..=j].to_vec()
296}
297
298fn flush_normal_run(t: &mut Tokenizer<'_>) {
299    if t.i > t.normal_start {
300        t.template
301            .push_str(checked_query_slice(t.query, t.normal_start, t.i));
302    }
303}
304
305fn flush_pending(t: &mut Tokenizer<'_>) {
306    match t.state {
307        State::InString | State::InDollarQuote => {
308            // Flush remaining content from the unflushed &str slice
309            t.current_value
310                .push_str(checked_query_slice(t.query, t.value_start, t.bytes.len()));
311            t.params.push(std::mem::take(&mut t.current_value));
312            t.template.push('?');
313        }
314        State::InNumber => {
315            t.params.push(std::mem::take(&mut t.current_value));
316            t.template.push('?');
317        }
318        State::Normal | State::InDoubleQuote | State::InBacktick => {
319            let len = t.bytes.len();
320            if len > t.normal_start {
321                t.template
322                    .push_str(checked_query_slice(t.query, t.normal_start, len));
323            }
324        }
325    }
326}
327
328fn is_in_keyword(i: usize, bytes: &[u8]) -> bool {
329    let b = bytes[i];
330    (b == b'I' || b == b'i')
331        && i + 1 < bytes.len()
332        && (bytes[i + 1] == b'N' || bytes[i + 1] == b'n')
333        && (i == 0 || bytes[i - 1].is_ascii_whitespace())
334        && (i + 2 >= bytes.len() || !bytes[i + 2].is_ascii_alphanumeric())
335}
336
337fn collapse_in_lists(template: String, has_in_list: bool, params: Vec<String>) -> SqlNormalized {
338    let template = if has_in_list {
339        match IN_LIST_RE.replace_all(&template, "IN (?)") {
340            Cow::Borrowed(_) => template,
341            Cow::Owned(s) => s,
342        }
343    } else {
344        template
345    };
346    SqlNormalized { template, params }
347}
348
349/// Check if the byte before position `i` is part of an identifier
350/// (letter, digit, or underscore), meaning the digit at `i` is NOT
351/// the start of a standalone numeric literal.
352fn is_identifier_byte_before(i: usize, bytes: &[u8]) -> bool {
353    if i == 0 {
354        return false;
355    }
356    let prev = bytes[i - 1];
357    prev.is_ascii_alphanumeric() || prev == b'_'
358}
359
360#[cfg(test)]
361mod tests {
362    use super::*;
363
364    #[test]
365    fn numeric_literal() {
366        let r = normalize_sql("SELECT * FROM order_item WHERE order_id = 42");
367        assert_eq!(r.template, "SELECT * FROM order_item WHERE order_id = ?");
368        assert_eq!(r.params, vec!["42"]);
369    }
370
371    #[test]
372    fn float_literal() {
373        let r = normalize_sql("SELECT * FROM t WHERE price > 3.14");
374        assert_eq!(r.template, "SELECT * FROM t WHERE price > ?");
375        assert_eq!(r.params, vec!["3.14"]);
376    }
377
378    #[test]
379    fn string_literal() {
380        let r = normalize_sql("SELECT * FROM users WHERE name = 'Alice'");
381        assert_eq!(r.template, "SELECT * FROM users WHERE name = ?");
382        assert_eq!(r.params, vec!["Alice"]);
383    }
384
385    #[test]
386    fn uuid_in_string() {
387        let r = normalize_sql("SELECT * FROM t WHERE id = 'a1b2c3d4-e5f6-7890-abcd-ef1234567890'");
388        assert_eq!(r.template, "SELECT * FROM t WHERE id = ?");
389        assert_eq!(r.params, vec!["a1b2c3d4-e5f6-7890-abcd-ef1234567890"]);
390    }
391
392    #[test]
393    fn in_list_collapsed() {
394        let r = normalize_sql("SELECT * FROM t WHERE id IN (1, 2, 3)");
395        assert_eq!(r.template, "SELECT * FROM t WHERE id IN (?)");
396        assert_eq!(r.params, vec!["1", "2", "3"]);
397    }
398
399    #[test]
400    fn in_list_strings_collapsed() {
401        let r = normalize_sql("SELECT * FROM t WHERE name IN ('a', 'b', 'c')");
402        assert_eq!(r.template, "SELECT * FROM t WHERE name IN (?)");
403        assert_eq!(r.params, vec!["a", "b", "c"]);
404    }
405
406    #[test]
407    fn escaped_quotes() {
408        let r = normalize_sql("SELECT * FROM t WHERE name = 'O''Brien'");
409        assert_eq!(r.template, "SELECT * FROM t WHERE name = ?");
410        assert_eq!(r.params, vec!["O'Brien"]);
411    }
412
413    #[test]
414    fn table_names_with_digits_preserved() {
415        let r = normalize_sql("SELECT * FROM order_item2 WHERE id = 1");
416        assert_eq!(r.template, "SELECT * FROM order_item2 WHERE id = ?");
417        assert_eq!(r.params, vec!["1"]);
418    }
419
420    #[test]
421    fn join_query() {
422        let r = normalize_sql(
423            "SELECT p.name FROM order_item p JOIN orders g ON p.order_id = g.id WHERE g.id = 42",
424        );
425        assert_eq!(
426            r.template,
427            "SELECT p.name FROM order_item p JOIN orders g ON p.order_id = g.id WHERE g.id = ?"
428        );
429        assert_eq!(r.params, vec!["42"]);
430    }
431
432    #[test]
433    fn multiple_params() {
434        let r = normalize_sql("UPDATE t SET a = 1, b = 'foo' WHERE id = 99");
435        assert_eq!(r.template, "UPDATE t SET a = ?, b = ? WHERE id = ?");
436        assert_eq!(r.params, vec!["1", "foo", "99"]);
437    }
438
439    #[test]
440    fn no_literals() {
441        let r = normalize_sql("SELECT count(*) FROM users");
442        assert_eq!(r.template, "SELECT count(*) FROM users");
443        assert!(r.params.is_empty());
444    }
445
446    #[test]
447    fn multi_dot_number_rejected() {
448        // 1.2.3 should be treated as 1.2 then .3, not a single number
449        let r = normalize_sql("SELECT * FROM t WHERE x = 1.2.3");
450        assert_eq!(r.params[0], "1.2");
451    }
452
453    #[test]
454    fn unterminated_string_flushed() {
455        let r = normalize_sql("SELECT * FROM t WHERE name = 'unterminated");
456        assert_eq!(r.template, "SELECT * FROM t WHERE name = ?");
457        assert_eq!(r.params, vec!["unterminated"]);
458    }
459
460    #[test]
461    fn empty_query() {
462        let r = normalize_sql("");
463        assert_eq!(r.template, "");
464        assert!(r.params.is_empty());
465    }
466
467    #[test]
468    fn number_at_start_of_query() {
469        let r = normalize_sql("42");
470        assert_eq!(r.template, "?");
471        assert_eq!(r.params, vec!["42"]);
472    }
473
474    #[test]
475    fn multi_dot_full_template() {
476        // 1.2.3 -> "1.2" is a float, then ".3" remains: dot in template, 3 is a new number
477        let r = normalize_sql("SELECT * FROM t WHERE x = 1.2.3");
478        assert_eq!(r.template, "SELECT * FROM t WHERE x = ?.?");
479        assert_eq!(r.params, vec!["1.2", "3"]);
480    }
481
482    #[test]
483    fn empty_string_literal() {
484        let r = normalize_sql("SELECT * FROM t WHERE name = ''");
485        assert_eq!(r.template, "SELECT * FROM t WHERE name = ?");
486        assert_eq!(r.params, vec![""]);
487    }
488
489    #[test]
490    fn digit_in_string_literal() {
491        let r = normalize_sql("SELECT * FROM t WHERE code = '42'");
492        assert_eq!(r.template, "SELECT * FROM t WHERE code = ?");
493        assert_eq!(r.params, vec!["42"]);
494    }
495
496    #[test]
497    fn underscore_before_digit_preserved() {
498        // col_1 is an identifier, the 1 is part of it
499        let r = normalize_sql("SELECT col_1 FROM t");
500        assert_eq!(r.template, "SELECT col_1 FROM t");
501        assert!(r.params.is_empty());
502    }
503
504    #[test]
505    fn number_only_query_at_eof() {
506        // Number flush at EOF
507        let r = normalize_sql("SELECT * FROM t LIMIT 100");
508        assert_eq!(r.template, "SELECT * FROM t LIMIT ?");
509        assert_eq!(r.params, vec!["100"]);
510    }
511
512    #[test]
513    fn cow_borrowed_path_no_in_list() {
514        // No IN clause -> Cow::Borrowed path
515        let r = normalize_sql("SELECT 1");
516        assert_eq!(r.template, "SELECT ?");
517        assert_eq!(r.params, vec!["1"]);
518    }
519
520    #[test]
521    fn negative_number_not_collapsed() {
522        // Minus sign is not part of the number token
523        let r = normalize_sql("SELECT * FROM t WHERE x = -5");
524        assert_eq!(r.template, "SELECT * FROM t WHERE x = -?");
525        assert_eq!(r.params, vec!["5"]);
526    }
527
528    // -- CTE support --
529
530    #[test]
531    fn cte_basic() {
532        let r = normalize_sql(
533            "WITH active AS (SELECT id FROM users WHERE status = 'active') \
534             SELECT * FROM orders WHERE user_id IN (SELECT id FROM active) AND total > 100",
535        );
536        assert_eq!(
537            r.template,
538            "WITH active AS (SELECT id FROM users WHERE status = ?) \
539             SELECT * FROM orders WHERE user_id IN (SELECT id FROM active) AND total > ?"
540        );
541        assert_eq!(r.params, vec!["active", "100"]);
542    }
543
544    #[test]
545    fn cte_nested() {
546        let r = normalize_sql(
547            "WITH a AS (SELECT 1), b AS (SELECT * FROM a WHERE x = 'test') \
548             SELECT * FROM b WHERE id = 42",
549        );
550        assert_eq!(
551            r.template,
552            "WITH a AS (SELECT ?), b AS (SELECT * FROM a WHERE x = ?) \
553             SELECT * FROM b WHERE id = ?"
554        );
555        assert_eq!(r.params, vec!["1", "test", "42"]);
556    }
557
558    // -- Double-quoted identifiers --
559
560    #[test]
561    fn double_quoted_identifier_preserved() {
562        let r = normalize_sql(r#"SELECT * FROM "MyTable" WHERE "Column" = 42"#);
563        assert_eq!(r.template, r#"SELECT * FROM "MyTable" WHERE "Column" = ?"#);
564        assert_eq!(r.params, vec!["42"]);
565    }
566
567    #[test]
568    fn double_quoted_with_digits_preserved() {
569        // Digits inside double quotes should NOT be treated as literals
570        let r = normalize_sql(r#"SELECT * FROM "table_2" WHERE "col_3" = 'value'"#);
571        assert_eq!(r.template, r#"SELECT * FROM "table_2" WHERE "col_3" = ?"#);
572        assert_eq!(r.params, vec!["value"]);
573    }
574
575    // -- MySQL backtick identifiers --
576
577    #[test]
578    fn backtick_identifier_preserved() {
579        let r = normalize_sql("SELECT `name` FROM `users` WHERE `id` = 42");
580        assert_eq!(r.template, "SELECT `name` FROM `users` WHERE `id` = ?");
581        assert_eq!(r.params, vec!["42"]);
582    }
583
584    #[test]
585    fn backtick_identifier_with_digits_preserved() {
586        // Digits inside backticks must NOT be extracted as literals.
587        let r = normalize_sql("SELECT `col2` FROM `table_3` WHERE `id` = 'x'");
588        assert_eq!(r.template, "SELECT `col2` FROM `table_3` WHERE `id` = ?");
589        assert_eq!(r.params, vec!["x"]);
590    }
591
592    #[test]
593    fn sqlserver_bracket_identifier_survives_without_special_handling() {
594        // `[` is not a special state, but common SQL Server identifiers still
595        // normalize cleanly: a digit after an identifier byte is kept (`Col1`),
596        // the literal outside the brackets becomes `?`.
597        let r = normalize_sql("SELECT [Col1] FROM [dbo].[Users] WHERE [Id] = 5");
598        assert_eq!(
599            r.template,
600            "SELECT [Col1] FROM [dbo].[Users] WHERE [Id] = ?"
601        );
602        assert_eq!(r.params, vec!["5"]);
603    }
604
605    #[test]
606    fn bracket_string_literals_are_redacted() {
607        // PostgreSQL array/subscript string values must be redacted, not
608        // leaked verbatim: `[` is a normal char, so `'...'` runs through the
609        // string path. Guards against PII/secret leakage into templates.
610        let r = normalize_sql("SELECT * FROM t WHERE tags = ARRAY['secret', 'pii']");
611        assert_eq!(r.template, "SELECT * FROM t WHERE tags = ARRAY[?, ?]");
612        assert_eq!(r.params, vec!["secret", "pii"]);
613    }
614
615    // -- PostgreSQL positional parameters ($1, $2) --
616
617    #[test]
618    fn dollar_param_single() {
619        let r = normalize_sql("SELECT * FROM order_items WHERE order_id = $1");
620        assert_eq!(r.template, "SELECT * FROM order_items WHERE order_id = $?");
621        assert!(
622            r.params.is_empty(),
623            "positional $1 is a placeholder, not a literal"
624        );
625    }
626
627    #[test]
628    fn dollar_param_multiple() {
629        let r = normalize_sql("SELECT * FROM t WHERE a = $1 AND b = $2");
630        assert_eq!(r.template, "SELECT * FROM t WHERE a = $? AND b = $?");
631        assert!(r.params.is_empty());
632    }
633
634    #[test]
635    fn dollar_param_two_digit_index() {
636        let r = normalize_sql("SELECT * FROM t WHERE id = $12");
637        assert_eq!(r.template, "SELECT * FROM t WHERE id = $?");
638        assert!(r.params.is_empty());
639    }
640
641    #[test]
642    fn dollar_param_does_not_break_dollar_quote() {
643        // $$ is a dollar-quote, not a $param. Ensure the dollar-param
644        // check (which runs first) does not swallow $$...$$.
645        let r = normalize_sql("SELECT $$hello$$");
646        assert_eq!(r.template, "SELECT ?");
647        assert_eq!(r.params, vec!["hello"]);
648    }
649
650    // -- Dollar-quoted strings (PostgreSQL) --
651
652    #[test]
653    fn dollar_quote_basic() {
654        let r = normalize_sql("SELECT $$hello world$$ AS greeting");
655        assert_eq!(r.template, "SELECT ? AS greeting");
656        assert_eq!(r.params, vec!["hello world"]);
657    }
658
659    #[test]
660    fn dollar_quote_tagged() {
661        let r = normalize_sql("SELECT $tag$some body$tag$ AS body");
662        assert_eq!(r.template, "SELECT ? AS body");
663        assert_eq!(r.params, vec!["some body"]);
664    }
665
666    #[test]
667    fn dollar_quote_in_function() {
668        let r = normalize_sql(
669            "CREATE FUNCTION foo() RETURNS void AS $$ BEGIN RAISE NOTICE 'hi'; END; $$ LANGUAGE plpgsql",
670        );
671        assert_eq!(
672            r.template,
673            "CREATE FUNCTION foo() RETURNS void AS ? LANGUAGE plpgsql"
674        );
675    }
676
677    // -- CALL statements --
678
679    #[test]
680    fn call_with_params() {
681        let r = normalize_sql("CALL process_order(42, 'rush', NOW())");
682        assert_eq!(r.template, "CALL process_order(?, ?, NOW())");
683        assert_eq!(r.params, vec!["42", "rush"]);
684    }
685
686    #[test]
687    fn call_with_interval() {
688        let r = normalize_sql("CALL schedule_task(1, INTERVAL '2 days')");
689        assert_eq!(r.template, "CALL schedule_task(?, INTERVAL ?)");
690        assert_eq!(r.params, vec!["1", "2 days"]);
691    }
692
693    // -- UTF-8 multi-byte support --
694
695    #[test]
696    fn utf8_in_string_literal() {
697        let r = normalize_sql("SELECT * FROM t WHERE name = 'caf\u{00e9}'");
698        assert_eq!(r.template, "SELECT * FROM t WHERE name = ?");
699        assert_eq!(r.params, vec!["caf\u{00e9}"]);
700    }
701
702    #[test]
703    fn utf8_emoji_in_string_literal() {
704        let r = normalize_sql("INSERT INTO t (msg) VALUES ('\u{1F600} hello')");
705        assert_eq!(r.template, "INSERT INTO t (msg) VALUES (?)");
706        assert_eq!(r.params, vec!["\u{1F600} hello"]);
707    }
708
709    #[test]
710    fn utf8_cjk_in_string_literal() {
711        let r = normalize_sql("SELECT * FROM t WHERE name = '\u{4F60}\u{597D}'");
712        assert_eq!(r.template, "SELECT * FROM t WHERE name = ?");
713        assert_eq!(r.params, vec!["\u{4F60}\u{597D}"]);
714    }
715
716    #[test]
717    fn utf8_in_dollar_quote() {
718        let r = normalize_sql("SELECT $$caf\u{00e9} au lait$$ AS drink");
719        assert_eq!(r.template, "SELECT ? AS drink");
720        assert_eq!(r.params, vec!["caf\u{00e9} au lait"]);
721    }
722
723    #[test]
724    fn utf8_with_escaped_quotes() {
725        let r = normalize_sql("SELECT * FROM t WHERE name = 'caf\u{00e9} d''or'");
726        assert_eq!(r.template, "SELECT * FROM t WHERE name = ?");
727        assert_eq!(r.params, vec!["caf\u{00e9} d'or"]);
728    }
729
730    // -- char-boundary slice guard --
731
732    #[test]
733    fn checked_query_slice_returns_substring_on_char_boundaries() {
734        // "x\u{00e9}y": x=0, e-acute=1..3 (2 bytes), y=3. Bounds 1 and 3
735        // are char boundaries, so the slice is the middle character.
736        assert_eq!(checked_query_slice("x\u{00e9}y", 1, 3), "\u{00e9}");
737    }
738
739    #[cfg(debug_assertions)]
740    #[test]
741    #[should_panic(expected = "must anchor on ASCII delimiters")]
742    fn checked_query_slice_rejects_non_char_boundary() {
743        // End bound 2 falls inside the 2-byte e-acute (bytes 1..3): not a
744        // char boundary. The debug-only guard must fire with the tokenizer
745        // message before the raw slice would panic.
746        let _ = checked_query_slice("x\u{00e9}y", 1, 2);
747    }
748
749    // -- SQL comments (pass through as-is, no special handling) --
750
751    #[test]
752    fn line_comment_passes_through() {
753        let r = normalize_sql("SELECT 1 -- this is a comment");
754        assert_eq!(r.template, "SELECT ? -- this is a comment");
755        assert_eq!(r.params, vec!["1"]);
756    }
757
758    #[test]
759    fn block_comment_passes_through() {
760        let r = normalize_sql("SELECT /* comment */ 1 FROM t");
761        assert_eq!(r.template, "SELECT /* comment */ ? FROM t");
762    }
763
764    #[test]
765    fn comment_inside_string_not_treated_as_comment() {
766        let r = normalize_sql("SELECT * FROM t WHERE name = 'value -- not a comment'");
767        assert_eq!(r.template, "SELECT * FROM t WHERE name = ?");
768        assert_eq!(r.params, vec!["value -- not a comment"]);
769    }
770
771    // -- Unterminated constructs --
772
773    #[test]
774    fn unterminated_dollar_quote_flushed() {
775        let r = normalize_sql("SELECT $$incomplete");
776        assert_eq!(r.template, "SELECT ?");
777        assert_eq!(r.params, vec!["incomplete"]);
778    }
779
780    #[test]
781    fn unterminated_double_quote_flushed() {
782        let r = normalize_sql("SELECT \"unterminated");
783        assert_eq!(r.template, "SELECT \"unterminated");
784        assert!(r.params.is_empty());
785    }
786
787    // -- Double-quoted identifiers with escaped quotes --
788
789    #[test]
790    fn double_quoted_identifier_with_digits() {
791        let r = normalize_sql(r#"SELECT "col123" FROM "table456" WHERE id = 1"#);
792        assert_eq!(
793            r.template,
794            r#"SELECT "col123" FROM "table456" WHERE id = ?"#
795        );
796        assert_eq!(r.params, vec!["1"]);
797    }
798
799    // -- Empty dollar-quoted strings --
800
801    #[test]
802    fn empty_dollar_quoted_string() {
803        let r = normalize_sql("SELECT $$$$ AS empty");
804        assert_eq!(r.template, "SELECT ? AS empty");
805        assert_eq!(r.params, vec![""]);
806    }
807
808    // -- Query length truncation --
809
810    #[test]
811    fn long_query_truncated_at_max() {
812        let long_query = format!("SELECT * FROM t WHERE name = '{}'", "a".repeat(70_000));
813        let r = normalize_sql(&long_query);
814        // Template should be truncated, not the full 70k+ length
815        assert!(r.template.len() <= 65_536 + 10); // some overhead for template chars
816    }
817
818    // -- Whitespace-only string literal --
819
820    #[test]
821    fn whitespace_only_string_literal() {
822        let r = normalize_sql("SELECT * FROM t WHERE name = '   '");
823        assert_eq!(r.template, "SELECT * FROM t WHERE name = ?");
824        assert_eq!(r.params, vec!["   "]);
825    }
826
827    // -- Four consecutive quotes (escaped empty + close) --
828
829    #[test]
830    fn four_consecutive_quotes() {
831        let r = normalize_sql("SELECT * FROM t WHERE name = ''''");
832        assert_eq!(r.template, "SELECT * FROM t WHERE name = ?");
833        assert_eq!(r.params, vec!["'"]);
834    }
835}
sentinel_core/normalize/sql.rs

sentinel_core/normalize/
sql.rs
