rupg_sql/
filter.rs

1//! The filter between the lexer and the parser, a port of `base_yylex` in `parser.c`.
2//!
3//! The grammar is LALR(1), so it sees one token ahead. Some constructs need two. For them the filter looks at the next token and gives a different token for the current one:
4//!
5//! - `NOT` before `BETWEEN`, `IN`, `LIKE`, `ILIKE` or `SIMILAR` is `NOT_LA`.
6//! - `NULLS` before `FIRST` or `LAST` is `NULLS_LA`.
7//! - `WITH` before `TIME` or `ORDINALITY` is `WITH_LA`.
8//! - `WITHOUT` before `TIME` is `WITHOUT_LA`.
9//! - `FORMAT` before `JSON` is `FORMAT_LA`.
10//!
11//! The filter also makes `U&"..."` and `U&'...'`, with an optional `UESCAPE '...'` after them, into `IDENT` and `SCONST`.
12//!
13//! Lifted from `crates/rudb-pgparse/src/filter.rs` of tamnd/rudb at f5f7065a (spec/04 section 4.9).
14
15use std::borrow::Cow;
16
17use rupg_common::SqlState;
18
19use crate::error::{Error, Notice};
20use crate::generated::tables::token;
21use crate::lexer::{Lexer, Token, Value, truncate};
22
23/// The tokens that the parser reads.
24#[derive(Debug)]
25pub struct Tokens<'a> {
26    lexer: Lexer<'a>,
27    ahead: Option<Token<'a>>,
28}
29
30impl<'a> Tokens<'a> {
31    /// The tokens of `text`.
32    pub fn new(text: &'a str) -> Tokens<'a> {
33        Tokens { lexer: Lexer::new(text), ahead: None }
34    }
35
36    /// Takes the notices that the tokens until now gave.
37    pub fn take_notices(&mut self) -> Vec<Notice> {
38        self.lexer.take_notices()
39    }
40
41    /// The next token. After the last token it gives the end, token 0, again and again.
42    pub fn next_token(&mut self) -> Result<Token<'a>, Error> {
43        let mut current = match self.ahead.take() {
44            Some(token) => token,
45            None => self.lexer.next_token()?,
46        };
47        if !matches!(
48            current.kind,
49            token::FORMAT
50                | token::NOT
51                | token::NULLS_P
52                | token::WITH
53                | token::WITHOUT
54                | token::UIDENT
55                | token::USCONST
56        ) {
57            return Ok(current);
58        }
59        let next = self.lexer.next_token()?;
60        current.kind = match (current.kind, next.kind) {
61            (token::FORMAT, token::JSON) => token::FORMAT_LA,
62            (
63                token::NOT,
64                token::BETWEEN | token::IN_P | token::LIKE | token::ILIKE | token::SIMILAR,
65            ) => token::NOT_LA,
66            (token::NULLS_P, token::FIRST_P | token::LAST_P) => token::NULLS_LA,
67            (token::WITH, token::TIME | token::ORDINALITY) => token::WITH_LA,
68            (token::WITHOUT, token::TIME) => token::WITHOUT_LA,
69            (token::UIDENT | token::USCONST, _) => return self.unicode(current, next),
70            (kind, _) => kind,
71        };
72        self.ahead = Some(next);
73        Ok(current)
74    }
75
76    /// A `U&"..."` or a `U&'...'`, and the token after it.
77    fn unicode(&mut self, mut current: Token<'a>, next: Token<'a>) -> Result<Token<'a>, Error> {
78        let escape = if next.kind == token::UESCAPE {
79            let third = self.lexer.next_token()?;
80            let error = |message| {
81                Error::syntax(self.lexer.text().as_bytes(), message, third.start, third.end)
82            };
83            if third.kind != token::SCONST {
84                return Err(error("UESCAPE must be followed by a simple string literal"));
85            }
86            match third.text().map(str::as_bytes) {
87                Some(&[c]) if is_escape(c) => {
88                    // An error at the token after this one shows the text up to the end of the escape string, as it does in PostgreSQL.
89                    current.end = third.end;
90                    c
91                }
92                _ => return Err(error("invalid Unicode escape character")),
93            }
94        } else {
95            self.ahead = Some(next);
96            b'\\'
97        };
98        let text = current.text().unwrap_or_default();
99        let text = Cow::Owned(unescape(text, escape, current.start)?);
100        if current.kind == token::UIDENT {
101            current.value = Value::Text(truncate(text, self.lexer.notices()));
102            current.kind = token::IDENT;
103        } else {
104            current.value = Value::Text(text);
105            current.kind = token::SCONST;
106        }
107        Ok(current)
108    }
109}
110
111/// `check_uescapechar`: the escape character cannot be a hexadecimal digit, `+`, a quote, a double quote or a space.
112fn is_escape(c: u8) -> bool {
113    !(c.is_ascii_hexdigit()
114        || matches!(c, b'+' | b'\'' | b'"' | b' ' | b'\t' | b'\n' | b'\r' | 0x0b | 0x0c))
115}
116
117/// `str_udeescape`: makes the escapes of a `U&` string into characters. `position` is the offset of the `U&` token. An error points at the escape, at the offset in the value after `U&"`, as in PostgreSQL.
118fn unescape(text: &str, escape: u8, position: usize) -> Result<String, Error> {
119    let bytes = text.as_bytes();
120    let byte = |i: usize| bytes.get(i).copied().unwrap_or(0);
121    let hex = |from: usize, length: usize| {
122        let digits = bytes.get(from..from + length)?;
123        digits.iter().all(u8::is_ascii_hexdigit).then(|| {
124            digits.iter().fold(0u32, |v, &d| (v << 4) | char::from(d).to_digit(16).unwrap_or(0))
125        })
126    };
127    let location = |i: usize| position + 3 + i;
128    let pair =
129        |i: usize| Error::at(SqlState::SYNTAX_ERROR, "invalid Unicode surrogate pair", location(i));
130    let mut out = String::with_capacity(text.len());
131    let mut first = 0;
132    let mut i = 0;
133    while i < bytes.len() {
134        if bytes[i] != escape {
135            if first != 0 {
136                return Err(pair(i));
137            }
138            let length = text[i..].chars().next().map_or(1, char::len_utf8);
139            out.push_str(&text[i..i + length]);
140            i += length;
141            continue;
142        }
143        let (c, length) = if byte(i + 1) == escape {
144            if first != 0 {
145                return Err(pair(i));
146            }
147            out.push(char::from(escape));
148            i += 2;
149            continue;
150        } else if let Some(c) = hex(i + 1, 4) {
151            (c, 5)
152        } else if byte(i + 1) == b'+'
153            && let Some(c) = hex(i + 2, 6)
154        {
155            (c, 8)
156        } else {
157            return Err(Error {
158                code: SqlState::SYNTAX_ERROR,
159                message: "invalid Unicode escape".to_owned(),
160                detail: None,
161                hint: Some("Unicode escapes must be \\XXXX or \\+XXXXXX."),
162                location: Some(location(i)),
163            });
164        };
165        if c == 0 || c > 0x10ffff {
166            return Err(Error::at(
167                SqlState::SYNTAX_ERROR,
168                "invalid Unicode escape value",
169                location(i),
170            ));
171        }
172        let c = if first != 0 {
173            if !(0xdc00..=0xdfff).contains(&c) {
174                return Err(pair(i));
175            }
176            let c = ((first & 0x3ff) << 10) + (c & 0x3ff) + 0x10000;
177            first = 0;
178            c
179        } else if (0xdc00..=0xdfff).contains(&c) {
180            return Err(pair(i));
181        } else {
182            c
183        };
184        if (0xd800..=0xdbff).contains(&c) {
185            first = c;
186        } else if let Some(c) = char::from_u32(c) {
187            out.push(c);
188        }
189        i += length;
190    }
191    if first != 0 {
192        return Err(pair(i));
193    }
194    Ok(out)
195}
196
197#[cfg(test)]
198mod tests {
199    use super::*;
200
201    fn kinds(text: &str) -> Vec<u16> {
202        let mut tokens = Tokens::new(text);
203        let mut kinds = Vec::new();
204        loop {
205            match tokens.next_token().unwrap().kind {
206                0 => return kinds,
207                kind => kinds.push(kind),
208            }
209        }
210    }
211
212    #[test]
213    fn lookahead() {
214        use token::*;
215        assert_eq!(kinds("NOT IN NOT x"), [NOT_LA, IN_P, NOT, IDENT]);
216        assert_eq!(
217            kinds("NOT BETWEEN NOT LIKE NOT ILIKE NOT SIMILAR"),
218            [NOT_LA, BETWEEN, NOT_LA, LIKE, NOT_LA, ILIKE, NOT_LA, SIMILAR]
219        );
220        assert_eq!(
221            kinds("NULLS FIRST NULLS LAST NULLS x"),
222            [NULLS_LA, FIRST_P, NULLS_LA, LAST_P, NULLS_P, IDENT]
223        );
224        assert_eq!(
225            kinds("WITH TIME WITH ORDINALITY WITH"),
226            [WITH_LA, TIME, WITH_LA, ORDINALITY, WITH]
227        );
228        assert_eq!(kinds("WITHOUT TIME WITHOUT"), [WITHOUT_LA, TIME, WITHOUT]);
229        assert_eq!(kinds("FORMAT JSON FORMAT"), [FORMAT_LA, JSON, FORMAT]);
230        // Two lookahead tokens in a row.
231        assert_eq!(kinds("NOT NOT IN"), [NOT, NOT_LA, IN_P]);
232    }
233
234    #[test]
235    fn unicode() {
236        let mut tokens =
237            Tokens::new("U&'\\0041\\+01F600\\\\' U&'!0042' UESCAPE '!' U&\"\\00e9\" 1");
238        let a = tokens.next_token().unwrap();
239        assert_eq!((a.kind, a.text()), (token::SCONST, Some("A😀\\")));
240        let b = tokens.next_token().unwrap();
241        assert_eq!((b.kind, b.text()), (token::SCONST, Some("B")));
242        assert_eq!(&tokens.lexer.text()[b.start..b.end], "U&'!0042' UESCAPE '!'");
243        let c = tokens.next_token().unwrap();
244        assert_eq!((c.kind, c.text()), (token::IDENT, Some("é")));
245        assert_eq!(tokens.next_token().unwrap().kind, token::ICONST);
246        assert!(!is_escape(b'a') && !is_escape(b'+') && !is_escape(b' ') && is_escape(b'!'));
247    }
248}
rupg_sql/filter.rs

rupg_sql/
filter.rs
