sql_ddl_to_json_schema/
parser.rs

1//! The public `Parser`: streaming statement splitter plus statement parsing.
2
3use serde_json::{json, Value};
4
5use crate::compact;
6use crate::grammar::parse_statement;
7use crate::json_schema;
8
9/// Error returned by parser operations.
10///
11/// Marked `#[non_exhaustive]` so new variants can be added without a breaking
12/// change. Match arms on this type must include a wildcard.
13#[derive(Debug, Clone, PartialEq, Eq)]
14#[non_exhaustive]
15pub enum Error {
16    /// An unsupported SQL dialect was given to the constructor.
17    UnsupportedDialect(String),
18    /// A statement failed to parse. The message contains the line number in
19    /// stream coordinates.
20    Parse(String),
21    /// A formatter received JSON that was not a valid parse tree root.
22    Format(String),
23}
24
25impl std::fmt::Display for Error {
26    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
27        match self {
28            Error::UnsupportedDialect(d) => write!(
29                f,
30                "Unsupported SQL dialect given to parser: '{}'. Please provide 'mysql', 'mariadb' or none to use default.",
31                d
32            ),
33            Error::Parse(m) => write!(f, "{}", m),
34            Error::Format(m) => write!(f, "{}", m),
35        }
36    }
37}
38
39impl std::error::Error for Error {}
40
41/// Options for JSON Schema output.
42#[derive(Debug, Clone, Copy, PartialEq, Eq)]
43pub struct JsonSchemaOptions {
44    /// Whether to keep column schemas under `definitions` and reference them
45    /// with `$ref`. When false, schemas are flattened into `properties`.
46    pub use_ref: bool,
47}
48
49impl Default for JsonSchemaOptions {
50    fn default() -> Self {
51        JsonSchemaOptions { use_ref: true }
52    }
53}
54
55/// Parser for MySQL and MariaDB DDL.
56///
57/// Feed SQL in one or more chunks, then read the parse tree, the compact table
58/// model, or JSON Schema documents.
59///
60/// ```
61/// use sql_ddl_to_json_schema::Parser;
62///
63/// let mut parser = Parser::new("mysql").unwrap();
64/// parser.feed("CREATE TABLE t (id INT PRIMARY KEY);");
65/// let tables = parser.parse_compact().unwrap();
66/// assert_eq!(tables[0]["name"], "t");
67/// ```
68pub struct Parser {
69    statements: Vec<String>,
70    remains: String,
71    escaped: bool,
72    quoted: Option<char>,
73    comment: Option<Comment>,
74    previous_unit: Option<u16>,
75    second_previous_unit: Option<u16>,
76}
77
78#[derive(Debug, Clone, Copy, PartialEq, Eq)]
79enum Comment {
80    HashLine,
81    DashLine,
82    Block { previous_was_star: bool },
83}
84
85impl Parser {
86    /// Build a parser for the given dialect.
87    ///
88    /// An empty string, `"mysql"`, or `"mariadb"` selects the MySQL grammar.
89    /// Any other value returns [`Error::UnsupportedDialect`].
90    pub fn new(dialect: &str) -> Result<Self, Error> {
91        if dialect.is_empty() || dialect == "mysql" || dialect == "mariadb" {
92            Ok(Parser {
93                statements: Vec::new(),
94                remains: String::new(),
95                escaped: false,
96                quoted: None,
97                comment: None,
98                previous_unit: None,
99                second_previous_unit: None,
100            })
101        } else {
102            Err(Error::UnsupportedDialect(dialect.to_string()))
103        }
104    }
105
106    /// Feed a chunk of SQL. Splits input into statements at unquoted, unescaped
107    /// semicolons and keeps cross-chunk state. Returns `&mut Self` for chaining.
108    pub fn feed(&mut self, chunk: &str) -> &mut Self {
109        // Operate on UTF-16 code units to match the source string indexing.
110        let units: Vec<u16> = chunk.encode_utf16().collect();
111        let mut parsed: Vec<u16> = Vec::with_capacity(units.len());
112        let mut last_statement_index = 0usize;
113
114        for (i, &unit) in units.iter().enumerate() {
115            parsed.push(unit);
116            let ch = char::from_u32(unit as u32);
117            let previous_unit = self.previous_unit;
118            let second_previous_unit = self.second_previous_unit;
119
120            if let Some(comment) = self.comment {
121                match comment {
122                    Comment::HashLine | Comment::DashLine => {
123                        if matches!(ch, Some('\n') | Some('\r')) {
124                            self.comment = None;
125                        }
126                    }
127                    Comment::Block { previous_was_star } => {
128                        if unit == b'/' as u16 && previous_was_star {
129                            self.comment = None;
130                        } else {
131                            self.comment = Some(Comment::Block {
132                                previous_was_star: unit == b'*' as u16,
133                            });
134                        }
135                    }
136                }
137                self.escaped = false;
138            } else if ch == Some('\\') {
139                self.escaped = !self.escaped;
140            } else {
141                if !self.escaped && is_quote_char(ch) {
142                    let c = ch.unwrap();
143                    match self.quoted {
144                        Some(open) => {
145                            if open == c {
146                                self.quoted = None;
147                            }
148                        }
149                        None => self.quoted = Some(c),
150                    }
151                } else if self.quoted.is_none() {
152                    if ch == Some('#') {
153                        self.comment = Some(Comment::HashLine);
154                    } else if second_previous_unit == Some(b'-' as u16)
155                        && previous_unit == Some(b'-' as u16)
156                        && unit == b' ' as u16
157                    {
158                        self.comment = Some(Comment::DashLine);
159                    } else if previous_unit == Some(b'/' as u16) && unit == b'*' as u16 {
160                        self.comment = Some(Comment::Block {
161                            previous_was_star: false,
162                        });
163                    } else if ch == Some(';') {
164                        let slice = &parsed[last_statement_index..i + 1];
165                        let statement = format!("{}{}", self.remains, utf16_to_string(slice));
166                        self.statements.push(statement);
167                        self.remains.clear();
168                        last_statement_index = i + 1;
169                    }
170                }
171                self.escaped = false;
172            }
173
174            self.second_previous_unit = previous_unit;
175            self.previous_unit = Some(unit);
176        }
177
178        let tail = &parsed[last_statement_index..];
179        self.remains.push_str(&utf16_to_string(tail));
180        self
181    }
182
183    /// Drain the buffered statements and parse each into a `P_DDS` node.
184    ///
185    /// Returns the parse tree root `{ id: "MAIN", def: [...] }`. This consumes
186    /// the pending statements, so a second call with no new input returns an
187    /// empty tree. On a parse error the line number is reported in stream
188    /// coordinates and all state resets.
189    ///
190    /// ```
191    /// use sql_ddl_to_json_schema::Parser;
192    ///
193    /// let mut parser = Parser::new("mysql").unwrap();
194    /// parser.feed("CREATE TABLE t (id INT);");
195    /// let first = parser.parse().unwrap();
196    /// assert_eq!(first["def"].as_array().unwrap().len(), 1);
197    /// // The statements are drained. A second call sees nothing.
198    /// let second = parser.parse().unwrap();
199    /// assert!(second["def"].as_array().unwrap().is_empty());
200    /// ```
201    pub fn parse(&mut self) -> Result<Value, Error> {
202        let mut line_count = 1usize;
203        let mut results = Vec::new();
204
205        let statements = std::mem::take(&mut self.statements);
206        for statement in statements {
207            if statement.is_empty() {
208                break;
209            }
210            match parse_statement(&statement) {
211                Ok(node) => {
212                    line_count += count_line_breaks(&statement);
213                    results.push(node);
214                }
215                Err(e) => {
216                    let error_line = e.line;
217                    let new_count = line_count + error_line - 1;
218                    // Reset all state so a failed parse leaves a clean parser.
219                    self.statements.clear();
220                    self.remains.clear();
221                    self.escaped = false;
222                    self.quoted = None;
223                    self.comment = None;
224                    self.previous_unit = None;
225                    self.second_previous_unit = None;
226                    return Err(Error::Parse(format!(
227                        "invalid syntax at line {}",
228                        new_count
229                    )));
230                }
231            }
232        }
233
234        self.remains.clear();
235        self.escaped = false;
236        self.quoted = None;
237        self.comment = None;
238        self.previous_unit = None;
239        self.second_previous_unit = None;
240
241        Ok(json!({ "id": "MAIN", "def": results }))
242    }
243
244    /// Drain the buffered statements and build the compact table model.
245    ///
246    /// Consumes the pending statements. To format a tree you already hold, use
247    /// [`compact_from_tree`].
248    pub fn parse_compact(&mut self) -> Result<Vec<Value>, Error> {
249        let tree = self.parse()?;
250        compact_from_tree(&tree)
251    }
252
253    /// Drain the buffered statements and build JSON Schema draft-07 documents,
254    /// one per table.
255    ///
256    /// Consumes the pending statements. Pass [`JsonSchemaOptions::default`] for
257    /// the default `{ use_ref: true }`. To format tables you already hold, use
258    /// [`json_schema_from_tables`].
259    pub fn parse_json_schema(&mut self, options: JsonSchemaOptions) -> Result<Vec<Value>, Error> {
260        let tables = self.parse_compact()?;
261        Ok(json_schema_from_tables(&tables, options))
262    }
263}
264
265/// Build the compact table model from a `MAIN` parse tree.
266///
267/// Use this to format a tree you already hold, for example one returned by
268/// [`Parser::parse`]. Returns [`Error::Format`] if the root id is not `MAIN`.
269pub fn compact_from_tree(tree: &Value) -> Result<Vec<Value>, Error> {
270    compact::format(tree).map_err(Error::Format)
271}
272
273/// Build JSON Schema draft-07 documents from compact tables, one per table.
274///
275/// Use this to format tables you already hold, for example those returned by
276/// [`Parser::parse_compact`].
277#[must_use]
278pub fn json_schema_from_tables(tables: &[Value], options: JsonSchemaOptions) -> Vec<Value> {
279    json_schema::format(tables, options)
280}
281
282/// Whether a char opens or closes a quoted region.
283fn is_quote_char(ch: Option<char>) -> bool {
284    matches!(ch, Some('"') | Some('\'') | Some('`'))
285}
286
287/// Count `\r\n`, `\r`, and `\n` line breaks the way the source regex does.
288fn count_line_breaks(s: &str) -> usize {
289    let bytes = s.as_bytes();
290    let mut count = 0;
291    let mut i = 0;
292    while i < bytes.len() {
293        match bytes[i] {
294            b'\r' => {
295                if i + 1 < bytes.len() && bytes[i + 1] == b'\n' {
296                    i += 2;
297                } else {
298                    i += 1;
299                }
300                count += 1;
301            }
302            b'\n' => {
303                count += 1;
304                i += 1;
305            }
306            _ => i += 1,
307        }
308    }
309    count
310}
311
312/// Decode a UTF-16 slice into a string. Invalid units become the replacement
313/// character, which does not occur for real DDL input.
314fn utf16_to_string(units: &[u16]) -> String {
315    String::from_utf16_lossy(units)
316}
317
318#[cfg(test)]
319mod tests {
320    use super::*;
321
322    // The preparser tracks escape state, the open quote, and the count of split
323    // statements. These fields are private, so the checks live here.
324
325    #[test]
326    fn tracks_escape_quote_and_statements() {
327        let mut p = Parser::new("mysql").unwrap();
328
329        // Single char.
330        p.feed("a");
331        assert!(!p.escaped);
332        assert_eq!(p.quoted, None);
333
334        // Start escaping.
335        p.feed("\\");
336        assert!(p.escaped);
337        assert_eq!(p.quoted, None);
338
339        // Finish escaping.
340        p.feed("\\");
341        assert!(!p.escaped);
342        assert_eq!(p.quoted, None);
343
344        p.feed("\\");
345        assert!(p.escaped);
346        p.feed("n");
347        assert!(!p.escaped);
348        assert_eq!(p.quoted, None);
349
350        // Double quotes without escape.
351        p.feed("\"");
352        assert!(!p.escaped);
353        assert_eq!(p.quoted, Some('"'));
354        p.feed("\"");
355        assert_eq!(p.quoted, None);
356        p.feed("\"a");
357        assert_eq!(p.quoted, Some('"'));
358        p.feed("a\"");
359        assert_eq!(p.quoted, None);
360
361        // Single quotes without escape.
362        p.feed("'");
363        assert_eq!(p.quoted, Some('\''));
364        p.feed("'");
365        assert_eq!(p.quoted, None);
366        p.feed("'a");
367        assert_eq!(p.quoted, Some('\''));
368        p.feed("a'");
369        assert_eq!(p.quoted, None);
370
371        // Backticks without escape.
372        p.feed("`");
373        assert_eq!(p.quoted, Some('`'));
374        p.feed("`");
375        assert_eq!(p.quoted, None);
376        p.feed("`a");
377        assert_eq!(p.quoted, Some('`'));
378        p.feed("a`");
379        assert_eq!(p.quoted, None);
380
381        // Quoting with escape.
382        p.feed("`\\`");
383        assert!(!p.escaped);
384        assert_eq!(p.quoted, Some('`'));
385        p.feed("\\\\`");
386        assert_eq!(p.quoted, None);
387
388        p.feed("\"\\");
389        assert!(p.escaped);
390        assert_eq!(p.quoted, Some('"'));
391        p.feed("\"a`");
392        assert_eq!(p.quoted, Some('"'));
393        p.feed("'\\'");
394        assert_eq!(p.quoted, Some('"'));
395        p.feed("\"");
396        assert_eq!(p.quoted, None);
397
398        // Escaped semicolon does not split. The next char ends the escape.
399        p.feed("\\;");
400        assert!(!p.escaped);
401        assert_eq!(p.quoted, None);
402        assert_eq!(p.statements.len(), 1);
403
404        // Unescaped semicolon splits.
405        p.feed("a;");
406        assert_eq!(p.statements.len(), 2);
407
408        // Semicolon inside quotes does not split.
409        p.feed("a\";");
410        assert_eq!(p.quoted, Some('"'));
411        assert_eq!(p.statements.len(), 2);
412
413        p.feed("a\";");
414        assert_eq!(p.quoted, None);
415        assert_eq!(p.statements.len(), 3);
416    }
417
418    #[test]
419    fn splits_on_utf16_code_units() {
420        // Statement splitting counts UTF-16 code units. An astral char is two
421        // units, and it must not derail the split at the following semicolon.
422        let mut p = Parser::new("mysql").unwrap();
423        p.feed("USE \u{1F600};USE b;");
424        assert_eq!(p.statements.len(), 2);
425    }
426
427    #[test]
428    fn semicolon_in_block_comment_does_not_split_statement() {
429        let mut p = Parser::new("mysql").unwrap();
430        p.feed("CREATE TABLE c (id INT /* ; */);");
431
432        let tables = p.parse_compact().unwrap();
433
434        assert_eq!(tables[0]["name"], "c");
435    }
436
437    #[test]
438    fn comment_semicolons_do_not_split_at_chunk_boundaries() {
439        let scenarios: &[&[&str]] = &[
440            &["CREATE TABLE c (id INT /", "* ; *", "/);"],
441            &["CREATE TABLE c (id INT) -", "- ;\n;"],
442            &["CREATE TABLE c (id INT /*/ ; */);"],
443            &["CREATE TABLE c (id INT /* ; *", "/);"],
444        ];
445
446        for chunks in scenarios {
447            let mut p = Parser::new("mysql").unwrap();
448            for chunk in *chunks {
449                p.feed(chunk);
450            }
451
452            let tables = p.parse_compact().unwrap();
453
454            assert_eq!(tables[0]["name"], "c");
455        }
456    }
457}
sql_ddl_to_json_schema/parser.rs

sql_ddl_to_json_schema/
parser.rs
