databend_common_ast/parser/
parser.rs

1// Copyright 2021 Datafuse Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use derive_visitor::DriveMut;
16use derive_visitor::VisitorMut;
17use pretty_assertions::assert_eq;
18
19use crate::ast::ExplainKind;
20use crate::ast::Expr;
21use crate::ast::Identifier;
22use crate::ast::Literal;
23use crate::ast::SelectTarget;
24use crate::ast::Statement;
25use crate::ast::StatementWithFormat;
26use crate::parser::common::comma_separated_list0;
27use crate::parser::common::comma_separated_list1;
28use crate::parser::common::ident;
29use crate::parser::common::transform_span;
30use crate::parser::common::IResult;
31use crate::parser::error::display_parser_error;
32use crate::parser::expr::expr;
33use crate::parser::expr::values;
34use crate::parser::input::Dialect;
35use crate::parser::input::Input;
36use crate::parser::input::ParseMode;
37use crate::parser::statement::insert_stmt;
38use crate::parser::statement::replace_stmt;
39use crate::parser::statement::statement;
40use crate::parser::token::Token;
41use crate::parser::token::TokenKind;
42use crate::parser::token::Tokenizer;
43use crate::parser::Backtrace;
44use crate::ParseError;
45use crate::Range;
46use crate::Result;
47
48pub fn tokenize_sql(sql: &str) -> Result<Vec<Token>> {
49    Tokenizer::new(sql).collect::<Result<Vec<_>>>()
50}
51
52/// Parse a SQL string into `Statement`s.
53#[fastrace::trace]
54pub fn parse_sql(tokens: &[Token], dialect: Dialect) -> Result<(Statement, Option<String>)> {
55    let stmt = run_parser(tokens, dialect, ParseMode::Default, false, statement)?;
56
57    #[cfg(debug_assertions)]
58    assert_reparse(tokens[0].source, stmt.clone());
59
60    Ok((stmt.stmt, stmt.format))
61}
62
63/// Parse udf function into Expr
64pub fn parse_expr(tokens: &[Token], dialect: Dialect) -> Result<Expr> {
65    run_parser(tokens, dialect, ParseMode::Default, false, expr)
66}
67
68pub fn parse_comma_separated_exprs(tokens: &[Token], dialect: Dialect) -> Result<Vec<Expr>> {
69    run_parser(tokens, dialect, ParseMode::Default, true, |i| {
70        comma_separated_list0(expr)(i)
71    })
72}
73
74pub fn parse_comma_separated_idents(tokens: &[Token], dialect: Dialect) -> Result<Vec<Identifier>> {
75    run_parser(tokens, dialect, ParseMode::Default, true, |i| {
76        comma_separated_list1(ident)(i)
77    })
78}
79
80pub fn parse_values(tokens: &[Token], dialect: Dialect) -> Result<Vec<Expr>> {
81    run_parser(tokens, dialect, ParseMode::Default, false, values)
82}
83
84pub fn parse_raw_insert_stmt(
85    tokens: &[Token],
86    dialect: Dialect,
87    in_streaming_load: bool,
88) -> Result<Statement> {
89    run_parser(
90        tokens,
91        dialect,
92        ParseMode::Default,
93        false,
94        insert_stmt(true, in_streaming_load),
95    )
96}
97
98pub fn parse_raw_replace_stmt(tokens: &[Token], dialect: Dialect) -> Result<Statement> {
99    run_parser(
100        tokens,
101        dialect,
102        ParseMode::Default,
103        false,
104        replace_stmt(true),
105    )
106}
107
108pub fn run_parser<O>(
109    tokens: &[Token],
110    dialect: Dialect,
111    mode: ParseMode,
112    allow_partial: bool,
113    mut parser: impl FnMut(Input) -> IResult<O>,
114) -> Result<O> {
115    let backtrace = Backtrace::new();
116    let input = Input {
117        tokens,
118        dialect,
119        mode,
120        backtrace: &backtrace,
121    };
122    match parser(input) {
123        Ok((rest, res)) => {
124            let is_complete = rest[0].kind == TokenKind::EOI;
125            if is_complete || allow_partial {
126                Ok(res)
127            } else {
128                Err(ParseError(
129                    transform_span(&rest[..1]),
130                    format!(
131                        "unable to parse rest of the sql, rest tokens:  {:?} ",
132                        rest.tokens
133                    ),
134                ))
135            }
136        }
137        Err(nom::Err::Error(err) | nom::Err::Failure(err)) => {
138            let source = tokens[0].source;
139            Err(ParseError(None, display_parser_error(err, source)))
140        }
141        Err(nom::Err::Incomplete(_)) => unreachable!(),
142    }
143}
144
145/// Check that the statement can be displayed and reparsed without loss
146#[allow(dead_code)]
147fn assert_reparse(sql: &str, stmt: StatementWithFormat) {
148    let stmt = reset_ast(stmt);
149
150    let new_sql = stmt.to_string();
151    let new_tokens = crate::parser::tokenize_sql(&new_sql).unwrap();
152    let new_stmt = run_parser(
153        &new_tokens,
154        Dialect::PostgreSQL,
155        ParseMode::Default,
156        false,
157        statement,
158    )
159    .map_err(|err| panic!("{} in {}", err.1, new_sql))
160    .unwrap();
161
162    let new_stmt = reset_ast(new_stmt);
163    assert_eq!(stmt, new_stmt, "\nleft:\n{}\nright:\n{}", sql, new_sql);
164}
165
166#[allow(dead_code)]
167fn reset_ast(mut stmt: StatementWithFormat) -> StatementWithFormat {
168    #[derive(VisitorMut)]
169    #[visitor(Range(enter), Literal(enter), ExplainKind(enter), SelectTarget(enter))]
170    struct ResetAST;
171
172    impl ResetAST {
173        fn enter_range(&mut self, range: &mut Range) {
174            range.start = 0;
175            range.end = 0;
176        }
177
178        fn enter_literal(&mut self, literal: &mut Literal) {
179            *literal = Literal::Null;
180        }
181
182        fn enter_explain_kind(&mut self, kind: &mut ExplainKind) {
183            match kind {
184                ExplainKind::Ast(_) => *kind = ExplainKind::Ast("".to_string()),
185                ExplainKind::Syntax(_) => *kind = ExplainKind::Syntax("".to_string()),
186                ExplainKind::Memo(_) => *kind = ExplainKind::Memo("".to_string()),
187                _ => (),
188            }
189        }
190
191        fn enter_select_target(&mut self, target: &mut SelectTarget) {
192            if let SelectTarget::StarColumns { column_filter, .. } = target {
193                *column_filter = None
194            }
195        }
196    }
197
198    stmt.drive_mut(&mut ResetAST);
199
200    stmt
201}
databend_common_ast/parser/parser.rs

databend_common_ast/parser/
parser.rs
