Skip to main content

rok_db_codegen/
parse.rs

1//! Build the schema model and the query list from `.sql` files.
2
3use sqlparser::ast::{
4    ColumnOption, CommentObject, CreateTable, Expr, GeneratedAs, Ident, ObjectName, Statement,
5    TableConstraint, UserDefinedTypeRepresentation,
6};
7use sqlparser::dialect::PostgreSqlDialect;
8use sqlparser::parser::Parser;
9
10use crate::ir::{
11    Check, Column, EnumType, Extra, ForeignKey, Index, Query, QueryKind, Schema, Table, Unique,
12};
13use crate::split::{RawStatement, split};
14use crate::{Error, naming, types};
15
16/// One input file.
17#[derive(Debug, Clone)]
18pub struct SourceFile {
19    /// Path for messages (`db/user.sql`).
20    pub path: String,
21    /// Module name (from the file stem).
22    pub module: String,
23    /// File contents.
24    pub text: String,
25}
26
27/// The parsed input: the schema and the annotated queries.
28#[derive(Debug, Default)]
29pub struct Parsed {
30    /// Declared schema.
31    pub schema: Schema,
32    /// Annotated queries.
33    pub queries: Vec<Query>,
34    /// Modules, in file order (shared `_` files excluded).
35    pub modules: Vec<String>,
36}
37
38/// Parse every file. Files whose stem starts with `_` hold shared
39/// statements; their enums land in the `types` module.
40pub fn parse(files: &[SourceFile]) -> Result<Parsed, Error> {
41    let mut parsed = Parsed::default();
42    let mut comments: Vec<(String, Option<String>, Option<String>)> = Vec::new();
43    for file in files {
44        let shared = file.module.starts_with('_');
45        let module = if shared {
46            "types".to_owned()
47        } else {
48            file.module.clone()
49        };
50        if !shared && !parsed.modules.contains(&module) {
51            parsed.modules.push(module.clone());
52        }
53        let statements = split(&file.text);
54        let mut i = 0;
55        while i < statements.len() {
56            let stmt = &statements[i];
57            let at = format!("{}:{}", file.path, stmt.line);
58            let directives = Directives::read(&stmt.comments, &at)?;
59            if directives.down {
60                return Err(Error::at(
61                    &at,
62                    "`-- rok:down` must directly follow the statement it undoes",
63                ));
64            }
65            // A `-- rok:down` statement right after this one is its undo SQL.
66            let explicit_down = match statements.get(i + 1) {
67                Some(next) if Directives::read(&next.comments, &at)?.down => {
68                    i += 1;
69                    Some(next.sql.clone())
70                }
71                _ => None,
72            };
73            if let Some((name, kind)) = directives.query {
74                if shared {
75                    return Err(Error::at(&at, "queries can't live in a shared `_` file"));
76                }
77                parsed.queries.push(Query {
78                    name,
79                    module: module.clone(),
80                    kind,
81                    sql: stmt.sql.clone(),
82                    doc: directives.doc,
83                    param_names: directives.params,
84                    location: at,
85                });
86                i += 1;
87                continue;
88            }
89            schema_statement(
90                stmt,
91                &module,
92                directives.doc,
93                explicit_down,
94                &at,
95                &mut parsed.schema,
96                &mut comments,
97            )?;
98            i += 1;
99        }
100    }
101    apply_comments(&mut parsed.schema, &comments);
102    resolve_references(&mut parsed.schema);
103    check(&parsed)?;
104    Ok(parsed)
105}
106
107/// Directives and docs from a statement's comment block.
108#[derive(Default)]
109struct Directives {
110    query: Option<(String, QueryKind)>,
111    params: Vec<(usize, String)>,
112    down: bool,
113    doc: Vec<String>,
114}
115
116impl Directives {
117    fn read(comments: &[String], at: &str) -> Result<Self, Error> {
118        let mut d = Directives::default();
119        for line in comments {
120            let line = line.trim();
121            if let Some(rest) = line.strip_prefix("name:") {
122                let mut parts = rest.split_whitespace();
123                let (Some(name), Some(kind), None) = (parts.next(), parts.next(), parts.next())
124                else {
125                    return Err(Error::at(
126                        at,
127                        "expected `-- name: <function> <:one|:one!|:many|:stream|:exec>`",
128                    ));
129                };
130                let kind = QueryKind::parse(kind).ok_or_else(|| {
131                    Error::at(
132                        at,
133                        format!(
134                            "unknown query kind `{kind}`; use :one, :one!, :many, :stream or :exec"
135                        ),
136                    )
137                })?;
138                if naming::snake_ident(name) != name {
139                    return Err(Error::at(
140                        at,
141                        format!("query name `{name}` must be a snake_case Rust identifier"),
142                    ));
143                }
144                d.query = Some((name.to_owned(), kind));
145            } else if let Some(rest) = line.strip_prefix("param:") {
146                let mut parts = rest.split_whitespace();
147                let parsed = match (parts.next(), parts.next()) {
148                    (Some(p), Some(name)) => p
149                        .strip_prefix('$')
150                        .and_then(|n| n.parse::<usize>().ok())
151                        .map(|n| (n, naming::snake_ident(name))),
152                    _ => None,
153                };
154                d.params
155                    .push(parsed.ok_or_else(|| Error::at(at, "expected `-- param: $<n> <name>`"))?);
156            } else if line == "rok:down" {
157                d.down = true;
158            } else if let Some(directive) = line.strip_prefix("rok:") {
159                return Err(Error::at(
160                    at,
161                    format!("unknown directive `rok:{directive}` here"),
162                ));
163            } else {
164                d.doc.push(line.to_owned());
165            }
166        }
167        Ok(d)
168    }
169}
170
171fn parse_one(sql: &str) -> Option<Statement> {
172    let mut statements = Parser::parse_sql(&PostgreSqlDialect {}, sql).ok()?;
173    (statements.len() == 1).then(|| statements.remove(0))
174}
175
176fn schema_statement(
177    stmt: &RawStatement,
178    module: &str,
179    doc: Vec<String>,
180    explicit_down: Option<String>,
181    at: &str,
182    schema: &mut Schema,
183    comments: &mut Vec<(String, Option<String>, Option<String>)>,
184) -> Result<(), Error> {
185    let parsed = parse_one(&stmt.sql);
186    let looks_like_table = stmt
187        .sql
188        .split_whitespace()
189        .take(2)
190        .map(str::to_ascii_uppercase)
191        .eq(["CREATE", "TABLE"]);
192    match parsed {
193        Some(Statement::CreateTable(table)) => {
194            let table = table_from(&table, module, doc, &stmt.sql, at)?;
195            if schema.tables.iter().any(|t| t.name == table.name) {
196                return Err(Error::at(
197                    at,
198                    format!("table `{}` is declared twice", table.name),
199                ));
200            }
201            schema.tables.push(table);
202        }
203        Some(Statement::CreateType {
204            name,
205            representation: Some(UserDefinedTypeRepresentation::Enum { labels }),
206        }) => {
207            schema.enums.push(EnumType {
208                name: object_name(&name),
209                values: labels.iter().map(|l| l.value.clone()).collect(),
210                module: module.to_owned(),
211                doc,
212            });
213        }
214        Some(Statement::CreateIndex(index)) => {
215            let Some(name) = &index.name else {
216                return Err(Error::at(
217                    at,
218                    "give the index a name (`CREATE INDEX <name> ON ...`)",
219                ));
220            };
221            schema.indexes.push(Index {
222                name: object_name(name),
223                table: object_name(&index.table_name),
224                sql: Statement::CreateIndex(index.clone()).to_string(),
225            });
226        }
227        Some(Statement::Comment {
228            object_type,
229            object_name: target,
230            comment,
231            ..
232        }) if matches!(object_type, CommentObject::Table | CommentObject::Column) => {
233            let target = object_name(&target);
234            let (table, column) = match object_type {
235                CommentObject::Column => match target.rsplit_once('.') {
236                    Some((t, c)) => (t.to_owned(), Some(c.to_owned())),
237                    None => return Err(Error::at(at, "expected `COMMENT ON COLUMN table.column`")),
238                },
239                _ => (target, None),
240            };
241            let what = if column.is_some() { "COLUMN" } else { "TABLE" };
242            comments.push((table, column, comment.clone()));
243            schema.extras.push(Extra {
244                up: stmt.sql.clone(),
245                down: format!("COMMENT ON {what} {} IS NULL", stmt_target(&stmt.sql)),
246                before_tables: false,
247            });
248        }
249        Some(Statement::CreateExtension(ext)) => schema.extras.push(Extra {
250            up: stmt.sql.clone(),
251            down: explicit_down
252                .clone()
253                .unwrap_or_else(|| format!("DROP EXTENSION IF EXISTS {}", quote(&ext.name.value))),
254            before_tables: true,
255        }),
256        Some(Statement::CreateView(view)) if explicit_down.is_none() => schema.extras.push(Extra {
257            up: stmt.sql.clone(),
258            down: format!("DROP VIEW IF EXISTS {}", view.name),
259            before_tables: false,
260        }),
261        None if looks_like_table => {
262            return Err(Error::at(at, "couldn't parse this CREATE TABLE statement"));
263        }
264        _ => {
265            let Some(down) = explicit_down else {
266                return Err(Error::at(
267                    at,
268                    "rok-db-gen can't undo this statement by itself; follow it with \
269                     `-- rok:down` and the SQL that undoes it",
270                ));
271            };
272            schema.extras.push(Extra {
273                up: stmt.sql.clone(),
274                down,
275                before_tables: false,
276            });
277        }
278    }
279    Ok(())
280}
281
282/// `COMMENT ON COLUMN users.email IS '...'` -> `users.email`.
283fn stmt_target(sql: &str) -> String {
284    let words: Vec<&str> = sql.split_whitespace().collect();
285    words.get(3).copied().unwrap_or_default().to_owned()
286}
287
288fn table_from(
289    table: &CreateTable,
290    module: &str,
291    doc: Vec<String>,
292    sql: &str,
293    at: &str,
294) -> Result<Table, Error> {
295    let name = object_name(&table.name);
296    let short = naming::unqualified(&name).to_owned();
297    let column_notes = column_comments(sql);
298    let mut out = Table {
299        name: name.clone(),
300        module: module.to_owned(),
301        doc,
302        columns: Vec::new(),
303        primary_key: Vec::new(),
304        uniques: Vec::new(),
305        foreign_keys: Vec::new(),
306        checks: Vec::new(),
307    };
308    for def in &table.columns {
309        let column_name = ident(&def.name);
310        let mut column = Column {
311            name: column_name.clone(),
312            sql_type: types::canonical(&def.data_type.to_string()),
313            nullable: true,
314            default: None,
315            generated: None,
316            identity: None,
317            tenant: false,
318            doc: Vec::new(),
319        };
320        for option in &def.options {
321            match &option.option {
322                ColumnOption::NotNull => column.nullable = false,
323                ColumnOption::Null => column.nullable = true,
324                ColumnOption::Default(expr) => column.default = Some(expr.to_string()),
325                ColumnOption::PrimaryKey(_) => {
326                    if !out.primary_key.is_empty() {
327                        return Err(Error::at(
328                            at,
329                            format!("table `{name}` has two primary keys"),
330                        ));
331                    }
332                    out.primary_key = vec![column_name.clone()];
333                }
334                ColumnOption::Unique(_) => out.uniques.push(Unique {
335                    name: format!("{short}_{column_name}_key"),
336                    columns: vec![column_name.clone()],
337                }),
338                ColumnOption::ForeignKey(fk) => out.foreign_keys.push(ForeignKey {
339                    name: constraint_name(fk.name.as_ref(), || {
340                        format!("{short}_{column_name}_fkey")
341                    }),
342                    columns: vec![column_name.clone()],
343                    ref_table: object_name(&fk.foreign_table),
344                    ref_columns: fk.referred_columns.iter().map(ident).collect(),
345                    on_delete: fk.on_delete.as_ref().map(ToString::to_string),
346                    on_update: fk.on_update.as_ref().map(ToString::to_string),
347                }),
348                ColumnOption::Check(check) => out.checks.push(Check {
349                    name: constraint_name(check.name.as_ref(), || {
350                        format!("{short}_{column_name}_check")
351                    }),
352                    expr: check.expr.to_string(),
353                }),
354                ColumnOption::Generated {
355                    generated_as,
356                    generation_expr,
357                    ..
358                } => match generation_expr {
359                    Some(expr) => column.generated = Some(expr.to_string()),
360                    None => {
361                        column.identity = Some(
362                            match generated_as {
363                                GeneratedAs::Always => "ALWAYS",
364                                _ => "BY DEFAULT",
365                            }
366                            .to_owned(),
367                        );
368                    }
369                },
370                other => {
371                    return Err(Error::at(
372                        at,
373                        format!("column option `{other}` on `{column_name}` is not supported"),
374                    ));
375                }
376            }
377        }
378        if let Some((doc, tenant)) = column_notes
379            .iter()
380            .find_map(|(c, d, t)| (c.eq_ignore_ascii_case(&column_name)).then(|| (d.clone(), *t)))
381        {
382            column.doc = doc;
383            column.tenant = tenant;
384        }
385        out.columns.push(column);
386    }
387    let mut unnamed_checks = 0;
388    for constraint in &table.constraints {
389        match constraint {
390            TableConstraint::PrimaryKey(pk) => {
391                if !out.primary_key.is_empty() {
392                    return Err(Error::at(
393                        at,
394                        format!("table `{name}` has two primary keys"),
395                    ));
396                }
397                out.primary_key = pk
398                    .columns
399                    .iter()
400                    .map(|c| expr_ident(&c.column.expr))
401                    .collect();
402            }
403            TableConstraint::Unique(u) => {
404                let columns: Vec<String> = u
405                    .columns
406                    .iter()
407                    .map(|c| expr_ident(&c.column.expr))
408                    .collect();
409                out.uniques.push(Unique {
410                    name: constraint_name(u.name.as_ref(), || {
411                        format!("{short}_{}_key", columns.join("_"))
412                    }),
413                    columns,
414                });
415            }
416            TableConstraint::ForeignKey(fk) => {
417                let columns: Vec<String> = fk.columns.iter().map(ident).collect();
418                out.foreign_keys.push(ForeignKey {
419                    name: constraint_name(fk.name.as_ref(), || {
420                        format!("{short}_{}_fkey", columns.join("_"))
421                    }),
422                    columns,
423                    ref_table: object_name(&fk.foreign_table),
424                    ref_columns: fk.referred_columns.iter().map(ident).collect(),
425                    on_delete: fk.on_delete.as_ref().map(ToString::to_string),
426                    on_update: fk.on_update.as_ref().map(ToString::to_string),
427                });
428            }
429            TableConstraint::Check(check) => {
430                let name = constraint_name(check.name.as_ref(), || {
431                    unnamed_checks += 1;
432                    match unnamed_checks {
433                        1 => format!("{short}_check"),
434                        n => format!("{short}_check{}", n - 1),
435                    }
436                });
437                out.checks.push(Check {
438                    name,
439                    expr: check.expr.to_string(),
440                });
441            }
442            other => {
443                return Err(Error::at(
444                    at,
445                    format!("table constraint `{other}` is not supported"),
446                ));
447            }
448        }
449    }
450    // Primary key columns are NOT NULL.
451    for column in &mut out.columns {
452        if out.primary_key.contains(&column.name) {
453            column.nullable = false;
454        }
455    }
456    Ok(out)
457}
458
459/// `-- comment` lines above column definitions inside `CREATE TABLE`:
460/// (column, doc lines, `rok:tenant`).
461fn column_comments(sql: &str) -> Vec<(String, Vec<String>, bool)> {
462    let mut out = Vec::new();
463    let mut doc = Vec::new();
464    let mut tenant = false;
465    for line in sql.lines().skip(1) {
466        let line = line.trim();
467        if let Some(comment) = line.strip_prefix("--") {
468            let comment = comment.trim();
469            if comment == "rok:tenant" {
470                tenant = true;
471            } else {
472                doc.push(comment.to_owned());
473            }
474            continue;
475        }
476        if line.is_empty() {
477            continue;
478        }
479        if !doc.is_empty() || tenant {
480            let first = line
481                .split(|c: char| c.is_whitespace() || c == ',')
482                .next()
483                .unwrap_or_default();
484            let column = first.trim_matches('"').to_owned();
485            out.push((column, std::mem::take(&mut doc), tenant));
486            tenant = false;
487        }
488    }
489    out
490}
491
492/// `REFERENCES users` without columns points at the primary key.
493fn resolve_references(schema: &mut Schema) {
494    let keys: Vec<(String, Vec<String>)> = schema
495        .tables
496        .iter()
497        .map(|t| (t.name.clone(), t.primary_key.clone()))
498        .collect();
499    for table in &mut schema.tables {
500        for fk in &mut table.foreign_keys {
501            if fk.ref_columns.is_empty() {
502                if let Some((_, key)) = keys.iter().find(|(name, _)| *name == fk.ref_table) {
503                    fk.ref_columns = key.clone();
504                }
505            }
506        }
507    }
508}
509
510fn apply_comments(schema: &mut Schema, comments: &[(String, Option<String>, Option<String>)]) {
511    for (table, column, text) in comments {
512        let Some(t) = schema.tables.iter_mut().find(|t| &t.name == table) else {
513            continue;
514        };
515        let lines: Vec<String> = text
516            .iter()
517            .flat_map(|t| t.lines())
518            .map(str::to_owned)
519            .collect();
520        match column {
521            Some(c) => {
522                if let Some(col) = t.columns.iter_mut().find(|col| &col.name == c) {
523                    col.doc = lines;
524                }
525            }
526            None => t.doc = lines,
527        }
528    }
529}
530
531/// Cross-file checks: references and names.
532fn check(parsed: &Parsed) -> Result<(), Error> {
533    let schema = &parsed.schema;
534    for module in &parsed.modules {
535        if naming::RESERVED_MODULES.contains(&module.as_str()) {
536            return Err(Error::new(format!(
537                "`{module}.sql` would generate `{module}.rs`, which rok-db-gen writes itself; rename the file"
538            )));
539        }
540    }
541    for table in &schema.tables {
542        for fk in &table.foreign_keys {
543            let Some(target) = schema.tables.iter().find(|t| t.name == fk.ref_table) else {
544                return Err(Error::new(format!(
545                    "`{}` references `{}`, which no .sql file declares",
546                    table.name, fk.ref_table
547                )));
548            };
549            if fk.ref_columns.is_empty() && target.primary_key.is_empty() {
550                return Err(Error::new(format!(
551                    "`{}` references `{}`, which has no primary key",
552                    table.name, fk.ref_table
553                )));
554            }
555        }
556    }
557    let mut structs: Vec<(String, &str)> = Vec::new();
558    for table in &schema.tables {
559        let name = naming::struct_name(&table.name);
560        if let Some((_, other)) = structs.iter().find(|(n, _)| *n == name) {
561            return Err(Error::new(format!(
562                "tables `{other}` and `{}` both become `{name}`; rename one",
563                table.name
564            )));
565        }
566        structs.push((name, &table.name));
567    }
568    for e in &schema.enums {
569        let name = naming::pascal(naming::unqualified(&e.name));
570        if structs.iter().any(|(n, _)| *n == name) {
571            return Err(Error::new(format!(
572                "enum `{}` and a table both become `{name}`; rename one",
573                e.name
574            )));
575        }
576    }
577    let mut seen: Vec<(&str, &str)> = Vec::new();
578    for q in &parsed.queries {
579        if seen.contains(&(q.module.as_str(), q.name.as_str())) {
580            return Err(Error::at(
581                &q.location,
582                format!("a query named `{}` already exists in this file", q.name),
583            ));
584        }
585        seen.push((&q.module, &q.name));
586    }
587    Ok(())
588}
589
590fn constraint_name(name: Option<&Ident>, default: impl FnOnce() -> String) -> String {
591    name.map_or_else(default, ident)
592}
593
594/// An identifier as PostgreSQL stores it: unquoted names fold to lower case.
595pub(crate) fn ident(i: &Ident) -> String {
596    if i.quote_style.is_some() {
597        i.value.clone()
598    } else {
599        i.value.to_ascii_lowercase()
600    }
601}
602
603fn expr_ident(expr: &Expr) -> String {
604    match expr {
605        Expr::Identifier(i) => ident(i),
606        other => other.to_string(),
607    }
608}
609
610/// `"app"."Users"` -> `app.Users`; unquoted parts fold to lower case.
611pub(crate) fn object_name(name: &ObjectName) -> String {
612    name.0
613        .iter()
614        .map(|part| part.as_ident().map_or_else(|| part.to_string(), ident))
615        .collect::<Vec<_>>()
616        .join(".")
617}
618
619/// Quote a possibly schema-qualified name for SQL.
620pub(crate) fn quote(name: &str) -> String {
621    name.split('.')
622        .map(|part| format!("\"{}\"", part.replace('"', "\"\"")))
623        .collect::<Vec<_>>()
624        .join(".")
625}
626
627#[cfg(test)]
628mod tests {
629    use super::*;
630
631    fn parse_text(text: &str) -> Result<Parsed, Error> {
632        parse(&[SourceFile {
633            path: "db/user.sql".into(),
634            module: "user".into(),
635            text: text.into(),
636        }])
637    }
638
639    #[test]
640    fn tables_columns_and_constraints() {
641        let p = parse_text(
642            "CREATE TYPE mood AS ENUM ('happy', 'sad');
643             -- A user.
644             CREATE TABLE users (
645                 id BIGSERIAL PRIMARY KEY,
646                 -- Where we write.
647                 email VARCHAR(255) NOT NULL UNIQUE,
648                 manager_id BIGINT REFERENCES users (id) ON DELETE SET NULL,
649                 mood mood NOT NULL DEFAULT 'happy',
650                 -- rok:tenant
651                 org_id INT NOT NULL,
652                 lower_email TEXT GENERATED ALWAYS AS (lower(email)) STORED,
653                 CHECK (org_id > 0)
654             );
655             CREATE INDEX users_org_idx ON users (org_id);
656             -- name: find_by_email :one
657             SELECT * FROM users WHERE email = $1;",
658        )
659        .unwrap();
660        let t = &p.schema.tables[0];
661        assert_eq!(t.doc, ["A user."]);
662        assert_eq!(t.primary_key, ["id"]);
663        let email = t.column("email").unwrap();
664        assert_eq!(
665            (email.sql_type.as_str(), email.nullable),
666            ("VARCHAR(255)", false)
667        );
668        assert_eq!(email.doc, ["Where we write."]);
669        assert_eq!(t.uniques[0].name, "users_email_key");
670        assert_eq!(t.foreign_keys[0].name, "users_manager_id_fkey");
671        assert_eq!(t.foreign_keys[0].on_delete.as_deref(), Some("SET NULL"));
672        assert_eq!(
673            t.column("mood").unwrap().default.as_deref(),
674            Some("'happy'")
675        );
676        assert!(t.column("org_id").unwrap().tenant);
677        assert_eq!(
678            t.column("lower_email").unwrap().generated.as_deref(),
679            Some("lower(email)")
680        );
681        assert_eq!(t.checks[0].name, "users_check");
682        assert_eq!(p.schema.enums[0].values, ["happy", "sad"]);
683        assert_eq!(p.schema.indexes[0].name, "users_org_idx");
684        assert_eq!(p.queries[0].name, "find_by_email");
685        assert_eq!(p.queries[0].kind, QueryKind::One);
686    }
687
688    #[test]
689    fn extras_need_a_down() {
690        let err = parse_text("CREATE FUNCTION f() RETURNS int AS $$ SELECT 1 $$ LANGUAGE sql;")
691            .unwrap_err();
692        assert!(err.to_string().contains("rok:down"), "{err}");
693        let p = parse_text(
694            "CREATE FUNCTION f() RETURNS int AS $$ SELECT 1 $$ LANGUAGE sql;
695             -- rok:down
696             DROP FUNCTION f();",
697        )
698        .unwrap();
699        assert_eq!(p.schema.extras[0].down, "DROP FUNCTION f()");
700    }
701
702    #[test]
703    fn errors_point_at_the_file() {
704        let err =
705            parse_text("CREATE TABLE posts (id INT PRIMARY KEY, user_id INT REFERENCES users);")
706                .unwrap_err();
707        assert!(err.to_string().contains("users"), "{err}");
708        let err = parse_text("\n-- name: x :maybe\nSELECT 1;").unwrap_err();
709        assert!(err.to_string().contains("db/user.sql:3"), "{err}");
710    }
711}