1pub mod ast_canon;
8mod ast_resolution;
9pub mod builder;
10pub mod cluster;
11pub mod directives;
12pub mod error;
13pub mod normalize_body;
14pub mod normalize_expr;
15pub mod statement;
16
17use std::collections::{BTreeMap, HashMap};
18use std::path::{Path, PathBuf};
19
20pub use directives::{FileDirectives, extract_file_directives};
21pub use error::{ParseError, SourceLocation};
22pub use statement::Statement;
23
24use crate::identifier::{Identifier, QualifiedName};
25use crate::ir::IrError;
26use crate::ir::catalog::Catalog;
27use crate::ir::publication::Publication;
28use crate::ir::statistic::Statistic;
29use crate::ir::subscription::Subscription;
30
31pub fn parse_directory(root: &Path, ignores: &[glob::Pattern]) -> Result<Catalog, ParseError> {
41 parse_directory_with_locations(root, ignores).map(|(c, _)| c)
42}
43
44pub fn parse_directory_with_locations(
51 root: &Path,
52 ignores: &[glob::Pattern],
53) -> Result<(Catalog, HashMap<String, SourceLocation>), ParseError> {
54 let mut files: Vec<PathBuf> = Vec::new();
55 for entry in walkdir::WalkDir::new(root).sort_by_file_name() {
56 let entry = entry.map_err(|e| ParseError::Io {
57 path: e.path().map(Path::to_path_buf).unwrap_or_default(),
58 source: e
59 .into_io_error()
60 .unwrap_or_else(|| std::io::Error::other("walkdir error")),
61 })?;
62 if !entry.file_type().is_file() {
63 continue;
64 }
65 let path = entry.into_path();
66 if path.extension().and_then(|e| e.to_str()) != Some("sql") {
67 continue;
68 }
69 if ignores.iter().any(|pat| pat.matches_path(&path)) {
70 continue;
71 }
72 files.push(path);
73 }
74
75 let mut catalog = Catalog::default();
76 let mut locations: HashMap<String, SourceLocation> = HashMap::new();
77 let mut pending_fks: Vec<builder::alter_table_stmt::PendingFk> = Vec::new();
78 let mut pending_column_attrs: Vec<builder::alter_table_stmt::PendingColumnAttr> = Vec::new();
79 let mut pending_owners: Vec<builder::alter_table_stmt::PendingOwner> = Vec::new();
80 let mut pending_rls_toggles: Vec<builder::alter_table_stmt::PendingRlsToggle> = Vec::new();
81 let mut pending_rel_options: Vec<builder::alter_table_stmt::PendingRelOptions> = Vec::new();
82 let mut deferred_comments: Vec<(
83 pg_query::protobuf::CommentStmt,
84 SourceLocation,
85 Option<crate::identifier::Identifier>,
86 )> = Vec::new();
87 let mut publications: BTreeMap<Identifier, Publication> = BTreeMap::new();
91 let mut subscriptions: BTreeMap<Identifier, Subscription> = BTreeMap::new();
93 let mut statistics: BTreeMap<QualifiedName, Statistic> = BTreeMap::new();
95
96 for path in files {
97 let contents = std::fs::read_to_string(&path).map_err(|e| ParseError::Io {
98 path: path.clone(),
99 source: e,
100 })?;
101 process_file(
102 &path,
103 &contents,
104 &mut catalog,
105 &mut locations,
106 &mut pending_fks,
107 &mut pending_column_attrs,
108 &mut pending_owners,
109 &mut pending_rls_toggles,
110 &mut pending_rel_options,
111 &mut deferred_comments,
112 &mut publications,
113 &mut subscriptions,
114 &mut statistics,
115 )?;
116 }
117
118 for (stmt, location, default_schema) in deferred_comments {
121 builder::comment_stmt::apply_comment(
122 &stmt,
123 &mut catalog,
124 default_schema.as_ref(),
125 &location,
126 )?;
127 }
128
129 apply_pending_fks(&mut catalog, pending_fks)?;
131 apply_pending_column_attrs(&mut catalog, pending_column_attrs)?;
132 apply_pending_owners(&mut catalog, pending_owners)?;
133 apply_pending_rls_toggles(&mut catalog, pending_rls_toggles)?;
134 apply_pending_rel_options(&mut catalog, pending_rel_options)?;
135
136 catalog.publications = publications.into_values().collect();
138
139 catalog.subscriptions = subscriptions.into_values().collect();
141
142 catalog.statistics = statistics.into_values().collect();
144
145 ast_resolution::resolve(&catalog, &locations).map_err(ParseError::AstResolution)?;
148
149 if !catalog.views.is_empty() || !catalog.materialized_views.is_empty() {
153 ast_canon::canonicalize_view_bodies(&mut catalog).map_err(ParseError::AstCanon)?;
154 }
155
156 ast_canon::promote_mv_index_parents(&mut catalog);
162
163 let canonical = catalog
164 .canonicalize()
165 .map_err(|e: IrError| translate_canonicalize_error(e, &locations))?;
166 Ok((canonical, locations))
167}
168
169fn apply_statistics_comment(
175 stmt: &pg_query::protobuf::CommentStmt,
176 statistics: &mut BTreeMap<QualifiedName, Statistic>,
177 location: &SourceLocation,
178) -> Result<(), ParseError> {
179 use pg_query::NodeEnum;
180
181 let obj = stmt
183 .object
184 .as_ref()
185 .and_then(|o| o.node.as_ref())
186 .ok_or_else(|| ParseError::Structural {
187 location: location.clone(),
188 message: "COMMENT ON STATISTICS missing object reference".into(),
189 })?;
190 let NodeEnum::List(list) = obj else {
191 return Err(ParseError::Structural {
192 location: location.clone(),
193 message: format!(
194 "COMMENT ON STATISTICS expected a List node, got {:?}",
195 std::mem::discriminant(obj)
196 ),
197 });
198 };
199 let parts: Vec<String> = list
200 .items
201 .iter()
202 .filter_map(|n| {
203 if let Some(NodeEnum::String(s)) = n.node.as_ref() {
204 Some(s.sval.clone())
205 } else {
206 None
207 }
208 })
209 .collect();
210 let qname = match parts.as_slice() {
211 [schema, name] => QualifiedName::new(
212 builder::shared::ident(schema, location)?,
213 builder::shared::ident(name, location)?,
214 ),
215 [_name] => {
216 return Err(ParseError::UnqualifiedName {
217 location: location.clone(),
218 });
219 }
220 _ => {
221 return Err(ParseError::Structural {
222 location: location.clone(),
223 message: format!(
224 "COMMENT ON STATISTICS expected 1-2 qualified components, got {parts:?}"
225 ),
226 });
227 }
228 };
229
230 let comment = if stmt.comment.is_empty() {
231 None
232 } else {
233 Some(stmt.comment.clone())
234 };
235
236 let statistic = statistics.get_mut(&qname).ok_or_else(|| {
237 ParseError::CommentOnStatisticBeforeCreate(qname.clone(), location.clone())
238 })?;
239 statistic.comment = comment;
240 Ok(())
241}
242
243fn apply_pending_fks(
245 catalog: &mut Catalog,
246 pending_fks: Vec<builder::alter_table_stmt::PendingFk>,
247) -> Result<(), ParseError> {
248 for pending in pending_fks {
249 let table = catalog
250 .tables
251 .iter_mut()
252 .find(|t| t.qname == pending.target)
253 .ok_or_else(|| ParseError::Structural {
254 location: SourceLocation::new(PathBuf::new(), 0, 0),
255 message: format!("ALTER TABLE referenced unknown table {}", pending.target),
256 })?;
257 table.constraints.push(pending.constraint);
258 }
259 Ok(())
260}
261
262fn apply_pending_column_attrs(
264 catalog: &mut Catalog,
265 pending: Vec<builder::alter_table_stmt::PendingColumnAttr>,
266) -> Result<(), ParseError> {
267 use builder::alter_table_stmt::PendingColumnAttrKind;
268 for attr in pending {
269 let table = catalog
270 .tables
271 .iter_mut()
272 .find(|t| t.qname == attr.target)
273 .ok_or_else(|| ParseError::Structural {
274 location: SourceLocation::new(PathBuf::new(), 0, 0),
275 message: format!(
276 "ALTER TABLE ALTER COLUMN referenced unknown table {}",
277 attr.target
278 ),
279 })?;
280 let col = table
281 .columns
282 .iter_mut()
283 .find(|c| c.name == attr.column)
284 .ok_or_else(|| ParseError::Structural {
285 location: SourceLocation::new(PathBuf::new(), 0, 0),
286 message: format!(
287 "ALTER TABLE ALTER COLUMN referenced unknown column {}.{}",
288 attr.target, attr.column
289 ),
290 })?;
291 match attr.kind {
292 PendingColumnAttrKind::Storage(s) => {
293 col.storage = Some(s);
294 }
295 PendingColumnAttrKind::Compression(c) => {
296 col.compression = c;
297 }
298 }
299 }
300 Ok(())
301}
302
303fn apply_pending_rel_options(
308 catalog: &mut Catalog,
309 pending: Vec<builder::alter_table_stmt::PendingRelOptions>,
310) -> Result<(), ParseError> {
311 let loc = SourceLocation::new(PathBuf::new(), 0, 0);
312 builder::alter_table_stmt::apply_pending_rel_options(catalog, pending, &loc)
313}
314
315fn apply_pending_rls_toggles(
319 catalog: &mut Catalog,
320 pending: Vec<builder::alter_table_stmt::PendingRlsToggle>,
321) -> Result<(), ParseError> {
322 let loc = SourceLocation::new(PathBuf::new(), 0, 0);
323 builder::alter_table_stmt::apply_pending_rls_toggles(catalog, pending, &loc)
324}
325
326fn apply_pending_owners(
330 catalog: &mut Catalog,
331 pending: Vec<builder::alter_table_stmt::PendingOwner>,
332) -> Result<(), ParseError> {
333 let loc = SourceLocation::new(PathBuf::new(), 0, 0);
334 for po in pending {
335 builder::alter_table_stmt::apply_pending_owners(catalog, vec![po], &loc)?;
336 }
337 Ok(())
338}
339
340#[allow(clippy::too_many_lines, clippy::too_many_arguments)]
341fn process_file(
342 path: &Path,
343 contents: &str,
344 catalog: &mut Catalog,
345 locations: &mut HashMap<String, SourceLocation>,
346 pending_fks: &mut Vec<builder::alter_table_stmt::PendingFk>,
347 pending_column_attrs: &mut Vec<builder::alter_table_stmt::PendingColumnAttr>,
348 pending_owners: &mut Vec<builder::alter_table_stmt::PendingOwner>,
349 pending_rls_toggles: &mut Vec<builder::alter_table_stmt::PendingRlsToggle>,
350 pending_rel_options: &mut Vec<builder::alter_table_stmt::PendingRelOptions>,
351 deferred_comments: &mut Vec<(
352 pg_query::protobuf::CommentStmt,
353 SourceLocation,
354 Option<crate::identifier::Identifier>,
355 )>,
356 publications: &mut BTreeMap<Identifier, Publication>,
357 subscriptions: &mut BTreeMap<Identifier, Subscription>,
358 statistics: &mut BTreeMap<QualifiedName, Statistic>,
359) -> Result<(), ParseError> {
360 let directives = directives::extract_file_directives(contents, path)?;
361 let parsed = pg_query::parse(contents).map_err(|e| ParseError::PgQuery {
362 location: SourceLocation::new(path.to_path_buf(), 1, 1),
363 message: e.to_string(),
364 })?;
365
366 for raw in parsed.protobuf.stmts {
367 let location = stmt_location(path, contents, raw.stmt_location);
368 let Some(node) = raw.stmt.and_then(|n| n.node) else {
369 continue;
370 };
371 let stmt = Statement::classify(node, location.clone())?;
372 match stmt {
373 Statement::CreateSchema(s) => {
374 let schema = builder::create_schema_stmt::build_schema(&s, &location)?;
375 let schema_qname = QualifiedName::new(schema.name.clone(), schema.name.clone()); if let Some(prior) = locations.get(&schema.name.to_string()) {
377 return Err(ParseError::DuplicateObject {
378 qname: schema.name.to_string(),
379 first: prior.clone(),
380 second: location,
381 });
382 }
383 locations.insert(schema.name.to_string(), location.clone());
384 catalog.schemas.push(schema);
385 let _ = schema_qname;
386 }
387 Statement::CreateTable(s) => {
388 let mut table =
389 builder::create_stmt::build_table(&s, directives.schema.as_ref(), &location)?;
390 let serial_seqs =
391 builder::desugar_serial::desugar_serials_in_table(&mut table, &location)?;
392 if let Some(prior) = locations.get(&table.qname.to_string()) {
393 return Err(ParseError::DuplicateObject {
394 qname: table.qname.to_string(),
395 first: prior.clone(),
396 second: location,
397 });
398 }
399 locations.insert(table.qname.to_string(), location.clone());
400 catalog.tables.push(table);
401 for seq in serial_seqs {
402 if let Some(prior) = locations.get(&seq.qname.to_string()) {
403 return Err(ParseError::DuplicateObject {
404 qname: seq.qname.to_string(),
405 first: prior.clone(),
406 second: location.clone(),
407 });
408 }
409 locations.insert(seq.qname.to_string(), location.clone());
410 catalog.sequences.push(seq);
411 }
412 }
413 Statement::CreateSequence(s) => {
414 let seq = builder::create_seq_stmt::build_sequence(
415 &s,
416 directives.schema.as_ref(),
417 &location,
418 )?;
419 if let Some(prior) = locations.get(&seq.qname.to_string()) {
420 return Err(ParseError::DuplicateObject {
421 qname: seq.qname.to_string(),
422 first: prior.clone(),
423 second: location,
424 });
425 }
426 locations.insert(seq.qname.to_string(), location.clone());
427 catalog.sequences.push(seq);
428 }
429 Statement::CreateIndex(s) => {
430 let idx =
431 builder::index_stmt::build_index(&s, directives.schema.as_ref(), &location)?;
432 if let Some(prior) = locations.get(&idx.qname.to_string()) {
433 return Err(ParseError::DuplicateObject {
434 qname: idx.qname.to_string(),
435 first: prior.clone(),
436 second: location,
437 });
438 }
439 locations.insert(idx.qname.to_string(), location.clone());
440 catalog.indexes.push(idx);
441 }
442 Statement::AlterTable(s) => {
443 let alter_out = builder::alter_table_stmt::build_alter_table(
444 &s,
445 directives.schema.as_ref(),
446 &location,
447 )?;
448 pending_fks.extend(alter_out.pending_fks);
449 pending_column_attrs.extend(alter_out.pending_column_attrs);
450 pending_owners.extend(alter_out.pending_owners);
451 pending_rls_toggles.extend(alter_out.pending_rls_toggles);
452 pending_rel_options.extend(alter_out.pending_rel_options);
453 }
454 Statement::Comment(s) => {
455 use pg_query::protobuf::ObjectType;
456 let kind = ObjectType::try_from(s.objtype).unwrap_or(ObjectType::Undefined);
457 if matches!(kind, ObjectType::ObjectStatisticExt) {
458 apply_statistics_comment(&s, statistics, &location)?;
462 } else {
463 deferred_comments.push((s, location, directives.schema.clone()));
464 }
465 }
466 Statement::CreateView(s) => {
467 let view = builder::create_view_stmt::build_view(
468 &s,
469 directives.schema.as_ref(),
470 &location,
471 )?;
472 if let Some(prior) = locations.get(&view.qname.to_string()) {
473 return Err(ParseError::DuplicateObject {
474 qname: view.qname.to_string(),
475 first: prior.clone(),
476 second: location,
477 });
478 }
479 locations.insert(view.qname.to_string(), location.clone());
480 catalog.views.push(view);
481 }
482 Statement::CreateMaterializedView(s) => {
483 let mv = builder::create_materialized_view_stmt::build_materialized_view(
484 &s,
485 directives.schema.as_ref(),
486 &location,
487 )?;
488 if let Some(prior) = locations.get(&mv.qname.to_string()) {
489 return Err(ParseError::DuplicateObject {
490 qname: mv.qname.to_string(),
491 first: prior.clone(),
492 second: location,
493 });
494 }
495 locations.insert(mv.qname.to_string(), location.clone());
496 catalog.materialized_views.push(mv);
497 }
498 Statement::CreateEnum(s) => {
499 let ut = builder::create_enum_stmt::build_enum(
500 &s,
501 directives.schema.as_ref(),
502 &location,
503 )?;
504 if let Some(prior) = locations.get(&ut.qname.to_string()) {
505 return Err(ParseError::DuplicateObject {
506 qname: ut.qname.to_string(),
507 first: prior.clone(),
508 second: location,
509 });
510 }
511 locations.insert(ut.qname.to_string(), location.clone());
512 catalog.types.push(ut);
513 }
514 Statement::CreateDomain(s) => {
515 let ut = builder::create_domain_stmt::build_domain(
516 &s,
517 directives.schema.as_ref(),
518 &location,
519 )?;
520 if let Some(prior) = locations.get(&ut.qname.to_string()) {
521 return Err(ParseError::DuplicateObject {
522 qname: ut.qname.to_string(),
523 first: prior.clone(),
524 second: location,
525 });
526 }
527 locations.insert(ut.qname.to_string(), location.clone());
528 catalog.types.push(ut);
529 }
530 Statement::CreateCompositeType(s) => {
531 let ut = builder::create_composite_type_stmt::build_composite(
532 &s,
533 directives.schema.as_ref(),
534 &location,
535 )?;
536 if let Some(prior) = locations.get(&ut.qname.to_string()) {
537 return Err(ParseError::DuplicateObject {
538 qname: ut.qname.to_string(),
539 first: prior.clone(),
540 second: location,
541 });
542 }
543 locations.insert(ut.qname.to_string(), location.clone());
544 catalog.types.push(ut);
545 }
546 Statement::CreateFunction(s) => {
547 let routine = builder::create_function_stmt::build_function_or_procedure(
548 &s,
549 directives.schema.as_ref(),
550 &location,
551 )?;
552 let builder::create_function_stmt::Routine::Function(f) = routine else {
553 return Err(ParseError::Structural {
554 location,
555 message: "internal error: expected Function from non-procedure stmt".into(),
556 });
557 };
558 let arg_sig = f
559 .args
560 .iter()
561 .filter(|a| {
562 matches!(
563 a.mode,
564 crate::ir::function::ArgMode::In
565 | crate::ir::function::ArgMode::InOut
566 | crate::ir::function::ArgMode::Variadic
567 )
568 })
569 .map(|a| a.ty.render_sql())
570 .collect::<Vec<_>>()
571 .join(",");
572 let key = format!("functions.{}({arg_sig})", f.qname);
573 if let Some(prior) = locations.get(&key) {
574 return Err(ParseError::DuplicateObject {
575 qname: key,
576 first: prior.clone(),
577 second: location,
578 });
579 }
580 locations.insert(key, location.clone());
581 catalog.functions.push(f);
582 }
583 Statement::CreateProcedure(s) => {
584 let routine = builder::create_function_stmt::build_function_or_procedure(
585 &s,
586 directives.schema.as_ref(),
587 &location,
588 )?;
589 let builder::create_function_stmt::Routine::Procedure(p) = routine else {
590 return Err(ParseError::Structural {
591 location,
592 message: "internal error: expected Procedure from procedure stmt".into(),
593 });
594 };
595 let key = format!("procedures.{}", p.qname);
601 if let Some(prior) = locations.get(&key) {
602 return Err(ParseError::DuplicateObject {
603 qname: key,
604 first: prior.clone(),
605 second: location,
606 });
607 }
608 locations.insert(key, location.clone());
609 catalog.procedures.push(p);
610 }
611 Statement::CreateExtension(s) => {
612 let ext = builder::create_extension_stmt::build_extension(&s, &location)?;
613 let key = format!("extensions.{}", ext.name);
614 if let Some(prior) = locations.get(&key) {
615 return Err(ParseError::DuplicateObject {
616 qname: key,
617 first: prior.clone(),
618 second: location,
619 });
620 }
621 locations.insert(key, location.clone());
622 catalog.extensions.push(ext);
623 }
624 Statement::CreateTrigger(s) => {
625 let trigger = builder::create_trigger_stmt::build_trigger(&s, &location)?;
626 let key = format!("triggers.{}", trigger.qname);
627 if let Some(prior) = locations.get(&key) {
628 return Err(ParseError::DuplicateObject {
629 qname: key,
630 first: prior.clone(),
631 second: location,
632 });
633 }
634 locations.insert(key, location.clone());
635 catalog.triggers.push(trigger);
636 }
637 Statement::AlterTableAttachPartition(s) => {
638 let attach = builder::alter_table_attach_partition::build_attach_partition(
639 &s,
640 directives.schema.as_ref(),
641 &location,
642 )?;
643 let child_table = catalog
644 .tables
645 .iter_mut()
646 .find(|t| t.qname == attach.child)
647 .ok_or_else(|| ParseError::Structural {
648 location: location.clone(),
649 message: format!(
650 "ATTACH PARTITION {} must follow its CREATE TABLE statement",
651 attach.child
652 ),
653 })?;
654 if child_table.partition_of.is_some() {
655 return Err(ParseError::Structural {
656 location,
657 message: format!("table {} already declared as a partition", attach.child),
658 });
659 }
660 child_table.partition_of = Some(attach.partition_of);
661 }
662 Statement::Grant(s) => {
663 builder::grants::apply(&s, catalog, &location)?;
664 }
665 Statement::AlterOwner(s) => {
666 builder::owner_stmt::apply(&s, catalog, &location)?;
667 }
668 Statement::AlterDefaultPrivileges(s) => {
669 builder::default_privileges::apply(&s, catalog, &location)?;
670 }
671 Statement::CreatePolicy(s) => {
672 builder::policy_stmt::apply(&s, catalog, &location)?;
673 }
674 Statement::CreatePublication(s) => {
675 builder::publication_stmt::parse_create_publication(&s, location, publications)?;
676 }
677 Statement::AlterPublication(s) => {
678 builder::publication_stmt::parse_alter_publication(&s, location, publications)?;
679 }
680 Statement::CreateSubscription(s) => {
681 builder::subscription_stmt::parse_create_subscription(&s, location, subscriptions)?;
682 }
683 Statement::AlterSubscription(s) => {
684 builder::subscription_stmt::parse_alter_subscription(&s, location, subscriptions)?;
685 }
686 Statement::CreateStatistics(s) => {
687 builder::statistic_stmt::parse_create_statistics(&s, location, statistics)?;
688 }
689 Statement::AlterStatistics(s) => {
690 builder::statistic_stmt::parse_alter_statistics(&s, &location, statistics)?;
691 }
692 }
693 }
694 Ok(())
695}
696
697fn stmt_location(path: &Path, contents: &str, byte_offset: i32) -> SourceLocation {
699 let offset = usize::try_from(byte_offset).unwrap_or(0);
700 let mut line = 1usize;
701 let mut col = 1usize;
702 for (i, c) in contents.char_indices() {
703 if i >= offset {
704 break;
705 }
706 if c == '\n' {
707 line += 1;
708 col = 1;
709 } else {
710 col += 1;
711 }
712 }
713 SourceLocation::new(path.to_path_buf(), line, col)
714}
715
716fn translate_canonicalize_error(
717 e: IrError,
718 locations: &HashMap<String, SourceLocation>,
719) -> ParseError {
720 if let IrError::InvalidIdentifier(msg) = &e {
721 if let Some(rest) = msg.strip_prefix("duplicate ")
723 && let Some((_, qname)) = rest.split_once(": ")
724 && let Some(loc) = locations.get(qname)
725 {
726 return ParseError::DuplicateObject {
727 qname: qname.to_string(),
728 first: loc.clone(),
729 second: loc.clone(),
730 };
731 }
732 }
733 let placeholder = SourceLocation::new(PathBuf::new(), 0, 0);
734 ParseError::Ir {
735 location: placeholder,
736 source: e,
737 }
738}
739
740#[cfg(test)]
742pub(crate) fn smoke_parse(sql: &str) -> Result<pg_query::ParseResult, pg_query::Error> {
743 pg_query::parse(sql)
744}
745
746#[cfg(test)]
747mod tests {
748 use super::*;
749
750 #[test]
751 fn pg_query_round_trips_a_create_table() {
752 let sql = "CREATE TABLE app.users (id integer);";
753 let result = smoke_parse(sql).expect("pg_query parses");
754 assert!(!result.protobuf.stmts.is_empty());
756 }
757
758 #[test]
759 fn pg_query_reports_syntax_errors() {
760 let sql = "CREATE TABLE !bad!;";
761 assert!(smoke_parse(sql).is_err());
762 }
763}