1use crate::trilogy_parser::{parse_bounded, Rule};
12use pest::iterators::Pair;
13use std::path::{Path, PathBuf};
14use thiserror::Error;
15
16#[derive(Debug, Clone, PartialEq, Eq, Hash)]
17pub struct ImportStatement {
18 pub raw_path: String,
19 pub parent_dirs: usize,
20 pub alias: Option<String>,
21 pub is_stdlib: bool,
22}
23
24impl ImportStatement {
25 pub fn resolve(&self, working_dir: &Path) -> Option<PathBuf> {
26 if self.is_stdlib {
27 return None;
28 }
29
30 let mut base = working_dir.to_path_buf();
31 for _ in 0..self.parent_dirs {
32 base = base.parent()?.to_path_buf();
33 }
34 for part in self.raw_path.split('.') {
35 base.push(part);
36 }
37 base.set_extension("preql");
38 Some(base)
39 }
40
41 pub fn effective_alias(&self) -> &str {
42 self.alias
43 .as_deref()
44 .unwrap_or_else(|| self.raw_path.split('.').last().unwrap_or(&self.raw_path))
45 }
46}
47
48#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
52pub enum AddressKind {
53 Literal,
55 Templated,
57 Query,
59 File,
61}
62
63impl AddressKind {
64 pub fn as_str(&self) -> &'static str {
65 match self {
66 AddressKind::Literal => "literal",
67 AddressKind::Templated => "templated",
68 AddressKind::Query => "query",
69 AddressKind::File => "file",
70 }
71 }
72}
73
74impl std::fmt::Display for AddressKind {
75 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
76 write!(f, "{}", self.as_str())
77 }
78}
79
80#[derive(Debug, Clone, PartialEq, Eq, Hash)]
81pub struct DatasourceDeclaration {
82 pub name: String,
83 pub address: Option<String>,
86 pub address_kind: AddressKind,
87 pub is_root: bool,
89 pub is_partial: bool,
92 pub is_partitioned: bool,
94}
95
96#[derive(Debug, Clone, PartialEq, Eq, Hash)]
97pub struct PersistStatement {
98 pub mode: PersistMode,
99 pub target_datasource: String,
100}
101
102#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
103pub enum PersistMode {
104 Append,
105 Overwrite,
106 Persist,
107}
108
109impl std::fmt::Display for PersistMode {
110 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
111 match self {
112 PersistMode::Append => write!(f, "append"),
113 PersistMode::Overwrite => write!(f, "overwrite"),
114 PersistMode::Persist => write!(f, "persist"),
115 }
116 }
117}
118
119#[derive(Debug, Clone, Default)]
120pub struct ParsedFile {
121 pub imports: Vec<ImportStatement>,
122 pub datasources: Vec<DatasourceDeclaration>,
123 pub persists: Vec<PersistStatement>,
124}
125
126#[derive(Error, Debug)]
127pub enum ParseError {
128 #[error("Failed to parse file: {0}")]
129 PestError(#[from] pest::error::Error<Rule>),
130
131 #[error("Invalid import statement structure")]
132 InvalidImportStructure,
133
134 #[error("Invalid datasource statement structure")]
135 InvalidDatasourceStructure,
136
137 #[error("Invalid persist statement structure")]
138 InvalidPersistStructure,
139}
140
141pub fn parse_file(content: &str) -> Result<ParsedFile, ParseError> {
142 let mut pairs = parse_bounded(content).map_err(|e| *e)?;
143 let start = pairs
144 .next()
145 .ok_or(ParseError::InvalidImportStructure)?;
146
147 let mut result = ParsedFile::default();
148 for top in start.into_inner() {
149 if top.as_rule() != Rule::block {
150 continue;
151 }
152 for stmt in top.into_inner() {
156 match stmt.as_rule() {
157 Rule::import_statement | Rule::selective_import_statement => {
163 result.imports.push(extract_import(stmt)?);
164 }
165 Rule::datasource => {
166 if let Some(declaration) = extract_datasource(stmt)? {
167 result.datasources.push(declaration);
168 }
169 }
170 Rule::persist_statement => {
171 result.persists.push(extract_persist(stmt)?);
172 }
173 _ => {}
174 }
175 }
176 }
177
178 Ok(result)
179}
180
181pub fn parse_imports(content: &str) -> Result<Vec<ImportStatement>, ParseError> {
182 Ok(parse_file(content)?.imports)
183}
184
185fn extract_import(pair: Pair<Rule>) -> Result<ImportStatement, ParseError> {
189 let full_text = pair.as_str();
190 let mut n_dots = 0usize;
191 let mut idents: Vec<String> = Vec::new();
192 for child in pair.into_inner() {
193 match child.as_rule() {
194 Rule::IMPORT_DOT => n_dots += 1,
195 Rule::IDENTIFIER => idents.push(child.as_str().to_string()),
196 _ => {}
197 }
198 }
199 if idents.is_empty() {
200 return Err(ParseError::InvalidImportStructure);
201 }
202
203 let has_alias = full_text
206 .split_ascii_whitespace()
207 .any(|tok| tok.eq_ignore_ascii_case("as"));
208 let alias = if has_alias && idents.len() >= 2 {
209 Some(idents.pop().unwrap())
210 } else {
211 None
212 };
213
214 let raw_path = idents.join(".");
215 let is_stdlib = raw_path == "std" || raw_path.starts_with("std.");
216 let parent_dirs = n_dots.saturating_sub(1);
220
221 Ok(ImportStatement {
222 raw_path,
223 parent_dirs,
224 alias,
225 is_stdlib,
226 })
227}
228
229fn extract_datasource(pair: Pair<Rule>) -> Result<Option<DatasourceDeclaration>, ParseError> {
235 let mut name: Option<String> = None;
236 let mut address: Option<String> = None;
237 let mut address_kind: Option<AddressKind> = None;
238 let mut is_root = false;
239 let mut is_template = false;
240 let mut is_partial = false;
241 let mut is_partitioned = false;
242
243 for child in pair.into_inner() {
244 match child.as_rule() {
245 Rule::DATASOURCE_ROOT => is_root = true,
246 Rule::DATASOURCE_TEMPLATE => is_template = true,
247 Rule::DATASOURCE_PARTIAL => is_partial = true,
248 Rule::IDENTIFIER if name.is_none() => name = Some(child.as_str().to_string()),
249 Rule::address => {
251 let tok = child
252 .into_inner()
253 .next()
254 .ok_or(ParseError::InvalidDatasourceStructure)?;
255 match tok.as_rule() {
256 Rule::F_QUOTED_ADDRESS => {
257 address_kind = Some(AddressKind::Templated);
260 address = Some(
261 tok.as_str()
262 .trim_start_matches(['f', 'F'])
263 .trim_matches('`')
264 .to_string(),
265 );
266 }
267 Rule::QUOTED_ADDRESS => {
268 address_kind = Some(AddressKind::Literal);
270 address = Some(
271 tok.as_str().trim_matches('`').trim_matches('\'').to_string(),
272 );
273 }
274 _ => {
275 address_kind = Some(AddressKind::Literal);
276 address = Some(tok.as_str().to_string());
277 }
278 }
279 }
280 Rule::query => address_kind = Some(AddressKind::Query),
281 Rule::file => {
282 address_kind = Some(AddressKind::File);
283 address = Some(child.as_str()[4..].trim().to_string());
285 }
286 Rule::datasource_partition_clause => is_partitioned = true,
287 _ => {}
288 }
289 }
290
291 if is_template {
292 return Ok(None);
293 }
294 match (name, address_kind) {
295 (Some(name), Some(address_kind)) => Ok(Some(DatasourceDeclaration {
296 name,
297 address,
298 address_kind,
299 is_root,
300 is_partial,
301 is_partitioned,
302 })),
303 _ => Err(ParseError::InvalidDatasourceStructure),
304 }
305}
306
307fn extract_persist(pair: Pair<Rule>) -> Result<PersistStatement, ParseError> {
308 let inner = pair
310 .into_inner()
311 .next()
312 .ok_or(ParseError::InvalidPersistStructure)?;
313 match inner.as_rule() {
314 Rule::auto_persist => extract_auto_persist(inner),
315 Rule::full_persist => extract_full_persist(inner),
316 _ => Err(ParseError::InvalidPersistStructure),
317 }
318}
319
320fn extract_auto_persist(pair: Pair<Rule>) -> Result<PersistStatement, ParseError> {
322 let mut mode: Option<PersistMode> = None;
323 let mut target: Option<String> = None;
324 for child in pair.into_inner() {
325 match child.as_rule() {
326 Rule::PERSIST_MODE => mode = Some(parse_persist_mode(child.as_str())),
327 Rule::IDENTIFIER if target.is_none() => {
328 target = Some(child.as_str().to_string());
329 }
330 _ => {}
331 }
332 }
333 match (mode, target) {
334 (Some(mode), Some(target_datasource)) => Ok(PersistStatement {
335 mode,
336 target_datasource,
337 }),
338 _ => Err(ParseError::InvalidPersistStructure),
339 }
340}
341
342fn extract_full_persist(pair: Pair<Rule>) -> Result<PersistStatement, ParseError> {
347 let mut mode: Option<PersistMode> = None;
348 let mut last_ident: Option<String> = None;
349 for child in pair.into_inner() {
350 match child.as_rule() {
351 Rule::PERSIST_MODE => mode = Some(parse_persist_mode(child.as_str())),
352 Rule::IDENTIFIER => last_ident = Some(child.as_str().to_string()),
353 _ => {}
354 }
355 }
356 match (mode, last_ident) {
357 (Some(mode), Some(target_datasource)) => Ok(PersistStatement {
358 mode,
359 target_datasource,
360 }),
361 _ => Err(ParseError::InvalidPersistStructure),
362 }
363}
364
365fn parse_persist_mode(s: &str) -> PersistMode {
366 match s.to_ascii_lowercase().as_str() {
367 "append" => PersistMode::Append,
368 "overwrite" => PersistMode::Overwrite,
369 _ => PersistMode::Persist,
370 }
371}
372
373#[cfg(test)]
374mod tests {
375 use super::*;
376
377 #[test]
378 fn test_simple_import() {
379 let parsed = parse_file("import models.customer;").unwrap();
380 assert_eq!(parsed.imports.len(), 1);
381 assert_eq!(parsed.imports[0].raw_path, "models.customer");
382 assert_eq!(parsed.imports[0].parent_dirs, 0);
383 assert!(parsed.imports[0].alias.is_none());
384 }
385
386 #[test]
387 fn test_import_with_alias() {
388 let parsed = parse_file("import models.customer as cust;").unwrap();
389 assert_eq!(parsed.imports.len(), 1);
390 assert_eq!(parsed.imports[0].raw_path, "models.customer");
391 assert_eq!(parsed.imports[0].alias, Some("cust".to_string()));
392 }
393
394 #[test]
395 fn test_relative_import() {
396 let parsed = parse_file("import ..models.customer;").unwrap();
397 assert_eq!(parsed.imports.len(), 1);
398 assert_eq!(parsed.imports[0].raw_path, "models.customer");
399 assert_eq!(parsed.imports[0].parent_dirs, 1);
400 }
401
402 #[test]
403 fn test_sibling_relative_import() {
404 let parsed = parse_file("import .customer;").unwrap();
405 assert_eq!(parsed.imports[0].raw_path, "customer");
406 assert_eq!(parsed.imports[0].parent_dirs, 0);
407 }
408
409 #[test]
410 fn test_stdlib_import() {
411 let parsed = parse_file("import std.aggregates;").unwrap();
412 assert!(parsed.imports[0].is_stdlib);
413 }
414
415 #[test]
416 fn test_datasource_simple() {
417 let content = r#"
418 key order_id int;
419 datasource orders (
420 order_id: order_id,
421 amount: amount
422 )
423 grain (order_id)
424 address my_database.orders;
425 "#;
426 let parsed = parse_file(content).unwrap();
427 assert_eq!(parsed.datasources.len(), 1);
428 let ds = &parsed.datasources[0];
429 assert_eq!(ds.name, "orders");
430 assert_eq!(ds.address.as_deref(), Some("my_database.orders"));
431 assert_eq!(ds.address_kind, AddressKind::Literal);
432 assert!(!ds.is_root);
433 assert!(!ds.is_partitioned);
434 }
435
436 #[test]
437 fn test_datasource_with_quoted_address() {
438 let content = r#"
439 key customer_id int;
440 datasource customers (
441 id: customer_id,
442 name: customer_name
443 )
444 grain (customer_id)
445 address `my_db.customers`;
446 "#;
447 let parsed = parse_file(content).unwrap();
448 assert_eq!(parsed.datasources.len(), 1);
449 let ds = &parsed.datasources[0];
450 assert_eq!(ds.name, "customers");
451 assert_eq!(ds.address.as_deref(), Some("my_db.customers"));
452 assert_eq!(ds.address_kind, AddressKind::Literal);
453 }
454
455 #[test]
456 fn test_root_partitioned_datasource() {
457 let content = r#"
458 key event_id int;
459 root datasource events (
460 event_id: event_id
461 )
462 grain (event_id)
463 address analytics.events
464 partition by event_id;
465 "#;
466 let parsed = parse_file(content).unwrap();
467 let ds = &parsed.datasources[0];
468 assert!(ds.is_root);
469 assert!(ds.is_partitioned);
470 assert_eq!(ds.address.as_deref(), Some("analytics.events"));
471 }
472
473 #[test]
474 fn test_templated_address_datasource() {
475 let content = r#"
476 key order_id int;
477 datasource orders (
478 order_id: order_id
479 )
480 grain (order_id)
481 address f`{{env}}.orders`;
482 "#;
483 let parsed = parse_file(content).unwrap();
484 let ds = &parsed.datasources[0];
485 assert_eq!(ds.address_kind, AddressKind::Templated);
486 assert_eq!(ds.address.as_deref(), Some("{{env}}.orders"));
488 }
489
490 #[test]
491 fn test_query_datasource_has_no_address() {
492 let content = r#"
493 key order_id int;
494 datasource order_view (
495 order_id: order_id
496 )
497 grain (order_id)
498 query '''select 1 as order_id''';
499 "#;
500 let parsed = parse_file(content).unwrap();
501 let ds = &parsed.datasources[0];
502 assert_eq!(ds.address_kind, AddressKind::Query);
503 assert!(ds.address.is_none());
504 }
505
506 #[test]
507 fn test_file_datasource() {
508 let content = r#"
509 key launch_id int;
510 datasource launches (
511 launch_id: launch_id
512 )
513 grain (launch_id)
514 file `gcs://bucket/launch_report/launch.parquet`;
515 "#;
516 let parsed = parse_file(content).unwrap();
517 let ds = &parsed.datasources[0];
518 assert_eq!(ds.address_kind, AddressKind::File);
519 assert_eq!(
522 ds.address.as_deref(),
523 Some("`gcs://bucket/launch_report/launch.parquet`")
524 );
525 }
526
527 #[test]
528 fn test_file_datasource_read_write_pair() {
529 let content = r#"
530 key launch_id int;
531 datasource launches (
532 launch_id: launch_id
533 )
534 grain (launch_id)
535 file `https://host/launch.parquet`:`gcs://bucket/launch.parquet`;
536 "#;
537 let parsed = parse_file(content).unwrap();
538 let ds = &parsed.datasources[0];
539 assert_eq!(ds.address_kind, AddressKind::File);
540 assert_eq!(
541 ds.address.as_deref(),
542 Some("`https://host/launch.parquet`:`gcs://bucket/launch.parquet`")
543 );
544 }
545
546 #[test]
547 fn test_root_partial_file_datasource() {
548 let content = r#"
549 key a_id int;
550 root partial datasource a_raw (
551 a_id: a_id
552 )
553 grain (a_id)
554 file `./a_raw_source.py`;
555 "#;
556 let parsed = parse_file(content).unwrap();
557 let ds = &parsed.datasources[0];
558 assert_eq!(ds.name, "a_raw");
559 assert!(ds.is_root);
560 assert!(ds.is_partial);
561 assert!(!ds.is_partitioned);
562 assert_eq!(ds.address_kind, AddressKind::File);
563 }
564
565 #[test]
566 fn test_partial_datasource_name_not_shadowed_by_modifier() {
567 let content = r#"
568 key customer_id int;
569 partial datasource customer_revenue (
570 customer_id: customer_id
571 )
572 grain (customer_id)
573 address db.customer_revenue;
574 "#;
575 let parsed = parse_file(content).unwrap();
576 let ds = &parsed.datasources[0];
577 assert_eq!(ds.name, "customer_revenue");
578 assert!(ds.is_partial);
579 assert!(!ds.is_root);
580 }
581
582 #[test]
583 fn test_multiple_datasources_keep_independent_flags() {
584 let content = r#"
585 key order_id int;
586 root datasource raw_orders (
587 order_id: order_id
588 )
589 grain (order_id)
590 address raw.orders;
591
592 datasource orders (
593 order_id: order_id
594 )
595 grain (order_id)
596 address db.orders;
597 "#;
598 let parsed = parse_file(content).unwrap();
599 assert_eq!(parsed.datasources.len(), 2);
600 assert!(parsed.datasources[0].is_root);
601 assert_eq!(parsed.datasources[0].address.as_deref(), Some("raw.orders"));
602 assert!(!parsed.datasources[1].is_root);
603 assert_eq!(parsed.datasources[1].address.as_deref(), Some("db.orders"));
604 }
605
606 #[test]
607 fn test_quoted_address_strips_inner_quote_layer() {
608 let content = r#"
609 key order_id int;
610 datasource orders (
611 order_id: order_id
612 )
613 grain (order_id)
614 address `'my project.my dataset.orders'`;
615 "#;
616 let parsed = parse_file(content).unwrap();
617 let ds = &parsed.datasources[0];
618 assert_eq!(ds.address_kind, AddressKind::Literal);
619 assert_eq!(
620 ds.address.as_deref(),
621 Some("my project.my dataset.orders")
622 );
623 }
624
625 #[test]
626 fn test_address_kind_strings_are_stable() {
627 assert_eq!(AddressKind::Literal.to_string(), "literal");
629 assert_eq!(AddressKind::Templated.to_string(), "templated");
630 assert_eq!(AddressKind::Query.to_string(), "query");
631 assert_eq!(AddressKind::File.to_string(), "file");
632 }
633
634 #[test]
635 fn test_selective_import_creates_edge() {
636 let parsed = parse_file("from models.customer import customer_id;").unwrap();
637 assert_eq!(parsed.imports.len(), 1);
638 assert_eq!(parsed.imports[0].raw_path, "models.customer");
639 assert!(parsed.imports[0].alias.is_none());
640 }
641
642 #[test]
643 fn test_selective_import_with_alias() {
644 let parsed = parse_file("from ..models.customer as cust import customer_id, name;").unwrap();
645 assert_eq!(parsed.imports.len(), 1);
646 assert_eq!(parsed.imports[0].raw_path, "models.customer");
647 assert_eq!(parsed.imports[0].alias, Some("cust".to_string()));
648 assert_eq!(parsed.imports[0].parent_dirs, 1);
649 }
650
651 #[test]
652 fn test_self_import_is_not_an_edge() {
653 let parsed = parse_file("self import as me;").unwrap();
654 assert!(parsed.imports.is_empty());
655 }
656
657 #[test]
658 fn test_auto_persist() {
659 let parsed = parse_file("persist orders;").unwrap();
660 assert_eq!(parsed.persists.len(), 1);
661 assert_eq!(parsed.persists[0].target_datasource, "orders");
662 assert_eq!(parsed.persists[0].mode, PersistMode::Persist);
663 }
664
665 #[test]
666 fn test_append_auto_persist() {
667 let parsed = parse_file("append orders;").unwrap();
668 assert_eq!(parsed.persists.len(), 1);
669 assert_eq!(parsed.persists[0].target_datasource, "orders");
670 assert_eq!(parsed.persists[0].mode, PersistMode::Append);
671 }
672
673 #[test]
674 fn test_full_persist() {
675 let content = r#"
676 key order_id int;
677 overwrite into target_orders from select order_id;
678 "#;
679 let parsed = parse_file(content).unwrap();
680 assert_eq!(parsed.persists.len(), 1);
681 assert_eq!(parsed.persists[0].target_datasource, "target_orders");
682 assert_eq!(parsed.persists[0].mode, PersistMode::Overwrite);
683 }
684
685 #[test]
686 fn test_multiple_imports() {
687 let content = r#"
688 import models.customer;
689 import models.orders as ord;
690 // comment
691 import ..shared.utils;
692 "#;
693 let parsed = parse_file(content).unwrap();
694 assert_eq!(parsed.imports.len(), 3);
695 assert_eq!(parsed.imports[1].alias, Some("ord".to_string()));
696 assert_eq!(parsed.imports[2].parent_dirs, 1);
697 }
698
699 #[test]
700 fn test_mixed_file() {
701 let content = r#"
702 import models.customer;
703
704 key order_id int;
705 datasource local_orders (
706 order_id: order_id
707 )
708 grain (order_id)
709 address local.orders;
710
711 persist local_orders;
712 "#;
713 let parsed = parse_file(content).unwrap();
714 assert_eq!(parsed.imports.len(), 1);
715 assert_eq!(parsed.datasources.len(), 1);
716 assert_eq!(parsed.persists.len(), 1);
717 assert_eq!(parsed.datasources[0].name, "local_orders");
718 assert_eq!(parsed.persists[0].target_datasource, "local_orders");
719 }
720}