1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
//! # inbq
//!
//! A library for parsing BigQuery queries and extracting schema-aware, column-level lineage.
//!
//! # Features
//!
//! - Parse BigQuery queries into well-structured ASTs with easy-to-navigate nodes.
//! - Extract schema-aware, column-level lineage.
//! - Trace data flow through nested structs and arrays.
//! - Capture referenced columns and the specific query components (e.g., select, where, join) they appear in.
//! - Process both single and multi-statement queries with procedural language constructs.
//! - Combines the performance of a Rust core with the ease of a Python API through efficient, low-overhead bindings.
//!
//! # Example
//!
//! ```rust,no_run
//! use inbq::{
//! lineage::{
//! catalog::{Catalog, Column, SchemaObject, SchemaObjectKind},
//! extract_lineage,
//! },
//! parser::Parser,
//! scanner::Scanner,
//! };
//!
//! fn column(name: &str, dtype: &str) -> Column {
//! Column {
//! name: name.to_owned(),
//! dtype: dtype.to_owned(),
//! }
//! }
//!
//! fn main() -> anyhow::Result<()> {
//! env_logger::init();
//!
//! let sql = r#""
//! declare default_val float64 default (select min(val) from project.dataset.out);
//!
//! insert into `project.dataset.out`
//! select
//! id,
//! if(x is null or s.x is null, default_val, x + s.x)
//! from `project.dataset.t1` inner join `project.dataset.t2` using (id)
//! where s.source = "baz";
//! ""#;
//! let mut scanner = Scanner::new(sql);
//! scanner.scan()?;
//! let mut parser = Parser::new(scanner.tokens());
//! let ast = parser.parse()?;
//! println!("Syntax Tree: {:?}", ast);
//!
//! let data_catalog = Catalog {
//! schema_objects: vec![
//! SchemaObject {
//! name: "project.dataset.out".to_owned(),
//! kind: SchemaObjectKind::Table {
//! columns: vec![column("id", "int64"), column("val", "int64")],
//! },
//! },
//! SchemaObject {
//! name: "project.dataset.t1".to_owned(),
//! kind: SchemaObjectKind::Table {
//! columns: vec![column("id", "int64"), column("x", "float64")],
//! },
//! },
//! SchemaObject {
//! name: "project.dataset.t2".to_owned(),
//! kind: SchemaObjectKind::Table {
//! columns: vec![
//! column("id", "int64"),
//! column("s", "struct<source string, x float64>"),
//! ],
//! },
//! },
//! ],
//! };
//!
//! let lineage = extract_lineage(&[&ast], &data_catalog, false, true)
//! .pop()
//! .unwrap()?;
//!
//! println!("\nLineage: {:?}", lineage.lineage);
//! println!("\nReferenced columns: {:?}", lineage.referenced_columns);
//! Ok(())
//! }
//! ```