fv_streams_sql/plan/mod.rs
1//! The typed streaming **logical plan** — our own IR, the seam between the SQL front-end and the
2//! `StageDef` lowering. Small and streaming-aware by construction (unlike DataFusion's `LogicalPlan`);
3//! a future optimiser is a pass over this, and a second dialect is another producer of it.
4
5use sqlparser::ast::{Expr, Query, SetExpr, Statement};
6use sqlparser::dialect::GenericDialect;
7use sqlparser::parser::Parser;
8
9mod cte;
10mod emit;
11mod functions;
12mod join;
13mod select;
14
15use cte::{bind_ctes, parse_ctes};
16use emit::{attach_trigger, split_emit};
17use select::plan_select;
18
19/// One projected output column.
20#[derive(Debug, Clone, PartialEq)]
21pub struct Proj {
22 /// The output column name.
23 pub out: String,
24 /// `Some(dialect_expr)` when computed or renamed (lowers to an `applyExpression`); `None` for a
25 /// bare pass-through column named `out` (kept by the `select`).
26 pub expr: Option<String>,
27}
28
29/// One aggregate in a `SELECT` over a window — `op` over `column` (empty for `count(*)`), out `alias`.
30#[derive(Debug, Clone, PartialEq)]
31pub struct AggSpec {
32 pub op: String,
33 pub column: String,
34 pub alias: String,
35 /// The call as written (`count(*)`, `sum(price)`), for `HAVING` to match by.
36 pub call: String,
37}
38
39/// The windowing a `GROUP BY <window-fn>(…)` selects. An optional trailing `INTERVAL` on any window
40/// function is the **allowed lateness** — how long a window stays open for late events after its end
41/// (the engine's `allowedLatenessMs`); `None` leaves it at the engine default. q4/q5 need a large
42/// lateness so no out-of-order bid is dropped from a closed window.
43#[derive(Debug, Clone, PartialEq)]
44pub enum Win {
45 /// `TUMBLE(ts, INTERVAL <size> [, INTERVAL <lateness>])` — fixed windows.
46 Tumble { size_ms: i64, lateness_ms: Option<i64> },
47 /// `HOP(ts, INTERVAL <slide>, INTERVAL <size> [, INTERVAL <lateness>])` — sliding windows.
48 Hop {
49 slide_ms: i64,
50 size_ms: i64,
51 lateness_ms: Option<i64>,
52 },
53 /// `SESSION(ts, INTERVAL <gap> [, INTERVAL <lateness>])` — dynamic session windows.
54 Session { gap_ms: i64, lateness_ms: Option<i64> },
55}
56
57/// The early-firing trigger a trailing `EMIT CHANGES EVERY …` clause selects — the 5a updating
58/// aggregate. sqlparser does not know `EMIT`, so a small lexical pre-pass ([`split_emit`]) strips
59/// and parses this clause before the rest of the statement reaches sqlparser (design §10 / A).
60#[derive(Debug, Clone, Copy, PartialEq)]
61pub enum Trigger {
62 EveryRows(u64),
63 EveryMs(i64),
64}
65
66/// How a two-input `JOIN` maps to an operator. An interval predicate (`ts BETWEEN …`) in the `ON`
67/// makes it a stream×stream interval join; a plain equi-join makes the right side a **bounded
68/// table** — the 5b lookup join.
69#[derive(Debug, Clone, PartialEq)]
70pub enum JoinKind {
71 /// `JOIN … ON k=k AND ts BETWEEN … ` → `streamJoin`.
72 Interval { time_column: String, window_ms: i64 },
73 /// `JOIN <bounded> ON k=k` → the 5b `lookupJoin` (right = the bounded table).
74 Lookup,
75}
76
77/// The typed logical plan. 5c-i covers the stateless shape (source → filter → project → sink);
78/// 5c-ii adds the windowed aggregate, the updating aggregate, and the joins.
79#[derive(Debug, Clone, PartialEq)]
80pub enum Plan {
81 /// `FROM <dataset>` — a source dataset by name.
82 Source(String),
83 /// `FROM <left> JOIN <right> ON <key> [interval]` — a two-input join.
84 Join {
85 left: Box<Plan>,
86 right: Box<Plan>,
87 join_key: String,
88 kind: JoinKind,
89 /// Right columns the query names with the right's qualifier → their output name (the
90 /// join's `rightRename`), so `a.ts` can never mean the left's `ts`.
91 right_rename: std::collections::BTreeMap<String, String>,
92 },
93 /// `WHERE <predicate>` — the predicate as a value-dialect string.
94 Filter { input: Box<Plan>, predicate: String },
95 /// `SELECT …`. `star` = `SELECT *` (pass-through, no step); else the projected columns.
96 Project {
97 input: Box<Plan>,
98 star: bool,
99 items: Vec<Proj>,
100 },
101 /// `SELECT … FROM … GROUP BY <window-fn>(ts,…), <keys>` — a windowed keyed aggregate. A
102 /// `Some(trigger)` makes it the **5a updating aggregate** (early firing).
103 Aggregate {
104 input: Box<Plan>,
105 time_column: String,
106 window: Win,
107 group_by: Vec<String>,
108 /// Pre-aggregate computed columns (an aggregate's expression argument, e.g. a `CASE` fed to
109 /// `count(DISTINCT …)`) — lowered to `applyExpression` steps BEFORE the aggregate.
110 pre: Vec<Proj>,
111 aggs: Vec<AggSpec>,
112 /// Post-aggregate computed columns — a `SELECT` expression over the window/group/aggregates
113 /// (e.g. `day = substring(epochToIso(windowStart), 1, 10)`), lowered to `applyExpression`
114 /// steps AFTER the aggregate (the q15/q16 day/minute columns).
115 post: Vec<Proj>,
116 /// `HAVING <predicate>` — a filter over the aggregate's output rows (the window, the group
117 /// columns, the aggregate aliases), lowered to a `filter` step AFTER the aggregate and its
118 /// post columns.
119 having: Option<String>,
120 trigger: Option<Trigger>,
121 },
122 /// `SELECT … FROM (SELECT …, ROW_NUMBER() OVER (PARTITION BY g ORDER BY c [DESC]) AS rn FROM s)
123 /// WHERE rn <= n` — the streaming top-N idiom → the `topN` operator.
124 TopN {
125 input: Box<Plan>,
126 group_by: Vec<String>,
127 order_by: String,
128 /// A second `ORDER BY` column → the tie-break (`tieBy`, ascending).
129 tie_by: Option<String>,
130 descending: bool,
131 n: usize,
132 },
133 /// A CTE (`WITH <name> AS (<subquery>)`): the sub-plan `input` materialises as its own stage
134 /// under `name`, and downstream references to `name` read that stage. The multi-stage boundary —
135 /// the SQL equivalent of a named intermediate dataset in the TOML `[[transforms]]` DAG.
136 Named { name: String, input: Box<Plan> },
137 /// `INSERT INTO <dataset>` — the terminal sink dataset by name.
138 Sink { input: Box<Plan>, name: String },
139}
140
141/// Parse one streaming SQL statement into a [`Plan`]. Fails LOUDLY, streaming-aware, on anything
142/// outside the supported subset.
143pub fn parse(sql: &str) -> Result<Plan, String> {
144 // Strip a trailing `EMIT CHANGES EVERY …` (sqlparser does not know it) before parsing the rest.
145 let (sql, trigger) = split_emit(sql)?;
146 let plan = parse_statement(&sql)?;
147 match trigger {
148 Some(t) => attach_trigger(plan, t),
149 None => Ok(plan),
150 }
151}
152
153fn parse_statement(sql: &str) -> Result<Plan, String> {
154 let mut stmts = Parser::parse_sql(&GenericDialect {}, sql).map_err(|e| format!("SQL parse error: {e}"))?;
155 if stmts.len() != 1 {
156 return Err(format!("expected exactly one statement, got {}", stmts.len()));
157 }
158 match stmts.pop().expect("one") {
159 Statement::Insert(insert) => {
160 let sink = object_name(&insert.table.to_string());
161 let query = insert
162 .source
163 .ok_or("INSERT INTO <sink> needs a SELECT (`INSERT INTO out SELECT …`)")?;
164 let input = plan_query(&query)?;
165 Ok(Plan::Sink {
166 input: Box::new(input),
167 name: sink,
168 })
169 }
170 Statement::Query(_) => Err("a streaming statement must write to a sink: `INSERT INTO <sink> SELECT …`".into()),
171 other => Err(format!(
172 "unsupported statement `{}` — the supported shape is `INSERT INTO <sink> SELECT … FROM … [WHERE …]`",
173 statement_kind(&other)
174 )),
175 }
176}
177
178fn plan_query(query: &Query) -> Result<Plan, String> {
179 // `WITH name AS (subquery), …` — each CTE becomes a named intermediate stream (the multi-stage
180 // boundary). SQL scoping: a CTE sees only the CTEs declared before it, so bodies are bound in
181 // order against the ones already built; the outer body binds against all of them.
182 let ctes = parse_ctes(query.with.as_ref())?;
183 if query.order_by.is_some() {
184 return Err("ORDER BY over an unbounded stream has no streaming meaning — use `ROW_NUMBER() OVER (PARTITION BY …) <= n` for a top-N".into());
185 }
186 if query.limit_clause.is_some() {
187 return Err("LIMIT over an unbounded stream is not supported — use a top-N (`ROW_NUMBER() … <= n`)".into());
188 }
189 let select = match query.body.as_ref() {
190 SetExpr::Select(s) => s.as_ref(),
191 SetExpr::SetOperation { .. } => {
192 return Err("UNION / INTERSECT / EXCEPT are not supported — a stage reads one input; write one pipeline per branch into the same sink (`INSERT INTO out SELECT … FROM a` and `INSERT INTO out SELECT … FROM b`), which is UNION ALL by construction".into())
193 }
194 _ => return Err("only a SELECT body is supported".into()),
195 };
196 Ok(bind_ctes(plan_select(select)?, &ctes))
197}
198
199/// The output name a bare projected expression takes (a column keeps its own name; a compound
200/// `a.b` takes `b`). `None` for a computed expression that needs an explicit alias.
201fn column_name_of(e: &Expr) -> Option<String> {
202 match e {
203 Expr::Identifier(id) => Some(id.value.clone()),
204 Expr::CompoundIdentifier(parts) => parts.last().map(|p| p.value.clone()),
205 _ => None,
206 }
207}
208
209/// An object name (`schema.name` → `name`) reduced to the last part — datasets are referenced by a
210/// single name in the pipeline config.
211fn object_name(raw: &str) -> String {
212 raw.rsplit('.').next().unwrap_or(raw).trim_matches('"').to_string()
213}
214
215fn statement_kind(s: &Statement) -> &'static str {
216 match s {
217 Statement::Insert(_) => "INSERT",
218 Statement::Query(_) => "SELECT",
219 Statement::CreateTable(_) => "CREATE TABLE",
220 _ => "statement",
221 }
222}