Skip to main content

fv_streams_sql/plan/
mod.rs

1//! The typed streaming **logical plan** — our own IR, the seam between the SQL front-end and the
2//! `StageDef` lowering. Small and streaming-aware by construction (unlike DataFusion's `LogicalPlan`);
3//! a future optimiser is a pass over this, and a second dialect is another producer of it.
4
5use sqlparser::ast::{Expr, Query, SetExpr, Statement};
6use sqlparser::dialect::GenericDialect;
7use sqlparser::parser::Parser;
8
9mod cte;
10mod emit;
11mod functions;
12mod join;
13mod select;
14
15use cte::{bind_ctes, parse_ctes};
16use emit::{attach_trigger, split_emit};
17use select::plan_select;
18
19/// One projected output column.
20#[derive(Debug, Clone, PartialEq)]
21pub struct Proj {
22    /// The output column name.
23    pub out: String,
24    /// `Some(dialect_expr)` when computed or renamed (lowers to an `applyExpression`); `None` for a
25    /// bare pass-through column named `out` (kept by the `select`).
26    pub expr: Option<String>,
27}
28
29/// One aggregate in a `SELECT` over a window — `op` over `column` (empty for `count(*)`), out `alias`.
30#[derive(Debug, Clone, PartialEq)]
31pub struct AggSpec {
32    pub op: String,
33    pub column: String,
34    pub alias: String,
35    /// The call as written (`count(*)`, `sum(price)`), for `HAVING` to match by.
36    pub call: String,
37}
38
39/// The windowing a `GROUP BY <window-fn>(…)` selects. An optional trailing `INTERVAL` on any window
40/// function is the **allowed lateness** — how long a window stays open for late events after its end
41/// (the engine's `allowedLatenessMs`); `None` leaves it at the engine default. q4/q5 need a large
42/// lateness so no out-of-order bid is dropped from a closed window.
43#[derive(Debug, Clone, PartialEq)]
44pub enum Win {
45    /// `TUMBLE(ts, INTERVAL <size> [, INTERVAL <lateness>])` — fixed windows.
46    Tumble { size_ms: i64, lateness_ms: Option<i64> },
47    /// `HOP(ts, INTERVAL <slide>, INTERVAL <size> [, INTERVAL <lateness>])` — sliding windows.
48    Hop {
49        slide_ms: i64,
50        size_ms: i64,
51        lateness_ms: Option<i64>,
52    },
53    /// `SESSION(ts, INTERVAL <gap> [, INTERVAL <lateness>])` — dynamic session windows.
54    Session { gap_ms: i64, lateness_ms: Option<i64> },
55}
56
57/// The early-firing trigger a trailing `EMIT CHANGES EVERY …` clause selects — the 5a updating
58/// aggregate. sqlparser does not know `EMIT`, so a small lexical pre-pass ([`split_emit`]) strips
59/// and parses this clause before the rest of the statement reaches sqlparser (design §10 / A).
60#[derive(Debug, Clone, Copy, PartialEq)]
61pub enum Trigger {
62    EveryRows(u64),
63    EveryMs(i64),
64}
65
66/// How a two-input `JOIN` maps to an operator. An interval predicate (`ts BETWEEN …`) in the `ON`
67/// makes it a stream×stream interval join; a plain equi-join makes the right side a **bounded
68/// table** — the 5b lookup join.
69#[derive(Debug, Clone, PartialEq)]
70pub enum JoinKind {
71    /// `JOIN … ON k=k AND ts BETWEEN … ` → `streamJoin`.
72    Interval { time_column: String, window_ms: i64 },
73    /// `JOIN <bounded> ON k=k` → the 5b `lookupJoin` (right = the bounded table).
74    Lookup,
75}
76
77/// The typed logical plan. 5c-i covers the stateless shape (source → filter → project → sink);
78/// 5c-ii adds the windowed aggregate, the updating aggregate, and the joins.
79#[derive(Debug, Clone, PartialEq)]
80pub enum Plan {
81    /// `FROM <dataset>` — a source dataset by name.
82    Source(String),
83    /// `FROM <left> JOIN <right> ON <key> [interval]` — a two-input join.
84    Join {
85        left: Box<Plan>,
86        right: Box<Plan>,
87        join_key: String,
88        kind: JoinKind,
89        /// Right columns the query names with the right's qualifier → their output name (the
90        /// join's `rightRename`), so `a.ts` can never mean the left's `ts`.
91        right_rename: std::collections::BTreeMap<String, String>,
92    },
93    /// `WHERE <predicate>` — the predicate as a value-dialect string.
94    Filter { input: Box<Plan>, predicate: String },
95    /// `SELECT …`. `star` = `SELECT *` (pass-through, no step); else the projected columns.
96    Project {
97        input: Box<Plan>,
98        star: bool,
99        items: Vec<Proj>,
100    },
101    /// `SELECT … FROM … GROUP BY <window-fn>(ts,…), <keys>` — a windowed keyed aggregate. A
102    /// `Some(trigger)` makes it the **5a updating aggregate** (early firing).
103    Aggregate {
104        input: Box<Plan>,
105        time_column: String,
106        window: Win,
107        group_by: Vec<String>,
108        /// Pre-aggregate computed columns (an aggregate's expression argument, e.g. a `CASE` fed to
109        /// `count(DISTINCT …)`) — lowered to `applyExpression` steps BEFORE the aggregate.
110        pre: Vec<Proj>,
111        aggs: Vec<AggSpec>,
112        /// Post-aggregate computed columns — a `SELECT` expression over the window/group/aggregates
113        /// (e.g. `day = substring(epochToIso(windowStart), 1, 10)`), lowered to `applyExpression`
114        /// steps AFTER the aggregate (the q15/q16 day/minute columns).
115        post: Vec<Proj>,
116        /// `HAVING <predicate>` — a filter over the aggregate's output rows (the window, the group
117        /// columns, the aggregate aliases), lowered to a `filter` step AFTER the aggregate and its
118        /// post columns.
119        having: Option<String>,
120        trigger: Option<Trigger>,
121    },
122    /// `SELECT … FROM (SELECT …, ROW_NUMBER() OVER (PARTITION BY g ORDER BY c [DESC]) AS rn FROM s)
123    /// WHERE rn <= n` — the streaming top-N idiom → the `topN` operator.
124    TopN {
125        input: Box<Plan>,
126        group_by: Vec<String>,
127        order_by: String,
128        /// A second `ORDER BY` column → the tie-break (`tieBy`, ascending).
129        tie_by: Option<String>,
130        descending: bool,
131        n: usize,
132    },
133    /// A CTE (`WITH <name> AS (<subquery>)`): the sub-plan `input` materialises as its own stage
134    /// under `name`, and downstream references to `name` read that stage. The multi-stage boundary —
135    /// the SQL equivalent of a named intermediate dataset in the TOML `[[transforms]]` DAG.
136    Named { name: String, input: Box<Plan> },
137    /// `INSERT INTO <dataset>` — the terminal sink dataset by name.
138    Sink { input: Box<Plan>, name: String },
139}
140
141/// Parse one streaming SQL statement into a [`Plan`]. Fails LOUDLY, streaming-aware, on anything
142/// outside the supported subset.
143pub fn parse(sql: &str) -> Result<Plan, String> {
144    // Strip a trailing `EMIT CHANGES EVERY …` (sqlparser does not know it) before parsing the rest.
145    let (sql, trigger) = split_emit(sql)?;
146    let plan = parse_statement(&sql)?;
147    match trigger {
148        Some(t) => attach_trigger(plan, t),
149        None => Ok(plan),
150    }
151}
152
153fn parse_statement(sql: &str) -> Result<Plan, String> {
154    let mut stmts = Parser::parse_sql(&GenericDialect {}, sql).map_err(|e| format!("SQL parse error: {e}"))?;
155    if stmts.len() != 1 {
156        return Err(format!("expected exactly one statement, got {}", stmts.len()));
157    }
158    match stmts.pop().expect("one") {
159        Statement::Insert(insert) => {
160            let sink = object_name(&insert.table.to_string());
161            let query = insert
162                .source
163                .ok_or("INSERT INTO <sink> needs a SELECT (`INSERT INTO out SELECT …`)")?;
164            let input = plan_query(&query)?;
165            Ok(Plan::Sink {
166                input: Box::new(input),
167                name: sink,
168            })
169        }
170        Statement::Query(_) => Err("a streaming statement must write to a sink: `INSERT INTO <sink> SELECT …`".into()),
171        other => Err(format!(
172            "unsupported statement `{}` — the supported shape is `INSERT INTO <sink> SELECT … FROM … [WHERE …]`",
173            statement_kind(&other)
174        )),
175    }
176}
177
178fn plan_query(query: &Query) -> Result<Plan, String> {
179    // `WITH name AS (subquery), …` — each CTE becomes a named intermediate stream (the multi-stage
180    // boundary). SQL scoping: a CTE sees only the CTEs declared before it, so bodies are bound in
181    // order against the ones already built; the outer body binds against all of them.
182    let ctes = parse_ctes(query.with.as_ref())?;
183    if query.order_by.is_some() {
184        return Err("ORDER BY over an unbounded stream has no streaming meaning — use `ROW_NUMBER() OVER (PARTITION BY …) <= n` for a top-N".into());
185    }
186    if query.limit_clause.is_some() {
187        return Err("LIMIT over an unbounded stream is not supported — use a top-N (`ROW_NUMBER() … <= n`)".into());
188    }
189    let select = match query.body.as_ref() {
190        SetExpr::Select(s) => s.as_ref(),
191        SetExpr::SetOperation { .. } => {
192            return Err("UNION / INTERSECT / EXCEPT are not supported — a stage reads one input; write one pipeline per branch into the same sink (`INSERT INTO out SELECT … FROM a` and `INSERT INTO out SELECT … FROM b`), which is UNION ALL by construction".into())
193        }
194        _ => return Err("only a SELECT body is supported".into()),
195    };
196    Ok(bind_ctes(plan_select(select)?, &ctes))
197}
198
199/// The output name a bare projected expression takes (a column keeps its own name; a compound
200/// `a.b` takes `b`). `None` for a computed expression that needs an explicit alias.
201fn column_name_of(e: &Expr) -> Option<String> {
202    match e {
203        Expr::Identifier(id) => Some(id.value.clone()),
204        Expr::CompoundIdentifier(parts) => parts.last().map(|p| p.value.clone()),
205        _ => None,
206    }
207}
208
209/// An object name (`schema.name` → `name`) reduced to the last part — datasets are referenced by a
210/// single name in the pipeline config.
211fn object_name(raw: &str) -> String {
212    raw.rsplit('.').next().unwrap_or(raw).trim_matches('"').to_string()
213}
214
215fn statement_kind(s: &Statement) -> &'static str {
216    match s {
217        Statement::Insert(_) => "INSERT",
218        Statement::Query(_) => "SELECT",
219        Statement::CreateTable(_) => "CREATE TABLE",
220        _ => "statement",
221    }
222}