Skip to main content

fv_streams_sql/
plan.rs

1//! The typed streaming **logical plan** — our own IR, the seam between the SQL front-end and the
2//! `StageDef` lowering. Small and streaming-aware by construction (unlike DataFusion's `LogicalPlan`);
3//! a future optimiser is a pass over this, and a second dialect is another producer of it.
4
5use sqlparser::ast::{
6    BinaryOperator, DateTimeField, DuplicateTreatment, Expr, Function, FunctionArg, FunctionArgExpr, FunctionArguments,
7    GroupByExpr, JoinConstraint, JoinOperator, Query, Select, SelectItem, SetExpr, Statement, TableFactor,
8    TableWithJoins, Value,
9};
10use sqlparser::dialect::GenericDialect;
11use sqlparser::parser::Parser;
12
13use crate::expr;
14
15/// One projected output column.
16#[derive(Debug, Clone, PartialEq)]
17pub struct Proj {
18    /// The output column name.
19    pub out: String,
20    /// `Some(dialect_expr)` when computed or renamed (lowers to an `applyExpression`); `None` for a
21    /// bare pass-through column named `out` (kept by the `select`).
22    pub expr: Option<String>,
23}
24
25/// One aggregate in a `SELECT` over a window — `op` over `column` (empty for `count(*)`), out `alias`.
26#[derive(Debug, Clone, PartialEq)]
27pub struct AggSpec {
28    pub op: String,
29    pub column: String,
30    pub alias: String,
31}
32
33/// The windowing a `GROUP BY <window-fn>(…)` selects. An optional trailing `INTERVAL` on any window
34/// function is the **allowed lateness** — how long a window stays open for late events after its end
35/// (the engine's `allowedLatenessMs`); `None` leaves it at the engine default. q4/q5 need a large
36/// lateness so no out-of-order bid is dropped from a closed window.
37#[derive(Debug, Clone, PartialEq)]
38pub enum Win {
39    /// `TUMBLE(ts, INTERVAL <size> [, INTERVAL <lateness>])` — fixed windows.
40    Tumble { size_ms: i64, lateness_ms: Option<i64> },
41    /// `HOP(ts, INTERVAL <slide>, INTERVAL <size> [, INTERVAL <lateness>])` — sliding windows.
42    Hop {
43        slide_ms: i64,
44        size_ms: i64,
45        lateness_ms: Option<i64>,
46    },
47    /// `SESSION(ts, INTERVAL <gap> [, INTERVAL <lateness>])` — dynamic session windows.
48    Session { gap_ms: i64, lateness_ms: Option<i64> },
49}
50
51/// The early-firing trigger a trailing `EMIT CHANGES EVERY …` clause selects — the 5a updating
52/// aggregate. sqlparser does not know `EMIT`, so a small lexical pre-pass ([`split_emit`]) strips
53/// and parses this clause before the rest of the statement reaches sqlparser (design §10 / A).
54#[derive(Debug, Clone, Copy, PartialEq)]
55pub enum Trigger {
56    EveryRows(u64),
57    EveryMs(i64),
58}
59
60/// How a two-input `JOIN` maps to an operator. An interval predicate (`ts BETWEEN …`) in the `ON`
61/// makes it a stream×stream interval join; a plain equi-join makes the right side a **bounded
62/// table** — the 5b lookup join.
63#[derive(Debug, Clone, PartialEq)]
64pub enum JoinKind {
65    /// `JOIN … ON k=k AND ts BETWEEN … ` → `streamJoin`.
66    Interval { time_column: String, window_ms: i64 },
67    /// `JOIN <bounded> ON k=k` → the 5b `lookupJoin` (right = the bounded table).
68    Lookup,
69}
70
71/// The typed logical plan. 5c-i covers the stateless shape (source → filter → project → sink);
72/// 5c-ii adds the windowed aggregate, the updating aggregate, and the joins.
73#[derive(Debug, Clone, PartialEq)]
74pub enum Plan {
75    /// `FROM <dataset>` — a source dataset by name.
76    Source(String),
77    /// `FROM <left> JOIN <right> ON <key> [interval]` — a two-input join.
78    Join {
79        left: Box<Plan>,
80        right: Box<Plan>,
81        join_key: String,
82        kind: JoinKind,
83    },
84    /// `WHERE <predicate>` — the predicate as a value-dialect string.
85    Filter { input: Box<Plan>, predicate: String },
86    /// `SELECT …`. `star` = `SELECT *` (pass-through, no step); else the projected columns.
87    Project {
88        input: Box<Plan>,
89        star: bool,
90        items: Vec<Proj>,
91    },
92    /// `SELECT … FROM … GROUP BY <window-fn>(ts,…), <keys>` — a windowed keyed aggregate. A
93    /// `Some(trigger)` makes it the **5a updating aggregate** (early firing).
94    Aggregate {
95        input: Box<Plan>,
96        time_column: String,
97        window: Win,
98        group_by: Vec<String>,
99        /// Pre-aggregate computed columns (an aggregate's expression argument, e.g. a `CASE` fed to
100        /// `count(DISTINCT …)`) — lowered to `applyExpression` steps BEFORE the aggregate.
101        pre: Vec<Proj>,
102        aggs: Vec<AggSpec>,
103        /// Post-aggregate computed columns — a `SELECT` expression over the window/group/aggregates
104        /// (e.g. `day = substring(epochToIso(windowStart), 0, 10)`), lowered to `applyExpression`
105        /// steps AFTER the aggregate (the q15/q16 day/minute columns).
106        post: Vec<Proj>,
107        trigger: Option<Trigger>,
108    },
109    /// `SELECT … FROM (SELECT …, ROW_NUMBER() OVER (PARTITION BY g ORDER BY c [DESC]) AS rn FROM s)
110    /// WHERE rn <= n` — the streaming top-N idiom → the `topN` operator.
111    TopN {
112        input: Box<Plan>,
113        group_by: Vec<String>,
114        order_by: String,
115        /// A second `ORDER BY` column → the tie-break (`tieBy`, ascending).
116        tie_by: Option<String>,
117        descending: bool,
118        n: usize,
119    },
120    /// A CTE (`WITH <name> AS (<subquery>)`): the sub-plan `input` materialises as its own stage
121    /// under `name`, and downstream references to `name` read that stage. The multi-stage boundary —
122    /// the SQL equivalent of a named intermediate dataset in the TOML `[[transforms]]` DAG.
123    Named { name: String, input: Box<Plan> },
124    /// `INSERT INTO <dataset>` — the terminal sink dataset by name.
125    Sink { input: Box<Plan>, name: String },
126}
127
128/// Parse one streaming SQL statement into a [`Plan`]. Fails LOUDLY, streaming-aware, on anything
129/// outside the supported subset.
130pub fn parse(sql: &str) -> Result<Plan, String> {
131    // Strip a trailing `EMIT CHANGES EVERY …` (sqlparser does not know it) before parsing the rest.
132    let (sql, trigger) = split_emit(sql)?;
133    let plan = parse_statement(&sql)?;
134    match trigger {
135        Some(t) => attach_trigger(plan, t),
136        None => Ok(plan),
137    }
138}
139
140fn parse_statement(sql: &str) -> Result<Plan, String> {
141    let mut stmts = Parser::parse_sql(&GenericDialect {}, sql).map_err(|e| format!("SQL parse error: {e}"))?;
142    if stmts.len() != 1 {
143        return Err(format!("expected exactly one statement, got {}", stmts.len()));
144    }
145    match stmts.pop().expect("one") {
146        Statement::Insert(insert) => {
147            let sink = object_name(&insert.table.to_string());
148            let query = insert
149                .source
150                .ok_or("INSERT INTO <sink> needs a SELECT (`INSERT INTO out SELECT …`)")?;
151            let input = plan_query(&query)?;
152            Ok(Plan::Sink {
153                input: Box::new(input),
154                name: sink,
155            })
156        }
157        Statement::Query(_) => Err("a streaming statement must write to a sink: `INSERT INTO <sink> SELECT …`".into()),
158        other => Err(format!(
159            "unsupported statement `{}` — the supported shape is `INSERT INTO <sink> SELECT … FROM … [WHERE …]`",
160            statement_kind(&other)
161        )),
162    }
163}
164
165fn plan_query(query: &Query) -> Result<Plan, String> {
166    // `WITH name AS (subquery), …` — each CTE becomes a named intermediate stream (the multi-stage
167    // boundary). SQL scoping: a CTE sees only the CTEs declared before it, so bodies are bound in
168    // order against the ones already built; the outer body binds against all of them.
169    let ctes = parse_ctes(query.with.as_ref())?;
170    if query.order_by.is_some() {
171        return Err("ORDER BY over an unbounded stream has no streaming meaning — use `ROW_NUMBER() OVER (PARTITION BY …) <= n` for a top-N".into());
172    }
173    if query.limit_clause.is_some() {
174        return Err("LIMIT over an unbounded stream is not supported — use a top-N (`ROW_NUMBER() … <= n`)".into());
175    }
176    let select = match query.body.as_ref() {
177        SetExpr::Select(s) => s.as_ref(),
178        SetExpr::SetOperation { .. } => {
179            return Err("UNION / INTERSECT / EXCEPT are not supported — express as separate pipelines".into())
180        }
181        _ => return Err("only a SELECT body is supported".into()),
182    };
183    Ok(bind_ctes(plan_select(select)?, &ctes))
184}
185
186/// Parse a `WITH` clause into `(name, plan)` in declaration order. Each CTE body is planned and then
187/// bound against the CTEs declared before it (SQL scoping), so later stages can read earlier ones.
188/// Recursive CTEs (`WITH RECURSIVE`) are not supported — there is no streaming fixpoint.
189fn parse_ctes(with: Option<&sqlparser::ast::With>) -> Result<Vec<(String, Plan)>, String> {
190    let Some(with) = with else { return Ok(Vec::new()) };
191    if with.recursive {
192        return Err("WITH RECURSIVE is not supported — a streaming CTE has no fixpoint".into());
193    }
194    let mut bound: Vec<(String, Plan)> = Vec::new();
195    for cte in &with.cte_tables {
196        let name = object_name(&cte.alias.name.value);
197        let plan = bind_ctes(plan_query(&cte.query)?, &bound);
198        bound.push((name, plan));
199    }
200    Ok(bound)
201}
202
203/// Replace every `Source(name)` that names a CTE with a `Named { name, <cte plan> }` boundary, so
204/// lowering materialises it as its own stage. Recurses through the whole plan; a non-CTE `Source`
205/// (a base dataset) is left untouched.
206fn bind_ctes(plan: Plan, ctes: &[(String, Plan)]) -> Plan {
207    if ctes.is_empty() {
208        return plan;
209    }
210    let boxed = |p: Box<Plan>| Box::new(bind_ctes(*p, ctes));
211    match plan {
212        Plan::Source(name) => match ctes.iter().find(|(n, _)| *n == name) {
213            Some((_, cte)) => Plan::Named {
214                name,
215                input: Box::new(cte.clone()),
216            },
217            None => Plan::Source(name),
218        },
219        Plan::Named { name, input } => Plan::Named {
220            name,
221            input: boxed(input),
222        },
223        Plan::Join {
224            left,
225            right,
226            join_key,
227            kind,
228        } => Plan::Join {
229            left: boxed(left),
230            right: boxed(right),
231            join_key,
232            kind,
233        },
234        Plan::Filter { input, predicate } => Plan::Filter {
235            input: boxed(input),
236            predicate,
237        },
238        Plan::Project { input, star, items } => Plan::Project {
239            input: boxed(input),
240            star,
241            items,
242        },
243        Plan::Aggregate {
244            input,
245            time_column,
246            window,
247            group_by,
248            pre,
249            aggs,
250            post,
251            trigger,
252        } => Plan::Aggregate {
253            input: boxed(input),
254            time_column,
255            window,
256            group_by,
257            pre,
258            aggs,
259            post,
260            trigger,
261        },
262        Plan::TopN {
263            input,
264            group_by,
265            order_by,
266            tie_by,
267            descending,
268            n,
269        } => Plan::TopN {
270            input: boxed(input),
271            group_by,
272            order_by,
273            tie_by,
274            descending,
275            n,
276        },
277        Plan::Sink { input, name } => Plan::Sink {
278            input: boxed(input),
279            name,
280        },
281    }
282}
283
284fn plan_select(select: &Select) -> Result<Plan, String> {
285    // The ROW_NUMBER top-N idiom (a subquery + `WHERE rn <= n`) → the topN operator, projected by
286    // the outer SELECT (the rank filter is consumed by the operator).
287    if let Some(topn) = detect_topn(select)? {
288        return parse_projection(topn, &select.projection);
289    }
290
291    // FROM: one table, optionally with one JOIN.
292    if select.from.len() != 1 {
293        return Err(format!(
294            "expected one FROM item (comma joins are not supported — use `JOIN … ON …`), got {}",
295            select.from.len()
296        ));
297    }
298    let source = parse_from(&select.from[0])?;
299    if select.having.is_some() {
300        return Err("HAVING is not supported yet".into());
301    }
302
303    // WHERE → filter (reads the input columns, so it sits below the aggregate/projection).
304    let filtered = match &select.selection {
305        Some(pred) => Plan::Filter {
306            input: Box::new(source),
307            predicate: expr::to_dialect(pred)?,
308        },
309        None => source,
310    };
311
312    // GROUP BY <window-fn>(…), <keys> → a windowed keyed aggregate.
313    let group_exprs = match &select.group_by {
314        GroupByExpr::Expressions(exprs, _) if !exprs.is_empty() => Some(exprs.as_slice()),
315        GroupByExpr::All(_) => return Err("GROUP BY ALL is not supported".into()),
316        _ => None,
317    };
318    if let Some(gexprs) = group_exprs {
319        return plan_aggregate(filtered, gexprs, &select.projection);
320    }
321
322    parse_projection(filtered, &select.projection)
323}
324
325/// The `SELECT` projection over an already-built input → a `Project` node.
326fn parse_projection(input: Plan, projection: &[SelectItem]) -> Result<Plan, String> {
327    let mut star = false;
328    let mut items = Vec::new();
329    for item in projection {
330        match item {
331            SelectItem::Wildcard(_) => star = true,
332            SelectItem::UnnamedExpr(e) => {
333                let out = column_name_of(e).ok_or("a projected expression needs an alias (`<expr> AS name`)")?;
334                match e {
335                    Expr::Identifier(_) | Expr::CompoundIdentifier(_) => items.push(Proj { out, expr: None }),
336                    _ => items.push(Proj {
337                        out,
338                        expr: Some(expr::to_dialect(e)?),
339                    }),
340                }
341            }
342            SelectItem::ExprWithAlias { expr: e, alias } => {
343                items.push(Proj {
344                    out: alias.value.clone(),
345                    expr: Some(expr::to_dialect(e)?),
346                });
347            }
348            _ => return Err("`table.*` / multi-alias projections are not supported — list the columns".into()),
349        }
350    }
351    if star && items.iter().any(|p| p.expr.is_none()) {
352        return Err("`SELECT *, <bare column>` is redundant — `*` already keeps every column; with `*` add only computed columns (`*, <expr> AS name`)".into());
353    }
354    Ok(Plan::Project {
355        input: Box::new(input),
356        star,
357        items,
358    })
359}
360
361/// Recognise `FROM (SELECT …, ROW_NUMBER() OVER (PARTITION BY g ORDER BY c [DESC]) AS rn FROM s)
362/// WHERE rn <= n` → a `TopN` over the inner source. `Ok(None)` when it is not that shape.
363fn detect_topn(select: &Select) -> Result<Option<Plan>, String> {
364    if select.from.len() != 1 || !select.from[0].joins.is_empty() {
365        return Ok(None);
366    }
367    let TableFactor::Derived { subquery, .. } = &select.from[0].relation else {
368        return Ok(None);
369    };
370    let SetExpr::Select(inner) = subquery.body.as_ref() else {
371        return Ok(None);
372    };
373    // find ROW_NUMBER() OVER (…) AS <rn>
374    let mut rn_fn = None;
375    for item in &inner.projection {
376        if let SelectItem::ExprWithAlias {
377            expr: Expr::Function(f),
378            alias,
379        } = item
380        {
381            if f.name.to_string().eq_ignore_ascii_case("row_number") {
382                rn_fn = Some((f, alias.value.clone()));
383            }
384        }
385    }
386    let Some((f, rn)) = rn_fn else { return Ok(None) }; // no ROW_NUMBER → not a top-N
387    let Some(sqlparser::ast::WindowType::WindowSpec(ws)) = &f.over else {
388        return Err("ROW_NUMBER() needs `OVER (PARTITION BY … ORDER BY …)`".into());
389    };
390    let group_by: Vec<String> = ws.partition_by.iter().filter_map(column_name_of).collect();
391    let ob = ws
392        .order_by
393        .first()
394        .ok_or("ROW_NUMBER() OVER needs an ORDER BY column")?;
395    let order_by = column_name_of(&ob.expr).ok_or("ROW_NUMBER() ORDER BY must be a column")?;
396    let descending = ob.options.asc == Some(false); // DESC
397                                                    // a second ORDER BY column is the tie-break (ascending, like the operator's `tieBy`).
398    let tie_by = match ws.order_by.get(1) {
399        Some(t) => Some(column_name_of(&t.expr).ok_or("ROW_NUMBER() tie-break must be a column")?),
400        None => None,
401    };
402    if inner.from.len() != 1 || !inner.from[0].joins.is_empty() {
403        return Err("the ROW_NUMBER subquery must read exactly one source".into());
404    }
405    let source = table_source(&inner.from[0].relation)?;
406    // the subquery's WHERE (e.g. `type = 'bid'`) filters before the rank.
407    let input = match &inner.selection {
408        Some(pred) => Plan::Filter {
409            input: Box::new(source),
410            predicate: expr::to_dialect(pred)?,
411        },
412        None => source,
413    };
414    let n = rank_limit(select.selection.as_ref(), &rn)?;
415    Ok(Some(Plan::TopN {
416        input: Box::new(input),
417        group_by,
418        order_by,
419        tie_by,
420        descending,
421        n,
422    }))
423}
424
425/// The `n` from an outer `WHERE <rn> <= n` (or `<rn> = 1`) top-N filter.
426fn rank_limit(where_: Option<&Expr>, rn: &str) -> Result<usize, String> {
427    let e = where_.ok_or_else(|| format!("a ROW_NUMBER top-N needs an outer `WHERE {rn} <= n`"))?;
428    let is_rn = |x: &Expr| column_name_of(x).as_deref() == Some(rn);
429    match e {
430        Expr::BinaryOp {
431            left,
432            op: BinaryOperator::LtEq,
433            right,
434        } if is_rn(left) => literal_usize(right),
435        Expr::BinaryOp {
436            left,
437            op: BinaryOperator::Eq,
438            right,
439        } if is_rn(left) => match literal_usize(right)? {
440            1 => Ok(1),
441            _ => Err("a `= n` top-N filter only supports n = 1 — use `<= n`".into()),
442        },
443        _ => Err(format!("the top-N filter must be `{rn} <= n`")),
444    }
445}
446
447fn literal_usize(e: &Expr) -> Result<usize, String> {
448    match e {
449        Expr::Value(v) => match &v.value {
450            Value::Number(s, _) => s.trim().parse().map_err(|_| format!("`{s}` is not a whole number")),
451            other => Err(format!("expected a number, got `{other}`")),
452        },
453        other => Err(format!("expected a number, got `{other}`")),
454    }
455}
456
457/// `GROUP BY <window-fn>(ts,…), <keys>` + aggregate projections → a windowed keyed aggregate.
458fn plan_aggregate(input: Plan, group_exprs: &[Expr], projection: &[SelectItem]) -> Result<Plan, String> {
459    let mut window: Option<(String, Win)> = None;
460    let mut group_by = Vec::new();
461    for g in group_exprs {
462        match g {
463            Expr::Function(f) if is_window_fn(f) => {
464                if window.is_some() {
465                    return Err("only one window function may appear in GROUP BY".into());
466                }
467                window = Some(parse_window_fn(f)?);
468            }
469            Expr::Identifier(id) => group_by.push(id.value.clone()),
470            Expr::CompoundIdentifier(p) => group_by.push(p.last().expect("non-empty").value.clone()),
471            other => {
472                return Err(format!(
473                    "unsupported GROUP BY item `{other}` — a window function (`TUMBLE/HOP/SESSION`) or a plain column"
474                ))
475            }
476        }
477    }
478    let (time_column, window) = window.ok_or(
479        "a streaming GROUP BY needs a window: `GROUP BY TUMBLE/HOP/SESSION(ts, INTERVAL …)` (an unbounded aggregate needs `… EMIT CHANGES` — the 5a updating aggregate)",
480    )?;
481    let (pre, aggs, post) = parse_agg_select(projection)?;
482    if aggs.is_empty() {
483        return Err("a windowed GROUP BY needs at least one aggregate in SELECT (count/sum/avg/min/max)".into());
484    }
485    Ok(Plan::Aggregate {
486        input: Box::new(input),
487        time_column,
488        window,
489        group_by,
490        pre,
491        aggs,
492        post,
493        trigger: None,
494    })
495}
496
497/// Strip a trailing `EMIT CHANGES EVERY <n> ROWS|MS` clause (case-insensitive, outside string
498/// literals). Returns the SQL without it and the parsed trigger, if any.
499fn split_emit(sql: &str) -> Result<(String, Option<Trigger>), String> {
500    let lower = sql.to_ascii_lowercase();
501    let bytes = lower.as_bytes();
502    let is_word = |b: u8| b.is_ascii_alphanumeric() || b == b'_';
503    let mut in_str = false;
504    let mut at = None;
505    let mut i = 0usize;
506    while i < bytes.len() {
507        if bytes[i] == b'\'' {
508            in_str = !in_str;
509            i += 1;
510            continue;
511        }
512        if !in_str
513            && lower[i..].starts_with("emit")
514            && (i == 0 || !is_word(bytes[i - 1]))
515            && (i + 4 >= bytes.len() || !is_word(bytes[i + 4]))
516        {
517            at = Some(i); // the last unquoted `emit` wins (it is a trailing clause)
518        }
519        i += 1;
520    }
521    let Some(at) = at else {
522        return Ok((sql.to_string(), None));
523    };
524    let head = sql[..at].trim_end().to_string();
525    let trigger = parse_emit(&sql[at..])?;
526    Ok((head, Some(trigger)))
527}
528
529fn parse_emit(clause: &str) -> Result<Trigger, String> {
530    let toks: Vec<String> = clause.split_whitespace().map(|s| s.to_ascii_lowercase()).collect();
531    let shape = "`EMIT CHANGES EVERY <n> ROWS` or `EMIT CHANGES EVERY <n> MS`";
532    if toks.len() < 2 || toks[0] != "emit" || toks[1] != "changes" {
533        return Err(format!("unrecognised EMIT clause — use {shape}"));
534    }
535    if toks.len() == 2 {
536        return Err(format!("`EMIT CHANGES` needs a cadence: {shape}"));
537    }
538    if toks.len() != 5 || toks[2] != "every" {
539        return Err(format!("EMIT clause: expected {shape}"));
540    }
541    let n: u64 = toks[3]
542        .parse()
543        .map_err(|_| format!("EMIT cadence `{}` is not a whole number", toks[3]))?;
544    match toks[4].as_str() {
545        "rows" | "row" => Ok(Trigger::EveryRows(n)),
546        "ms" | "milliseconds" | "millisecond" => Ok(Trigger::EveryMs(n as i64)),
547        other => Err(format!("EMIT cadence unit `{other}` — use ROWS or MS")),
548    }
549}
550
551/// Attach a trigger to the plan's windowed aggregate (the 5a updating aggregate). The trigger is
552/// valid only on a windowed (`TUMBLE`/`HOP`) aggregate — the operator that early-fires.
553fn attach_trigger(plan: Plan, trigger: Trigger) -> Result<Plan, String> {
554    match plan {
555        Plan::Sink { input, name } => Ok(Plan::Sink {
556            input: Box::new(attach_trigger(*input, trigger)?),
557            name,
558        }),
559        Plan::Aggregate {
560            input,
561            time_column,
562            window,
563            group_by,
564            pre,
565            aggs,
566            post,
567            trigger: _,
568        } => {
569            if matches!(window, Win::Session { .. }) {
570                return Err("EMIT CHANGES is supported on windowed aggregates (TUMBLE / HOP), not sessions".into());
571            }
572            Ok(Plan::Aggregate {
573                input,
574                time_column,
575                window,
576                group_by,
577                pre,
578                aggs,
579                post,
580                trigger: Some(trigger),
581            })
582        }
583        _ => Err("EMIT CHANGES needs a windowed GROUP BY aggregate (`GROUP BY <keys>, TUMBLE(ts, INTERVAL …)`)".into()),
584    }
585}
586
587fn is_window_fn(f: &Function) -> bool {
588    matches!(f.name.to_string().to_uppercase().as_str(), "TUMBLE" | "HOP" | "SESSION")
589}
590
591/// `TUMBLE(ts, INTERVAL <size>)` / `HOP(ts, INTERVAL <slide>, INTERVAL <size>)` /
592/// `SESSION(ts, INTERVAL <gap>)` → (event-time column, window).
593fn parse_window_fn(f: &Function) -> Result<(String, Win), String> {
594    let name = f.name.to_string().to_uppercase();
595    let args = fn_args(f)?;
596    let time_col = |e: &Expr| -> Result<String, String> {
597        match e {
598            Expr::Identifier(id) => Ok(id.value.clone()),
599            Expr::CompoundIdentifier(p) => Ok(p.last().expect("non-empty").value.clone()),
600            _ => Err(format!("{name}(…)'s first argument must be the event-time column")),
601        }
602    };
603    // an optional trailing INTERVAL beyond the base arity is the allowed lateness.
604    let lateness = |base: usize| -> Result<Option<i64>, String> {
605        match args.len() {
606            n if n == base => Ok(None),
607            n if n == base + 1 => Ok(Some(interval_ms(args[base])?)),
608            _ => Err(String::new()), // arity handled by the arm guards below
609        }
610    };
611    match name.as_str() {
612        "TUMBLE" if args.len() == 2 || args.len() == 3 => Ok((
613            time_col(args[0])?,
614            Win::Tumble {
615                size_ms: interval_ms(args[1])?,
616                lateness_ms: lateness(2)?,
617            },
618        )),
619        "HOP" if args.len() == 3 || args.len() == 4 => Ok((
620            time_col(args[0])?,
621            Win::Hop {
622                slide_ms: interval_ms(args[1])?,
623                size_ms: interval_ms(args[2])?,
624                lateness_ms: lateness(3)?,
625            },
626        )),
627        "SESSION" if args.len() == 2 || args.len() == 3 => Ok((
628            time_col(args[0])?,
629            Win::Session {
630                gap_ms: interval_ms(args[1])?,
631                lateness_ms: lateness(2)?,
632            },
633        )),
634        "TUMBLE" | "SESSION" => Err(format!(
635            "{name}(ts, INTERVAL <size> [, INTERVAL <lateness>]) takes 2 or 3 arguments"
636        )),
637        "HOP" => {
638            Err("HOP(ts, INTERVAL <slide>, INTERVAL <size> [, INTERVAL <lateness>]) takes 3 or 4 arguments".into())
639        }
640        other => Err(format!("unsupported window function `{other}`")),
641    }
642}
643
644fn is_agg_name(name: &str) -> bool {
645    matches!(name, "count" | "sum" | "avg" | "min" | "max")
646}
647
648/// Classify a windowed `SELECT`'s projections: aggregate calls (→ `aggs`, with any expression
649/// arguments hoisted to `pre` computed columns), bare group columns (skipped — GROUP BY captured
650/// them), and other computed columns over the window/group/aggregates (→ `post` applyExpression
651/// steps after the aggregate, e.g. `day = substring(epochToIso(windowStart), 0, 10)`).
652#[allow(clippy::type_complexity)]
653fn parse_agg_select(projection: &[SelectItem]) -> Result<(Vec<Proj>, Vec<AggSpec>, Vec<Proj>), String> {
654    let mut pre: Vec<Proj> = Vec::new();
655    let mut aggs = Vec::new();
656    let mut post: Vec<Proj> = Vec::new();
657    for item in projection {
658        let (e, alias) = match item {
659            SelectItem::ExprWithAlias { expr, alias } => (expr, Some(alias.value.clone())),
660            SelectItem::UnnamedExpr(e) => (e, None),
661            SelectItem::Wildcard(_) => return Err("`SELECT *` is not allowed with GROUP BY".into()),
662            _ => return Err("unsupported projection in a GROUP BY query".into()),
663        };
664        match e {
665            Expr::Function(f) if is_agg_name(&f.name.to_string().to_lowercase()) => {
666                aggs.push(agg_of(f, alias, &mut pre)?);
667            }
668            // a bare group column — already in the aggregate's output.
669            Expr::Identifier(_) | Expr::CompoundIdentifier(_) => {}
670            // a computed column over the window/group/aggregates → a post-aggregate applyExpression.
671            _ => {
672                let out =
673                    alias.ok_or("a computed SELECT column in a GROUP BY query needs an alias (`<expr> AS name`)")?;
674                post.push(Proj {
675                    out,
676                    expr: Some(expr::to_dialect(e)?),
677                });
678            }
679        }
680    }
681    Ok((pre, aggs, post))
682}
683
684fn agg_of(f: &Function, alias: Option<String>, pre: &mut Vec<Proj>) -> Result<AggSpec, String> {
685    let name = f.name.to_string().to_lowercase();
686    let alias = alias.ok_or_else(|| format!("aggregate `{name}(…)` needs an alias (`{name}(…) AS name`)"))?;
687    let distinct = matches!(
688        &f.args,
689        FunctionArguments::List(l) if l.duplicate_treatment == Some(DuplicateTreatment::Distinct)
690    );
691    let op = match (name.as_str(), distinct) {
692        ("count", false) => "count",
693        ("count", true) => "countDistinct",
694        ("sum", false) => "sum",
695        ("avg", false) => "avg",
696        ("min", false) => "min",
697        ("max", false) => "max",
698        (other, true) => return Err(format!("DISTINCT is only supported on `count()`, not `{other}()`")),
699        (other, false) => {
700            return Err(format!(
701                "unsupported aggregate `{other}` — count/sum/avg/min/max/count(DISTINCT …)"
702            ))
703        }
704    };
705    let args = fn_args(f)?;
706    // count(*) has no column; a bare column feeds the aggregate directly; an EXPRESSION argument
707    // (CASE, arithmetic) becomes a synthetic pre-aggregate column (the q15/q16 null-mask shape).
708    let column = match args.first() {
709        None => String::new(),
710        Some(Expr::Identifier(id)) => id.value.clone(),
711        Some(Expr::CompoundIdentifier(p)) => p.last().expect("non-empty").value.clone(),
712        Some(other) => {
713            let synth = format!("_fv_agg{}", pre.len());
714            pre.push(Proj {
715                out: synth.clone(),
716                expr: Some(expr::to_dialect(other)?),
717            });
718            synth
719        }
720    };
721    Ok(AggSpec {
722        op: op.to_string(),
723        column,
724        alias,
725    })
726}
727
728/// The plain `Expr` arguments of a function call (a `*` wildcard contributes none — `count(*)`).
729fn fn_args(f: &Function) -> Result<Vec<&Expr>, String> {
730    let list = match &f.args {
731        FunctionArguments::List(l) => &l.args,
732        FunctionArguments::None => return Ok(Vec::new()),
733        FunctionArguments::Subquery(_) => return Err("subquery arguments are not supported".into()),
734    };
735    let mut out = Vec::new();
736    for a in list {
737        match a {
738            FunctionArg::Unnamed(FunctionArgExpr::Expr(e)) => out.push(e),
739            FunctionArg::Unnamed(FunctionArgExpr::Wildcard) => {}
740            _ => return Err("unsupported function argument (named / qualified wildcard)".into()),
741        }
742    }
743    Ok(out)
744}
745
746/// An `INTERVAL '<n>' <unit>` as milliseconds.
747fn interval_ms(e: &Expr) -> Result<i64, String> {
748    let Expr::Interval(iv) = e else {
749        return Err(format!("expected an INTERVAL (e.g. `INTERVAL '10' SECOND`), got `{e}`"));
750    };
751    let n: i64 = match iv.value.as_ref() {
752        Expr::Value(v) => match &v.value {
753            Value::SingleQuotedString(s) | Value::Number(s, _) => s
754                .trim()
755                .parse()
756                .map_err(|_| format!("interval value `{s}` is not an integer"))?,
757            other => return Err(format!("unsupported interval value `{other}`")),
758        },
759        other => return Err(format!("unsupported interval value `{other}`")),
760    };
761    let unit = match &iv.leading_field {
762        Some(DateTimeField::Millisecond) => 1,
763        Some(DateTimeField::Second) => 1_000,
764        Some(DateTimeField::Minute) => 60_000,
765        Some(DateTimeField::Hour) => 3_600_000,
766        Some(DateTimeField::Day) => 86_400_000,
767        other => {
768            return Err(format!(
769                "unsupported interval unit `{other:?}` — use MILLISECOND / SECOND / MINUTE / HOUR / DAY"
770            ))
771        }
772    };
773    Ok(n * unit)
774}
775
776/// `FROM <left> [JOIN <right> ON …]` → a `Source` or a `Join`.
777fn parse_from(twj: &TableWithJoins) -> Result<Plan, String> {
778    let left = table_source(&twj.relation)?;
779    match twj.joins.len() {
780        0 => Ok(left),
781        1 => {
782            let j = &twj.joins[0];
783            let right = table_source(&j.relation)?;
784            let (join_key, kind) = parse_join_operator(&j.join_operator)?;
785            Ok(Plan::Join {
786                left: Box::new(left),
787                right: Box::new(right),
788                join_key,
789                kind,
790            })
791        }
792        _ => Err("only a single JOIN is supported".into()),
793    }
794}
795
796fn table_source(tf: &TableFactor) -> Result<Plan, String> {
797    match tf {
798        TableFactor::Table { name, .. } => Ok(Plan::Source(object_name(&name.to_string()))),
799        other => Err(format!("unsupported FROM item `{other}` — a plain dataset name")),
800    }
801}
802
803/// `JOIN … ON a.k = b.k [AND ts BETWEEN … ]` → (join key, kind). An interval predicate makes it a
804/// stream×stream interval join; without one, the right side is a bounded table (the 5b lookup join).
805fn parse_join_operator(op: &JoinOperator) -> Result<(String, JoinKind), String> {
806    let on = match op {
807        JoinOperator::Inner(JoinConstraint::On(e)) | JoinOperator::Join(JoinConstraint::On(e)) => e,
808        _ => return Err("only `INNER JOIN … ON <a.k = b.k> [AND <ts> BETWEEN …]` is supported".into()),
809    };
810    let mut join_key: Option<String> = None;
811    let mut interval: Option<(String, i64)> = None;
812    for c in split_and(on) {
813        match c {
814            Expr::BinaryOp {
815                left,
816                op: BinaryOperator::Eq,
817                right,
818            } => {
819                join_key = Some(same_column(left, right)?);
820            }
821            Expr::Between {
822                expr,
823                negated: false,
824                low,
825                high,
826            } => {
827                interval = Some(between_interval(expr, low, high)?);
828            }
829            other => {
830                return Err(format!(
831                    "unsupported JOIN ON predicate `{other}` — an equi-key `a.k = b.k` and optionally `ts BETWEEN …`"
832                ))
833            }
834        }
835    }
836    let join_key = join_key.ok_or("JOIN ON needs an equi-key (`a.k = b.k`, same column name on both sides)")?;
837    let kind = match interval {
838        Some((time_column, window_ms)) => JoinKind::Interval { time_column, window_ms },
839        None => JoinKind::Lookup,
840    };
841    Ok((join_key, kind))
842}
843
844/// Flatten an `AND` chain into its conjuncts.
845fn split_and(e: &Expr) -> Vec<&Expr> {
846    match e {
847        Expr::BinaryOp {
848            left,
849            op: BinaryOperator::And,
850            right,
851        } => {
852            let mut v = split_and(left);
853            v.extend(split_and(right));
854            v
855        }
856        Expr::Nested(inner) => split_and(inner),
857        other => vec![other],
858    }
859}
860
861/// `a.k = b.k` → `k`. The two sides must name the SAME column (the join key present on both inputs).
862fn same_column(l: &Expr, r: &Expr) -> Result<String, String> {
863    let lc = column_name_of(l);
864    let rc = column_name_of(r);
865    match (lc, rc) {
866        (Some(a), Some(b)) if a == b => Ok(a),
867        (Some(a), Some(b)) => Err(format!(
868            "the JOIN key columns must share a name (`{a}` vs `{b}`) — rename one upstream"
869        )),
870        _ => Err("JOIN key must be `a.col = b.col` (plain columns)".into()),
871    }
872}
873
874/// `ts BETWEEN b.ts - INTERVAL … AND b.ts + INTERVAL …` → (event-time column, window ms).
875fn between_interval(expr: &Expr, low: &Expr, high: &Expr) -> Result<(String, i64), String> {
876    let time_column = column_name_of(expr)
877        .ok_or("the interval join's BETWEEN must be over an event-time column (`a.ts BETWEEN …`)")?;
878    // the window is the magnitude of the interval on either bound (symmetric ±window).
879    let ms = interval_in(high).or_else(|_| interval_in(low))?;
880    Ok((time_column, ms))
881}
882
883/// Find an `INTERVAL` inside a bound like `b.ts + INTERVAL '5' SECOND` and return it in ms.
884fn interval_in(e: &Expr) -> Result<i64, String> {
885    match e {
886        Expr::Interval(_) => interval_ms(e),
887        Expr::BinaryOp { left, right, .. } => interval_in(right).or_else(|_| interval_in(left)),
888        Expr::Nested(inner) => interval_in(inner),
889        _ => Err("the interval join's BETWEEN needs an `INTERVAL` bound (`b.ts + INTERVAL '5' SECOND`)".into()),
890    }
891}
892
893/// The output name a bare projected expression takes (a column keeps its own name; a compound
894/// `a.b` takes `b`). `None` for a computed expression that needs an explicit alias.
895fn column_name_of(e: &Expr) -> Option<String> {
896    match e {
897        Expr::Identifier(id) => Some(id.value.clone()),
898        Expr::CompoundIdentifier(parts) => parts.last().map(|p| p.value.clone()),
899        _ => None,
900    }
901}
902
903/// An object name (`schema.name` → `name`) reduced to the last part — datasets are referenced by a
904/// single name in the pipeline config.
905fn object_name(raw: &str) -> String {
906    raw.rsplit('.').next().unwrap_or(raw).trim_matches('"').to_string()
907}
908
909fn statement_kind(s: &Statement) -> &'static str {
910    match s {
911        Statement::Insert(_) => "INSERT",
912        Statement::Query(_) => "SELECT",
913        Statement::CreateTable(_) => "CREATE TABLE",
914        _ => "statement",
915    }
916}