use std::collections::HashSet;
use fv_streams_types::StageDef;
use serde_json::{json, Value};
use crate::plan::{AggSpec, JoinKind, Plan, Proj, Trigger, Win};
pub fn lower(plan: &Plan) -> Result<Vec<StageDef>, String> {
let (body, sink) = match plan {
Plan::Sink { input, name } => (input.as_ref(), name.clone()),
_ => return Err("a streaming statement must terminate in a sink (`INSERT INTO <sink>`)".into()),
};
let mut ctx = Ctx::default();
ctx.emit(body, sink)?;
Ok(ctx.stages)
}
#[derive(Default)]
struct Ctx {
stages: Vec<StageDef>,
emitted: HashSet<String>,
next: usize,
}
enum Base {
Source(String),
Op { inputs: Vec<String>, steps: Vec<Value> },
}
impl Ctx {
fn fresh(&mut self) -> String {
let n = format!("_fv_stage{}", self.next);
self.next += 1;
n
}
fn materialize(&mut self, plan: &Plan) -> Result<String, String> {
match plan {
Plan::Source(name) => Ok(name.clone()),
Plan::Named { name, input } => {
if self.emitted.insert(name.clone()) {
self.emit(input, name.clone())?;
}
Ok(name.clone())
}
other => {
let name = self.fresh();
self.emit(other, name.clone())?;
Ok(name)
}
}
}
fn emit(&mut self, body: &Plan, out: String) -> Result<(), String> {
let stage = match body {
Plan::Project { input, star, items } => self.lower_stateless(input, *star, items, out)?,
Plan::Aggregate {
input,
time_column,
window,
group_by,
pre,
aggs,
post,
having,
trigger,
} => self.lower_aggregate(
input,
time_column,
window,
group_by,
pre,
aggs,
post,
having.as_deref(),
*trigger,
out,
)?,
Plan::Source(_) | Plan::Named { .. } => {
let input = self.materialize(body)?;
StageDef {
steps: vec![],
inputs: vec![input],
output: out,
}
}
_ => return Err("expected a SELECT projection or a GROUP BY aggregate".into()),
};
self.stages.push(stage);
Ok(())
}
fn base_and_filter(&mut self, body: &Plan) -> Result<(Base, Option<String>), String> {
let (inner, pred) = match body {
Plan::Filter { input, predicate } => (input.as_ref(), Some(predicate.clone())),
other => (other, None),
};
let base = match inner {
Plan::Source(_) | Plan::Named { .. } => Base::Source(self.materialize(inner)?),
Plan::Join {
left,
right,
join_key,
kind,
right_rename,
} => {
let step = match kind {
JoinKind::Interval { time_column, window_ms } => json!({
"op": "streamJoin",
"joinKey": join_key,
"timeColumn": time_column,
"windowMs": window_ms,
"keyBy": true,
}),
JoinKind::Lookup => json!({ "op": "lookupJoin", "joinKey": join_key }),
};
let mut step = step;
if !right_rename.is_empty() {
step["rightRename"] = json!(right_rename);
}
let inputs = vec![self.materialize(left)?, self.materialize(right)?];
Base::Op {
inputs,
steps: vec![step],
}
}
Plan::TopN {
input,
group_by,
order_by,
tie_by,
descending,
n,
} => {
let (base, pre) = self.base_and_filter(input)?;
let Base::Source(source) = base else {
return Err("a ROW_NUMBER top-N must read a single source or CTE".into());
};
let mut steps = Vec::new();
if let Some(pred) = pre {
steps.push(json!({ "op": "filter", "expression": pred }));
}
let mut topn = json!({
"op": "topN",
"n": n,
"orderBy": order_by,
"direction": if *descending { "desc" } else { "asc" },
"keyBy": true,
"groupBy": group_by,
});
if let Some(tie) = tie_by {
topn.as_object_mut().expect("obj").insert("tieBy".into(), json!(tie));
}
steps.push(topn);
Base::Op {
inputs: vec![source],
steps,
}
}
_ => return Err("expected a source, a join, or a top-N under the projection".into()),
};
Ok((base, pred))
}
fn lower_stateless(&mut self, input: &Plan, star: bool, items: &[Proj], sink: String) -> Result<StageDef, String> {
let (base, predicate) = self.base_and_filter(input)?;
let mut steps: Vec<Value> = Vec::new();
let inputs = match base {
Base::Source(name) => vec![name],
Base::Op {
inputs,
steps: op_steps,
} => {
steps.extend(op_steps);
inputs
}
};
if let Some(pred) = predicate {
steps.push(json!({ "op": "filter", "expression": pred }));
}
for Proj { out, expr } in items {
if let Some(expr) = expr {
steps.push(json!({ "op": "applyExpression", "column": out, "expression": expr }));
}
}
if !star {
let columns: Vec<&str> = items.iter().map(|p| p.out.as_str()).collect();
steps.push(json!({ "op": "select", "columns": columns }));
}
Ok(StageDef {
steps,
inputs,
output: sink,
})
}
#[allow(clippy::too_many_arguments)]
fn lower_aggregate(
&mut self,
input: &Plan,
time_column: &str,
window: &Win,
group_by: &[String],
pre: &[Proj],
aggs: &[AggSpec],
post: &[Proj],
having: Option<&str>,
trigger: Option<Trigger>,
sink: String,
) -> Result<StageDef, String> {
let (source, predicate) = match input {
Plan::Filter { input, predicate } => (self.materialize(input)?, Some(predicate.clone())),
other => (self.materialize(other)?, None),
};
let mut steps: Vec<Value> = Vec::new();
if let Some(pred) = predicate {
steps.push(json!({ "op": "filter", "expression": pred }));
}
for Proj { out, expr } in pre {
if let Some(expr) = expr {
steps.push(json!({ "op": "applyExpression", "column": out, "expression": expr }));
}
}
let agg_json: Vec<Value> = aggs
.iter()
.map(|a| json!({ "op": a.op, "column": a.column, "alias": a.alias }))
.collect();
let mut step = match window {
Win::Tumble { size_ms, .. } => json!({
"op": "windowedAggregate",
"timeColumn": time_column,
"windowMs": size_ms,
"keyBy": true,
"groupBy": group_by,
"aggs": agg_json,
}),
Win::Hop { slide_ms, size_ms, .. } => json!({
"op": "windowedAggregate",
"timeColumn": time_column,
"windowMs": size_ms,
"slideMs": slide_ms,
"keyBy": true,
"groupBy": group_by,
"aggs": agg_json,
}),
Win::Session { gap_ms, .. } => json!({
"op": "sessionAggregate",
"timeColumn": time_column,
"gapMs": gap_ms,
"keyBy": true,
"groupBy": group_by,
"aggs": agg_json,
}),
};
let lateness = match window {
Win::Tumble { lateness_ms, .. } | Win::Hop { lateness_ms, .. } | Win::Session { lateness_ms, .. } => {
*lateness_ms
}
};
if let Some(ms) = lateness {
step.as_object_mut()
.expect("agg step is an object")
.insert("allowedLatenessMs".into(), json!(ms));
}
if let Some(trigger) = trigger {
let emit = match trigger {
Trigger::EveryRows(n) => json!({ "everyRows": n }),
Trigger::EveryMs(ms) => json!({ "everyMs": ms }),
};
step.as_object_mut()
.expect("agg step is an object")
.insert("emit".into(), emit);
}
steps.push(step);
for Proj { out, expr } in post {
if let Some(expr) = expr {
steps.push(json!({ "op": "applyExpression", "column": out, "expression": expr }));
}
}
if let Some(pred) = having {
steps.push(json!({ "op": "filter", "expression": pred }));
}
Ok(StageDef {
steps,
inputs: vec![source],
output: sink,
})
}
}