use fv_streams_types::Topology;
mod expr;
mod lower;
mod plan;
pub use plan::Plan;
pub fn compile(sql: &str) -> Result<Topology, String> {
let plan = plan::parse(sql)?;
let plan = optimize(plan);
let stages = lower::lower(&plan)?;
Ok(Topology { stages })
}
fn optimize(plan: Plan) -> Plan {
plan
}
#[cfg(test)]
pub(crate) fn compile_stages(sql: &str) -> Result<Vec<fv_streams_types::StageDef>, String> {
lower::lower(&optimize(plan::parse(sql)?))
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn a_filter_projection_lowers_to_the_same_steps_the_toml_would() {
let stages =
compile_stages("INSERT INTO out SELECT auction, price * 0.908 AS eur FROM bids WHERE type = 'bid'")
.unwrap();
assert_eq!(stages.len(), 1);
let s = &stages[0];
assert_eq!(s.inputs, vec!["bids".to_string()]);
assert_eq!(s.output, "out");
assert_eq!(
s.steps,
vec![
json!({"op": "filter", "expression": "(type == 'bid')"}),
json!({"op": "applyExpression", "column": "eur", "expression": "(price * 0.908)"}),
json!({"op": "select", "columns": ["auction", "eur"]}),
]
);
}
#[test]
fn select_star_is_a_pass_through_no_steps() {
let s = &compile_stages("INSERT INTO out SELECT * FROM bids").unwrap()[0];
assert!(s.steps.is_empty());
assert_eq!(s.inputs, vec!["bids".to_string()]);
assert_eq!(s.output, "out");
}
#[test]
fn star_plus_a_computed_column_keeps_all_columns_the_q1_shape() {
let s = &compile_stages("INSERT INTO out SELECT *, price * 0.908 AS priceEur FROM events WHERE type = 'bid'")
.unwrap()[0];
assert_eq!(
s.steps,
vec![
json!({"op": "filter", "expression": "(type == 'bid')"}),
json!({"op": "applyExpression", "column": "priceEur", "expression": "(price * 0.908)"}),
]
);
assert!(compile("INSERT INTO o SELECT *, auction FROM b").is_err());
}
#[test]
fn boolean_comparison_and_case_render_in_the_value_dialect() {
let s = &compile_stages(
"INSERT INTO out SELECT CASE WHEN price < 100 THEN 1 ELSE 0 END AS band FROM b WHERE a > 1 AND c = 2",
)
.unwrap()[0];
assert_eq!(
s.steps[0],
json!({"op": "filter", "expression": "((a > 1) && (c == 2))"})
);
assert_eq!(
s.steps[1],
json!({"op": "applyExpression", "column": "band", "expression": "((price < 100) ? 1 : 0)"})
);
}
#[test]
fn a_bare_rename_and_a_function_call_lower_correctly() {
let s = &compile_stages("INSERT INTO out SELECT bidder AS who, round(price * 0.9) AS p FROM b").unwrap()[0];
assert_eq!(
s.steps[0],
json!({"op": "applyExpression", "column": "who", "expression": "bidder"})
);
assert_eq!(
s.steps[1],
json!({"op": "applyExpression", "column": "p", "expression": "round((price * 0.9))"})
);
assert_eq!(s.steps[2], json!({"op": "select", "columns": ["who", "p"]}));
}
#[test]
fn the_unsupported_shapes_are_rejected_with_streaming_aware_errors() {
assert!(compile("SELECT * FROM b").unwrap_err().contains("INSERT INTO"));
assert!(compile("INSERT INTO o SELECT count(*) AS n FROM b GROUP BY x")
.unwrap_err()
.contains("window"));
assert!(compile("INSERT INTO o SELECT * FROM a, b").is_err());
assert!(compile("INSERT INTO o SELECT * FROM a UNION SELECT * FROM b")
.unwrap_err()
.contains("UNION"));
assert!(compile("INSERT INTO o SELECT * FROM b ORDER BY price")
.unwrap_err()
.contains("ROW_NUMBER"));
}
#[test]
fn a_tumbling_windowed_aggregate_lowers_to_the_windowedaggregate_step() {
let s = &compile_stages(
"INSERT INTO out SELECT auction, count(*) AS n, max(price) AS top \
FROM bids WHERE type = 'bid' GROUP BY auction, TUMBLE(dateTime, INTERVAL '10' SECOND)",
)
.unwrap()[0];
assert_eq!(s.inputs, vec!["bids".to_string()]);
assert_eq!(s.output, "out");
assert_eq!(s.steps[0], json!({"op": "filter", "expression": "(type == 'bid')"}));
assert_eq!(
s.steps[1],
json!({
"op": "windowedAggregate",
"timeColumn": "dateTime",
"windowMs": 10000,
"keyBy": true,
"groupBy": ["auction"],
"aggs": [
{"op": "count", "column": "", "alias": "n"},
{"op": "max", "column": "price", "alias": "top"},
],
})
);
}
#[test]
fn hop_is_a_sliding_window_and_session_is_a_session_aggregate() {
let hop = &compile_stages(
"INSERT INTO o SELECT auction, count(*) AS n FROM b GROUP BY auction, HOP(ts, INTERVAL '2' SECOND, INTERVAL '10' SECOND)",
)
.unwrap()[0];
assert_eq!(
hop.steps[0],
json!({"op": "windowedAggregate", "timeColumn": "ts", "windowMs": 10000, "slideMs": 2000,
"keyBy": true, "groupBy": ["auction"], "aggs": [{"op": "count", "column": "", "alias": "n"}]})
);
let sess = &compile_stages(
"INSERT INTO o SELECT bidder, sum(price) AS s FROM b GROUP BY bidder, SESSION(ts, INTERVAL '5' SECOND)",
)
.unwrap()[0];
assert_eq!(
sess.steps[0],
json!({"op": "sessionAggregate", "timeColumn": "ts", "gapMs": 5000,
"keyBy": true, "groupBy": ["bidder"], "aggs": [{"op": "sum", "column": "price", "alias": "s"}]})
);
}
#[test]
fn emit_changes_makes_it_the_updating_aggregate_q17_shape() {
let s = &compile_stages(
"INSERT INTO out SELECT auction, count(*) AS n FROM bids WHERE type = 'bid' \
GROUP BY auction, TUMBLE(dateTime, INTERVAL '1' DAY) EMIT CHANGES EVERY 5000 ROWS",
)
.unwrap()[0];
assert_eq!(s.steps[0], json!({"op": "filter", "expression": "(type == 'bid')"}));
assert_eq!(
s.steps[1],
json!({
"op": "windowedAggregate",
"timeColumn": "dateTime",
"windowMs": 86_400_000,
"keyBy": true,
"groupBy": ["auction"],
"aggs": [{"op": "count", "column": "", "alias": "n"}],
"emit": {"everyRows": 5000},
}),
"EMIT CHANGES EVERY n ROWS → the emit block the engine's parse_trigger reads"
);
let ms = &compile_stages(
"INSERT INTO o SELECT k, count(*) AS n FROM b GROUP BY k, TUMBLE(ts, INTERVAL '10' SECOND) EMIT CHANGES EVERY 500 MS",
)
.unwrap()[0];
assert_eq!(ms.steps[0]["emit"], json!({"everyMs": 500}));
}
#[test]
fn emit_is_rejected_where_it_has_no_meaning_and_needs_a_cadence() {
assert!(compile("INSERT INTO o SELECT * FROM b EMIT CHANGES EVERY 1 ROWS")
.unwrap_err()
.contains("windowed"));
assert!(compile(
"INSERT INTO o SELECT k, count(*) AS n FROM b GROUP BY k, TUMBLE(ts, INTERVAL '1' SECOND) EMIT CHANGES"
)
.unwrap_err()
.to_lowercase()
.contains("cadence"));
assert!(compile("INSERT INTO o SELECT k, count(*) AS n FROM b GROUP BY k, SESSION(ts, INTERVAL '5' SECOND) EMIT CHANGES EVERY 1 ROWS")
.unwrap_err()
.to_lowercase()
.contains("session"));
}
#[test]
fn the_word_emit_inside_a_string_literal_is_not_a_trigger() {
let s = &compile_stages("INSERT INTO o SELECT channel FROM b WHERE channel = 'emit changes'").unwrap()[0];
assert_eq!(
s.steps[0],
json!({"op": "filter", "expression": "(channel == 'emit changes')"})
);
assert_eq!(
s.steps.last().unwrap(),
&json!({"op": "select", "columns": ["channel"]})
);
}
#[test]
fn count_distinct_and_a_case_argument_become_a_pre_column_the_q15_shape() {
let s = &compile_stages(
"INSERT INTO o SELECT auction, count(*) AS total, count(DISTINCT bidder) AS bidders, \
count(DISTINCT CASE WHEN price < 10000 THEN bidder END) AS r1bidders \
FROM b GROUP BY auction, TUMBLE(ts, INTERVAL '1' DAY)",
)
.unwrap()[0];
assert_eq!(
s.steps[0],
json!({"op": "applyExpression", "column": "_fv_agg0", "expression": "((price < 10000) ? bidder : null)"})
);
assert_eq!(
s.steps[1],
json!({
"op": "windowedAggregate", "timeColumn": "ts", "windowMs": 86_400_000, "keyBy": true,
"groupBy": ["auction"],
"aggs": [
{"op": "count", "column": "", "alias": "total"},
{"op": "countDistinct", "column": "bidder", "alias": "bidders"},
{"op": "countDistinct", "column": "_fv_agg0", "alias": "r1bidders"},
],
})
);
assert!(compile(
"INSERT INTO o SELECT sum(DISTINCT x) AS s FROM b GROUP BY k, TUMBLE(ts, INTERVAL '1' SECOND)"
)
.is_err());
}
#[test]
fn a_post_aggregate_computed_column_lowers_after_the_aggregate_the_q15_day_shape() {
let s = &compile_stages(
"INSERT INTO out SELECT count(*) AS total, count(DISTINCT CASE WHEN price < 10000 THEN bidder END) AS r1, \
substring(epochToIso(windowStart), 0, 10) AS day \
FROM events WHERE type = 'bid' GROUP BY TUMBLE(dateTime, INTERVAL '1' DAY)",
)
.unwrap()[0];
assert_eq!(s.steps[0], json!({"op": "filter", "expression": "(type == 'bid')"}));
assert_eq!(
s.steps[1],
json!({"op": "applyExpression", "column": "_fv_agg0", "expression": "((price < 10000) ? bidder : null)"})
);
assert_eq!(s.steps[2]["op"], "windowedAggregate");
assert_eq!(
s.steps[2]["aggs"][1],
json!({"op": "countDistinct", "column": "_fv_agg0", "alias": "r1"})
);
assert_eq!(
s.steps[3],
json!({"op": "applyExpression", "column": "day", "expression": "substring(epochToIso(windowStart), 0, 10)"})
);
}
#[test]
fn a_group_by_without_a_window_is_rejected_with_a_pointer_to_windows_or_emit() {
let e = compile("INSERT INTO o SELECT auction, count(*) AS n FROM b GROUP BY auction").unwrap_err();
assert!(e.contains("window") || e.contains("EMIT"), "got: {e}");
assert!(compile("INSERT INTO o SELECT count(*) AS n FROM b GROUP BY TUMBLE(ts, INTERVAL '10' YEAR)").is_err());
}
#[test]
fn a_plain_equi_join_lowers_to_the_5b_lookup_join() {
let s = &compile_stages(
"INSERT INTO out SELECT bids.auction, side.value FROM bids JOIN side ON bids.key = side.key",
)
.unwrap()[0];
assert_eq!(s.inputs, vec!["bids".to_string(), "side".to_string()]);
assert_eq!(s.steps[0], json!({"op": "lookupJoin", "joinKey": "key"}));
assert_eq!(
s.steps.last().unwrap(),
&json!({"op": "select", "columns": ["auction", "value"]})
);
}
#[test]
fn a_join_with_a_time_interval_lowers_to_the_stream_join() {
let s = &compile_stages(
"INSERT INTO out SELECT a.x, b.y FROM a JOIN b \
ON a.k = b.k AND a.ts BETWEEN b.ts - INTERVAL '5' SECOND AND b.ts + INTERVAL '5' SECOND",
)
.unwrap()[0];
assert_eq!(s.inputs, vec!["a".to_string(), "b".to_string()]);
assert_eq!(
s.steps[0],
json!({"op": "streamJoin", "joinKey": "k", "timeColumn": "ts", "windowMs": 5000, "keyBy": true})
);
}
#[test]
fn row_number_top_n_lowers_to_the_topn_operator() {
let s = &compile_stages(
"INSERT INTO out SELECT auction, price FROM ( \
SELECT auction, price, ROW_NUMBER() OVER (PARTITION BY auction ORDER BY price DESC) AS rn FROM bids \
) WHERE rn <= 1",
)
.unwrap()[0];
assert_eq!(s.inputs, vec!["bids".to_string()]);
assert_eq!(
s.steps[0],
json!({"op": "topN", "n": 1, "orderBy": "price", "direction": "desc", "keyBy": true, "groupBy": ["auction"]})
);
assert_eq!(
s.steps.last().unwrap(),
&json!({"op": "select", "columns": ["auction", "price"]})
);
let asc = &compile_stages(
"INSERT INTO o SELECT k FROM (SELECT k, v, ROW_NUMBER() OVER (PARTITION BY k ORDER BY v) AS rn FROM t) WHERE rn <= 10",
)
.unwrap()[0];
assert_eq!(asc.steps[0]["direction"], "asc");
assert_eq!(asc.steps[0]["n"], 10);
}
#[test]
fn a_join_key_with_mismatched_column_names_is_a_loud_error() {
assert!(compile("INSERT INTO o SELECT * FROM a JOIN b ON a.x = b.y")
.unwrap_err()
.contains("share a name"));
}
#[test]
fn compile_wraps_the_stages_in_a_topology() {
let topo = compile("INSERT INTO out SELECT * FROM bids").unwrap();
assert_eq!(topo.stages.len(), 1);
assert_eq!(topo.stages[0].output, "out");
}
#[test]
fn a_cte_computing_a_key_then_a_lookup_join_is_two_stages_the_q13_shape() {
let stages = compile_stages(
"INSERT INTO out \
WITH bids AS (SELECT *, auction % 30011 AS key FROM events WHERE type = 'bid') \
SELECT * FROM bids JOIN side ON bids.key = side.key",
)
.unwrap();
assert_eq!(stages.len(), 2, "the CTE is its own stage, then the join");
assert_eq!(stages[0].output, "bids");
assert_eq!(stages[0].inputs, vec!["events".to_string()]);
assert_eq!(
stages[0].steps,
vec![
json!({"op": "filter", "expression": "(type == 'bid')"}),
json!({"op": "applyExpression", "column": "key", "expression": "(auction % 30011)"}),
]
);
assert_eq!(stages[1].output, "out");
assert_eq!(stages[1].inputs, vec!["bids".to_string(), "side".to_string()]);
assert_eq!(stages[1].steps, vec![json!({"op": "lookupJoin", "joinKey": "key"})]);
}
#[test]
fn a_windowed_count_cte_then_a_per_window_top_n_is_two_stages_the_q5_shape() {
let stages = compile_stages(
"INSERT INTO out \
WITH counts AS ( \
SELECT auction, count(*) AS n FROM events WHERE type = 'bid' \
GROUP BY auction, HOP(dateTime, INTERVAL '2' SECOND, INTERVAL '10' SECOND) \
) \
SELECT auction FROM ( \
SELECT auction, n, ROW_NUMBER() OVER (PARTITION BY windowStart ORDER BY n DESC, auction) AS rn \
FROM counts \
) WHERE rn <= 1",
)
.unwrap();
assert_eq!(stages.len(), 2);
assert_eq!(stages[0].output, "counts");
assert_eq!(stages[0].inputs, vec!["events".to_string()]);
assert_eq!(
stages[0].steps[0],
json!({"op": "filter", "expression": "(type == 'bid')"})
);
assert_eq!(
stages[0].steps[1],
json!({"op": "windowedAggregate", "timeColumn": "dateTime", "windowMs": 10000, "slideMs": 2000,
"keyBy": true, "groupBy": ["auction"], "aggs": [{"op": "count", "column": "", "alias": "n"}]})
);
assert_eq!(stages[1].output, "out");
assert_eq!(stages[1].inputs, vec!["counts".to_string()]);
assert_eq!(
stages[1].steps[0],
json!({"op": "topN", "n": 1, "orderBy": "n", "direction": "desc", "keyBy": true,
"groupBy": ["windowStart"], "tieBy": "auction"})
);
assert_eq!(stages[1].steps[1], json!({"op": "select", "columns": ["auction"]}));
}
#[test]
fn a_cte_feeding_an_aggregate_nests_two_aggregates_the_q4_shape() {
let stages = compile_stages(
"INSERT INTO out \
WITH perAuction AS ( \
SELECT category, max(price) AS finalPrice FROM catBids \
GROUP BY category, TUMBLE(dateTime, INTERVAL '10' SECOND) \
) \
SELECT category, avg(finalPrice) AS avgFinal FROM perAuction \
GROUP BY category, TUMBLE(windowStart, INTERVAL '10' SECOND)",
)
.unwrap();
assert_eq!(stages.len(), 2);
assert_eq!(stages[0].output, "perAuction");
assert_eq!(stages[0].inputs, vec!["catBids".to_string()]);
assert_eq!(stages[0].steps[0]["op"], "windowedAggregate");
assert_eq!(
stages[0].steps[0]["aggs"],
json!([{"op": "max", "column": "price", "alias": "finalPrice"}])
);
assert_eq!(stages[1].output, "out");
assert_eq!(stages[1].inputs, vec!["perAuction".to_string()]);
assert_eq!(
stages[1].steps[0]["aggs"],
json!([{"op": "avg", "column": "finalPrice", "alias": "avgFinal"}])
);
}
#[test]
fn a_chain_of_ctes_wires_stage_to_stage_in_dependency_order() {
let stages = compile_stages(
"INSERT INTO out \
WITH a AS (SELECT bidder, price FROM events WHERE type = 'bid'), \
b AS (SELECT bidder, price * 0.9 AS net FROM a WHERE price > 100) \
SELECT * FROM b",
)
.unwrap();
assert_eq!(
stages.iter().map(|s| s.output.as_str()).collect::<Vec<_>>(),
vec!["a", "b", "out"]
);
assert_eq!(stages[0].inputs, vec!["events".to_string()]);
assert_eq!(stages[1].inputs, vec!["a".to_string()]);
assert_eq!(stages[2].inputs, vec!["b".to_string()]);
}
#[test]
fn a_cte_referenced_twice_is_still_one_stage() {
let stages = compile_stages(
"INSERT INTO out \
WITH e AS (SELECT bidder, auction, dateTime FROM events WHERE type = 'bid') \
SELECT l.bidder FROM e AS l JOIN e AS r \
ON l.auction = r.auction AND l.dateTime BETWEEN r.dateTime - INTERVAL '5' SECOND AND r.dateTime + INTERVAL '5' SECOND",
)
.unwrap();
assert_eq!(stages.iter().filter(|s| s.output == "e").count(), 1);
assert_eq!(stages.len(), 2);
assert_eq!(stages[1].inputs, vec!["e".to_string(), "e".to_string()]);
}
#[test]
fn a_trailing_interval_on_a_window_is_the_allowed_lateness() {
let hop = &compile_stages(
"INSERT INTO o SELECT k, count(*) AS n FROM b \
GROUP BY k, HOP(ts, INTERVAL '2' SECOND, INTERVAL '10' SECOND, INTERVAL '5000' SECOND)",
)
.unwrap()[0];
assert_eq!(hop.steps[0]["allowedLatenessMs"], json!(5_000_000));
assert_eq!(hop.steps[0]["windowMs"], json!(10_000));
assert_eq!(hop.steps[0]["slideMs"], json!(2_000));
let tumble = &compile_stages(
"INSERT INTO o SELECT k, max(v) AS m FROM b GROUP BY k, TUMBLE(ts, INTERVAL '10' SECOND, INTERVAL '30' SECOND)",
)
.unwrap()[0];
assert_eq!(tumble.steps[0]["allowedLatenessMs"], json!(30_000));
let plain = &compile_stages(
"INSERT INTO o SELECT k, count(*) AS n FROM b GROUP BY k, TUMBLE(ts, INTERVAL '10' SECOND)",
)
.unwrap()[0];
assert_eq!(plain.steps[0].get("allowedLatenessMs"), None);
assert!(compile("INSERT INTO o SELECT k, count(*) AS n FROM b GROUP BY k, TUMBLE(ts, INTERVAL '1' SECOND, INTERVAL '2' SECOND, INTERVAL '3' SECOND)").is_err());
}
#[test]
fn recursive_ctes_are_rejected() {
assert!(
compile("INSERT INTO o WITH RECURSIVE r AS (SELECT * FROM b) SELECT * FROM r")
.unwrap_err()
.contains("RECURSIVE")
);
}
}