fv_streams/
control_plane.rs1use super::pipeline::Pipeline;
4use super::serve;
5use async_trait::async_trait;
6use fv_streams_engine::{Binding, BuildSignal, ControlPlane, StageDef, Topology};
7use serde_json::{json, Value as J};
8use std::sync::atomic::{AtomicBool, Ordering};
9use std::sync::Arc;
10
11pub struct FileControlPlane {
15 pub(crate) def: J,
16 pub(crate) stop: Arc<AtomicBool>,
17 pub(crate) live: tokio::sync::watch::Sender<J>,
19}
20
21impl FileControlPlane {
22 pub fn new(pipeline: &Pipeline, stop: Arc<AtomicBool>) -> Self {
23 Self {
24 def: pipeline.def.clone(),
25 stop,
26 live: tokio::sync::watch::channel(json!({ "status": "STARTING", "metrics": {} })).0,
27 }
28 }
29
30 pub fn live(&self) -> serve::Live {
32 self.live.subscribe()
33 }
34}
35
36#[async_trait]
37impl ControlPlane for FileControlPlane {
38 async fn topology(&self, _pipeline: &str) -> Result<Topology, String> {
39 if let Some(sql) = self.def["sql"].as_str() {
43 return fv_streams_sql::compile(sql).map_err(|e| format!("pipeline SQL: {e}"));
44 }
45 let transforms = self.def["transforms"]
46 .as_array()
47 .ok_or("pipeline.toml: needs at least one [[transforms]] table (or a `sql = \"…\"` field)")?;
48 let stages = transforms
49 .iter()
50 .map(|t| StageDef {
51 steps: t["steps"].as_array().cloned().unwrap_or_default(),
52 inputs: t["inputs"]
53 .as_array()
54 .map(|a| a.iter().filter_map(|v| v.as_str().map(String::from)).collect())
55 .unwrap_or_default(),
56 output: t["output"].as_str().unwrap_or("").to_string(),
57 })
58 .collect();
59 Ok(Topology { stages })
60 }
61
62 async fn resolve_dataset(&self, dataset: &str) -> Result<Binding, String> {
63 let ds = &self.def["datasets"][dataset];
64 if ds.is_null() {
65 return Err(format!(
66 "pipeline.toml: dataset '{dataset}' is not defined under [datasets] — every transform input/output must be"
67 ));
68 }
69 let cx = fv_streams_connectors::Ctx {
73 dataset,
74 ds,
75 brokers: &self.def["brokers"],
76 };
77 for warning in fv_streams_connectors::check_dataset(&cx)? {
78 eprintln!("fv-streams: {warning}");
79 }
80 Ok(Binding {
81 api_name: dataset.to_string(),
82 source: fv_streams_connectors::resolve_source(&cx)?,
83 sink: fv_streams_connectors::resolve_sink(&cx)?,
84 })
85 }
86
87 async fn publishes(&self, dataset: &str) -> bool {
88 let ds = &self.def["datasets"][dataset];
89 ds.get("sink").is_some() || ds.get("topic").is_some()
90 }
91
92 async fn heartbeat(&self, _build_id: &str) -> BuildSignal {
93 if self.stop.load(Ordering::Relaxed) {
94 BuildSignal::Stop
95 } else {
96 BuildSignal::Continue
97 }
98 }
99
100 async fn put_record(&self, _build_id: &str, record: &J) {
101 let _ = self.live.send(record.clone());
103 println!("{}", progress_line(record));
104 }
105}
106
107pub(crate) fn progress_line(record: &J) -> String {
110 let m = &record["metrics"];
111 let n = |k: &str| m[k].as_u64().unwrap_or(0);
112 let status = record["status"].as_str().unwrap_or("RUNNING");
113 let mut line = format!(
114 "▸ {status} consumed={} emitted={} dropped={}",
115 n("consumed"),
116 n("emitted"),
117 n("dropped")
118 );
119 if n("pending") > 0 {
120 line.push_str(&format!(" pending={}", n("pending")));
121 }
122 line
123}