Skip to main content

fv_streams/
control_plane.rs

1//! The file-defined host: the engine's control plane answered from a pipeline file.
2
3use super::pipeline::Pipeline;
4use super::serve;
5use async_trait::async_trait;
6use fv_streams_engine::{Binding, BuildSignal, ControlPlane, StageDef, Topology};
7use serde_json::{json, Value as J};
8use std::sync::atomic::{AtomicBool, Ordering};
9use std::sync::Arc;
10
11/// The file-defined host: the engine's [`ControlPlane`] answered from a [`Pipeline`] definition.
12/// Heartbeat = run until the stop flag; the build record prints as a status line (there is no
13/// server to send it to).
14pub struct FileControlPlane {
15    pub(crate) def: J,
16    pub(crate) stop: Arc<AtomicBool>,
17    /// The live build record, for the observability surface (`--serve`).
18    pub(crate) live: tokio::sync::watch::Sender<J>,
19}
20
21impl FileControlPlane {
22    pub fn new(pipeline: &Pipeline, stop: Arc<AtomicBool>) -> Self {
23        Self {
24            def: pipeline.def.clone(),
25            stop,
26            live: tokio::sync::watch::channel(json!({ "status": "STARTING", "metrics": {} })).0,
27        }
28    }
29
30    /// A receiver of the live build record (every heartbeat's record).
31    pub fn live(&self) -> serve::Live {
32        self.live.subscribe()
33    }
34}
35
36#[async_trait]
37impl ControlPlane for FileControlPlane {
38    async fn topology(&self, _pipeline: &str) -> Result<Topology, String> {
39        // A `sql = "…"` field is the SQL front door: it compiles to the SAME stages a
40        // `[[transforms]]` block would (one runtime, two front doors). Datasets/brokers/settings
41        // stay as config; the SQL only replaces the transforms.
42        if let Some(sql) = self.def["sql"].as_str() {
43            return fv_streams_sql::compile(sql).map_err(|e| format!("pipeline SQL: {e}"));
44        }
45        let transforms = self.def["transforms"]
46            .as_array()
47            .ok_or("pipeline.toml: needs at least one [[transforms]] table (or a `sql = \"…\"` field)")?;
48        let stages = transforms
49            .iter()
50            .map(|t| StageDef {
51                steps: t["steps"].as_array().cloned().unwrap_or_default(),
52                inputs: t["inputs"]
53                    .as_array()
54                    .map(|a| a.iter().filter_map(|v| v.as_str().map(String::from)).collect())
55                    .unwrap_or_default(),
56                output: t["output"].as_str().unwrap_or("").to_string(),
57            })
58            .collect();
59        Ok(Topology { stages })
60    }
61
62    async fn resolve_dataset(&self, dataset: &str) -> Result<Binding, String> {
63        let ds = &self.def["datasets"][dataset];
64        if ds.is_null() {
65            return Err(format!(
66                "pipeline.toml: dataset '{dataset}' is not defined under [datasets] — every transform input/output must be"
67            ));
68        }
69        // The connectors crate's registry resolves both halves: `connector = "…"` names the
70        // source (Kafka when absent), `sink = "…"` the sink (Kafka when absent). An input's sink
71        // and an output's source are resolved and never opened.
72        let cx = fv_streams_connectors::Ctx {
73            dataset,
74            ds,
75            brokers: &self.def["brokers"],
76        };
77        for warning in fv_streams_connectors::check_dataset(&cx)? {
78            eprintln!("fv-streams: {warning}");
79        }
80        Ok(Binding {
81            api_name: dataset.to_string(),
82            source: fv_streams_connectors::resolve_source(&cx)?,
83            sink: fv_streams_connectors::resolve_sink(&cx)?,
84        })
85    }
86
87    async fn publishes(&self, dataset: &str) -> bool {
88        let ds = &self.def["datasets"][dataset];
89        ds.get("sink").is_some() || ds.get("topic").is_some()
90    }
91
92    async fn heartbeat(&self, _build_id: &str) -> BuildSignal {
93        if self.stop.load(Ordering::Relaxed) {
94            BuildSignal::Stop
95        } else {
96            BuildSignal::Continue
97        }
98    }
99
100    async fn put_record(&self, _build_id: &str, record: &J) {
101        // No server to persist to — publish to the observability surface and print the counters.
102        let _ = self.live.send(record.clone());
103        println!("{}", progress_line(record));
104    }
105}
106
107/// The line a heartbeat prints: the status and the counters, and how many messages the sinks are
108/// waiting on when there are any, so a run that is not delivering shows it.
109pub(crate) fn progress_line(record: &J) -> String {
110    let m = &record["metrics"];
111    let n = |k: &str| m[k].as_u64().unwrap_or(0);
112    let status = record["status"].as_str().unwrap_or("RUNNING");
113    let mut line = format!(
114        "▸ {status}  consumed={} emitted={} dropped={}",
115        n("consumed"),
116        n("emitted"),
117        n("dropped")
118    );
119    if n("pending") > 0 {
120        line.push_str(&format!(" pending={}", n("pending")));
121    }
122    line
123}