fv-streams 0.4.0

The FusionVault Streams standalone runner: run a stream pipeline defined in one TOML file against Kafka or Redpanda with no control plane, including the built-in Nexmark event generator.
Documentation
//! The ARROW-NATIVE source (5d): a `connector = "arrow"` dataset reads Arrow IPC `RecordBatch`es from
//! a file and feeds them straight into the runtime — no JSON decode. The counterpart of the Arrow
//! sink; together they are the Arrow edges that let an Arrow-speaking peer skip the JSON tax.
//!
//! ```toml
//! [datasets.events]
//! connector = "arrow"
//! path = "events.arrow"     # an Arrow IPC stream file — or an Arrow sink's output directory
//!                           # (`<out>-part<n>/`: its committed parts are read in manifest order)
//! partitions = 4            # task count it is sized for (STREAM_CONSUMERS overrides)
//! ```
//!
//! Task `t` of `tasks` owns the batches whose index is `t` modulo `tasks` (any task count reads the
//! same stream). Each row leaves under a deterministic `__fv_key` (its global offset), so replay
//! re-emits identical rows.

use std::fs::File;

use arrow::array::{Array, Int64Array, RecordBatch};
use arrow::ipc::reader::StreamReader;
use serde_json::Value as J;

/// Parsed `connector = "arrow"` config.
#[derive(Clone, Debug)]
pub struct ArrowCfg {
    pub dataset: String,
    pub path: String,
    pub partitions: usize,
}

/// Read every batch of an Arrow output: a single IPC stream file, or an Arrow sink's directory of
/// committed parts (concatenated in manifest order — exactly the committed rows, once).
pub fn read_output(path: &str) -> Result<Vec<RecordBatch>, String> {
    let p = std::path::Path::new(path);
    let files: Vec<std::path::PathBuf> = if p.is_dir() {
        fv_streams_engine::file_sink::committed_parts(p).map_err(|e| e.to_string())?
    } else {
        vec![p.to_path_buf()]
    };
    let mut out = Vec::new();
    for f in files {
        let file = File::open(&f).map_err(|e| format!("cannot open {}: {e}", f.display()))?;
        let reader = StreamReader::try_new(file, None).map_err(|e| format!("{}: {e}", f.display()))?;
        for b in reader {
            out.push(b.map_err(|e| format!("{}: {e}", f.display()))?);
        }
    }
    Ok(out)
}

/// Recognise an `arrow` source dataset (returns `None` for anything else).
pub fn parse(dataset: &str, ds: &J) -> Option<Result<ArrowCfg, String>> {
    if ds["connector"].as_str() != Some("arrow") {
        return None;
    }
    let path = match ds["path"].as_str() {
        Some(p) => p.to_string(),
        None => {
            return Some(Err(format!(
                "pipeline.toml: [datasets.{dataset}] connector = \"arrow\" needs `path = \"…\"`"
            )))
        }
    };
    // Validate up front: a source is configuration, so a missing or unreadable input fails the build
    // before anything runs — like every other dataset — not inside a task. A sink directory is
    // readable when its manifest is; a file when its stream header is.
    let p = std::path::Path::new(&path);
    let readable = if p.is_dir() {
        fv_streams_engine::file_sink::read_manifest(p)
            .map(|_| ())
            .map_err(|e| e.to_string())
    } else {
        File::open(p)
            .map_err(|e| e.to_string())
            .and_then(|f| StreamReader::try_new(f, None).map(|_| ()).map_err(|e| e.to_string()))
    };
    if let Err(e) = readable {
        return Some(Err(format!(
            "pipeline.toml: [datasets.{dataset}] connector = \"arrow\": cannot read {path}: {e}"
        )));
    }
    Some(Ok(ArrowCfg {
        dataset: dataset.to_string(),
        path,
        partitions: ds["partitions"].as_i64().unwrap_or(4).max(1) as usize,
    }))
}

/// The host handle: one per `connector = "arrow"` dataset.
pub struct ArrowInline {
    pub cfg: ArrowCfg,
}

impl fv_streams_engine::InlineSource for ArrowInline {
    fn name(&self) -> String {
        format!("arrow:{}", self.cfg.dataset)
    }
    fn tasks(&self) -> usize {
        self.cfg.partitions.max(1)
    }
    fn open(
        &self,
        task: usize,
        tasks: usize,
        start: Option<i64>,
        watermark_column: Option<&str>,
    ) -> Result<Box<dyn fv_streams_engine::dataflow::Source + Send>, String> {
        // read the whole input up front (bounded — one owner per task by batch index). `parse`
        // validated it; a failure here (it vanished since) is reported, not a panic.
        let path = &self.cfg.path;
        let batches = read_output(path).map_err(|e| format!("arrow source {path}: {e}"))?;
        // a per-batch base offset so keys are globally unique across batches.
        let mut base = 0i64;
        let offsets: Vec<i64> = batches
            .iter()
            .map(|b| {
                let o = base;
                base += b.num_rows() as i64;
                o
            })
            .collect();
        Ok(Box::new(ArrowFileSource {
            name: self.name(),
            batches,
            offsets,
            next: start.map(|s| s.max(0) as usize).unwrap_or(0),
            task,
            tasks,
            watermark_column: watermark_column.map(String::from),
            done: false,
        }))
    }
}

/// One source task reading its share of the IPC file's batches.
struct ArrowFileSource {
    /// The name the checkpoint records this source's positions under (`arrow:<dataset>`).
    name: String,
    batches: Vec<RecordBatch>,
    offsets: Vec<i64>,
    next: usize,
    task: usize,
    tasks: usize,
    watermark_column: Option<String>,
    done: bool,
}

impl fv_streams_engine::dataflow::Source for ArrowFileSource {
    fn poll(&mut self, out: &mut fv_streams_engine::dataflow::Out) -> fv_streams_engine::dataflow::Poll {
        use fv_streams_engine::dataflow::Poll;
        if self.done {
            return Poll::Done;
        }
        while self.next < self.batches.len() {
            let i = self.next;
            self.next += 1;
            if i % self.tasks != self.task {
                continue;
            }
            let batch = &self.batches[i];
            if batch.num_rows() == 0 {
                continue;
            }
            // deterministic keys: the row's global offset (so replay re-keys identically).
            let base = self.offsets[i];
            let keys: Vec<Option<String>> = (0..batch.num_rows())
                .map(|r| Some((base + r as i64).to_string()))
                .collect();
            let stamped = fv_streams_engine::decode::with_partition(
                &fv_streams_engine::decode::with_keys(batch, &keys),
                self.task as i32,
            );
            // stamp a watermark from the event-time column (the max in this batch), when asked.
            if let Some(wc) = &self.watermark_column {
                if let Some(max) = max_i64(batch, wc) {
                    out.watermark = Some(Some(max));
                }
            }
            out.push(stamped);
            return Poll::Again;
        }
        self.done = true;
        Poll::Done
    }

    fn on_barrier(&mut self, _epoch: u64) -> Vec<u8> {
        fv_streams_engine::dataflow::position_record(&self.name, self.task as i32, self.next as i64)
    }
    fn position(&mut self) -> Vec<u8> {
        fv_streams_engine::dataflow::position_record(&self.name, self.task as i32, self.next as i64)
    }
}

/// The maximum of an Int64 column in a batch (the event-time watermark), if present.
fn max_i64(batch: &RecordBatch, column: &str) -> Option<i64> {
    let col = batch.column_by_name(column)?;
    let a = col.as_any().downcast_ref::<Int64Array>()?;
    (0..a.len()).filter(|&i| a.is_valid(i)).map(|i| a.value(i)).max()
}

#[cfg(test)]
mod tests {
    use super::*;
    use serde_json::json;

    #[test]
    fn the_position_record_names_the_source_and_a_restore_there_reads_nothing_twice() {
        // The checkpoint keys a source's positions by its name: a record without it is never
        // stored, and a restored run would read the file from the start again.
        use arrow::datatypes::{DataType, Field, Schema};
        use fv_streams_engine::dataflow::{Out, Poll};
        use fv_streams_engine::InlineSource;
        use std::sync::Arc;
        let p = std::env::temp_dir().join(format!("fv-arrow-source-pos-{}.arrow", std::process::id()));
        let schema = Arc::new(Schema::new(vec![Field::new("v", DataType::Int64, false)]));
        let mut w = arrow::ipc::writer::StreamWriter::try_new(File::create(&p).unwrap(), &schema).unwrap();
        for i in 0..3 {
            w.write(&RecordBatch::try_new(schema.clone(), vec![Arc::new(Int64Array::from(vec![i]))]).unwrap())
                .unwrap();
        }
        w.finish().unwrap();
        let src = ArrowInline {
            cfg: ArrowCfg {
                dataset: "events".into(),
                path: p.to_string_lossy().to_string(),
                partitions: 1,
            },
        };
        let drain = |start: Option<i64>| -> (usize, Vec<u8>) {
            let mut s = src.open(0, 1, start, None).unwrap();
            let mut rows = 0;
            loop {
                let mut out = Out::default();
                match s.poll(&mut out) {
                    Poll::Again | Poll::Idle => rows += out.batches.iter().map(|b| b.num_rows()).sum::<usize>(),
                    Poll::Done => break (rows, s.position()),
                    Poll::Failed(e) => panic!("{e}"),
                }
            }
        };
        let (rows, pos) = drain(None);
        assert_eq!(rows, 3);
        let v: serde_json::Value = serde_json::from_slice(&pos).unwrap();
        assert_eq!(v["topic"], "arrow:events");
        assert_eq!(v["in"]["0"], 3);
        assert_eq!(drain(Some(3)).0, 0, "restored at the end: nothing again");
        assert_eq!(drain(Some(1)).0, 2);
        std::fs::remove_file(&p).ok();
    }

    #[test]
    fn parse_validates_the_file_up_front_so_a_bad_source_fails_the_build_not_a_task() {
        // a missing file names the dataset, the path and the OS error — at config time.
        let ds = json!({ "connector": "arrow", "path": "/nonexistent/events.arrow" });
        let err = parse("events", &ds)
            .expect("recognised as an arrow source")
            .unwrap_err();
        assert!(
            err.contains("[datasets.events]") && err.contains("cannot read /nonexistent/events.arrow"),
            "{err}"
        );
        // a file that is not an Arrow IPC stream is refused the same way.
        let p = std::env::temp_dir().join(format!("fv-arrow-source-{}.bin", std::process::id()));
        std::fs::write(&p, b"not an arrow ipc stream").unwrap();
        let ds = json!({ "connector": "arrow", "path": p.to_string_lossy() });
        let err = parse("events", &ds).expect("recognised").unwrap_err();
        std::fs::remove_file(&p).ok();
        assert!(err.contains("cannot read"), "{err}");
        // not an arrow dataset at all: not ours to judge.
        assert!(parse("k", &json!({ "connector": "kafka" })).is_none());
    }
}