fv-streams 0.2.0

The FusionVault Streams standalone runner: run a stream pipeline defined in one TOML file against Kafka or Redpanda with no control plane, including the built-in Nexmark event generator.
Documentation
//! The ARROW-NATIVE source (5d): a `connector = "arrow"` dataset reads Arrow IPC `RecordBatch`es from
//! a file and feeds them straight into the runtime — no JSON decode. The counterpart of the Arrow
//! sink; together they are the Arrow edges that let an Arrow-speaking peer skip the JSON tax.
//!
//! ```toml
//! [datasets.events]
//! connector = "arrow"
//! path = "events.arrow"     # an Arrow IPC stream file (e.g. written by the Arrow sink)
//! partitions = 4            # task count it is sized for (STREAM_CONSUMERS overrides)
//! ```
//!
//! Task `t` of `tasks` owns the batches whose index is `t` modulo `tasks` (any task count reads the
//! same stream). Each row leaves under a deterministic `__fv_key` (its global offset), so replay
//! re-emits identical rows.

use std::fs::File;

use arrow::array::{Array, Int64Array, RecordBatch};
use arrow::ipc::reader::StreamReader;
use serde_json::{json, Value as J};

/// Parsed `connector = "arrow"` config.
#[derive(Clone)]
pub struct ArrowCfg {
    pub dataset: String,
    pub path: String,
    pub partitions: usize,
}

/// Recognise an `arrow` source dataset (returns `None` for anything else).
pub fn parse(dataset: &str, ds: &J) -> Option<Result<ArrowCfg, String>> {
    if ds["connector"].as_str() != Some("arrow") {
        return None;
    }
    let path = match ds["path"].as_str() {
        Some(p) => p.to_string(),
        None => {
            return Some(Err(format!(
                "pipeline.toml: [datasets.{dataset}] connector = \"arrow\" needs `path = \"…\"`"
            )))
        }
    };
    Some(Ok(ArrowCfg {
        dataset: dataset.to_string(),
        path,
        partitions: ds["partitions"].as_i64().unwrap_or(4).max(1) as usize,
    }))
}

/// The host handle: one per `connector = "arrow"` dataset.
pub struct ArrowInline {
    pub cfg: ArrowCfg,
}

impl fv_streams_engine::InlineSource for ArrowInline {
    fn name(&self) -> String {
        format!("arrow:{}", self.cfg.dataset)
    }
    fn tasks(&self) -> usize {
        self.cfg.partitions.max(1)
    }
    fn open(
        &self,
        task: usize,
        tasks: usize,
        start: Option<i64>,
        watermark_column: Option<&str>,
    ) -> Box<dyn fv_streams_engine::dataflow::Source + Send> {
        // read the whole IPC stream up front (a bounded file — one owner per task by batch index).
        let file = File::open(&self.cfg.path)
            .unwrap_or_else(|e| panic!("arrow source open {}: {e}", self.cfg.path));
        let reader = StreamReader::try_new(file, None)
            .unwrap_or_else(|e| panic!("arrow source {}: {e}", self.cfg.path));
        let batches: Vec<RecordBatch> = reader
            .collect::<Result<_, _>>()
            .unwrap_or_else(|e| panic!("arrow source {}: {e}", self.cfg.path));
        // a per-batch base offset so keys are globally unique across batches.
        let mut base = 0i64;
        let offsets: Vec<i64> = batches
            .iter()
            .map(|b| {
                let o = base;
                base += b.num_rows() as i64;
                o
            })
            .collect();
        Box::new(ArrowFileSource {
            batches,
            offsets,
            next: start.map(|s| s.max(0) as usize).unwrap_or(0),
            task,
            tasks,
            watermark_column: watermark_column.map(String::from),
            done: false,
        })
    }
}

/// One source task reading its share of the IPC file's batches.
struct ArrowFileSource {
    batches: Vec<RecordBatch>,
    offsets: Vec<i64>,
    next: usize,
    task: usize,
    tasks: usize,
    watermark_column: Option<String>,
    done: bool,
}

impl fv_streams_engine::dataflow::Source for ArrowFileSource {
    fn poll(&mut self, out: &mut fv_streams_engine::dataflow::Out) -> fv_streams_engine::dataflow::Poll {
        use fv_streams_engine::dataflow::Poll;
        if self.done {
            return Poll::Done;
        }
        while self.next < self.batches.len() {
            let i = self.next;
            self.next += 1;
            if i % self.tasks != self.task {
                continue;
            }
            let batch = &self.batches[i];
            if batch.num_rows() == 0 {
                continue;
            }
            // deterministic keys: the row's global offset (so replay re-keys identically).
            let base = self.offsets[i];
            let keys: Vec<Option<String>> = (0..batch.num_rows()).map(|r| Some((base + r as i64).to_string())).collect();
            let stamped = fv_streams_engine::decode::with_partition(
                &fv_streams_engine::decode::with_keys(batch, &keys),
                self.task as i32,
            );
            // stamp a watermark from the event-time column (the max in this batch), when asked.
            if let Some(wc) = &self.watermark_column {
                if let Some(max) = max_i64(batch, wc) {
                    out.watermark = Some(Some(max));
                }
            }
            out.push(stamped);
            return Poll::Again;
        }
        self.done = true;
        Poll::Done
    }

    fn on_barrier(&mut self, _epoch: u64) -> Vec<u8> {
        serde_json::to_vec(&json!({ "in": { self.task.to_string(): self.next } })).unwrap_or_default()
    }
}

/// The maximum of an Int64 column in a batch (the event-time watermark), if present.
fn max_i64(batch: &RecordBatch, column: &str) -> Option<i64> {
    let col = batch.column_by_name(column)?;
    let a = col.as_any().downcast_ref::<Int64Array>()?;
    (0..a.len()).filter(|&i| a.is_valid(i)).map(|i| a.value(i)).max()
}