fv-plan 0.4.0

The FusionVault transform executor on Apache DataFusion: inline row steps and relational reshapes (join, aggregate, window, union) over Arrow, with the value dialect translated into the plan, spatial SQL, file sources, and build orchestration checked against the published conformance vectors.
Documentation
//! Compute steps: running a Kinetics transform from a pipeline.
//!
//! A `wasm` or `container` step selects a transform from a [`fv_compute::Runtime`] by `ref`
//! (`id` or `id@version`). [`KineticsRunner`] is the [`ComputeRunner`] that bridges the row-based
//! build plane to the Arrow-typed transform contract: rows are typed into a `RecordBatch` per
//! declared input using the manifest's signature, the transform runs, and its output batch comes
//! back as rows. The runtime's backends decide *how* the transform runs; this module never links
//! one.
//!
//! ```no_run
//! use std::sync::Arc;
//! use fv_compute::Runtime;
//! use fv_plan::{Build, KineticsRunner};
//! # use fv_compute::{Compute, ComputeError, ImplKind, TransformBackend, TransformManifest};
//! # struct Noop;
//! # impl TransformBackend for Noop {
//! #     fn kind(&self) -> ImplKind { ImplKind::Wasm }
//! #     fn load(&self, _: &TransformManifest, _: &std::path::Path) -> Result<Box<dyn Compute>, ComputeError> { unimplemented!() }
//! # }
//! # let pipeline = serde_json::json!({});
//! # let datasets: Vec<serde_json::Value> = vec![];
//!
//! let runtime = Arc::new(Runtime::builder().root("./transforms").backend(Noop).build()?);
//! let kinetics = KineticsRunner::new(runtime);
//!
//! # tokio::runtime::Runtime::new()?.block_on(async {
//! let record = Build::new(&pipeline, &datasets).compute(&kinetics).run().await;
//! # });
//! # Ok::<(), Box<dyn std::error::Error>>(())
//! ```

use std::sync::Arc;

use datafusion::arrow::array::{
    ArrayRef, BooleanArray, Float32Array, Float64Array, Int32Array, Int64Array, RecordBatch, StringArray,
};
use fv_compute::{FvType, Runtime, SchemaSpec};
use fv_value::Value;
use serde_json::Value as J;

use crate::build::ComputeRunner;
use crate::convert::batch_to_rows;
use crate::row::Row;

/// Runs `wasm` / `container` steps on a shared [`fv_compute::Runtime`].
#[derive(Clone)]
pub struct KineticsRunner {
    runtime: Arc<Runtime>,
}

impl KineticsRunner {
    pub fn new(runtime: Arc<Runtime>) -> Self {
        Self { runtime }
    }

    /// The runtime this runner dispatches to.
    pub fn runtime(&self) -> &Arc<Runtime> {
        &self.runtime
    }

    /// The transform a compute step selects: its `ref` (or legacy `transform`) field.
    pub fn selector(step: &J) -> Result<&str, String> {
        step["ref"]
            .as_str()
            .or_else(|| step["transform"].as_str())
            .ok_or_else(|| "compute step missing `ref` (the transform id or id@version)".to_string())
    }

    /// Resolve and load the step's transform without running it. Call this when a pipeline or
    /// stream is configured, so a missing or broken transform fails there rather than per batch.
    pub fn ensure_loadable(&self, step: &J) -> Result<(), String> {
        self.runtime
            .ensure_loadable(Self::selector(step)?)
            .map_err(|e| e.to_string())
    }

    /// Run a compute step synchronously. The transform backends are synchronous (a component
    /// call, or a blocking child process), so this needs no async runtime; the async
    /// [`ComputeRunner`] impl is a thin wrapper.
    pub fn run_sync(&self, step: &J, inputs: &[(String, Vec<Row>)]) -> Result<Vec<Row>, String> {
        let selector = Self::selector(step)?;
        let compute = self.runtime.load(selector).map_err(|e| e.to_string())?;
        let manifest = compute.manifest();
        if !manifest.inputs.is_empty() && manifest.inputs.len() != inputs.len() {
            return Err(format!(
                "transform `{selector}` declares {} input(s), the pipeline supplied {}",
                manifest.inputs.len(),
                inputs.len()
            ));
        }
        let batches: Vec<RecordBatch> = manifest
            .inputs
            .iter()
            .zip(inputs.iter())
            .map(|(signature, (_, rows))| rows_to_typed_batch(signature, rows))
            .collect::<Result<_, _>>()?;
        let out = compute.run(&batches).map_err(|e| e.to_string())?;
        batch_to_rows(&out)
    }
}

#[async_trait::async_trait]
impl ComputeRunner for KineticsRunner {
    async fn run(&self, step: &J, inputs: &[(String, Vec<Row>)]) -> Result<Vec<Row>, String> {
        self.run_sync(step, inputs)
    }
}

/// Type rows into a `RecordBatch` by a declared signature (a manifest input), rather than by
/// inference: each column gets the Arrow type the signature names, and a value that does not fit
/// lands null.
pub fn rows_to_typed_batch(signature: &SchemaSpec, rows: &[Row]) -> Result<RecordBatch, String> {
    let arrays: Vec<ArrayRef> = signature
        .columns
        .iter()
        .map(|c| typed_column(c.dtype, &c.name, rows))
        .collect::<Result<_, _>>()?;
    RecordBatch::try_new(signature.to_arrow_schema_ref(), arrays).map_err(|e| e.to_string())
}

fn number(v: &Value) -> Option<f64> {
    match v {
        Value::Num(n) => Some(*n),
        _ => None,
    }
}

fn text(v: &Value) -> Option<String> {
    match v {
        Value::Null => None,
        Value::Str(s) => Some(s.clone()),
        Value::Bool(b) => Some(if *b { "true" } else { "false" }.into()),
        Value::Num(n) => Some(if n.fract() == 0.0 && n.is_finite() {
            format!("{}", *n as i64)
        } else {
            format!("{n}")
        }),
        other => Some(format!("{other:?}")),
    }
}

fn typed_column(dtype: FvType, name: &str, rows: &[Row]) -> Result<ArrayRef, String> {
    let cell = |r: &Row| r.get(name);
    Ok(match dtype {
        FvType::Int64 => Arc::new(
            rows.iter()
                .map(|r| number(&cell(r)).map(|n| n as i64))
                .collect::<Int64Array>(),
        ),
        FvType::Int32 => Arc::new(
            rows.iter()
                .map(|r| number(&cell(r)).map(|n| n as i32))
                .collect::<Int32Array>(),
        ),
        FvType::Float64 => Arc::new(rows.iter().map(|r| number(&cell(r))).collect::<Float64Array>()),
        FvType::Float32 => Arc::new(
            rows.iter()
                .map(|r| number(&cell(r)).map(|n| n as f32))
                .collect::<Float32Array>(),
        ),
        FvType::Bool => Arc::new(
            rows.iter()
                .map(|r| match cell(r) {
                    Value::Bool(b) => Some(b),
                    _ => None,
                })
                .collect::<BooleanArray>(),
        ),
        FvType::Utf8 => Arc::new(rows.iter().map(|r| text(&cell(r))).collect::<StringArray>()),
        other => {
            return Err(format!(
                "compute step: column `{name}` has type {other:?}, which the row bridge does not carry yet"
            ))
        }
    })
}