use std::sync::Arc;
use datafusion::arrow::array::{
ArrayRef, BooleanArray, Float32Array, Float64Array, Int32Array, Int64Array, RecordBatch, StringArray,
};
use fv_compute::{FvType, Runtime, SchemaSpec};
use fv_value::Value;
use serde_json::Value as J;
use crate::build::ComputeRunner;
use crate::convert::batch_to_rows;
use crate::row::Row;
#[derive(Clone)]
pub struct KineticsRunner {
runtime: Arc<Runtime>,
}
impl KineticsRunner {
pub fn new(runtime: Arc<Runtime>) -> Self {
Self { runtime }
}
pub fn runtime(&self) -> &Arc<Runtime> {
&self.runtime
}
pub fn selector(step: &J) -> Result<&str, String> {
step["ref"]
.as_str()
.or_else(|| step["transform"].as_str())
.ok_or_else(|| "compute step missing `ref` (the transform id or id@version)".to_string())
}
pub fn ensure_loadable(&self, step: &J) -> Result<(), String> {
self.runtime
.ensure_loadable(Self::selector(step)?)
.map_err(|e| e.to_string())
}
pub fn run_sync(&self, step: &J, inputs: &[(String, Vec<Row>)]) -> Result<Vec<Row>, String> {
let selector = Self::selector(step)?;
let compute = self.runtime.load(selector).map_err(|e| e.to_string())?;
let manifest = compute.manifest();
if !manifest.inputs.is_empty() && manifest.inputs.len() != inputs.len() {
return Err(format!(
"transform `{selector}` declares {} input(s), the pipeline supplied {}",
manifest.inputs.len(),
inputs.len()
));
}
let batches: Vec<RecordBatch> = manifest
.inputs
.iter()
.zip(inputs.iter())
.map(|(signature, (_, rows))| rows_to_typed_batch(signature, rows))
.collect::<Result<_, _>>()?;
let out = compute.run(&batches).map_err(|e| e.to_string())?;
batch_to_rows(&out)
}
}
#[async_trait::async_trait]
impl ComputeRunner for KineticsRunner {
async fn run(&self, step: &J, inputs: &[(String, Vec<Row>)]) -> Result<Vec<Row>, String> {
self.run_sync(step, inputs)
}
}
pub fn rows_to_typed_batch(signature: &SchemaSpec, rows: &[Row]) -> Result<RecordBatch, String> {
let arrays: Vec<ArrayRef> = signature
.columns
.iter()
.map(|c| typed_column(c.dtype, &c.name, rows))
.collect::<Result<_, _>>()?;
RecordBatch::try_new(signature.to_arrow_schema_ref(), arrays).map_err(|e| e.to_string())
}
fn number(v: &Value) -> Option<f64> {
match v {
Value::Num(n) => Some(*n),
_ => None,
}
}
fn text(v: &Value) -> Option<String> {
match v {
Value::Null => None,
Value::Str(s) => Some(s.clone()),
Value::Bool(b) => Some(if *b { "true" } else { "false" }.into()),
Value::Num(n) => Some(if n.fract() == 0.0 && n.is_finite() {
format!("{}", *n as i64)
} else {
format!("{n}")
}),
other => Some(format!("{other:?}")),
}
}
fn typed_column(dtype: FvType, name: &str, rows: &[Row]) -> Result<ArrayRef, String> {
let cell = |r: &Row| r.get(name);
Ok(match dtype {
FvType::Int64 => Arc::new(
rows.iter()
.map(|r| number(&cell(r)).map(|n| n as i64))
.collect::<Int64Array>(),
),
FvType::Int32 => Arc::new(
rows.iter()
.map(|r| number(&cell(r)).map(|n| n as i32))
.collect::<Int32Array>(),
),
FvType::Float64 => Arc::new(rows.iter().map(|r| number(&cell(r))).collect::<Float64Array>()),
FvType::Float32 => Arc::new(
rows.iter()
.map(|r| number(&cell(r)).map(|n| n as f32))
.collect::<Float32Array>(),
),
FvType::Bool => Arc::new(
rows.iter()
.map(|r| match cell(r) {
Value::Bool(b) => Some(b),
_ => None,
})
.collect::<BooleanArray>(),
),
FvType::Utf8 => Arc::new(rows.iter().map(|r| text(&cell(r))).collect::<StringArray>()),
other => {
return Err(format!(
"compute step: column `{name}` has type {other:?}, which the row bridge does not carry yet"
))
}
})
}