fv-compute-container 0.2.0

The container backend for fv-compute: a transform is a process or image that reads Arrow IPC (or row JSON) on stdin and writes it on stdout — for heavy, GPU or arbitrary-dependency transforms.
Documentation
//! The `json` container protocol (L3 / row-JSON) — RecordBatch <-> `{col: scalar}` rows.
//! Stdlib-friendly: a transform needs no Arrow library, so it runs under the stripped-env isolation
//! (the meridian riskBand transform uses this). Slower than the Arrow-IPC handoff; the manifest opts in.

use arrow::array::{
    Array, ArrayRef, BooleanArray, Float32Array, Float64Array, Int32Array, Int64Array, RecordBatch, StringArray,
};
use fv_compute::{ComputeError, FvType, SchemaSpec};
use serde_json::{Map, Value as J};
use std::sync::Arc;

fn schema_err(id: &str, msg: impl Into<String>) -> ComputeError {
    ComputeError::Schema {
        id: id.to_string(),
        msg: msg.into(),
    }
}

fn cell_to_json(col: &ArrayRef, i: usize) -> J {
    if col.is_null(i) {
        return J::Null;
    }
    use arrow::datatypes::DataType as D;
    let a = col.as_any();
    match col.data_type() {
        D::Boolean => J::Bool(a.downcast_ref::<BooleanArray>().unwrap().value(i)),
        D::Int64 => J::from(a.downcast_ref::<Int64Array>().unwrap().value(i)),
        D::Int32 => J::from(a.downcast_ref::<Int32Array>().unwrap().value(i)),
        D::Float64 => serde_json::Number::from_f64(a.downcast_ref::<Float64Array>().unwrap().value(i))
            .map(J::Number)
            .unwrap_or(J::Null),
        D::Float32 => serde_json::Number::from_f64(a.downcast_ref::<Float32Array>().unwrap().value(i) as f64)
            .map(J::Number)
            .unwrap_or(J::Null),
        D::Utf8 => J::String(a.downcast_ref::<StringArray>().unwrap().value(i).to_string()),
        _ => J::Null,
    }
}

/// A RecordBatch as `[{col: scalar}]` rows (the request shape).
pub fn batch_to_json_rows(rb: &RecordBatch) -> Vec<J> {
    let schema = rb.schema();
    (0..rb.num_rows())
        .map(|i| {
            let mut m = Map::new();
            for (c, f) in schema.fields().iter().enumerate() {
                m.insert(f.name().clone(), cell_to_json(rb.column(c), i));
            }
            J::Object(m)
        })
        .collect()
}

fn build_col(id: &str, dtype: FvType, name: &str, rows: &[J]) -> Result<ArrayRef, ComputeError> {
    let get = |r: &J| r.get(name).cloned().unwrap_or(J::Null);
    Ok(match dtype {
        FvType::Int64 => Arc::new(rows.iter().map(|r| get(r).as_i64()).collect::<Int64Array>()),
        FvType::Int32 => Arc::new(
            rows.iter()
                .map(|r| get(r).as_i64().map(|n| n as i32))
                .collect::<Int32Array>(),
        ),
        FvType::Float64 => Arc::new(rows.iter().map(|r| get(r).as_f64()).collect::<Float64Array>()),
        FvType::Float32 => Arc::new(
            rows.iter()
                .map(|r| get(r).as_f64().map(|n| n as f32))
                .collect::<Float32Array>(),
        ),
        FvType::Bool => Arc::new(rows.iter().map(|r| get(r).as_bool()).collect::<BooleanArray>()),
        FvType::Utf8 => Arc::new(
            rows.iter()
                .map(|r| get(r).as_str().map(String::from))
                .collect::<StringArray>(),
        ),
        other => {
            return Err(schema_err(
                id,
                format!("json protocol: column `{name}` type {other:?} unsupported"),
            ))
        }
    })
}

/// `[{col: scalar}]` rows -> a RecordBatch typed by the declared output signature.
pub fn json_rows_to_batch(id: &str, rows: &[J], out: &SchemaSpec) -> Result<RecordBatch, ComputeError> {
    let arrays: Vec<ArrayRef> = out
        .columns
        .iter()
        .map(|c| build_col(id, c.dtype, &c.name, rows))
        .collect::<Result<_, _>>()?;
    RecordBatch::try_new(out.to_arrow_schema_ref(), arrays).map_err(|e| schema_err(id, e.to_string()))
}

#[cfg(test)]
mod tests {
    use super::*;
    use arrow::datatypes::{DataType, Field, Schema};
    use fv_compute::ColumnSpec;

    #[test]
    fn batch_to_rows_and_back() {
        let schema = Arc::new(Schema::new(vec![
            Field::new("id", DataType::Int64, true),
            Field::new("name", DataType::Utf8, true),
        ]));
        let rb = RecordBatch::try_new(
            schema,
            vec![
                Arc::new(Int64Array::from(vec![1, 2])),
                Arc::new(StringArray::from(vec!["a", "b"])),
            ],
        )
        .unwrap();
        let rows = batch_to_json_rows(&rb);
        assert_eq!(rows[0]["id"], J::from(1));
        assert_eq!(rows[1]["name"], J::from("b"));
        let out = SchemaSpec {
            name: None,
            columns: vec![
                ColumnSpec {
                    name: "id".into(),
                    dtype: FvType::Int64,
                },
                ColumnSpec {
                    name: "name".into(),
                    dtype: FvType::Utf8,
                },
            ],
        };
        assert_eq!(json_rows_to_batch("t", &rows, &out).unwrap(), rb);
    }
}