fv-compute-container 0.2.0

The container backend for fv-compute: a transform is a process or image that reads Arrow IPC (or row JSON) on stdin and writes it on stdout — for heavy, GPU or arbitrary-dependency transforms.
Documentation
//! Throughput bench for the container backend: rows/sec through the Arrow-IPC container
//! path (process spawn + IPC) vs native, at 100k/500k. Compare against the wasm backend
//! (`transform-wasm-backend-*.json`).
//! Arrow-IPC container should crush row-JSON container (no per-row JSON), while wasm avoids the spawn.
//!
//!   cargo run --release -p fv-compute-container --bin fvc-bench

use arrow::array::{Float64Array, Int64Array, RecordBatch};
use arrow::datatypes::{DataType, Field, Schema};
use fv_compute::{parse_and_validate, TransformBackend};
use fv_compute_container::ContainerBackend;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Instant;

fn refguest_path() -> PathBuf {
    // Sibling of this bench binary in the same target dir.
    std::env::current_exe().unwrap().parent().unwrap().join("fvc-refguest")
}

fn input(n: usize) -> RecordBatch {
    let schema = Arc::new(Schema::new(vec![
        Field::new("id", DataType::Int64, true),
        Field::new("amount", DataType::Float64, true),
    ]));
    RecordBatch::try_new(
        schema,
        vec![
            Arc::new(Int64Array::from((0..n as i64).collect::<Vec<_>>())),
            Arc::new(Float64Array::from((0..n).map(|x| x as f64 * 1.5).collect::<Vec<_>>())),
        ],
    )
    .unwrap()
}

fn manifest_toml(entry: &str) -> String {
    format!(
        "id = \"spikeTotal\"\nversion = \"0.1.0\"\nimpl = \"container\"\nentry = \"{entry}\"\n\
         [[inputs]]\ncolumns = [{{ name = \"id\", type = \"int64\" }}, {{ name = \"amount\", type = \"float64\" }}]\n\
         [output]\ncolumns = [{{ name = \"id\", type = \"int64\" }}, {{ name = \"amount\", type = \"float64\" }}, {{ name = \"total\", type = \"float64\" }}]\n"
    )
}

fn best_rps(n: usize, iters: usize, mut f: impl FnMut()) -> f64 {
    let mut best = f64::MAX;
    for _ in 0..iters {
        let t = Instant::now();
        f();
        best = best.min(t.elapsed().as_secs_f64());
    }
    n as f64 / best
}

fn main() {
    let entry = refguest_path();
    let manifest = parse_and_validate(&manifest_toml(&entry.to_string_lossy())).unwrap();
    let compute = ContainerBackend::new()
        .load(&manifest, &std::env::current_dir().unwrap())
        .unwrap();

    for &n in &[100_000usize, 500_000] {
        let rb = input(n);
        let _ = compute.run(std::slice::from_ref(&rb)).unwrap(); // warm (page caches etc.)
        let container_rps = best_rps(n, 8, || {
            std::hint::black_box(compute.run(std::slice::from_ref(&rb)).unwrap());
        });
        println!("=== {n} rows ===");
        println!("  container (Arrow IPC, spawn+IPC) {container_rps:>14.0} rows/s");

        let json = format!("{{\n  \"rows\": {n},\n  \"container_arrow_ipc_rows_per_s\": {container_rps}\n}}\n");
        let dir = std::env::var_os("FV_BENCH_OUT")
            .map(PathBuf::from)
            .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target/bench-results"));
        let _ = std::fs::create_dir_all(&dir);
        let path = dir.join(format!("transform-container-{n}.json"));
        std::fs::write(&path, json).unwrap();
        println!("  wrote {}", path.display());
    }
}