fv-compute-container 0.2.0

The container backend for fv-compute: a transform is a process or image that reads Arrow IPC (or row JSON) on stdin and writes it on stdout — for heavy, GPU or arbitrary-dependency transforms.
Documentation
//! # fv-compute-container — the container backend
//!
//! The second IMPL behind the same registry + contract: for heavy / GPU / arbitrary-dependency
//! transforms that don't fit the wasm sandbox. A transform is a process/image that reads Arrow IPC
//! on stdin and writes Arrow IPC on stdout (the Arrow handoff), run in the isolated runtime.
//! Heavier per invocation than wasm (process spawn) but the only option for arbitrary native deps.
//!
//! Register it on a [`fv_compute::Backends`] set for `impl = "container"` manifests. Single-input
//! today; multi-input container joins are a later extension.

/// Every Rust code block in the README is compiled and run as a doctest.
#[cfg(doctest)]
#[doc = include_str!("../README.md")]
pub struct ReadmeDoctests;

use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};

use arrow::array::RecordBatch;
use arrow::compute::concat_batches;
use fv_compute::{Compute, ComputeError, ContainerProtocol, ImplKind, TransformBackend, TransformManifest};

mod ipc;
mod json;

/// The container backend. Stateless — each `load` returns a [`ContainerCompute`] that spawns the
/// transform's process per `run`.
#[derive(Default)]
pub struct ContainerBackend;

impl ContainerBackend {
    pub fn new() -> Self {
        Self
    }
}

impl TransformBackend for ContainerBackend {
    fn kind(&self) -> ImplKind {
        ImplKind::Container
    }

    fn load(&self, manifest: &TransformManifest, root: &Path) -> Result<Box<dyn Compute>, ComputeError> {
        let entry = manifest
            .entry
            .as_deref()
            .filter(|s| !s.trim().is_empty())
            .ok_or_else(|| ComputeError::Load {
                id: manifest.id.clone(),
                msg: "container impl needs an `entry` command (e.g. \"python3 t.py\" or \"docker run …\")".into(),
            })?
            .to_string();
        Ok(Box::new(ContainerCompute {
            manifest: manifest.clone(),
            entry,
            cwd: root.to_path_buf(),
        }))
    }
}

/// A loaded container transform. Each `run` spawns the `entry` process with cwd = the transform's
/// directory (so relative paths in the command resolve against it) and speaks Arrow IPC over stdio.
pub struct ContainerCompute {
    manifest: TransformManifest,
    entry: String,
    cwd: PathBuf,
}

impl ContainerCompute {
    fn command(&self) -> Result<Command, ComputeError> {
        let mut parts = self.entry.split_whitespace();
        let prog = parts.next().ok_or_else(|| ComputeError::Load {
            id: self.manifest.id.clone(),
            msg: "empty container `entry`".into(),
        })?;
        let mut cmd = Command::new(prog);
        cmd.args(parts).current_dir(&self.cwd);
        Ok(cmd)
    }

    /// Spawn the transform process, write `payload` to stdin (on a thread, draining stdout to avoid a
    /// pipe deadlock), return stdout bytes. Fail-closed on spawn failure / non-zero exit.
    fn exec(&self, id: &str, payload: Vec<u8>) -> Result<Vec<u8>, ComputeError> {
        let mut child = self
            .command()?
            .stdin(Stdio::piped())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped())
            .spawn()
            .map_err(|e| ComputeError::Run {
                id: id.to_string(),
                msg: format!("spawn `{}`: {e}", self.entry),
            })?;

        let mut stdin = child.stdin.take().ok_or_else(|| ComputeError::Run {
            id: id.to_string(),
            msg: "no stdin".into(),
        })?;
        let writer = std::thread::spawn(move || stdin.write_all(&payload));

        let mut stdout_buf = Vec::new();
        let mut stdout = child.stdout.take().ok_or_else(|| ComputeError::Run {
            id: id.to_string(),
            msg: "no stdout".into(),
        })?;
        stdout.read_to_end(&mut stdout_buf).map_err(|e| ComputeError::Run {
            id: id.to_string(),
            msg: format!("read stdout: {e}"),
        })?;
        let _ = writer.join();

        let output = child.wait_with_output().map_err(|e| ComputeError::Run {
            id: id.to_string(),
            msg: format!("wait: {e}"),
        })?;
        if !output.status.success() {
            let stderr = String::from_utf8_lossy(&output.stderr);
            let tail = stderr.lines().last().unwrap_or("").trim();
            return Err(ComputeError::Run {
                id: id.to_string(),
                msg: format!("container exited {} ({tail})", output.status),
            });
        }
        Ok(stdout_buf)
    }

    fn check_output(&self, out: &RecordBatch) -> Result<(), ComputeError> {
        let expected = self.manifest.output.to_arrow_schema();
        let got = out.schema();
        let mismatch = expected.fields().len() != got.fields().len()
            || expected
                .fields()
                .iter()
                .zip(got.fields().iter())
                .any(|(e, g)| e.name() != g.name() || e.data_type() != g.data_type());
        if mismatch {
            return Err(ComputeError::Schema {
                id: self.manifest.id.clone(),
                msg: format!("output schema {got:?} != declared {expected:?}"),
            });
        }
        Ok(())
    }
}

impl Compute for ContainerCompute {
    fn manifest(&self) -> &TransformManifest {
        &self.manifest
    }

    fn run(&self, inputs: &[RecordBatch]) -> Result<RecordBatch, ComputeError> {
        let id = self.manifest.id.clone();
        if inputs.len() != 1 {
            return Err(ComputeError::Schema {
                id,
                msg: format!("container backend supports a single input (got {})", inputs.len()),
            });
        }
        let out = match self.manifest.protocol {
            ContainerProtocol::ArrowIpc => {
                let payload = ipc::write_ipc(&id, &inputs[0].schema(), &inputs[..1])?;
                let bytes = self.exec(&id, payload)?;
                let batches = ipc::read_ipc(&id, &bytes)?;
                if batches.is_empty() {
                    return Err(ComputeError::Run {
                        id,
                        msg: "container produced no Arrow batches".into(),
                    });
                }
                let schema = batches[0].schema();
                concat_batches(&schema, &batches).map_err(|e| ComputeError::Run {
                    id: id.clone(),
                    msg: e.to_string(),
                })?
            }
            ContainerProtocol::Json => {
                let columns: Vec<String> = inputs[0].schema().fields().iter().map(|f| f.name().clone()).collect();
                let request = serde_json::json!({
                    "ref": id,
                    "config": {},
                    "mode": "batch",
                    "inputs": [{ "dataset": "in", "columns": columns, "rows": json::batch_to_json_rows(&inputs[0]) }],
                });
                let payload = serde_json::to_vec(&request).map_err(|e| ComputeError::Run {
                    id: id.clone(),
                    msg: e.to_string(),
                })?;
                let bytes = self.exec(&id, payload)?;
                let resp: serde_json::Value = serde_json::from_slice(&bytes).map_err(|e| ComputeError::Run {
                    id: id.clone(),
                    msg: format!("response is not JSON: {e}"),
                })?;
                if let Some(err) = resp.get("error").filter(|e| !e.is_null()) {
                    return Err(ComputeError::Run {
                        id,
                        msg: format!("transform error: {}", err),
                    });
                }
                let rows = resp["rows"].as_array().ok_or_else(|| ComputeError::Run {
                    id: id.clone(),
                    msg: "response missing a `rows` array".into(),
                })?;
                json::json_rows_to_batch(&id, rows, &self.manifest.output)?
            }
        };
        self.check_output(&out)?;
        Ok(out)
    }
}