#[cfg(doctest)]
#[doc = include_str!("../README.md")]
pub struct ReadmeDoctests;
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use arrow::array::RecordBatch;
use arrow::compute::concat_batches;
use fv_compute::{Compute, ComputeError, ContainerProtocol, ImplKind, TransformBackend, TransformManifest};
mod ipc;
mod json;
#[derive(Default)]
pub struct ContainerBackend;
impl ContainerBackend {
pub fn new() -> Self {
Self
}
}
impl TransformBackend for ContainerBackend {
fn kind(&self) -> ImplKind {
ImplKind::Container
}
fn load(&self, manifest: &TransformManifest, root: &Path) -> Result<Box<dyn Compute>, ComputeError> {
let entry = manifest
.entry
.as_deref()
.filter(|s| !s.trim().is_empty())
.ok_or_else(|| ComputeError::Load {
id: manifest.id.clone(),
msg: "container impl needs an `entry` command (e.g. \"python3 t.py\" or \"docker run …\")".into(),
})?
.to_string();
Ok(Box::new(ContainerCompute {
manifest: manifest.clone(),
entry,
cwd: root.to_path_buf(),
}))
}
}
pub struct ContainerCompute {
manifest: TransformManifest,
entry: String,
cwd: PathBuf,
}
impl ContainerCompute {
fn command(&self) -> Result<Command, ComputeError> {
let mut parts = self.entry.split_whitespace();
let prog = parts.next().ok_or_else(|| ComputeError::Load {
id: self.manifest.id.clone(),
msg: "empty container `entry`".into(),
})?;
let mut cmd = Command::new(prog);
cmd.args(parts).current_dir(&self.cwd);
Ok(cmd)
}
fn exec(&self, id: &str, payload: Vec<u8>) -> Result<Vec<u8>, ComputeError> {
let mut child = self
.command()?
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.map_err(|e| ComputeError::Run {
id: id.to_string(),
msg: format!("spawn `{}`: {e}", self.entry),
})?;
let mut stdin = child.stdin.take().ok_or_else(|| ComputeError::Run {
id: id.to_string(),
msg: "no stdin".into(),
})?;
let writer = std::thread::spawn(move || stdin.write_all(&payload));
let mut stdout_buf = Vec::new();
let mut stdout = child.stdout.take().ok_or_else(|| ComputeError::Run {
id: id.to_string(),
msg: "no stdout".into(),
})?;
stdout.read_to_end(&mut stdout_buf).map_err(|e| ComputeError::Run {
id: id.to_string(),
msg: format!("read stdout: {e}"),
})?;
let _ = writer.join();
let output = child.wait_with_output().map_err(|e| ComputeError::Run {
id: id.to_string(),
msg: format!("wait: {e}"),
})?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
let tail = stderr.lines().last().unwrap_or("").trim();
return Err(ComputeError::Run {
id: id.to_string(),
msg: format!("container exited {} ({tail})", output.status),
});
}
Ok(stdout_buf)
}
fn check_output(&self, out: &RecordBatch) -> Result<(), ComputeError> {
let expected = self.manifest.output.to_arrow_schema();
let got = out.schema();
let mismatch = expected.fields().len() != got.fields().len()
|| expected
.fields()
.iter()
.zip(got.fields().iter())
.any(|(e, g)| e.name() != g.name() || e.data_type() != g.data_type());
if mismatch {
return Err(ComputeError::Schema {
id: self.manifest.id.clone(),
msg: format!("output schema {got:?} != declared {expected:?}"),
});
}
Ok(())
}
}
impl Compute for ContainerCompute {
fn manifest(&self) -> &TransformManifest {
&self.manifest
}
fn run(&self, inputs: &[RecordBatch]) -> Result<RecordBatch, ComputeError> {
let id = self.manifest.id.clone();
if inputs.len() != 1 {
return Err(ComputeError::Schema {
id,
msg: format!("container backend supports a single input (got {})", inputs.len()),
});
}
let out = match self.manifest.protocol {
ContainerProtocol::ArrowIpc => {
let payload = ipc::write_ipc(&id, &inputs[0].schema(), &inputs[..1])?;
let bytes = self.exec(&id, payload)?;
let batches = ipc::read_ipc(&id, &bytes)?;
if batches.is_empty() {
return Err(ComputeError::Run {
id,
msg: "container produced no Arrow batches".into(),
});
}
let schema = batches[0].schema();
concat_batches(&schema, &batches).map_err(|e| ComputeError::Run {
id: id.clone(),
msg: e.to_string(),
})?
}
ContainerProtocol::Json => {
let columns: Vec<String> = inputs[0].schema().fields().iter().map(|f| f.name().clone()).collect();
let request = serde_json::json!({
"ref": id,
"config": {},
"mode": "batch",
"inputs": [{ "dataset": "in", "columns": columns, "rows": json::batch_to_json_rows(&inputs[0]) }],
});
let payload = serde_json::to_vec(&request).map_err(|e| ComputeError::Run {
id: id.clone(),
msg: e.to_string(),
})?;
let bytes = self.exec(&id, payload)?;
let resp: serde_json::Value = serde_json::from_slice(&bytes).map_err(|e| ComputeError::Run {
id: id.clone(),
msg: format!("response is not JSON: {e}"),
})?;
if let Some(err) = resp.get("error").filter(|e| !e.is_null()) {
return Err(ComputeError::Run {
id,
msg: format!("transform error: {}", err),
});
}
let rows = resp["rows"].as_array().ok_or_else(|| ComputeError::Run {
id: id.clone(),
msg: "response missing a `rows` array".into(),
})?;
json::json_rows_to_batch(&id, rows, &self.manifest.output)?
}
};
self.check_output(&out)?;
Ok(out)
}
}