Skip to main content

fv_compute_container/
lib.rs

1//! # fv-compute-container — the container backend
2//!
3//! The second IMPL behind the same registry + contract: for heavy / GPU / arbitrary-dependency
4//! transforms that don't fit the wasm sandbox. A transform is a process/image that reads Arrow IPC
5//! on stdin and writes Arrow IPC on stdout (the Arrow handoff), run in the isolated runtime.
6//! Heavier per invocation than wasm (process spawn) but the only option for arbitrary native deps.
7//!
8//! Register it on a [`fv_compute::Backends`] set for `impl = "container"` manifests. Single-input
9//! today; multi-input container joins are a later extension.
10
11/// Every Rust code block in the README is compiled and run as a doctest.
12#[cfg(doctest)]
13#[doc = include_str!("../README.md")]
14pub struct ReadmeDoctests;
15
16use std::io::{Read, Write};
17use std::path::{Path, PathBuf};
18use std::process::{Command, Stdio};
19
20use arrow::array::RecordBatch;
21use arrow::compute::concat_batches;
22use fv_compute::{Compute, ComputeError, ContainerProtocol, ImplKind, TransformBackend, TransformManifest};
23
24mod ipc;
25mod json;
26
27/// The container backend. Stateless — each `load` returns a [`ContainerCompute`] that spawns the
28/// transform's process per `run`.
29#[derive(Default)]
30pub struct ContainerBackend;
31
32impl ContainerBackend {
33    pub fn new() -> Self {
34        Self
35    }
36}
37
38impl TransformBackend for ContainerBackend {
39    fn kind(&self) -> ImplKind {
40        ImplKind::Container
41    }
42
43    fn load(&self, manifest: &TransformManifest, root: &Path) -> Result<Box<dyn Compute>, ComputeError> {
44        let entry = manifest
45            .entry
46            .as_deref()
47            .filter(|s| !s.trim().is_empty())
48            .ok_or_else(|| ComputeError::Load {
49                id: manifest.id.clone(),
50                msg: "container impl needs an `entry` command (e.g. \"python3 t.py\" or \"docker run …\")".into(),
51            })?
52            .to_string();
53        Ok(Box::new(ContainerCompute {
54            manifest: manifest.clone(),
55            entry,
56            cwd: root.to_path_buf(),
57        }))
58    }
59}
60
61/// A loaded container transform. Each `run` spawns the `entry` process with cwd = the transform's
62/// directory (so relative paths in the command resolve against it) and speaks Arrow IPC over stdio.
63pub struct ContainerCompute {
64    manifest: TransformManifest,
65    entry: String,
66    cwd: PathBuf,
67}
68
69impl ContainerCompute {
70    fn command(&self) -> Result<Command, ComputeError> {
71        let mut parts = self.entry.split_whitespace();
72        let prog = parts.next().ok_or_else(|| ComputeError::Load {
73            id: self.manifest.id.clone(),
74            msg: "empty container `entry`".into(),
75        })?;
76        let mut cmd = Command::new(prog);
77        cmd.args(parts).current_dir(&self.cwd);
78        Ok(cmd)
79    }
80
81    /// Spawn the transform process, write `payload` to stdin (on a thread, draining stdout to avoid a
82    /// pipe deadlock), return stdout bytes. Fail-closed on spawn failure / non-zero exit.
83    fn exec(&self, id: &str, payload: Vec<u8>) -> Result<Vec<u8>, ComputeError> {
84        let mut child = self
85            .command()?
86            .stdin(Stdio::piped())
87            .stdout(Stdio::piped())
88            .stderr(Stdio::piped())
89            .spawn()
90            .map_err(|e| ComputeError::Run {
91                id: id.to_string(),
92                msg: format!("spawn `{}`: {e}", self.entry),
93            })?;
94
95        let mut stdin = child.stdin.take().ok_or_else(|| ComputeError::Run {
96            id: id.to_string(),
97            msg: "no stdin".into(),
98        })?;
99        let writer = std::thread::spawn(move || stdin.write_all(&payload));
100
101        let mut stdout_buf = Vec::new();
102        let mut stdout = child.stdout.take().ok_or_else(|| ComputeError::Run {
103            id: id.to_string(),
104            msg: "no stdout".into(),
105        })?;
106        stdout.read_to_end(&mut stdout_buf).map_err(|e| ComputeError::Run {
107            id: id.to_string(),
108            msg: format!("read stdout: {e}"),
109        })?;
110        let _ = writer.join();
111
112        let output = child.wait_with_output().map_err(|e| ComputeError::Run {
113            id: id.to_string(),
114            msg: format!("wait: {e}"),
115        })?;
116        if !output.status.success() {
117            let stderr = String::from_utf8_lossy(&output.stderr);
118            let tail = stderr.lines().last().unwrap_or("").trim();
119            return Err(ComputeError::Run {
120                id: id.to_string(),
121                msg: format!("container exited {} ({tail})", output.status),
122            });
123        }
124        Ok(stdout_buf)
125    }
126
127    fn check_output(&self, out: &RecordBatch) -> Result<(), ComputeError> {
128        let expected = self.manifest.output.to_arrow_schema();
129        let got = out.schema();
130        let mismatch = expected.fields().len() != got.fields().len()
131            || expected
132                .fields()
133                .iter()
134                .zip(got.fields().iter())
135                .any(|(e, g)| e.name() != g.name() || e.data_type() != g.data_type());
136        if mismatch {
137            return Err(ComputeError::Schema {
138                id: self.manifest.id.clone(),
139                msg: format!("output schema {got:?} != declared {expected:?}"),
140            });
141        }
142        Ok(())
143    }
144}
145
146impl Compute for ContainerCompute {
147    fn manifest(&self) -> &TransformManifest {
148        &self.manifest
149    }
150
151    fn run(&self, inputs: &[RecordBatch]) -> Result<RecordBatch, ComputeError> {
152        let id = self.manifest.id.clone();
153        if inputs.len() != 1 {
154            return Err(ComputeError::Schema {
155                id,
156                msg: format!("container backend supports a single input (got {})", inputs.len()),
157            });
158        }
159        let out = match self.manifest.protocol {
160            ContainerProtocol::ArrowIpc => {
161                let payload = ipc::write_ipc(&id, &inputs[0].schema(), &inputs[..1])?;
162                let bytes = self.exec(&id, payload)?;
163                let batches = ipc::read_ipc(&id, &bytes)?;
164                if batches.is_empty() {
165                    return Err(ComputeError::Run {
166                        id,
167                        msg: "container produced no Arrow batches".into(),
168                    });
169                }
170                let schema = batches[0].schema();
171                concat_batches(&schema, &batches).map_err(|e| ComputeError::Run {
172                    id: id.clone(),
173                    msg: e.to_string(),
174                })?
175            }
176            ContainerProtocol::Json => {
177                let columns: Vec<String> = inputs[0].schema().fields().iter().map(|f| f.name().clone()).collect();
178                let request = serde_json::json!({
179                    "ref": id,
180                    "config": {},
181                    "mode": "batch",
182                    "inputs": [{ "dataset": "in", "columns": columns, "rows": json::batch_to_json_rows(&inputs[0]) }],
183                });
184                let payload = serde_json::to_vec(&request).map_err(|e| ComputeError::Run {
185                    id: id.clone(),
186                    msg: e.to_string(),
187                })?;
188                let bytes = self.exec(&id, payload)?;
189                let resp: serde_json::Value = serde_json::from_slice(&bytes).map_err(|e| ComputeError::Run {
190                    id: id.clone(),
191                    msg: format!("response is not JSON: {e}"),
192                })?;
193                if let Some(err) = resp.get("error").filter(|e| !e.is_null()) {
194                    return Err(ComputeError::Run {
195                        id,
196                        msg: format!("transform error: {}", err),
197                    });
198                }
199                let rows = resp["rows"].as_array().ok_or_else(|| ComputeError::Run {
200                    id: id.clone(),
201                    msg: "response missing a `rows` array".into(),
202                })?;
203                json::json_rows_to_batch(&id, rows, &self.manifest.output)?
204            }
205        };
206        self.check_output(&out)?;
207        Ok(out)
208    }
209}