fv_compute_container/
lib.rs1#[cfg(doctest)]
13#[doc = include_str!("../README.md")]
14pub struct ReadmeDoctests;
15
16use std::io::{Read, Write};
17use std::path::{Path, PathBuf};
18use std::process::{Command, Stdio};
19
20use arrow::array::RecordBatch;
21use arrow::compute::concat_batches;
22use fv_compute::{Compute, ComputeError, ContainerProtocol, ImplKind, TransformBackend, TransformManifest};
23
24mod ipc;
25mod json;
26
27#[derive(Default)]
30pub struct ContainerBackend;
31
32impl ContainerBackend {
33 pub fn new() -> Self {
34 Self
35 }
36}
37
38impl TransformBackend for ContainerBackend {
39 fn kind(&self) -> ImplKind {
40 ImplKind::Container
41 }
42
43 fn load(&self, manifest: &TransformManifest, root: &Path) -> Result<Box<dyn Compute>, ComputeError> {
44 let entry = manifest
45 .entry
46 .as_deref()
47 .filter(|s| !s.trim().is_empty())
48 .ok_or_else(|| ComputeError::Load {
49 id: manifest.id.clone(),
50 msg: "container impl needs an `entry` command (e.g. \"python3 t.py\" or \"docker run …\")".into(),
51 })?
52 .to_string();
53 Ok(Box::new(ContainerCompute {
54 manifest: manifest.clone(),
55 entry,
56 cwd: root.to_path_buf(),
57 }))
58 }
59}
60
61pub struct ContainerCompute {
64 manifest: TransformManifest,
65 entry: String,
66 cwd: PathBuf,
67}
68
69impl ContainerCompute {
70 fn command(&self) -> Result<Command, ComputeError> {
71 let mut parts = self.entry.split_whitespace();
72 let prog = parts.next().ok_or_else(|| ComputeError::Load {
73 id: self.manifest.id.clone(),
74 msg: "empty container `entry`".into(),
75 })?;
76 let mut cmd = Command::new(prog);
77 cmd.args(parts).current_dir(&self.cwd);
78 Ok(cmd)
79 }
80
81 fn exec(&self, id: &str, payload: Vec<u8>) -> Result<Vec<u8>, ComputeError> {
84 let mut child = self
85 .command()?
86 .stdin(Stdio::piped())
87 .stdout(Stdio::piped())
88 .stderr(Stdio::piped())
89 .spawn()
90 .map_err(|e| ComputeError::Run {
91 id: id.to_string(),
92 msg: format!("spawn `{}`: {e}", self.entry),
93 })?;
94
95 let mut stdin = child.stdin.take().ok_or_else(|| ComputeError::Run {
96 id: id.to_string(),
97 msg: "no stdin".into(),
98 })?;
99 let writer = std::thread::spawn(move || stdin.write_all(&payload));
100
101 let mut stdout_buf = Vec::new();
102 let mut stdout = child.stdout.take().ok_or_else(|| ComputeError::Run {
103 id: id.to_string(),
104 msg: "no stdout".into(),
105 })?;
106 stdout.read_to_end(&mut stdout_buf).map_err(|e| ComputeError::Run {
107 id: id.to_string(),
108 msg: format!("read stdout: {e}"),
109 })?;
110 let _ = writer.join();
111
112 let output = child.wait_with_output().map_err(|e| ComputeError::Run {
113 id: id.to_string(),
114 msg: format!("wait: {e}"),
115 })?;
116 if !output.status.success() {
117 let stderr = String::from_utf8_lossy(&output.stderr);
118 let tail = stderr.lines().last().unwrap_or("").trim();
119 return Err(ComputeError::Run {
120 id: id.to_string(),
121 msg: format!("container exited {} ({tail})", output.status),
122 });
123 }
124 Ok(stdout_buf)
125 }
126
127 fn check_output(&self, out: &RecordBatch) -> Result<(), ComputeError> {
128 let expected = self.manifest.output.to_arrow_schema();
129 let got = out.schema();
130 let mismatch = expected.fields().len() != got.fields().len()
131 || expected
132 .fields()
133 .iter()
134 .zip(got.fields().iter())
135 .any(|(e, g)| e.name() != g.name() || e.data_type() != g.data_type());
136 if mismatch {
137 return Err(ComputeError::Schema {
138 id: self.manifest.id.clone(),
139 msg: format!("output schema {got:?} != declared {expected:?}"),
140 });
141 }
142 Ok(())
143 }
144}
145
146impl Compute for ContainerCompute {
147 fn manifest(&self) -> &TransformManifest {
148 &self.manifest
149 }
150
151 fn run(&self, inputs: &[RecordBatch]) -> Result<RecordBatch, ComputeError> {
152 let id = self.manifest.id.clone();
153 if inputs.len() != 1 {
154 return Err(ComputeError::Schema {
155 id,
156 msg: format!("container backend supports a single input (got {})", inputs.len()),
157 });
158 }
159 let out = match self.manifest.protocol {
160 ContainerProtocol::ArrowIpc => {
161 let payload = ipc::write_ipc(&id, &inputs[0].schema(), &inputs[..1])?;
162 let bytes = self.exec(&id, payload)?;
163 let batches = ipc::read_ipc(&id, &bytes)?;
164 if batches.is_empty() {
165 return Err(ComputeError::Run {
166 id,
167 msg: "container produced no Arrow batches".into(),
168 });
169 }
170 let schema = batches[0].schema();
171 concat_batches(&schema, &batches).map_err(|e| ComputeError::Run {
172 id: id.clone(),
173 msg: e.to_string(),
174 })?
175 }
176 ContainerProtocol::Json => {
177 let columns: Vec<String> = inputs[0].schema().fields().iter().map(|f| f.name().clone()).collect();
178 let request = serde_json::json!({
179 "ref": id,
180 "config": {},
181 "mode": "batch",
182 "inputs": [{ "dataset": "in", "columns": columns, "rows": json::batch_to_json_rows(&inputs[0]) }],
183 });
184 let payload = serde_json::to_vec(&request).map_err(|e| ComputeError::Run {
185 id: id.clone(),
186 msg: e.to_string(),
187 })?;
188 let bytes = self.exec(&id, payload)?;
189 let resp: serde_json::Value = serde_json::from_slice(&bytes).map_err(|e| ComputeError::Run {
190 id: id.clone(),
191 msg: format!("response is not JSON: {e}"),
192 })?;
193 if let Some(err) = resp.get("error").filter(|e| !e.is_null()) {
194 return Err(ComputeError::Run {
195 id,
196 msg: format!("transform error: {}", err),
197 });
198 }
199 let rows = resp["rows"].as_array().ok_or_else(|| ComputeError::Run {
200 id: id.clone(),
201 msg: "response missing a `rows` array".into(),
202 })?;
203 json::json_rows_to_batch(&id, rows, &self.manifest.output)?
204 }
205 };
206 self.check_output(&out)?;
207 Ok(out)
208 }
209}