Skip to main content

rightkit_sync/helper/
process.rs

1//! Child-process line-JSON client. One long-lived helper, respawned after any
2//! failure (timeout, crash, protocol error); the helper keeps no state that
3//! matters across restarts because the cursor lives in the ledger.
4
5use std::collections::VecDeque;
6use std::io::{BufRead, BufReader, Read, Write};
7use std::path::PathBuf;
8use std::process::{Child, ChildStdin, Command, Stdio};
9use std::sync::atomic::{AtomicU64, Ordering};
10use std::sync::mpsc::{self, Receiver, RecvTimeoutError};
11use std::sync::{Arc, Mutex};
12use std::time::Duration;
13
14use serde_json::{Map, Value};
15
16use super::wire::{Response, PROTOCOL_VERSION};
17use crate::error::{SyncError, SyncResult};
18
19/// Largest accepted response line. A line over the cap kills the helper.
20const MAX_LINE_BYTES: usize = 128 * 1024 * 1024;
21/// Largest piece of one helper stderr line kept in memory; the tail keeps the last 20 pieces.
22const MAX_STDERR_LINE_BYTES: usize = 64 * 1024;
23
24#[derive(Debug, Clone)]
25pub struct HelperConfig {
26    pub program: PathBuf,
27    pub args: Vec<String>,
28    pub envs: Vec<(String, String)>,
29    /// Per-request deadline (donor: 45 s; CloudKit batches can be slow).
30    pub request_timeout: Duration,
31}
32
33impl HelperConfig {
34    pub fn new(program: impl Into<PathBuf>) -> Self {
35        Self {
36            program: program.into(),
37            args: Vec::new(),
38            envs: Vec::new(),
39            request_timeout: Duration::from_secs(120),
40        }
41    }
42    pub fn arg(mut self, a: impl Into<String>) -> Self {
43        self.args.push(a.into());
44        self
45    }
46    pub fn env(mut self, k: impl Into<String>, v: impl Into<String>) -> Self {
47        self.envs.push((k.into(), v.into()));
48        self
49    }
50    pub fn timeout(mut self, d: Duration) -> Self {
51        self.request_timeout = d;
52        self
53    }
54    /// Arguments the real `rightkit-cloudkit-helper` needs.
55    pub fn cloudkit(self, container: &str, zone: &str, subscription: &str) -> Self {
56        self.arg("--container")
57            .arg(container)
58            .arg("--zone")
59            .arg(zone)
60            .arg("--subscription")
61            .arg(subscription)
62    }
63}
64
65struct Live {
66    child: Child,
67    stdin: ChildStdin,
68    lines: Receiver<Result<String, String>>,
69}
70
71pub struct HelperProcess {
72    cfg: HelperConfig,
73    live: Mutex<Option<Live>>,
74    next_id: AtomicU64,
75    stderr_tail: Arc<Mutex<VecDeque<String>>>,
76    spawns: AtomicU64,
77}
78
79impl HelperProcess {
80    pub fn new(cfg: HelperConfig) -> Self {
81        Self {
82            cfg,
83            live: Mutex::new(None),
84            next_id: AtomicU64::new(1),
85            stderr_tail: Arc::new(Mutex::new(VecDeque::new())),
86            spawns: AtomicU64::new(0),
87        }
88    }
89
90    /// How many helper processes were started (observability and tests).
91    pub fn spawn_count(&self) -> u64 {
92        self.spawns.load(Ordering::Relaxed)
93    }
94
95    fn spawn(&self) -> SyncResult<Live> {
96        let mut cmd = Command::new(&self.cfg.program);
97        cmd.args(&self.cfg.args)
98            .stdin(Stdio::piped())
99            .stdout(Stdio::piped())
100            .stderr(Stdio::piped());
101        #[cfg(windows)]
102        {
103            use std::os::windows::process::CommandExt;
104            cmd.creation_flags(0x0800_0000); // CREATE_NO_WINDOW: never flash a console from a GUI host
105        }
106        for (k, v) in &self.cfg.envs {
107            cmd.env(k, v);
108        }
109        let mut child = cmd.spawn().map_err(|e| {
110            SyncError::Unavailable(format!(
111                "sync helper {} could not start: {e}",
112                self.cfg.program.display()
113            ))
114        })?;
115        self.spawns.fetch_add(1, Ordering::Relaxed);
116        let stdin = child.stdin.take().expect("piped stdin");
117        let stdout = child.stdout.take().expect("piped stdout");
118        let stderr = child.stderr.take().expect("piped stderr");
119        // One line of queue: the reader waits for the caller before it reads further, so a helper
120        // that keeps writing cannot queue response lines without bound (at most two lines of at
121        // most `MAX_LINE_BYTES` are held: one queued, one being read).
122        let (tx, rx) = mpsc::sync_channel(1);
123        std::thread::spawn(move || {
124            let mut reader = BufReader::new(stdout);
125            loop {
126                let mut line = String::new();
127                match reader
128                    .by_ref()
129                    .take(MAX_LINE_BYTES as u64 + 1)
130                    .read_line(&mut line)
131                {
132                    Ok(0) => {
133                        let _ = tx.send(Err("helper closed stdout".to_string()));
134                        return;
135                    }
136                    Ok(_) if line.len() > MAX_LINE_BYTES => {
137                        let _ = tx.send(Err("helper response line too large".to_string()));
138                        return;
139                    }
140                    Ok(_) => {
141                        if tx.send(Ok(line)).is_err() {
142                            return;
143                        }
144                    }
145                    Err(e) => {
146                        let _ = tx.send(Err(e.to_string()));
147                        return;
148                    }
149                }
150            }
151        });
152        let tail = Arc::clone(&self.stderr_tail);
153        std::thread::spawn(move || {
154            // Bounded like the stdout reader: a line longer than the cap is read as several
155            // lines, so a helper that never writes a newline cannot grow this buffer.
156            let mut reader = BufReader::new(stderr);
157            loop {
158                let mut line = String::new();
159                match reader
160                    .by_ref()
161                    .take(MAX_STDERR_LINE_BYTES as u64)
162                    .read_line(&mut line)
163                {
164                    Ok(0) | Err(_) => return,
165                    Ok(_) => {
166                        let mut t = tail.lock().unwrap();
167                        if t.len() >= 20 {
168                            t.pop_front();
169                        }
170                        t.push_back(line.trim_end_matches(['\r', '\n']).to_string());
171                    }
172                }
173            }
174        });
175        Ok(Live {
176            child,
177            stdin,
178            lines: rx,
179        })
180    }
181
182    fn stderr_summary(&self) -> String {
183        let t = self.stderr_tail.lock().unwrap();
184        t.back().cloned().unwrap_or_default()
185    }
186
187    fn kill(live: Option<Live>) {
188        if let Some(mut l) = live {
189            let _ = l.child.kill();
190            let _ = l.child.wait();
191        }
192    }
193
194    /// Stop the helper (also happens on drop).
195    pub fn shutdown(&self) {
196        let live = self.live.lock().unwrap().take();
197        Self::kill(live);
198    }
199
200    /// One request, one response. Transport-level failures (spawn, write,
201    /// timeout, EOF, malformed line) kill the helper and surface as retryable
202    /// errors; helper-reported errors come back as `Ok(Response{ok:false})`
203    /// mapped by the caller.
204    pub fn call(&self, cmd: &str, params: Map<String, Value>) -> SyncResult<Response> {
205        let id = self.next_id.fetch_add(1, Ordering::Relaxed);
206        let mut req = params;
207        req.insert("v".into(), PROTOCOL_VERSION.into());
208        req.insert("id".into(), id.into());
209        req.insert("cmd".into(), cmd.into());
210        let line = Value::Object(req).to_string();
211
212        let mut guard = self.live.lock().unwrap();
213        if guard.is_none() {
214            *guard = Some(self.spawn()?);
215        }
216        let outcome = (|| -> Result<Response, String> {
217            let live = guard.as_mut().expect("spawned");
218            live.stdin
219                .write_all(line.as_bytes())
220                .and_then(|()| live.stdin.write_all(b"\n"))
221                .and_then(|()| live.stdin.flush())
222                .map_err(|e| format!("write to helper failed: {e}"))?;
223            let deadline = std::time::Instant::now() + self.cfg.request_timeout;
224            loop {
225                let left = deadline.saturating_duration_since(std::time::Instant::now());
226                match live.lines.recv_timeout(left) {
227                    Ok(Ok(text)) => {
228                        let resp: Response = serde_json::from_str(text.trim())
229                            .map_err(|e| format!("malformed helper response: {e}"))?;
230                        if resp.id == id {
231                            return Ok(resp);
232                        }
233                        // Stale id from a request we already abandoned.
234                    }
235                    Ok(Err(e)) => return Err(e),
236                    Err(RecvTimeoutError::Timeout) => {
237                        return Err(format!(
238                            "helper timed out after {:?}",
239                            self.cfg.request_timeout
240                        ))
241                    }
242                    Err(RecvTimeoutError::Disconnected) => {
243                        return Err("helper reader stopped".to_string())
244                    }
245                }
246            }
247        })();
248        match outcome {
249            Ok(resp) => Ok(resp),
250            Err(why) => {
251                Self::kill(guard.take());
252                let tail = self.stderr_summary();
253                let detail = if tail.is_empty() {
254                    why
255                } else {
256                    format!("{why} ({tail})")
257                };
258                Err(SyncError::RetryAfter { secs: 15.0, detail })
259            }
260        }
261    }
262}
263
264impl Drop for HelperProcess {
265    fn drop(&mut self) {
266        let live = self.live.get_mut().ok().and_then(Option::take);
267        Self::kill(live);
268    }
269}