1#![allow(
13 clippy::duration_suboptimal_units,
14 reason = "alternatives (from_mins/from_hours) require Rust 1.91; this crate maintains a lower MSRV"
15)]
16
17use crate::config::Config;
18use crate::ocr::OcrError;
19use std::collections::VecDeque;
20use std::io::{BufRead, BufReader, Read};
21use std::path::{Path, PathBuf};
22use std::process::{Command, Stdio};
23use std::sync::{Arc, Mutex};
24use std::time::{Duration, Instant};
25
26const READY_TIMEOUT: Duration = Duration::from_secs(120);
28const POLL_INTERVAL: Duration = Duration::from_millis(250);
30const HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(2);
33const TAIL_CAPACITY: usize = 40;
36
37#[derive(Clone, Default)]
40struct OutputTail(Arc<Mutex<VecDeque<String>>>);
41
42impl OutputTail {
43 fn push(&self, line: String) {
44 let mut lines = self
45 .0
46 .lock()
47 .unwrap_or_else(std::sync::PoisonError::into_inner);
48 if lines.len() >= TAIL_CAPACITY {
49 lines.pop_front();
50 }
51 lines.push_back(line);
52 }
53
54 fn snapshot(&self) -> String {
56 let lines = self
57 .0
58 .lock()
59 .unwrap_or_else(std::sync::PoisonError::into_inner);
60 lines.iter().cloned().collect::<Vec<_>>().join("\n")
61 }
62
63 fn drain(self, reader: impl Read) {
66 let mut reader = BufReader::new(reader);
67 let mut line = String::new();
68 loop {
69 line.clear();
70 match reader.read_line(&mut line) {
71 Ok(0) | Err(_) => break,
72 Ok(_) => {
73 let trimmed = line.trim_end_matches(['\n', '\r']);
74 self.push(trimmed.to_string());
75 }
76 }
77 }
78 }
79}
80
81const CTX_SIZE: u32 = 12000;
86
87#[must_use]
103pub fn build_argv(cfg: &Config, port: u16) -> Vec<String> {
104 vec![
105 "--model".into(),
106 cfg.model.display().to_string(),
107 "--mmproj".into(),
108 cfg.mmproj.display().to_string(),
109 "--port".into(),
110 port.to_string(),
111 "--host".into(),
112 "127.0.0.1".into(),
113 "--ctx-size".into(),
114 CTX_SIZE.to_string(),
115 "--parallel".into(),
116 "1".into(),
117 "--no-warmup".into(),
118 "--flash-attn".into(),
119 "off".into(),
120 ]
121}
122
123pub fn find_binary(
131 cfg: &Config,
132 path_var: Option<&str>,
133 exists: &dyn Fn(&Path) -> bool,
134) -> Result<PathBuf, OcrError> {
135 if let Some(explicit) = &cfg.binary {
136 return if exists(explicit) {
137 Ok(explicit.clone())
138 } else {
139 Err(OcrError::BinaryMissing)
140 };
141 }
142 path_var
143 .unwrap_or_default()
144 .split(':')
145 .filter(|d| !d.is_empty())
146 .map(|d| Path::new(d).join("llama-server"))
147 .find(|c| exists(c))
148 .ok_or(OcrError::BinaryMissing)
149}
150
151pub fn free_port() -> Result<u16, OcrError> {
156 let listener = std::net::TcpListener::bind("127.0.0.1:0")
157 .map_err(|e| OcrError::SpawnFailed(format!("could not reserve a port: {e}")))?;
158 listener
159 .local_addr()
160 .map(|a| a.port())
161 .map_err(|e| OcrError::SpawnFailed(format!("could not read the reserved port: {e}")))
162}
163
164pub struct Child {
166 proc: std::process::Child,
167 port: u16,
168 last_used: Instant,
169 idle: Duration,
170 tail: OutputTail,
171 busy: bool,
174}
175
176impl Child {
177 pub fn spawn(cfg: &Config) -> Result<Self, OcrError> {
185 if !cfg.model.exists() || !cfg.mmproj.exists() {
186 return Err(OcrError::WeightsMissing {
187 model: cfg.model.clone(),
188 mmproj: cfg.mmproj.clone(),
189 auto_download: cfg.auto_download,
190 });
191 }
192 let path_var = std::env::var("PATH").ok();
193 let bin = find_binary(cfg, path_var.as_deref(), &|p| p.exists())?;
194 let port = free_port()?;
195
196 let mut proc = Command::new(&bin)
197 .args(build_argv(cfg, port))
198 .stdin(Stdio::null())
199 .stdout(Stdio::piped())
200 .stderr(Stdio::piped())
201 .spawn()
202 .map_err(|e| OcrError::SpawnFailed(format!("{}: {e}", bin.display())))?;
203
204 let tail = OutputTail::default();
205 if let Some(stdout) = proc.stdout.take() {
206 let tail = tail.clone();
207 std::thread::spawn(move || tail.drain(stdout));
208 }
209 if let Some(stderr) = proc.stderr.take() {
210 let tail = tail.clone();
211 std::thread::spawn(move || tail.drain(stderr));
212 }
213
214 let mut child = Self {
215 proc,
216 port,
217 last_used: Instant::now(),
218 idle: Duration::from_secs(cfg.idle_secs),
219 tail,
220 busy: false,
221 };
222 child.await_ready()?;
223 Ok(child)
224 }
225
226 fn await_ready(&mut self) -> Result<(), OcrError> {
228 let url = format!("{}/health", self.base_url());
229 let deadline = Instant::now() + READY_TIMEOUT;
230 while Instant::now() < deadline {
231 if let Some(status) = self.proc.try_wait().ok().flatten() {
232 return Err(OcrError::NotReady(format!(
233 "the process exited early with {status}\n{}",
234 self.tail.snapshot()
235 )));
236 }
237 if ureq::get(&url)
238 .config()
239 .timeout_global(Some(HEALTH_PROBE_TIMEOUT))
240 .build()
241 .call()
242 .is_ok()
243 {
244 return Ok(());
245 }
246 std::thread::sleep(POLL_INTERVAL);
247 }
248 Err(OcrError::NotReady(format!(
249 "no answer from {url} after {}s\n{}",
250 READY_TIMEOUT.as_secs(),
251 self.tail.snapshot()
252 )))
253 }
254
255 #[must_use]
257 pub fn base_url(&self) -> String {
258 format!("http://127.0.0.1:{}", self.port)
259 }
260
261 pub fn touch(&mut self) {
263 self.last_used = Instant::now();
264 }
265
266 #[must_use]
268 pub fn idle_expired(&self, now: Instant) -> bool {
269 expired(self.last_used, self.idle, now)
270 }
271
272 pub fn set_busy(&mut self, busy: bool) {
276 self.busy = busy;
277 }
278
279 #[must_use]
282 pub fn should_reap(&self, now: Instant) -> bool {
283 should_reap(self.busy, self.last_used, self.idle, now)
284 }
285}
286
287fn expired(last_used: Instant, idle: Duration, now: Instant) -> bool {
291 now.duration_since(last_used) >= idle
292}
293
294fn should_reap(busy: bool, last_used: Instant, idle: Duration, now: Instant) -> bool {
298 !busy && expired(last_used, idle, now)
299}
300
301impl Drop for Child {
302 fn drop(&mut self) {
303 let _ = self.proc.kill();
304 let _ = self.proc.wait();
305 }
306}
307
308#[cfg(test)]
309mod tests {
310 use super::*;
311
312 fn cfg() -> Config {
313 Config {
314 model: PathBuf::from("/w/glm-ocr.gguf"),
315 mmproj: PathBuf::from("/w/glm-ocr-mmproj.gguf"),
316 binary: None,
317 idle_secs: 300,
318 max_tokens: 1024,
319 auto_download: false,
320 }
321 }
322
323 #[test]
326 fn argv_disables_flash_attention() {
327 let argv = build_argv(&cfg(), 8123);
328 let joined = argv.join(" ");
329 assert!(joined.contains("--flash-attn off"), "got: {joined}");
330 }
331
332 #[test]
333 fn argv_names_model_projector_and_port() {
334 let argv = build_argv(&cfg(), 8123);
335 let pair = |flag: &str| {
336 argv.iter()
337 .position(|a| a == flag)
338 .map(|i| argv[i + 1].clone())
339 };
340 assert_eq!(pair("--model").as_deref(), Some("/w/glm-ocr.gguf"));
341 assert_eq!(pair("--mmproj").as_deref(), Some("/w/glm-ocr-mmproj.gguf"));
342 assert_eq!(pair("--port").as_deref(), Some("8123"));
343 assert!(argv.iter().any(|a| a == "--no-warmup"));
344 }
345
346 #[test]
353 fn argv_bounds_context_and_slots() {
354 let argv = build_argv(&cfg(), 8123);
355 let pair = |flag: &str| {
356 argv.iter()
357 .position(|a| a == flag)
358 .map(|i| argv[i + 1].clone())
359 };
360 assert_eq!(pair("--ctx-size").as_deref(), Some("12000"));
361 assert_eq!(pair("--parallel").as_deref(), Some("1"));
362 }
363
364 #[test]
365 fn configured_binary_wins_over_path() {
366 let mut c = cfg();
367 c.binary = Some(PathBuf::from("/opt/llama-server"));
368 let found = find_binary(&c, Some("/usr/bin:/bin"), &|p| {
369 p == Path::new("/opt/llama-server")
370 });
371 assert_eq!(found.unwrap(), PathBuf::from("/opt/llama-server"));
372 }
373
374 #[test]
375 fn path_is_searched_when_no_binary_configured() {
376 let found = find_binary(&cfg(), Some("/usr/bin:/opt/bin"), &|p| {
377 p == Path::new("/opt/bin/llama-server")
378 });
379 assert_eq!(found.unwrap(), PathBuf::from("/opt/bin/llama-server"));
380 }
381
382 #[test]
383 fn missing_binary_is_an_error() {
384 let e = find_binary(&cfg(), Some("/usr/bin"), &|_| false).unwrap_err();
385 assert_eq!(e, OcrError::BinaryMissing);
386
387 let mut c = cfg();
388 c.binary = Some(PathBuf::from("/nope/llama-server"));
389 assert_eq!(
390 find_binary(&c, None, &|_| false).unwrap_err(),
391 OcrError::BinaryMissing
392 );
393 }
394
395 #[test]
396 fn free_port_is_usable_and_nonzero() {
397 let p = free_port().unwrap();
398 assert!(p > 0);
399 assert!(std::net::TcpListener::bind(("127.0.0.1", p)).is_ok());
401 }
402
403 #[test]
406 fn idle_expiry_is_measured_from_the_last_use() {
407 let idle = Duration::from_secs(300);
408 let last_used = Instant::now();
409 let at = |elapsed: Duration| last_used + elapsed;
410 assert!(!expired(last_used, idle, at(Duration::from_secs(299))));
411 assert!(expired(last_used, idle, at(Duration::from_secs(300))));
412 assert!(expired(last_used, idle, at(Duration::from_secs(3600))));
413 }
414
415 #[test]
419 fn busy_child_is_never_reaped_even_when_idle_expired() {
420 let idle = Duration::from_secs(300);
421 let last_used = Instant::now();
422 let long_after = last_used + Duration::from_secs(3600);
423 assert!(!should_reap(true, last_used, idle, long_after));
424 assert!(should_reap(false, last_used, idle, long_after));
425 assert!(!should_reap(false, last_used, idle, last_used));
426 }
427
428 #[test]
431 fn output_tail_keeps_only_the_most_recent_lines() {
432 let tail = OutputTail::default();
433 for i in 0..(TAIL_CAPACITY + 10) {
434 tail.push(format!("line {i}"));
435 }
436 let snapshot = tail.snapshot();
437 let lines: Vec<&str> = snapshot.lines().collect();
438 assert_eq!(lines.len(), TAIL_CAPACITY);
439 assert_eq!(lines.first(), Some(&"line 10"));
440 assert_eq!(
441 lines.last().copied(),
442 Some(format!("line {}", TAIL_CAPACITY + 9)).as_deref()
443 );
444 }
445}