Skip to main content

ocr_mcp/
server.rs

1// Copyright (c) 2026 Enzo Lombardi
2// SPDX-License-Identifier: MIT
3
4//! Owning the `llama-server` child process.
5//!
6//! Argument-vector construction, binary lookup and port selection are pure and
7//! tested directly; spawning is exercised separately with a fake binary. The
8//! child's stdout and stderr are always piped, never inherited: this process
9//! speaks JSON-RPC on stdout, and an inherited child's startup banner would
10//! corrupt the framing and wedge the host.
11
12#![allow(
13    clippy::duration_suboptimal_units,
14    reason = "alternatives (from_mins/from_hours) require Rust 1.91; this crate maintains a lower MSRV"
15)]
16
17use crate::config::Config;
18use crate::ocr::OcrError;
19use std::collections::VecDeque;
20use std::io::{BufRead, BufReader, Read};
21use std::path::{Path, PathBuf};
22use std::process::{Command, Stdio};
23use std::sync::{Arc, Mutex};
24use std::time::{Duration, Instant};
25
26/// How long to wait for `/health` before giving up on a freshly spawned child.
27const READY_TIMEOUT: Duration = Duration::from_secs(120);
28/// Gap between `/health` polls.
29const POLL_INTERVAL: Duration = Duration::from_millis(250);
30/// Timeout for a single `/health` liveness probe: it must never let one slow
31/// call push a poll iteration past `READY_TIMEOUT`.
32const HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(2);
33/// How many of the most recent lines of child output to retain for error
34/// messages.
35const TAIL_CAPACITY: usize = 40;
36
37/// A bounded, thread-safe tail of the most recent lines written by the
38/// child's stdout/stderr, used only to enrich error text.
39#[derive(Clone, Default)]
40struct OutputTail(Arc<Mutex<VecDeque<String>>>);
41
42impl OutputTail {
43    fn push(&self, line: String) {
44        let mut lines = self
45            .0
46            .lock()
47            .unwrap_or_else(std::sync::PoisonError::into_inner);
48        if lines.len() >= TAIL_CAPACITY {
49            lines.pop_front();
50        }
51        lines.push_back(line);
52    }
53
54    /// Renders the retained lines, oldest first, joined by newlines.
55    fn snapshot(&self) -> String {
56        let lines = self
57            .0
58            .lock()
59            .unwrap_or_else(std::sync::PoisonError::into_inner);
60        lines.iter().cloned().collect::<Vec<_>>().join("\n")
61    }
62
63    /// Drains `reader` line by line into this tail until EOF. Runs on its
64    /// own thread so the child's pipes never fill and deadlock the process.
65    fn drain(self, reader: impl Read) {
66        let mut reader = BufReader::new(reader);
67        let mut line = String::new();
68        loop {
69            line.clear();
70            match reader.read_line(&mut line) {
71                Ok(0) | Err(_) => break,
72                Ok(_) => {
73                    let trimmed = line.trim_end_matches(['\n', '\r']);
74                    self.push(trimmed.to_string());
75                }
76            }
77        }
78    }
79}
80
81/// Context window to allocate. One image is roughly 2700 tokens and the reply
82/// is capped well below `max_tokens`, so this is generous — but it must be set
83/// explicitly, because llama-server otherwise allocates KV for the model's
84/// full 131072-token context. See `build_argv`.
85const CTX_SIZE: u32 = 12000;
86
87/// Builds the `llama-server` argument vector.
88///
89/// `--flash-attn off` is mandatory: GLM-OCR does not currently work with
90/// flash attention enabled. `--no-warmup` keeps first-spawn latency down,
91/// since the very next thing that happens is a real request.
92///
93/// `--ctx-size` and `--parallel` exist to bound memory, and they matter more
94/// than they look. Left to its defaults llama-server reserves KV for the full
95/// 131072-token context across four parallel slots: 9.78 GB resident for a
96/// 1.4 GB model, held for the entire idle window after every call. This server
97/// is a guest — it typically shares a machine with the far larger model that
98/// called it — and reclaiming that memory when idle is the whole reason it
99/// runs as a separate process. Bounding both measured 2.06 GB with identical
100/// transcriptions and unchanged latency. We serve one request at a time, so
101/// the extra slots buy nothing.
102#[must_use]
103pub fn build_argv(cfg: &Config, port: u16) -> Vec<String> {
104    vec![
105        "--model".into(),
106        cfg.model.display().to_string(),
107        "--mmproj".into(),
108        cfg.mmproj.display().to_string(),
109        "--port".into(),
110        port.to_string(),
111        "--host".into(),
112        "127.0.0.1".into(),
113        "--ctx-size".into(),
114        CTX_SIZE.to_string(),
115        "--parallel".into(),
116        "1".into(),
117        "--no-warmup".into(),
118        "--flash-attn".into(),
119        "off".into(),
120    ]
121}
122
123/// Resolves the `llama-server` binary: the configured path first, then `PATH`.
124///
125/// `exists` is injected so the search is testable without touching the
126/// filesystem.
127///
128/// # Errors
129/// Returns [`OcrError::BinaryMissing`] when neither lookup finds a file.
130pub fn find_binary(
131    cfg: &Config,
132    path_var: Option<&str>,
133    exists: &dyn Fn(&Path) -> bool,
134) -> Result<PathBuf, OcrError> {
135    if let Some(explicit) = &cfg.binary {
136        return if exists(explicit) {
137            Ok(explicit.clone())
138        } else {
139            Err(OcrError::BinaryMissing)
140        };
141    }
142    path_var
143        .unwrap_or_default()
144        .split(':')
145        .filter(|d| !d.is_empty())
146        .map(|d| Path::new(d).join("llama-server"))
147        .find(|c| exists(c))
148        .ok_or(OcrError::BinaryMissing)
149}
150
151/// Asks the OS for a free localhost port by binding and immediately dropping.
152///
153/// # Errors
154/// Returns [`OcrError::SpawnFailed`] when no port can be bound.
155pub fn free_port() -> Result<u16, OcrError> {
156    let listener = std::net::TcpListener::bind("127.0.0.1:0")
157        .map_err(|e| OcrError::SpawnFailed(format!("could not reserve a port: {e}")))?;
158    listener
159        .local_addr()
160        .map(|a| a.port())
161        .map_err(|e| OcrError::SpawnFailed(format!("could not read the reserved port: {e}")))
162}
163
164/// A running `llama-server`, killed on `Drop` so no orphan outlives us.
165pub struct Child {
166    proc: std::process::Child,
167    port: u16,
168    last_used: Instant,
169    idle: Duration,
170    tail: OutputTail,
171    /// Set for the duration of an in-flight request so the idle reaper never
172    /// drops a child a request is actively using.
173    busy: bool,
174}
175
176impl Child {
177    /// Spawns `llama-server` and waits until `/health` answers.
178    ///
179    /// # Errors
180    /// [`OcrError::WeightsMissing`] when the GGUFs are absent,
181    /// [`OcrError::BinaryMissing`] when the binary cannot be found,
182    /// [`OcrError::SpawnFailed`] when the process will not start, and
183    /// [`OcrError::NotReady`] when `/health` never answers in time.
184    pub fn spawn(cfg: &Config) -> Result<Self, OcrError> {
185        if !cfg.model.exists() || !cfg.mmproj.exists() {
186            return Err(OcrError::WeightsMissing {
187                model: cfg.model.clone(),
188                mmproj: cfg.mmproj.clone(),
189                auto_download: cfg.auto_download,
190            });
191        }
192        let path_var = std::env::var("PATH").ok();
193        let bin = find_binary(cfg, path_var.as_deref(), &|p| p.exists())?;
194        let port = free_port()?;
195
196        let mut proc = Command::new(&bin)
197            .args(build_argv(cfg, port))
198            .stdin(Stdio::null())
199            .stdout(Stdio::piped())
200            .stderr(Stdio::piped())
201            .spawn()
202            .map_err(|e| OcrError::SpawnFailed(format!("{}: {e}", bin.display())))?;
203
204        let tail = OutputTail::default();
205        if let Some(stdout) = proc.stdout.take() {
206            let tail = tail.clone();
207            std::thread::spawn(move || tail.drain(stdout));
208        }
209        if let Some(stderr) = proc.stderr.take() {
210            let tail = tail.clone();
211            std::thread::spawn(move || tail.drain(stderr));
212        }
213
214        let mut child = Self {
215            proc,
216            port,
217            last_used: Instant::now(),
218            idle: Duration::from_secs(cfg.idle_secs),
219            tail,
220            busy: false,
221        };
222        child.await_ready()?;
223        Ok(child)
224    }
225
226    /// Polls `/health` until it answers or the timeout expires.
227    fn await_ready(&mut self) -> Result<(), OcrError> {
228        let url = format!("{}/health", self.base_url());
229        let deadline = Instant::now() + READY_TIMEOUT;
230        while Instant::now() < deadline {
231            if let Some(status) = self.proc.try_wait().ok().flatten() {
232                return Err(OcrError::NotReady(format!(
233                    "the process exited early with {status}\n{}",
234                    self.tail.snapshot()
235                )));
236            }
237            if ureq::get(&url)
238                .config()
239                .timeout_global(Some(HEALTH_PROBE_TIMEOUT))
240                .build()
241                .call()
242                .is_ok()
243            {
244                return Ok(());
245            }
246            std::thread::sleep(POLL_INTERVAL);
247        }
248        Err(OcrError::NotReady(format!(
249            "no answer from {url} after {}s\n{}",
250            READY_TIMEOUT.as_secs(),
251            self.tail.snapshot()
252        )))
253    }
254
255    /// The `http://127.0.0.1:<port>` root this child serves.
256    #[must_use]
257    pub fn base_url(&self) -> String {
258        format!("http://127.0.0.1:{}", self.port)
259    }
260
261    /// Records that the child was just used, restarting the idle countdown.
262    pub fn touch(&mut self) {
263        self.last_used = Instant::now();
264    }
265
266    /// True when the child has gone unused for longer than the idle timeout.
267    #[must_use]
268    pub fn idle_expired(&self, now: Instant) -> bool {
269        expired(self.last_used, self.idle, now)
270    }
271
272    /// Marks the child as actively serving (or having finished) a request.
273    /// While `true`, [`Child::should_reap`] always returns `false`: a request
274    /// in flight must never have its child dropped out from under it.
275    pub fn set_busy(&mut self, busy: bool) {
276        self.busy = busy;
277    }
278
279    /// True when this child is safe to drop: idle-expired and not currently
280    /// serving a request.
281    #[must_use]
282    pub fn should_reap(&self, now: Instant) -> bool {
283        should_reap(self.busy, self.last_used, self.idle, now)
284    }
285}
286
287/// The idle-expiry rule: has `idle` elapsed since `last_used`, as of `now`?
288/// Pulled out of [`Child::idle_expired`] so it can be exercised without a
289/// process.
290fn expired(last_used: Instant, idle: Duration, now: Instant) -> bool {
291    now.duration_since(last_used) >= idle
292}
293
294/// The full reaping decision: idle-expired *and* not mid-request. Pulled out
295/// of [`Child::should_reap`] so the reaper/request race can be exercised
296/// without a process.
297fn should_reap(busy: bool, last_used: Instant, idle: Duration, now: Instant) -> bool {
298    !busy && expired(last_used, idle, now)
299}
300
301impl Drop for Child {
302    fn drop(&mut self) {
303        let _ = self.proc.kill();
304        let _ = self.proc.wait();
305    }
306}
307
308#[cfg(test)]
309mod tests {
310    use super::*;
311
312    fn cfg() -> Config {
313        Config {
314            model: PathBuf::from("/w/glm-ocr.gguf"),
315            mmproj: PathBuf::from("/w/glm-ocr-mmproj.gguf"),
316            binary: None,
317            idle_secs: 300,
318            max_tokens: 1024,
319            auto_download: false,
320        }
321    }
322
323    /// GLM-OCR does not work under llama-server with flash attention on. This
324    /// assertion is the whole reason the argv is a pure function.
325    #[test]
326    fn argv_disables_flash_attention() {
327        let argv = build_argv(&cfg(), 8123);
328        let joined = argv.join(" ");
329        assert!(joined.contains("--flash-attn off"), "got: {joined}");
330    }
331
332    #[test]
333    fn argv_names_model_projector_and_port() {
334        let argv = build_argv(&cfg(), 8123);
335        let pair = |flag: &str| {
336            argv.iter()
337                .position(|a| a == flag)
338                .map(|i| argv[i + 1].clone())
339        };
340        assert_eq!(pair("--model").as_deref(), Some("/w/glm-ocr.gguf"));
341        assert_eq!(pair("--mmproj").as_deref(), Some("/w/glm-ocr-mmproj.gguf"));
342        assert_eq!(pair("--port").as_deref(), Some("8123"));
343        assert!(argv.iter().any(|a| a == "--no-warmup"));
344    }
345
346    /// The context and slot count must be bounded explicitly. Left to its
347    /// defaults, llama-server allocates KV for the model's full 131072-token
348    /// context across 4 parallel slots — measured at 9.78 GB resident for a
349    /// 1.4 GB model, held for the whole idle window. On a machine also hosting
350    /// an 81 GB ds4, that eviction is what drags the host model into swap.
351    /// Bounding it measured 2.06 GB with identical output and latency.
352    #[test]
353    fn argv_bounds_context_and_slots() {
354        let argv = build_argv(&cfg(), 8123);
355        let pair = |flag: &str| {
356            argv.iter()
357                .position(|a| a == flag)
358                .map(|i| argv[i + 1].clone())
359        };
360        assert_eq!(pair("--ctx-size").as_deref(), Some("12000"));
361        assert_eq!(pair("--parallel").as_deref(), Some("1"));
362    }
363
364    #[test]
365    fn configured_binary_wins_over_path() {
366        let mut c = cfg();
367        c.binary = Some(PathBuf::from("/opt/llama-server"));
368        let found = find_binary(&c, Some("/usr/bin:/bin"), &|p| {
369            p == Path::new("/opt/llama-server")
370        });
371        assert_eq!(found.unwrap(), PathBuf::from("/opt/llama-server"));
372    }
373
374    #[test]
375    fn path_is_searched_when_no_binary_configured() {
376        let found = find_binary(&cfg(), Some("/usr/bin:/opt/bin"), &|p| {
377            p == Path::new("/opt/bin/llama-server")
378        });
379        assert_eq!(found.unwrap(), PathBuf::from("/opt/bin/llama-server"));
380    }
381
382    #[test]
383    fn missing_binary_is_an_error() {
384        let e = find_binary(&cfg(), Some("/usr/bin"), &|_| false).unwrap_err();
385        assert_eq!(e, OcrError::BinaryMissing);
386
387        let mut c = cfg();
388        c.binary = Some(PathBuf::from("/nope/llama-server"));
389        assert_eq!(
390            find_binary(&c, None, &|_| false).unwrap_err(),
391            OcrError::BinaryMissing
392        );
393    }
394
395    #[test]
396    fn free_port_is_usable_and_nonzero() {
397        let p = free_port().unwrap();
398        assert!(p > 0);
399        // Binding it again must succeed: the probe socket has been dropped.
400        assert!(std::net::TcpListener::bind(("127.0.0.1", p)).is_ok());
401    }
402
403    /// The idle rule is arithmetic on an injected `now`, so it needs no
404    /// process and no sleeping.
405    #[test]
406    fn idle_expiry_is_measured_from_the_last_use() {
407        let idle = Duration::from_secs(300);
408        let last_used = Instant::now();
409        let at = |elapsed: Duration| last_used + elapsed;
410        assert!(!expired(last_used, idle, at(Duration::from_secs(299))));
411        assert!(expired(last_used, idle, at(Duration::from_secs(300))));
412        assert!(expired(last_used, idle, at(Duration::from_secs(3600))));
413    }
414
415    /// The reaper must never win a race against an in-flight request: a busy
416    /// child is never reaped no matter how idle-expired it looks, because
417    /// `last_used` is only bumped once the request finishes.
418    #[test]
419    fn busy_child_is_never_reaped_even_when_idle_expired() {
420        let idle = Duration::from_secs(300);
421        let last_used = Instant::now();
422        let long_after = last_used + Duration::from_secs(3600);
423        assert!(!should_reap(true, last_used, idle, long_after));
424        assert!(should_reap(false, last_used, idle, long_after));
425        assert!(!should_reap(false, last_used, idle, last_used));
426    }
427
428    /// Pushing more lines than the cap must keep only the most recent ones,
429    /// in order, with no process involved.
430    #[test]
431    fn output_tail_keeps_only_the_most_recent_lines() {
432        let tail = OutputTail::default();
433        for i in 0..(TAIL_CAPACITY + 10) {
434            tail.push(format!("line {i}"));
435        }
436        let snapshot = tail.snapshot();
437        let lines: Vec<&str> = snapshot.lines().collect();
438        assert_eq!(lines.len(), TAIL_CAPACITY);
439        assert_eq!(lines.first(), Some(&"line 10"));
440        assert_eq!(
441            lines.last().copied(),
442            Some(format!("line {}", TAIL_CAPACITY + 9)).as_deref()
443        );
444    }
445}