Skip to main content

visual_rubric/
pool.rs

1use std::ffi::OsString;
2use std::fs;
3use std::path::{Path, PathBuf};
4use std::sync::atomic::{AtomicUsize, Ordering};
5use std::sync::{Arc, Mutex, mpsc};
6use std::thread::{self, JoinHandle};
7use std::time::Duration;
8
9use rand::RngExt as _;
10use tempfile::TempDir;
11
12#[cfg(feature = "codex-acp")]
13mod codex_home;
14mod config;
15mod state;
16
17pub use config::{LogCaptureConfig, LogPathMode, PoolConfig, PoolStats};
18use state::SharedPoolState;
19
20use crate::{
21    AcpClient, DEFAULT_SYSTEM_PROMPT, PoolError, RateLimitEvent, RubricOptions, RubricVerdict,
22    encode_png, parse_verdict,
23};
24#[cfg(feature = "codex-acp")]
25use crate::{DEFAULT_CODEX_ACP_MODEL, DEFAULT_CODEX_ACP_REASONING_EFFORT};
26#[cfg(feature = "codex-acp")]
27use codex_home::seed_codex_home;
28
29const RECYCLE_SPAWN_ATTEMPTS: u32 = 2;
30
31/// Reusable worker pool for evaluating screenshot rubrics through Codex ACP.
32#[derive(Debug)]
33pub struct RubricPool {
34    senders: Vec<mpsc::Sender<Job>>,
35    handles: Mutex<Vec<JoinHandle<()>>>,
36    next: AtomicUsize,
37    config: PoolConfig,
38    shared: Arc<SharedPoolState>,
39}
40
41struct Job {
42    png_path: PathBuf,
43    question: String,
44    options: RubricOptions,
45    reply: mpsc::Sender<Result<RubricVerdict, PoolError>>,
46}
47
48impl RubricPool {
49    /// Starts a worker pool from the supplied configuration.
50    ///
51    /// # Errors
52    ///
53    /// Returns [`PoolError`] when configuration is invalid or worker startup
54    /// fails.
55    pub fn new(config: PoolConfig) -> Result<Self, PoolError> {
56        if config.workers == 0 {
57            return Err(PoolError::Spawn(
58                "workers must be greater than zero".to_string(),
59            ));
60        }
61        if config.workers > u64::BITS as usize {
62            return Err(PoolError::Spawn(format!(
63                "workers={} exceeds alive bitmask capacity {}",
64                config.workers,
65                u64::BITS
66            )));
67        }
68
69        let shared = Arc::new(SharedPoolState::default());
70        shared
71            .alive_mask
72            .store(alive_mask(config.workers), Ordering::Release);
73
74        let mut senders = Vec::with_capacity(config.workers);
75        let mut handles = Vec::with_capacity(config.workers);
76        for worker_id in 0..config.workers {
77            let (job_tx, job_rx) = mpsc::channel();
78            let (ready_tx, ready_rx) = mpsc::channel();
79            let worker = Worker {
80                id: worker_id,
81                config: config.clone(),
82                shared: Arc::clone(&shared),
83                jobs: job_rx,
84            };
85            let handle = thread::spawn(move || worker.run(ready_tx));
86            match ready_rx.recv() {
87                Ok(Ok(())) => {
88                    senders.push(job_tx);
89                    handles.push(handle);
90                }
91                Ok(Err(error)) => {
92                    shared
93                        .alive_mask
94                        .fetch_and(!worker_bit(worker_id), Ordering::AcqRel);
95                    drop(senders);
96                    join_handles(handles);
97                    let _ = handle.join();
98                    return Err(error);
99                }
100                Err(error) => {
101                    shared
102                        .alive_mask
103                        .fetch_and(!worker_bit(worker_id), Ordering::AcqRel);
104                    drop(senders);
105                    join_handles(handles);
106                    let _ = handle.join();
107                    return Err(PoolError::WorkerCrashed {
108                        worker_id,
109                        message: format!("worker exited before startup result: {error}"),
110                    });
111                }
112            }
113        }
114
115        Ok(Self {
116            senders,
117            handles: Mutex::new(handles),
118            next: AtomicUsize::new(0),
119            config,
120            shared,
121        })
122    }
123
124    /// Submits one PNG rubric job to a live worker.
125    ///
126    /// # Errors
127    ///
128    /// Returns [`PoolError`] for missing workers, worker crashes, timeouts, PNG
129    /// IO, Codex ACP failures, or verdict parsing failures.
130    pub fn submit(
131        &self,
132        png_path: &Path,
133        question: &str,
134        opts: RubricOptions,
135    ) -> Result<RubricVerdict, PoolError> {
136        if self.shared.fatal_quota.load(Ordering::Acquire) {
137            return Err(PoolError::QuotaExceeded);
138        }
139
140        let worker_id = self.next_live_worker()?;
141        let (reply_tx, reply_rx) = mpsc::channel();
142        let job = Job {
143            png_path: png_path.to_path_buf(),
144            question: question.to_string(),
145            options: merge_options(opts, &self.config.default_options),
146            reply: reply_tx,
147        };
148
149        self.senders[worker_id]
150            .send(job)
151            .map_err(|_| PoolError::WorkerCrashed {
152                worker_id,
153                message: "worker channel closed".to_string(),
154            })?;
155
156        match reply_rx.recv_timeout(self.config.submit_timeout) {
157            Ok(result) => result,
158            Err(mpsc::RecvTimeoutError::Timeout) => Err(PoolError::Timeout {
159                worker_id,
160                timeout: self.config.submit_timeout,
161            }),
162            Err(mpsc::RecvTimeoutError::Disconnected) => Err(PoolError::WorkerCrashed {
163                worker_id,
164                message: "worker dropped reply channel".to_string(),
165            }),
166        }
167    }
168
169    /// Stops workers and returns final pool statistics.
170    #[must_use]
171    pub fn shutdown(self) -> PoolStats {
172        let Self {
173            senders,
174            handles,
175            shared,
176            ..
177        } = self;
178        drop(senders);
179        if let Ok(handles) = handles.into_inner() {
180            join_handles(handles);
181        }
182        shared.stats()
183    }
184
185    /// Returns current pool statistics without shutting the pool down.
186    #[must_use]
187    pub fn stats(&self) -> PoolStats {
188        self.shared.stats()
189    }
190
191    fn next_live_worker(&self) -> Result<usize, PoolError> {
192        let worker_count = self.senders.len();
193        for _ in 0..worker_count {
194            let idx = self.next.fetch_add(1, Ordering::AcqRel) % worker_count;
195            let mask = self.shared.alive_mask.load(Ordering::Acquire);
196            if mask & worker_bit(idx) != 0 {
197                return Ok(idx);
198            }
199        }
200        Err(PoolError::NoLiveWorkers)
201    }
202}
203
204struct Worker {
205    id: usize,
206    config: PoolConfig,
207    shared: Arc<SharedPoolState>,
208    jobs: mpsc::Receiver<Job>,
209}
210
211struct WorkerRuntime {
212    acp: AcpClient,
213    _codex_home: Option<TempDir>,
214    prompts: u32,
215    model: String,
216    effort: String,
217}
218
219impl Worker {
220    fn run(self, ready: mpsc::Sender<Result<(), PoolError>>) {
221        let mut runtime = match self.spawn_runtime(&self.config.default_options) {
222            Ok(runtime) => {
223                let _ = ready.send(Ok(()));
224                runtime
225            }
226            Err(error) => {
227                self.mark_dead();
228                let _ = ready.send(Err(error));
229                return;
230            }
231        };
232
233        while let Ok(job) = self.jobs.recv() {
234            let result = self.handle_job(&mut runtime, &job);
235            // Publish the fatal state before replying, or a caller that
236            // resubmits right after the reply can still reach a worker.
237            if matches!(result, Err(PoolError::QuotaExceeded)) {
238                self.shared.fatal_quota.store(true, Ordering::Release);
239            }
240            let _ = job.reply.send(result);
241            if self.shared.alive_mask.load(Ordering::Acquire) & worker_bit(self.id) == 0 {
242                break;
243            }
244        }
245    }
246
247    fn handle_job(
248        &self,
249        runtime: &mut WorkerRuntime,
250        job: &Job,
251    ) -> Result<RubricVerdict, PoolError> {
252        let mut last_error = None;
253        for attempt in 0..=self.config.max_retries {
254            if !runtime.matches_options(&job.options) {
255                self.recycle_runtime(runtime, &job.options)?;
256            }
257
258            match self.evaluate_once(runtime, job) {
259                Ok(verdict) => {
260                    self.shared.completed.fetch_add(1, Ordering::AcqRel);
261                    runtime.prompts += 1;
262                    if runtime.prompts >= self.config.max_prompts_per_worker {
263                        self.recycle_runtime(runtime, &job.options)?;
264                    }
265                    return Ok(verdict);
266                }
267                Err(PoolError::QuotaExceeded) => {
268                    self.shared.failures.fetch_add(1, Ordering::AcqRel);
269                    return Err(PoolError::QuotaExceeded);
270                }
271                Err(PoolError::RateLimited { retry_after }) => {
272                    let delay =
273                        backoff_delay(attempt, self.config.backoff_base, self.config.backoff_cap);
274                    self.shared.push_rate_limit_event(RateLimitEvent {
275                        worker_id: self.id,
276                        attempt,
277                        delay,
278                        retry_after,
279                    });
280                    last_error = Some(PoolError::RateLimited { retry_after });
281                    if attempt < self.config.max_retries {
282                        thread::sleep(delay);
283                    }
284                }
285                Err(error) => {
286                    last_error = Some(error);
287                    self.recycle_runtime(runtime, &job.options)?;
288                }
289            }
290        }
291
292        self.shared.failures.fetch_add(1, Ordering::AcqRel);
293        Err(last_error.unwrap_or_else(|| PoolError::Rpc("retry loop exhausted".to_string())))
294    }
295
296    fn evaluate_once(
297        &self,
298        runtime: &mut WorkerRuntime,
299        job: &Job,
300    ) -> Result<RubricVerdict, PoolError> {
301        let b64 = encode_png(&job.png_path)?;
302        let system_prompt = job
303            .options
304            .system_prompt
305            .as_deref()
306            .map_or(DEFAULT_SYSTEM_PROMPT, |system_prompt| system_prompt);
307        let prompt = format!("{system_prompt}\n\nQuestion: {}", job.question);
308        let text = runtime.acp.prompt_image(&prompt, &b64)?;
309        parse_verdict(&text).map_err(|e| PoolError::ParseVerdict(format!("from {text:?}: {e}")))
310    }
311
312    fn recycle_runtime(
313        &self,
314        runtime: &mut WorkerRuntime,
315        options: &RubricOptions,
316    ) -> Result<(), PoolError> {
317        self.shared.worker_recycles.fetch_add(1, Ordering::AcqRel);
318        let mut last_error = None;
319        for _ in 0..RECYCLE_SPAWN_ATTEMPTS {
320            match self.spawn_runtime(options) {
321                Ok(new_runtime) => {
322                    *runtime = new_runtime;
323                    return Ok(());
324                }
325                Err(error) => {
326                    last_error = Some(error);
327                }
328            }
329        }
330        self.mark_dead();
331        Err(last_error.unwrap_or_else(|| PoolError::Spawn("recycle failed".to_string())))
332    }
333
334    fn spawn_runtime(&self, options: &RubricOptions) -> Result<WorkerRuntime, PoolError> {
335        let mut env = self.config.extra_env.clone();
336
337        let codex_home = self.maybe_seed_codex_home(&mut env)?;
338        self.maybe_setup_log_capture(&mut env)?;
339
340        let (model, effort) = self.resolve_model_effort(options);
341        let acp_args = self.resolve_acp_args();
342        let mut acp = AcpClient::spawn(&self.config.codex_acp_binary, &acp_args, &env, None)?;
343        acp.start_session(None, Some(model), Some(effort))?;
344
345        Ok(WorkerRuntime {
346            acp,
347            _codex_home: codex_home,
348            prompts: 0,
349            model: model.to_string(),
350            effort: effort.to_string(),
351        })
352    }
353
354    fn resolve_model_effort<'a>(&self, options: &'a RubricOptions) -> (&'a str, &'a str) {
355        let model = options.model.as_deref().unwrap_or({
356            #[cfg(feature = "codex-acp")]
357            {
358                DEFAULT_CODEX_ACP_MODEL
359            }
360            #[cfg(not(feature = "codex-acp"))]
361            {
362                "default"
363            }
364        });
365        let effort = options.effort.as_deref().unwrap_or({
366            #[cfg(feature = "codex-acp")]
367            {
368                DEFAULT_CODEX_ACP_REASONING_EFFORT
369            }
370            #[cfg(not(feature = "codex-acp"))]
371            {
372                "default"
373            }
374        });
375        (model, effort)
376    }
377
378    fn resolve_acp_args(&self) -> Vec<String> {
379        if !self.config.acp_args.is_empty() {
380            return self.config.acp_args.clone();
381        }
382        #[cfg(feature = "codex-acp")]
383        {
384            Vec::new()
385        }
386        #[cfg(not(feature = "codex-acp"))]
387        {
388            vec!["acp".to_string()]
389        }
390    }
391
392    #[allow(unused_variables, clippy::ptr_arg)]
393    fn maybe_seed_codex_home(
394        &self,
395        env: &mut Vec<(OsString, OsString)>,
396    ) -> Result<Option<TempDir>, PoolError> {
397        #[cfg(feature = "codex-acp")]
398        {
399            let codex_home =
400                TempDir::new().map_err(|e| PoolError::Spawn(format!("create CODEX_HOME: {e}")))?;
401            seed_codex_home(codex_home.path(), self.config.source_codex_home.as_deref())?;
402            env.push((
403                OsString::from("CODEX_HOME"),
404                codex_home.path().as_os_str().to_os_string(),
405            ));
406            Ok(Some(codex_home))
407        }
408        #[cfg(not(feature = "codex-acp"))]
409        Ok(None)
410    }
411
412    fn maybe_setup_log_capture(
413        &self,
414        env: &mut Vec<(OsString, OsString)>,
415    ) -> Result<(), PoolError> {
416        if let Some(log_capture) = &self.config.log_capture {
417            fs::create_dir_all(&log_capture.temp_dir).map_err(|e| {
418                PoolError::Spawn(format!(
419                    "create ACP TMPDIR {}: {e}",
420                    log_capture.temp_dir.display()
421                ))
422            })?;
423            env.push((
424                OsString::from("TMPDIR"),
425                log_capture.temp_dir.as_os_str().to_os_string(),
426            ));
427        }
428        Ok(())
429    }
430
431    fn mark_dead(&self) {
432        self.shared
433            .alive_mask
434            .fetch_and(!worker_bit(self.id), Ordering::AcqRel);
435    }
436}
437
438impl WorkerRuntime {
439    fn matches_options(&self, options: &RubricOptions) -> bool {
440        let model_ok =
441            !cfg!(feature = "codex-acp") || options.model.as_deref() == Some(self.model.as_str());
442        let effort_ok =
443            !cfg!(feature = "codex-acp") || options.effort.as_deref() == Some(self.effort.as_str());
444        model_ok && effort_ok
445    }
446}
447
448fn merge_options(mut opts: RubricOptions, defaults: &RubricOptions) -> RubricOptions {
449    if opts.model.is_none() {
450        opts.model.clone_from(&defaults.model);
451    }
452    if opts.effort.is_none() {
453        opts.effort.clone_from(&defaults.effort);
454    }
455    if opts.system_prompt.is_none() {
456        opts.system_prompt.clone_from(&defaults.system_prompt);
457    }
458    opts
459}
460
461fn backoff_delay(attempt: u32, base: Duration, cap: Duration) -> Duration {
462    let multiplier = 1u32 << attempt.min(6);
463    let capped = base.saturating_mul(multiplier).min(cap);
464    let capped_millis = u64::try_from(capped.as_millis()).unwrap_or(u64::MAX);
465    let jitter_cap = capped_millis / 4;
466    let jitter_ms = rand::rng().random_range(0..=jitter_cap);
467    capped.saturating_add(Duration::from_millis(jitter_ms))
468}
469
470fn alive_mask(workers: usize) -> u64 {
471    if workers == u64::BITS as usize {
472        u64::MAX
473    } else {
474        (1u64 << workers) - 1
475    }
476}
477
478fn worker_bit(worker_id: usize) -> u64 {
479    1u64 << worker_id
480}
481
482fn join_handles(handles: Vec<JoinHandle<()>>) {
483    for handle in handles {
484        let _ = handle.join();
485    }
486}