1use std::ffi::OsString;
2use std::fs;
3use std::path::{Path, PathBuf};
4use std::sync::atomic::{AtomicUsize, Ordering};
5use std::sync::{Arc, Mutex, mpsc};
6use std::thread::{self, JoinHandle};
7use std::time::Duration;
8
9use rand::RngExt as _;
10use tempfile::TempDir;
11
12#[cfg(feature = "codex-acp")]
13mod codex_home;
14mod config;
15mod state;
16
17pub use config::{LogCaptureConfig, LogPathMode, PoolConfig, PoolStats};
18use state::SharedPoolState;
19
20use crate::{
21 AcpClient, DEFAULT_SYSTEM_PROMPT, PoolError, RateLimitEvent, RubricOptions, RubricVerdict,
22 encode_png, parse_verdict,
23};
24#[cfg(feature = "codex-acp")]
25use crate::{DEFAULT_CODEX_ACP_MODEL, DEFAULT_CODEX_ACP_REASONING_EFFORT};
26#[cfg(feature = "codex-acp")]
27use codex_home::seed_codex_home;
28
29const RECYCLE_SPAWN_ATTEMPTS: u32 = 2;
30
31#[derive(Debug)]
33pub struct RubricPool {
34 senders: Vec<mpsc::Sender<Job>>,
35 handles: Mutex<Vec<JoinHandle<()>>>,
36 next: AtomicUsize,
37 config: PoolConfig,
38 shared: Arc<SharedPoolState>,
39}
40
41struct Job {
42 png_path: PathBuf,
43 question: String,
44 options: RubricOptions,
45 reply: mpsc::Sender<Result<RubricVerdict, PoolError>>,
46}
47
48impl RubricPool {
49 pub fn new(config: PoolConfig) -> Result<Self, PoolError> {
56 if config.workers == 0 {
57 return Err(PoolError::Spawn(
58 "workers must be greater than zero".to_string(),
59 ));
60 }
61 if config.workers > u64::BITS as usize {
62 return Err(PoolError::Spawn(format!(
63 "workers={} exceeds alive bitmask capacity {}",
64 config.workers,
65 u64::BITS
66 )));
67 }
68
69 let shared = Arc::new(SharedPoolState::default());
70 shared
71 .alive_mask
72 .store(alive_mask(config.workers), Ordering::Release);
73
74 let mut senders = Vec::with_capacity(config.workers);
75 let mut handles = Vec::with_capacity(config.workers);
76 for worker_id in 0..config.workers {
77 let (job_tx, job_rx) = mpsc::channel();
78 let (ready_tx, ready_rx) = mpsc::channel();
79 let worker = Worker {
80 id: worker_id,
81 config: config.clone(),
82 shared: Arc::clone(&shared),
83 jobs: job_rx,
84 };
85 let handle = thread::spawn(move || worker.run(ready_tx));
86 match ready_rx.recv() {
87 Ok(Ok(())) => {
88 senders.push(job_tx);
89 handles.push(handle);
90 }
91 Ok(Err(error)) => {
92 shared
93 .alive_mask
94 .fetch_and(!worker_bit(worker_id), Ordering::AcqRel);
95 drop(senders);
96 join_handles(handles);
97 let _ = handle.join();
98 return Err(error);
99 }
100 Err(error) => {
101 shared
102 .alive_mask
103 .fetch_and(!worker_bit(worker_id), Ordering::AcqRel);
104 drop(senders);
105 join_handles(handles);
106 let _ = handle.join();
107 return Err(PoolError::WorkerCrashed {
108 worker_id,
109 message: format!("worker exited before startup result: {error}"),
110 });
111 }
112 }
113 }
114
115 Ok(Self {
116 senders,
117 handles: Mutex::new(handles),
118 next: AtomicUsize::new(0),
119 config,
120 shared,
121 })
122 }
123
124 pub fn submit(
131 &self,
132 png_path: &Path,
133 question: &str,
134 opts: RubricOptions,
135 ) -> Result<RubricVerdict, PoolError> {
136 if self.shared.fatal_quota.load(Ordering::Acquire) {
137 return Err(PoolError::QuotaExceeded);
138 }
139
140 let worker_id = self.next_live_worker()?;
141 let (reply_tx, reply_rx) = mpsc::channel();
142 let job = Job {
143 png_path: png_path.to_path_buf(),
144 question: question.to_string(),
145 options: merge_options(opts, &self.config.default_options),
146 reply: reply_tx,
147 };
148
149 self.senders[worker_id]
150 .send(job)
151 .map_err(|_| PoolError::WorkerCrashed {
152 worker_id,
153 message: "worker channel closed".to_string(),
154 })?;
155
156 match reply_rx.recv_timeout(self.config.submit_timeout) {
157 Ok(result) => result,
158 Err(mpsc::RecvTimeoutError::Timeout) => Err(PoolError::Timeout {
159 worker_id,
160 timeout: self.config.submit_timeout,
161 }),
162 Err(mpsc::RecvTimeoutError::Disconnected) => Err(PoolError::WorkerCrashed {
163 worker_id,
164 message: "worker dropped reply channel".to_string(),
165 }),
166 }
167 }
168
169 #[must_use]
171 pub fn shutdown(self) -> PoolStats {
172 let Self {
173 senders,
174 handles,
175 shared,
176 ..
177 } = self;
178 drop(senders);
179 if let Ok(handles) = handles.into_inner() {
180 join_handles(handles);
181 }
182 shared.stats()
183 }
184
185 #[must_use]
187 pub fn stats(&self) -> PoolStats {
188 self.shared.stats()
189 }
190
191 fn next_live_worker(&self) -> Result<usize, PoolError> {
192 let worker_count = self.senders.len();
193 for _ in 0..worker_count {
194 let idx = self.next.fetch_add(1, Ordering::AcqRel) % worker_count;
195 let mask = self.shared.alive_mask.load(Ordering::Acquire);
196 if mask & worker_bit(idx) != 0 {
197 return Ok(idx);
198 }
199 }
200 Err(PoolError::NoLiveWorkers)
201 }
202}
203
204struct Worker {
205 id: usize,
206 config: PoolConfig,
207 shared: Arc<SharedPoolState>,
208 jobs: mpsc::Receiver<Job>,
209}
210
211struct WorkerRuntime {
212 acp: AcpClient,
213 _codex_home: Option<TempDir>,
214 prompts: u32,
215 model: String,
216 effort: String,
217}
218
219impl Worker {
220 fn run(self, ready: mpsc::Sender<Result<(), PoolError>>) {
221 let mut runtime = match self.spawn_runtime(&self.config.default_options) {
222 Ok(runtime) => {
223 let _ = ready.send(Ok(()));
224 runtime
225 }
226 Err(error) => {
227 self.mark_dead();
228 let _ = ready.send(Err(error));
229 return;
230 }
231 };
232
233 while let Ok(job) = self.jobs.recv() {
234 let result = self.handle_job(&mut runtime, &job);
235 if matches!(result, Err(PoolError::QuotaExceeded)) {
238 self.shared.fatal_quota.store(true, Ordering::Release);
239 }
240 let _ = job.reply.send(result);
241 if self.shared.alive_mask.load(Ordering::Acquire) & worker_bit(self.id) == 0 {
242 break;
243 }
244 }
245 }
246
247 fn handle_job(
248 &self,
249 runtime: &mut WorkerRuntime,
250 job: &Job,
251 ) -> Result<RubricVerdict, PoolError> {
252 let mut last_error = None;
253 for attempt in 0..=self.config.max_retries {
254 if !runtime.matches_options(&job.options) {
255 self.recycle_runtime(runtime, &job.options)?;
256 }
257
258 match self.evaluate_once(runtime, job) {
259 Ok(verdict) => {
260 self.shared.completed.fetch_add(1, Ordering::AcqRel);
261 runtime.prompts += 1;
262 if runtime.prompts >= self.config.max_prompts_per_worker {
263 self.recycle_runtime(runtime, &job.options)?;
264 }
265 return Ok(verdict);
266 }
267 Err(PoolError::QuotaExceeded) => {
268 self.shared.failures.fetch_add(1, Ordering::AcqRel);
269 return Err(PoolError::QuotaExceeded);
270 }
271 Err(PoolError::RateLimited { retry_after }) => {
272 let delay =
273 backoff_delay(attempt, self.config.backoff_base, self.config.backoff_cap);
274 self.shared.push_rate_limit_event(RateLimitEvent {
275 worker_id: self.id,
276 attempt,
277 delay,
278 retry_after,
279 });
280 last_error = Some(PoolError::RateLimited { retry_after });
281 if attempt < self.config.max_retries {
282 thread::sleep(delay);
283 }
284 }
285 Err(error) => {
286 last_error = Some(error);
287 self.recycle_runtime(runtime, &job.options)?;
288 }
289 }
290 }
291
292 self.shared.failures.fetch_add(1, Ordering::AcqRel);
293 Err(last_error.unwrap_or_else(|| PoolError::Rpc("retry loop exhausted".to_string())))
294 }
295
296 fn evaluate_once(
297 &self,
298 runtime: &mut WorkerRuntime,
299 job: &Job,
300 ) -> Result<RubricVerdict, PoolError> {
301 let b64 = encode_png(&job.png_path)?;
302 let system_prompt = job
303 .options
304 .system_prompt
305 .as_deref()
306 .map_or(DEFAULT_SYSTEM_PROMPT, |system_prompt| system_prompt);
307 let prompt = format!("{system_prompt}\n\nQuestion: {}", job.question);
308 let text = runtime.acp.prompt_image(&prompt, &b64)?;
309 parse_verdict(&text).map_err(|e| PoolError::ParseVerdict(format!("from {text:?}: {e}")))
310 }
311
312 fn recycle_runtime(
313 &self,
314 runtime: &mut WorkerRuntime,
315 options: &RubricOptions,
316 ) -> Result<(), PoolError> {
317 self.shared.worker_recycles.fetch_add(1, Ordering::AcqRel);
318 let mut last_error = None;
319 for _ in 0..RECYCLE_SPAWN_ATTEMPTS {
320 match self.spawn_runtime(options) {
321 Ok(new_runtime) => {
322 *runtime = new_runtime;
323 return Ok(());
324 }
325 Err(error) => {
326 last_error = Some(error);
327 }
328 }
329 }
330 self.mark_dead();
331 Err(last_error.unwrap_or_else(|| PoolError::Spawn("recycle failed".to_string())))
332 }
333
334 fn spawn_runtime(&self, options: &RubricOptions) -> Result<WorkerRuntime, PoolError> {
335 let mut env = self.config.extra_env.clone();
336
337 let codex_home = self.maybe_seed_codex_home(&mut env)?;
338 self.maybe_setup_log_capture(&mut env)?;
339
340 let (model, effort) = self.resolve_model_effort(options);
341 let acp_args = self.resolve_acp_args();
342 let mut acp = AcpClient::spawn(&self.config.codex_acp_binary, &acp_args, &env, None)?;
343 acp.start_session(None, Some(model), Some(effort))?;
344
345 Ok(WorkerRuntime {
346 acp,
347 _codex_home: codex_home,
348 prompts: 0,
349 model: model.to_string(),
350 effort: effort.to_string(),
351 })
352 }
353
354 fn resolve_model_effort<'a>(&self, options: &'a RubricOptions) -> (&'a str, &'a str) {
355 let model = options.model.as_deref().unwrap_or({
356 #[cfg(feature = "codex-acp")]
357 {
358 DEFAULT_CODEX_ACP_MODEL
359 }
360 #[cfg(not(feature = "codex-acp"))]
361 {
362 "default"
363 }
364 });
365 let effort = options.effort.as_deref().unwrap_or({
366 #[cfg(feature = "codex-acp")]
367 {
368 DEFAULT_CODEX_ACP_REASONING_EFFORT
369 }
370 #[cfg(not(feature = "codex-acp"))]
371 {
372 "default"
373 }
374 });
375 (model, effort)
376 }
377
378 fn resolve_acp_args(&self) -> Vec<String> {
379 if !self.config.acp_args.is_empty() {
380 return self.config.acp_args.clone();
381 }
382 #[cfg(feature = "codex-acp")]
383 {
384 Vec::new()
385 }
386 #[cfg(not(feature = "codex-acp"))]
387 {
388 vec!["acp".to_string()]
389 }
390 }
391
392 #[allow(unused_variables, clippy::ptr_arg)]
393 fn maybe_seed_codex_home(
394 &self,
395 env: &mut Vec<(OsString, OsString)>,
396 ) -> Result<Option<TempDir>, PoolError> {
397 #[cfg(feature = "codex-acp")]
398 {
399 let codex_home =
400 TempDir::new().map_err(|e| PoolError::Spawn(format!("create CODEX_HOME: {e}")))?;
401 seed_codex_home(codex_home.path(), self.config.source_codex_home.as_deref())?;
402 env.push((
403 OsString::from("CODEX_HOME"),
404 codex_home.path().as_os_str().to_os_string(),
405 ));
406 Ok(Some(codex_home))
407 }
408 #[cfg(not(feature = "codex-acp"))]
409 Ok(None)
410 }
411
412 fn maybe_setup_log_capture(
413 &self,
414 env: &mut Vec<(OsString, OsString)>,
415 ) -> Result<(), PoolError> {
416 if let Some(log_capture) = &self.config.log_capture {
417 fs::create_dir_all(&log_capture.temp_dir).map_err(|e| {
418 PoolError::Spawn(format!(
419 "create ACP TMPDIR {}: {e}",
420 log_capture.temp_dir.display()
421 ))
422 })?;
423 env.push((
424 OsString::from("TMPDIR"),
425 log_capture.temp_dir.as_os_str().to_os_string(),
426 ));
427 }
428 Ok(())
429 }
430
431 fn mark_dead(&self) {
432 self.shared
433 .alive_mask
434 .fetch_and(!worker_bit(self.id), Ordering::AcqRel);
435 }
436}
437
438impl WorkerRuntime {
439 fn matches_options(&self, options: &RubricOptions) -> bool {
440 let model_ok =
441 !cfg!(feature = "codex-acp") || options.model.as_deref() == Some(self.model.as_str());
442 let effort_ok =
443 !cfg!(feature = "codex-acp") || options.effort.as_deref() == Some(self.effort.as_str());
444 model_ok && effort_ok
445 }
446}
447
448fn merge_options(mut opts: RubricOptions, defaults: &RubricOptions) -> RubricOptions {
449 if opts.model.is_none() {
450 opts.model.clone_from(&defaults.model);
451 }
452 if opts.effort.is_none() {
453 opts.effort.clone_from(&defaults.effort);
454 }
455 if opts.system_prompt.is_none() {
456 opts.system_prompt.clone_from(&defaults.system_prompt);
457 }
458 opts
459}
460
461fn backoff_delay(attempt: u32, base: Duration, cap: Duration) -> Duration {
462 let multiplier = 1u32 << attempt.min(6);
463 let capped = base.saturating_mul(multiplier).min(cap);
464 let capped_millis = u64::try_from(capped.as_millis()).unwrap_or(u64::MAX);
465 let jitter_cap = capped_millis / 4;
466 let jitter_ms = rand::rng().random_range(0..=jitter_cap);
467 capped.saturating_add(Duration::from_millis(jitter_ms))
468}
469
470fn alive_mask(workers: usize) -> u64 {
471 if workers == u64::BITS as usize {
472 u64::MAX
473 } else {
474 (1u64 << workers) - 1
475 }
476}
477
478fn worker_bit(worker_id: usize) -> u64 {
479 1u64 << worker_id
480}
481
482fn join_handles(handles: Vec<JoinHandle<()>>) {
483 for handle in handles {
484 let _ = handle.join();
485 }
486}