1mod logs;
2mod recommendations;
3mod snapshot;
4
5#[cfg(test)]
6mod tests;
7
8use std::collections::VecDeque;
9use std::fs;
10use std::io::Write;
11use std::path::{Path, PathBuf};
12use std::sync::{Arc, Mutex, MutexGuard};
13use std::thread;
14use std::time::{SystemTime, UNIX_EPOCH};
15
16use serde::{Deserialize, Serialize};
17use sha2::{Digest, Sha256};
18use tempfile::NamedTempFile;
19
20use crate::{PoolConfig, PoolError, RubricOptions, RubricPool, RubricVerdict};
21
22use logs::copy_logs_from_config;
23use recommendations::{aggregate_status, classify_recommendations};
24
25pub use snapshot::{AssetChange, AssetSnapshot, SelectionMode, diff_snapshots, select_changed};
26
27const REPORT_SCHEMA_VERSION: u32 = 2;
28const CACHE_SCHEMA_VERSION: u32 = 1;
29const CACHE_KEY_DOMAIN: &[u8] = b"visual-rubric.batch.cache.v1";
30
31#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
33#[serde(rename_all = "snake_case")]
34pub enum AggregateStatus {
35 Error,
37 Fail,
39 Pass,
41 Skipped,
43}
44
45#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
47pub struct BatchRubricReport {
48 pub schema_version: u32,
50 pub aggregate_status: AggregateStatus,
52 pub started_at: String,
54 pub finished_at: String,
56 pub workers: usize,
58 #[serde(default)]
60 pub cache_hits: u64,
61 #[serde(default)]
63 pub cache_misses: u64,
64 pub options: RubricOptions,
66 pub logs: Vec<String>,
68 pub log_capture_error: Option<String>,
70 pub assets: Vec<AssetRubricReport>,
72 pub recommendations: Vec<IssueRecommendation>,
74}
75
76#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
78pub struct AssetRubricReport {
79 pub path: String,
81 pub change: String,
83 pub selected: bool,
85 pub result: AssetRubricResult,
87}
88
89#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
91#[serde(tag = "status", rename_all = "snake_case")]
92pub enum AssetRubricResult {
93 Pass {
95 reason: String,
97 anomalies: Vec<String>,
99 },
100 Fail {
102 reason: String,
104 anomalies: Vec<String>,
106 },
107 Error {
109 message: String,
111 },
112 NotEvaluatedAfterError {
114 root_error: String,
116 retry_hint: String,
118 },
119 Skipped {
121 reason: String,
123 },
124}
125
126#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord)]
128#[serde(rename_all = "snake_case")]
129pub enum RecommendationSeverity {
130 Low,
132 Medium,
134 High,
136}
137
138#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
140pub struct IssueRecommendation {
141 pub id: String,
143 pub class: String,
145 pub severity: RecommendationSeverity,
147 pub affected_assets: Vec<String>,
149 pub evidence: Vec<String>,
151 pub suggested_fix: String,
153 pub candidate_modules: Vec<String>,
155}
156
157#[derive(Debug)]
159pub struct IssueClassificationInput<'a> {
160 pub asset: &'a AssetRubricReport,
162 pub issue_text: &'a str,
164}
165
166pub trait IssueClassifier {
168 fn classify(&self, input: IssueClassificationInput<'_>) -> Vec<IssueRecommendation>;
170}
171
172pub struct BatchRubricConfig<'a> {
174 pub pool: PoolConfig,
176 pub question: String,
178 pub selection_mode: SelectionMode,
180 pub cache_dir: Option<PathBuf>,
182 pub classifier: Option<&'a dyn IssueClassifier>,
184}
185
186impl std::fmt::Debug for BatchRubricConfig<'_> {
187 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
188 f.debug_struct("BatchRubricConfig")
189 .field("pool", &self.pool)
190 .field("question", &self.question)
191 .field("selection_mode", &self.selection_mode)
192 .field("cache_dir", &self.cache_dir)
193 .field("classifier", &self.classifier.map(|_| "<classifier>"))
194 .finish()
195 }
196}
197
198#[derive(Debug)]
200pub struct BatchRubricRun<'a> {
201 config: BatchRubricConfig<'a>,
202}
203
204impl<'a> BatchRubricRun<'a> {
205 #[must_use]
207 pub fn new(config: BatchRubricConfig<'a>) -> Self {
208 Self { config }
209 }
210
211 #[must_use]
217 pub fn run(&self, changes: &[AssetChange]) -> BatchRubricReport {
218 self.run_with_evaluator(changes, None)
219 }
220
221 fn run_with_evaluator(
222 &self,
223 changes: &[AssetChange],
224 evaluator: Option<&dyn BatchEvaluator>,
225 ) -> BatchRubricReport {
226 let started_at = unix_timestamp();
227 let selected = select_changed(changes, self.config.selection_mode);
228 let mut assets = Vec::with_capacity(changes.len());
229 let selected_evaluation = if selected.is_empty() {
230 SelectedEvaluation::default()
231 } else if let Some(evaluator) = evaluator {
232 evaluate_selected(
233 evaluator,
234 changes,
235 &selected,
236 &self.config.question,
237 self.config.pool.workers,
238 self.config.cache_dir.as_deref(),
239 &self.config.pool.default_options,
240 )
241 } else {
242 match RubricPool::new(self.config.pool.clone()) {
243 Ok(pool) => {
244 let evaluation = evaluate_selected(
245 &pool,
246 changes,
247 &selected,
248 &self.config.question,
249 self.config.pool.workers,
250 self.config.cache_dir.as_deref(),
251 &self.config.pool.default_options,
252 );
253 if evaluation.aborted {
254 drop(pool);
255 } else {
256 let _stats = pool.shutdown();
257 }
258 evaluation
259 }
260 Err(error) => selected
261 .iter()
262 .map(|path| {
263 AssetRubricReport::selected(
264 path,
265 status_for_selected(path, changes),
266 AssetRubricResult::Error {
267 message: format!("start visual-rubric worker pool: {error}"),
268 },
269 )
270 })
271 .collect::<Vec<_>>()
272 .into(),
273 }
274 };
275 assets.extend(selected_evaluation.reports);
276 assets.extend(skipped_asset_reports(changes, self.config.selection_mode));
277 assets.sort_by(|left, right| left.path.cmp(&right.path));
278 let (logs, log_capture_error) =
279 match copy_logs_from_config(self.config.pool.log_capture.as_ref()) {
280 Ok(logs) => (logs, None),
281 Err(error) => (
282 Vec::new(),
283 Some(format!(
284 "copy configured ACP logs into batch report: {error}"
285 )),
286 ),
287 };
288 let recommendations = classify_recommendations(self.config.classifier, &assets);
289 let aggregate_status = aggregate_status(&assets);
290 BatchRubricReport {
291 schema_version: REPORT_SCHEMA_VERSION,
292 aggregate_status,
293 started_at,
294 finished_at: unix_timestamp(),
295 workers: self.config.pool.workers,
296 cache_hits: selected_evaluation.cache_hits,
297 cache_misses: selected_evaluation.cache_misses,
298 options: self.config.pool.default_options.clone(),
299 logs,
300 log_capture_error,
301 assets,
302 recommendations,
303 }
304 }
305}
306
307trait BatchEvaluator: Sync {
308 fn submit_asset(&self, png_path: &Path, question: &str) -> Result<RubricVerdict, PoolError>;
309}
310
311impl BatchEvaluator for RubricPool {
312 fn submit_asset(&self, png_path: &Path, question: &str) -> Result<RubricVerdict, PoolError> {
313 self.submit(png_path, question, RubricOptions::default())
314 }
315}
316
317#[derive(Default)]
318struct SelectedEvaluation {
319 reports: Vec<AssetRubricReport>,
320 aborted: bool,
321 cache_hits: u64,
322 cache_misses: u64,
323}
324
325impl From<Vec<AssetRubricReport>> for SelectedEvaluation {
326 fn from(reports: Vec<AssetRubricReport>) -> Self {
327 Self {
328 reports,
329 aborted: false,
330 cache_hits: 0,
331 cache_misses: 0,
332 }
333 }
334}
335
336fn evaluate_selected(
337 evaluator: &dyn BatchEvaluator,
338 changes: &[AssetChange],
339 selected: &[PathBuf],
340 question: &str,
341 workers: usize,
342 cache_dir: Option<&Path>,
343 cache_options: &RubricOptions,
344) -> SelectedEvaluation {
345 let worker_count = workers.max(1).min(selected.len());
346 let state = Arc::new(Mutex::new(EvaluationState {
347 pending: selected.iter().cloned().enumerate().collect(),
348 completed: Vec::with_capacity(selected.len()),
349 abort_message: None,
350 aborted: false,
351 cache_hits: 0,
352 cache_misses: 0,
353 }));
354
355 thread::scope(|scope| {
356 for _ in 0..worker_count {
357 let state = Arc::clone(&state);
358 scope.spawn(move || {
359 loop {
360 let next = {
361 let mut state = lock_state(&state);
362 if state.abort_message.is_some() {
363 None
364 } else {
365 state.pending.pop_front()
366 }
367 };
368 let Some((index, path)) = next else {
369 break;
370 };
371
372 let cache_key = if cache_dir.is_some() {
373 cache_key_for_asset(&path, question, cache_options)
374 } else {
375 None
376 };
377 if let (Some(dir), Some(key)) = (cache_dir, cache_key.as_deref()) {
378 if let Some(verdict) = read_cached_verdict(dir, key) {
379 let mut state = lock_state(&state);
380 state.cache_hits += 1;
381 state.completed.push(CompletedEvaluation {
382 index,
383 path,
384 result: result_from_verdict(verdict),
385 });
386 continue;
387 }
388 }
389 if cache_dir.is_some() {
390 let mut state = lock_state(&state);
391 state.cache_misses += 1;
392 }
393
394 let outcome = evaluator.submit_asset(&path, question);
395 if let (Some(dir), Some(key)) = (cache_dir, cache_key.as_deref()) {
396 if let Ok(verdict) = &outcome {
397 write_cached_verdict(dir, key, verdict);
398 }
399 }
400 let mut state = lock_state(&state);
401 match outcome {
402 Ok(verdict) => state.completed.push(CompletedEvaluation {
403 index,
404 path,
405 result: result_from_verdict(verdict),
406 }),
407 Err(error) => {
408 let message = error.to_string();
409 if should_abort_after_error(&error) && state.abort_message.is_none() {
410 state.aborted = true;
411 state.abort_message = Some(message.clone());
412 }
413 state.completed.push(CompletedEvaluation {
414 index,
415 path,
416 result: AssetRubricResult::Error { message },
417 });
418 }
419 }
420 }
421 });
422 }
423 });
424
425 let mut state = lock_state(&state);
426 let abort_message = state.abort_message.clone();
427 let mut completed = std::mem::take(&mut state.completed);
428 completed.sort_by_key(|evaluation| evaluation.index);
429 let mut reports = completed
430 .into_iter()
431 .map(|evaluation| {
432 AssetRubricReport::selected(
433 &evaluation.path,
434 status_for_selected(&evaluation.path, changes),
435 evaluation.result,
436 )
437 })
438 .collect::<Vec<_>>();
439
440 if let Some(message) = abort_message {
441 reports.extend(state.pending.drain(..).map(|(_, path)| {
442 AssetRubricReport::selected(
443 &path,
444 status_for_selected(&path, changes),
445 AssetRubricResult::NotEvaluatedAfterError {
446 root_error: message.clone(),
447 retry_hint: retry_hint_after_pool_error(&message).to_owned(),
448 },
449 )
450 }));
451 }
452
453 SelectedEvaluation {
454 reports,
455 aborted: state.aborted,
456 cache_hits: state.cache_hits,
457 cache_misses: state.cache_misses,
458 }
459}
460
461struct EvaluationState {
462 pending: VecDeque<(usize, PathBuf)>,
463 completed: Vec<CompletedEvaluation>,
464 abort_message: Option<String>,
465 aborted: bool,
466 cache_hits: u64,
467 cache_misses: u64,
468}
469
470struct CompletedEvaluation {
471 index: usize,
472 path: PathBuf,
473 result: AssetRubricResult,
474}
475
476fn lock_state<T>(state: &Mutex<T>) -> MutexGuard<'_, T> {
477 state.lock().unwrap_or_else(|error| error.into_inner())
478}
479
480fn result_from_verdict(verdict: RubricVerdict) -> AssetRubricResult {
481 if verdict.verdict.is_pass() {
482 AssetRubricResult::Pass {
483 reason: verdict.reason,
484 anomalies: verdict.anomalies,
485 }
486 } else {
487 AssetRubricResult::Fail {
488 reason: verdict.reason,
489 anomalies: verdict.anomalies,
490 }
491 }
492}
493
494#[derive(Debug, Deserialize, Serialize)]
495struct CachedVerdict {
496 schema_version: u32,
497 key: String,
498 verdict: RubricVerdict,
499}
500
501fn cache_key_for_asset(path: &Path, question: &str, options: &RubricOptions) -> Option<String> {
502 let image = fs::read(path).ok()?;
503 let options = serde_json::to_vec(options).ok()?;
504 let mut hasher = Sha256::new();
505 hasher.update(CACHE_KEY_DOMAIN);
506 update_key_part(&mut hasher, &image);
507 update_key_part(&mut hasher, question.as_bytes());
508 update_key_part(&mut hasher, &options);
509 Some(format!("{:x}", hasher.finalize()))
510}
511
512fn update_key_part(hasher: &mut Sha256, value: &[u8]) {
513 hasher.update((value.len() as u64).to_le_bytes());
514 hasher.update(value);
515}
516
517fn cached_verdict_path(cache_dir: &Path, key: &str) -> PathBuf {
518 cache_dir.join(format!("{key}.json"))
519}
520
521fn read_cached_verdict(cache_dir: &Path, key: &str) -> Option<RubricVerdict> {
522 let bytes = fs::read(cached_verdict_path(cache_dir, key)).ok()?;
523 let cached = serde_json::from_slice::<CachedVerdict>(&bytes).ok()?;
524 (cached.schema_version == CACHE_SCHEMA_VERSION && cached.key == key).then_some(cached.verdict)
525}
526
527fn write_cached_verdict(cache_dir: &Path, key: &str, verdict: &RubricVerdict) {
528 if fs::create_dir_all(cache_dir).is_err() {
529 return;
530 }
531 let Ok(mut temporary) = NamedTempFile::new_in(cache_dir) else {
532 return;
533 };
534 let cached = CachedVerdict {
535 schema_version: CACHE_SCHEMA_VERSION,
536 key: key.to_owned(),
537 verdict: verdict.clone(),
538 };
539 if serde_json::to_writer(&mut temporary, &cached).is_err()
540 || temporary.flush().is_err()
541 || temporary.as_file().sync_all().is_err()
542 {
543 return;
544 }
545 let _ = temporary.persist(cached_verdict_path(cache_dir, key));
546}
547
548fn should_abort_after_error(error: &PoolError) -> bool {
549 matches!(
550 error,
551 PoolError::Timeout { .. }
552 | PoolError::QuotaExceeded
553 | PoolError::WorkerCrashed { .. }
554 | PoolError::NoLiveWorkers
555 | PoolError::Closed
556 )
557}
558
559fn retry_hint_after_pool_error(message: &str) -> &'static str {
560 if message.contains("timed out") || message.contains("timeout") {
561 "Asset was not evaluated after an ACP timeout; inspect captured logs and rerun with fewer workers or a smaller asset scope."
562 } else if message.contains("quota") {
563 "Asset was not evaluated after an ACP quota error; inspect captured logs, wait for quota recovery or switch credentials, then rerun the batch."
564 } else {
565 "Asset was not evaluated after an ACP worker error; inspect captured logs and rerun with fewer workers or a smaller asset scope."
566 }
567}
568
569fn status_for_selected(path: &Path, changes: &[AssetChange]) -> &'static str {
570 changes
571 .iter()
572 .find_map(|change| (change.path() == path).then_some(change.status()))
573 .map_or("selected", |status| status)
574}
575
576fn skipped_asset_reports(
577 changes: &[AssetChange],
578 selection_mode: SelectionMode,
579) -> Vec<AssetRubricReport> {
580 let mut reports = Vec::with_capacity(changes.len());
581 for change in changes {
582 match (change, selection_mode) {
583 (AssetChange::Unchanged(path), SelectionMode::ChangedOnly) => {
584 reports.push(AssetRubricReport::skipped(
585 path,
586 "unchanged",
587 "asset content did not change during this command",
588 ));
589 }
590 (AssetChange::Deleted(path), _) => {
591 reports.push(AssetRubricReport::skipped(
592 path,
593 "deleted",
594 "asset was deleted before evaluation",
595 ));
596 }
597 (AssetChange::Added(_) | AssetChange::Changed(_) | AssetChange::Unchanged(_), _) => {}
598 }
599 }
600 reports
601}
602
603impl AssetRubricReport {
604 fn selected(path: &Path, change: &str, result: AssetRubricResult) -> Self {
605 Self {
606 path: path.to_string_lossy().into_owned(),
607 change: change.to_owned(),
608 selected: true,
609 result,
610 }
611 }
612
613 fn skipped(path: &Path, change: &str, reason: &str) -> Self {
614 Self {
615 path: path.to_string_lossy().into_owned(),
616 change: change.to_owned(),
617 selected: false,
618 result: AssetRubricResult::Skipped {
619 reason: reason.to_owned(),
620 },
621 }
622 }
623}
624
625fn unix_timestamp() -> String {
626 SystemTime::now()
627 .duration_since(UNIX_EPOCH)
628 .map(|duration| duration.as_secs().to_string())
629 .unwrap_or_else(|_| "0".to_owned())
630}