Skip to main content

visual_rubric/batch/
mod.rs

1mod logs;
2mod recommendations;
3mod snapshot;
4
5#[cfg(test)]
6mod tests;
7
8use std::collections::VecDeque;
9use std::fs;
10use std::io::Write;
11use std::path::{Path, PathBuf};
12use std::sync::{Arc, Mutex, MutexGuard};
13use std::thread;
14use std::time::{SystemTime, UNIX_EPOCH};
15
16use serde::{Deserialize, Serialize};
17use sha2::{Digest, Sha256};
18use tempfile::NamedTempFile;
19
20use crate::{PoolConfig, PoolError, RubricOptions, RubricPool, RubricVerdict};
21
22use logs::copy_logs_from_config;
23use recommendations::{aggregate_status, classify_recommendations};
24
25pub use snapshot::{AssetChange, AssetSnapshot, SelectionMode, diff_snapshots, select_changed};
26
27const REPORT_SCHEMA_VERSION: u32 = 2;
28const CACHE_SCHEMA_VERSION: u32 = 1;
29const CACHE_KEY_DOMAIN: &[u8] = b"visual-rubric.batch.cache.v1";
30
31/// Overall status for a batch rubric report.
32#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
33#[serde(rename_all = "snake_case")]
34pub enum AggregateStatus {
35    /// At least one selected asset had an evaluation error or was not evaluated after a pool error.
36    Error,
37    /// At least one selected asset failed the rubric.
38    Fail,
39    /// At least one selected asset passed and no selected asset failed or errored.
40    Pass,
41    /// No asset was selected for evaluation.
42    Skipped,
43}
44
45/// Serializable batch rubric report.
46#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
47pub struct BatchRubricReport {
48    /// Version of this JSON report schema.
49    pub schema_version: u32,
50    /// Aggregate status computed as `error > fail > pass > skipped`.
51    pub aggregate_status: AggregateStatus,
52    /// UNIX timestamp in seconds when evaluation started.
53    pub started_at: String,
54    /// UNIX timestamp in seconds when evaluation finished.
55    pub finished_at: String,
56    /// Worker count requested for the batch.
57    pub workers: usize,
58    /// Number of selected assets served by the persistent cache.
59    #[serde(default)]
60    pub cache_hits: u64,
61    /// Number of selected assets that missed the persistent cache.
62    #[serde(default)]
63    pub cache_misses: u64,
64    /// Effective default options supplied to the pool.
65    pub options: RubricOptions,
66    /// Copied ACP log paths.
67    pub logs: Vec<String>,
68    /// Log capture failure, when ACP logs were requested but could not be copied.
69    pub log_capture_error: Option<String>,
70    /// Per-asset results, including skipped unchanged/deleted assets.
71    pub assets: Vec<AssetRubricReport>,
72    /// Optional caller-classified recommendations derived from failed/error assets.
73    pub recommendations: Vec<IssueRecommendation>,
74}
75
76/// Per-asset report item.
77#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
78pub struct AssetRubricReport {
79    /// Asset path, using caller-provided path representation.
80    pub path: String,
81    /// Snapshot change status.
82    pub change: String,
83    /// Whether this asset was submitted to the rubric evaluator.
84    pub selected: bool,
85    /// Rubric evaluation result or skipped reason.
86    pub result: AssetRubricResult,
87}
88
89/// Per-asset rubric result.
90#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
91#[serde(tag = "status", rename_all = "snake_case")]
92pub enum AssetRubricResult {
93    /// Rubric passed.
94    Pass {
95        /// Verdict reason.
96        reason: String,
97        /// Reported anomalies.
98        anomalies: Vec<String>,
99    },
100    /// Rubric failed.
101    Fail {
102        /// Verdict reason.
103        reason: String,
104        /// Reported anomalies.
105        anomalies: Vec<String>,
106    },
107    /// Rubric evaluation errored.
108    Error {
109        /// Error message.
110        message: String,
111    },
112    /// Asset was selected but not evaluated after a fatal pool-level error.
113    NotEvaluatedAfterError {
114        /// Root error that aborted the remaining batch.
115        root_error: String,
116        /// Suggested retry action.
117        retry_hint: String,
118    },
119    /// Asset was not selected for evaluation.
120    Skipped {
121        /// Skip reason.
122        reason: String,
123    },
124}
125
126/// Severity for a caller-provided recommendation.
127#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord)]
128#[serde(rename_all = "snake_case")]
129pub enum RecommendationSeverity {
130    /// Low severity.
131    Low,
132    /// Medium severity.
133    Medium,
134    /// High severity.
135    High,
136}
137
138/// Caller-provided issue recommendation.
139#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
140pub struct IssueRecommendation {
141    /// Stable recommendation id.
142    pub id: String,
143    /// Machine-readable recommendation class.
144    pub class: String,
145    /// Recommendation severity.
146    pub severity: RecommendationSeverity,
147    /// Affected asset paths.
148    pub affected_assets: Vec<String>,
149    /// Short evidence snippets.
150    pub evidence: Vec<String>,
151    /// Suggested generic fix.
152    pub suggested_fix: String,
153    /// Candidate modules or subsystems, if known to the caller.
154    pub candidate_modules: Vec<String>,
155}
156
157/// Input passed to caller-provided issue classifiers.
158#[derive(Debug)]
159pub struct IssueClassificationInput<'a> {
160    /// Failed or errored asset report.
161    pub asset: &'a AssetRubricReport,
162    /// Combined failure text available for classification.
163    pub issue_text: &'a str,
164}
165
166/// Optional caller hook for mapping failed/error assets to reusable recommendations.
167pub trait IssueClassifier {
168    /// Classifies one failed/error asset.
169    fn classify(&self, input: IssueClassificationInput<'_>) -> Vec<IssueRecommendation>;
170}
171
172/// Batch rubric runner configuration.
173pub struct BatchRubricConfig<'a> {
174    /// Pool configuration.
175    pub pool: PoolConfig,
176    /// Question sent for every selected asset.
177    pub question: String,
178    /// Selection mode for unchanged assets.
179    pub selection_mode: SelectionMode,
180    /// Optional content-addressed cache directory for completed verdicts.
181    pub cache_dir: Option<PathBuf>,
182    /// Optional issue classifier.
183    pub classifier: Option<&'a dyn IssueClassifier>,
184}
185
186impl std::fmt::Debug for BatchRubricConfig<'_> {
187    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
188        f.debug_struct("BatchRubricConfig")
189            .field("pool", &self.pool)
190            .field("question", &self.question)
191            .field("selection_mode", &self.selection_mode)
192            .field("cache_dir", &self.cache_dir)
193            .field("classifier", &self.classifier.map(|_| "<classifier>"))
194            .finish()
195    }
196}
197
198/// Generic batch runner around [`RubricPool`].
199#[derive(Debug)]
200pub struct BatchRubricRun<'a> {
201    config: BatchRubricConfig<'a>,
202}
203
204impl<'a> BatchRubricRun<'a> {
205    /// Creates a batch runner.
206    #[must_use]
207    pub fn new(config: BatchRubricConfig<'a>) -> Self {
208        Self { config }
209    }
210
211    /// Evaluates selected assets from a snapshot diff.
212    ///
213    /// Pool startup and per-asset failures are represented inside the returned
214    /// report so callers can serialize the report before deciding whether to
215    /// fail their own command.
216    #[must_use]
217    pub fn run(&self, changes: &[AssetChange]) -> BatchRubricReport {
218        self.run_with_evaluator(changes, None)
219    }
220
221    fn run_with_evaluator(
222        &self,
223        changes: &[AssetChange],
224        evaluator: Option<&dyn BatchEvaluator>,
225    ) -> BatchRubricReport {
226        let started_at = unix_timestamp();
227        let selected = select_changed(changes, self.config.selection_mode);
228        let mut assets = Vec::with_capacity(changes.len());
229        let selected_evaluation = if selected.is_empty() {
230            SelectedEvaluation::default()
231        } else if let Some(evaluator) = evaluator {
232            evaluate_selected(
233                evaluator,
234                changes,
235                &selected,
236                &self.config.question,
237                self.config.pool.workers,
238                self.config.cache_dir.as_deref(),
239                &self.config.pool.default_options,
240            )
241        } else {
242            match RubricPool::new(self.config.pool.clone()) {
243                Ok(pool) => {
244                    let evaluation = evaluate_selected(
245                        &pool,
246                        changes,
247                        &selected,
248                        &self.config.question,
249                        self.config.pool.workers,
250                        self.config.cache_dir.as_deref(),
251                        &self.config.pool.default_options,
252                    );
253                    if evaluation.aborted {
254                        drop(pool);
255                    } else {
256                        let _stats = pool.shutdown();
257                    }
258                    evaluation
259                }
260                Err(error) => selected
261                    .iter()
262                    .map(|path| {
263                        AssetRubricReport::selected(
264                            path,
265                            status_for_selected(path, changes),
266                            AssetRubricResult::Error {
267                                message: format!("start visual-rubric worker pool: {error}"),
268                            },
269                        )
270                    })
271                    .collect::<Vec<_>>()
272                    .into(),
273            }
274        };
275        assets.extend(selected_evaluation.reports);
276        assets.extend(skipped_asset_reports(changes, self.config.selection_mode));
277        assets.sort_by(|left, right| left.path.cmp(&right.path));
278        let (logs, log_capture_error) =
279            match copy_logs_from_config(self.config.pool.log_capture.as_ref()) {
280                Ok(logs) => (logs, None),
281                Err(error) => (
282                    Vec::new(),
283                    Some(format!(
284                        "copy configured ACP logs into batch report: {error}"
285                    )),
286                ),
287            };
288        let recommendations = classify_recommendations(self.config.classifier, &assets);
289        let aggregate_status = aggregate_status(&assets);
290        BatchRubricReport {
291            schema_version: REPORT_SCHEMA_VERSION,
292            aggregate_status,
293            started_at,
294            finished_at: unix_timestamp(),
295            workers: self.config.pool.workers,
296            cache_hits: selected_evaluation.cache_hits,
297            cache_misses: selected_evaluation.cache_misses,
298            options: self.config.pool.default_options.clone(),
299            logs,
300            log_capture_error,
301            assets,
302            recommendations,
303        }
304    }
305}
306
307trait BatchEvaluator: Sync {
308    fn submit_asset(&self, png_path: &Path, question: &str) -> Result<RubricVerdict, PoolError>;
309}
310
311impl BatchEvaluator for RubricPool {
312    fn submit_asset(&self, png_path: &Path, question: &str) -> Result<RubricVerdict, PoolError> {
313        self.submit(png_path, question, RubricOptions::default())
314    }
315}
316
317#[derive(Default)]
318struct SelectedEvaluation {
319    reports: Vec<AssetRubricReport>,
320    aborted: bool,
321    cache_hits: u64,
322    cache_misses: u64,
323}
324
325impl From<Vec<AssetRubricReport>> for SelectedEvaluation {
326    fn from(reports: Vec<AssetRubricReport>) -> Self {
327        Self {
328            reports,
329            aborted: false,
330            cache_hits: 0,
331            cache_misses: 0,
332        }
333    }
334}
335
336fn evaluate_selected(
337    evaluator: &dyn BatchEvaluator,
338    changes: &[AssetChange],
339    selected: &[PathBuf],
340    question: &str,
341    workers: usize,
342    cache_dir: Option<&Path>,
343    cache_options: &RubricOptions,
344) -> SelectedEvaluation {
345    let worker_count = workers.max(1).min(selected.len());
346    let state = Arc::new(Mutex::new(EvaluationState {
347        pending: selected.iter().cloned().enumerate().collect(),
348        completed: Vec::with_capacity(selected.len()),
349        abort_message: None,
350        aborted: false,
351        cache_hits: 0,
352        cache_misses: 0,
353    }));
354
355    thread::scope(|scope| {
356        for _ in 0..worker_count {
357            let state = Arc::clone(&state);
358            scope.spawn(move || {
359                loop {
360                    let next = {
361                        let mut state = lock_state(&state);
362                        if state.abort_message.is_some() {
363                            None
364                        } else {
365                            state.pending.pop_front()
366                        }
367                    };
368                    let Some((index, path)) = next else {
369                        break;
370                    };
371
372                    let cache_key = if cache_dir.is_some() {
373                        cache_key_for_asset(&path, question, cache_options)
374                    } else {
375                        None
376                    };
377                    if let (Some(dir), Some(key)) = (cache_dir, cache_key.as_deref()) {
378                        if let Some(verdict) = read_cached_verdict(dir, key) {
379                            let mut state = lock_state(&state);
380                            state.cache_hits += 1;
381                            state.completed.push(CompletedEvaluation {
382                                index,
383                                path,
384                                result: result_from_verdict(verdict),
385                            });
386                            continue;
387                        }
388                    }
389                    if cache_dir.is_some() {
390                        let mut state = lock_state(&state);
391                        state.cache_misses += 1;
392                    }
393
394                    let outcome = evaluator.submit_asset(&path, question);
395                    if let (Some(dir), Some(key)) = (cache_dir, cache_key.as_deref()) {
396                        if let Ok(verdict) = &outcome {
397                            write_cached_verdict(dir, key, verdict);
398                        }
399                    }
400                    let mut state = lock_state(&state);
401                    match outcome {
402                        Ok(verdict) => state.completed.push(CompletedEvaluation {
403                            index,
404                            path,
405                            result: result_from_verdict(verdict),
406                        }),
407                        Err(error) => {
408                            let message = error.to_string();
409                            if should_abort_after_error(&error) && state.abort_message.is_none() {
410                                state.aborted = true;
411                                state.abort_message = Some(message.clone());
412                            }
413                            state.completed.push(CompletedEvaluation {
414                                index,
415                                path,
416                                result: AssetRubricResult::Error { message },
417                            });
418                        }
419                    }
420                }
421            });
422        }
423    });
424
425    let mut state = lock_state(&state);
426    let abort_message = state.abort_message.clone();
427    let mut completed = std::mem::take(&mut state.completed);
428    completed.sort_by_key(|evaluation| evaluation.index);
429    let mut reports = completed
430        .into_iter()
431        .map(|evaluation| {
432            AssetRubricReport::selected(
433                &evaluation.path,
434                status_for_selected(&evaluation.path, changes),
435                evaluation.result,
436            )
437        })
438        .collect::<Vec<_>>();
439
440    if let Some(message) = abort_message {
441        reports.extend(state.pending.drain(..).map(|(_, path)| {
442            AssetRubricReport::selected(
443                &path,
444                status_for_selected(&path, changes),
445                AssetRubricResult::NotEvaluatedAfterError {
446                    root_error: message.clone(),
447                    retry_hint: retry_hint_after_pool_error(&message).to_owned(),
448                },
449            )
450        }));
451    }
452
453    SelectedEvaluation {
454        reports,
455        aborted: state.aborted,
456        cache_hits: state.cache_hits,
457        cache_misses: state.cache_misses,
458    }
459}
460
461struct EvaluationState {
462    pending: VecDeque<(usize, PathBuf)>,
463    completed: Vec<CompletedEvaluation>,
464    abort_message: Option<String>,
465    aborted: bool,
466    cache_hits: u64,
467    cache_misses: u64,
468}
469
470struct CompletedEvaluation {
471    index: usize,
472    path: PathBuf,
473    result: AssetRubricResult,
474}
475
476fn lock_state<T>(state: &Mutex<T>) -> MutexGuard<'_, T> {
477    state.lock().unwrap_or_else(|error| error.into_inner())
478}
479
480fn result_from_verdict(verdict: RubricVerdict) -> AssetRubricResult {
481    if verdict.verdict.is_pass() {
482        AssetRubricResult::Pass {
483            reason: verdict.reason,
484            anomalies: verdict.anomalies,
485        }
486    } else {
487        AssetRubricResult::Fail {
488            reason: verdict.reason,
489            anomalies: verdict.anomalies,
490        }
491    }
492}
493
494#[derive(Debug, Deserialize, Serialize)]
495struct CachedVerdict {
496    schema_version: u32,
497    key: String,
498    verdict: RubricVerdict,
499}
500
501fn cache_key_for_asset(path: &Path, question: &str, options: &RubricOptions) -> Option<String> {
502    let image = fs::read(path).ok()?;
503    let options = serde_json::to_vec(options).ok()?;
504    let mut hasher = Sha256::new();
505    hasher.update(CACHE_KEY_DOMAIN);
506    update_key_part(&mut hasher, &image);
507    update_key_part(&mut hasher, question.as_bytes());
508    update_key_part(&mut hasher, &options);
509    Some(format!("{:x}", hasher.finalize()))
510}
511
512fn update_key_part(hasher: &mut Sha256, value: &[u8]) {
513    hasher.update((value.len() as u64).to_le_bytes());
514    hasher.update(value);
515}
516
517fn cached_verdict_path(cache_dir: &Path, key: &str) -> PathBuf {
518    cache_dir.join(format!("{key}.json"))
519}
520
521fn read_cached_verdict(cache_dir: &Path, key: &str) -> Option<RubricVerdict> {
522    let bytes = fs::read(cached_verdict_path(cache_dir, key)).ok()?;
523    let cached = serde_json::from_slice::<CachedVerdict>(&bytes).ok()?;
524    (cached.schema_version == CACHE_SCHEMA_VERSION && cached.key == key).then_some(cached.verdict)
525}
526
527fn write_cached_verdict(cache_dir: &Path, key: &str, verdict: &RubricVerdict) {
528    if fs::create_dir_all(cache_dir).is_err() {
529        return;
530    }
531    let Ok(mut temporary) = NamedTempFile::new_in(cache_dir) else {
532        return;
533    };
534    let cached = CachedVerdict {
535        schema_version: CACHE_SCHEMA_VERSION,
536        key: key.to_owned(),
537        verdict: verdict.clone(),
538    };
539    if serde_json::to_writer(&mut temporary, &cached).is_err()
540        || temporary.flush().is_err()
541        || temporary.as_file().sync_all().is_err()
542    {
543        return;
544    }
545    let _ = temporary.persist(cached_verdict_path(cache_dir, key));
546}
547
548fn should_abort_after_error(error: &PoolError) -> bool {
549    matches!(
550        error,
551        PoolError::Timeout { .. }
552            | PoolError::QuotaExceeded
553            | PoolError::WorkerCrashed { .. }
554            | PoolError::NoLiveWorkers
555            | PoolError::Closed
556    )
557}
558
559fn retry_hint_after_pool_error(message: &str) -> &'static str {
560    if message.contains("timed out") || message.contains("timeout") {
561        "Asset was not evaluated after an ACP timeout; inspect captured logs and rerun with fewer workers or a smaller asset scope."
562    } else if message.contains("quota") {
563        "Asset was not evaluated after an ACP quota error; inspect captured logs, wait for quota recovery or switch credentials, then rerun the batch."
564    } else {
565        "Asset was not evaluated after an ACP worker error; inspect captured logs and rerun with fewer workers or a smaller asset scope."
566    }
567}
568
569fn status_for_selected(path: &Path, changes: &[AssetChange]) -> &'static str {
570    changes
571        .iter()
572        .find_map(|change| (change.path() == path).then_some(change.status()))
573        .map_or("selected", |status| status)
574}
575
576fn skipped_asset_reports(
577    changes: &[AssetChange],
578    selection_mode: SelectionMode,
579) -> Vec<AssetRubricReport> {
580    let mut reports = Vec::with_capacity(changes.len());
581    for change in changes {
582        match (change, selection_mode) {
583            (AssetChange::Unchanged(path), SelectionMode::ChangedOnly) => {
584                reports.push(AssetRubricReport::skipped(
585                    path,
586                    "unchanged",
587                    "asset content did not change during this command",
588                ));
589            }
590            (AssetChange::Deleted(path), _) => {
591                reports.push(AssetRubricReport::skipped(
592                    path,
593                    "deleted",
594                    "asset was deleted before evaluation",
595                ));
596            }
597            (AssetChange::Added(_) | AssetChange::Changed(_) | AssetChange::Unchanged(_), _) => {}
598        }
599    }
600    reports
601}
602
603impl AssetRubricReport {
604    fn selected(path: &Path, change: &str, result: AssetRubricResult) -> Self {
605        Self {
606            path: path.to_string_lossy().into_owned(),
607            change: change.to_owned(),
608            selected: true,
609            result,
610        }
611    }
612
613    fn skipped(path: &Path, change: &str, reason: &str) -> Self {
614        Self {
615            path: path.to_string_lossy().into_owned(),
616            change: change.to_owned(),
617            selected: false,
618            result: AssetRubricResult::Skipped {
619                reason: reason.to_owned(),
620            },
621        }
622    }
623}
624
625fn unix_timestamp() -> String {
626    SystemTime::now()
627        .duration_since(UNIX_EPOCH)
628        .map(|duration| duration.as_secs().to_string())
629        .unwrap_or_else(|_| "0".to_owned())
630}