Skip to main content

visual_rubric/
lib.rs

1//! Shared AI visual-rubric runner for screenshot review.
2//!
3//! This crate owns the Codex ACP plumbing so browser screenshots, offscreen
4//! renderer captures, and VM/VNC screenshots can use one rubric path.
5//!
6//! Direct Codex ACP evaluation sends the rubric prompt and screenshot in one
7//! multimodal request. The crate also provides a two-stage pipeline: vision
8//! model extraction via an OpenAI-compatible HTTP API, then rubric scoring via
9//! ACP.
10#![warn(missing_docs)]
11#![allow(
12    clippy::io_other_error,
13    clippy::manual_checked_ops,
14    clippy::single_match
15)]
16
17#[cfg(feature = "acp")]
18mod acp;
19#[cfg(feature = "batch")]
20mod batch;
21#[cfg(feature = "audit")]
22mod browser;
23pub mod calibration;
24pub mod cli;
25mod config;
26mod configured_eval;
27pub mod coverage;
28pub mod deterministic;
29mod errors;
30pub mod manifest;
31#[cfg(feature = "pool")]
32mod pool;
33pub mod presets;
34pub mod qa;
35mod report;
36mod typed_strings;
37mod verdict;
38#[cfg(any(feature = "vision-api", feature = "http-rubric"))]
39pub mod vision;
40
41use std::path::Path;
42#[cfg(any(feature = "codex-acp", feature = "pipeline"))]
43use std::path::PathBuf;
44
45#[cfg(feature = "acp")]
46use acp::AcpClient;
47use base64::Engine as _;
48#[cfg(any(feature = "vision-api", feature = "http-rubric"))]
49use vision::VisionApiConfig;
50
51#[cfg(feature = "batch")]
52pub use batch::{
53    AggregateStatus, AssetChange, AssetRubricReport, AssetRubricResult, AssetSnapshot,
54    BatchRubricConfig, BatchRubricReport, BatchRubricRun, IssueClassificationInput,
55    IssueClassifier, IssueRecommendation, RecommendationSeverity, SelectionMode, diff_snapshots,
56    select_changed,
57};
58pub use calibration::{
59    CALIBRATION_CORPUS_SCHEMA_VERSION, CalibrationCaptureV1, CalibrationCorpusV1,
60    REQUIRED_CALIBRATION_CATEGORIES,
61};
62pub use cli::Cli;
63pub use config::{
64    ConfigMode, RubricOptions, RubricRunConfig, TomlConfig, TomlRubric, TomlSequence, TomlVision,
65    load_config_toml,
66};
67#[cfg(feature = "codex-acp")]
68pub use config::{default_codex_acp_binary, default_options, direct_codex_gpt_config};
69pub use configured_eval::{evaluate_configured, evaluate_configured_with_vision};
70pub use coverage::{
71    COVERAGE_CONTRACT_SCHEMA_VERSION, COVERAGE_REPORT_SCHEMA_VERSION, CoverageContractV1,
72    CoverageExclusionV1, CoverageReportV1, CoverageSurfaceV1, CoverageTransitionV1,
73};
74pub use deterministic::{
75    DETERMINISTIC_EVIDENCE_SCHEMA_VERSION, DeterministicEvidenceV1, DeterministicObservationV1,
76};
77pub use errors::{PoolError, RateLimitEvent, RubricError};
78pub use manifest::{
79    ArtifactDigest, CAPTURE_MANIFEST_SCHEMA_VERSION, CaptureCell, CaptureEnvironment,
80    CaptureManifest, ManifestError, Viewport,
81};
82#[cfg(feature = "pool")]
83pub use pool::{LogCaptureConfig, LogPathMode, PoolConfig, PoolStats, RubricPool};
84pub use qa::{
85    CALIBRATION_SENTINEL_SCHEMA_VERSION, CalibrationExpectation, CalibrationSentinelV1,
86    FINDING_SCHEMA_VERSION, FindingV1, OBSERVATION_SCHEMA_VERSION, ObservationV1,
87    VISUAL_RUN_REPORT_SCHEMA_VERSION, VisualRunReportV1, VisualRunStatus, finding_fingerprint,
88    validate_calibration_sentinels, validate_visual_run, validate_visual_run_against_manifest,
89};
90pub use report::{PageResult, RubricReport};
91pub use typed_strings::{RubricEffort, RubricVerdictStatus};
92pub use verdict::{RubricVerdict, assert_verdict, parse_verdict};
93
94/// Default system prompt used for screenshot rubric requests.
95///
96/// Shared with the `ui-regression` question preset.
97pub const DEFAULT_SYSTEM_PROMPT: &str = presets::UI_REGRESSION_SYSTEM_PROMPT;
98
99/// One ordered screenshot checkpoint in an interaction journey.
100#[cfg(any(feature = "codex-acp", feature = "pipeline"))]
101#[derive(Clone, Debug, PartialEq, Eq)]
102pub struct SequenceFrame {
103    /// Stable checkpoint label used in the rubric prompt and evidence.
104    pub label: String,
105    /// PNG artifact containing the checkpoint frame.
106    pub path: PathBuf,
107}
108
109/// Policy applied to one ordered screenshot sequence.
110#[derive(Clone, Copy, Debug, PartialEq, Eq)]
111pub struct SequenceOptions {
112    /// Maximum number of checkpoints accepted in one request.
113    pub max_frames: usize,
114    /// Whether the rubric must assess the semantic transition between
115    /// adjacent checkpoints. A one-frame sequence never has a transition.
116    pub require_transition: bool,
117}
118
119impl Default for SequenceOptions {
120    fn default() -> Self {
121        Self {
122            max_frames: 8,
123            require_transition: true,
124        }
125    }
126}
127
128impl SequenceOptions {
129    /// Returns an error when the policy cannot provide a bounded request.
130    pub fn validate(self) -> Result<(), String> {
131        if self.max_frames == 0 {
132            return Err("sequence max_frames must be greater than zero".to_owned());
133        }
134        if self.max_frames > 32 {
135            return Err("sequence max_frames must not exceed 32".to_owned());
136        }
137        Ok(())
138    }
139}
140
141/// Default Codex ACP model.
142#[cfg(feature = "codex-acp")]
143pub const DEFAULT_CODEX_ACP_MODEL: &str = "gpt-5.5";
144/// Default Codex ACP reasoning effort.
145#[cfg(feature = "codex-acp")]
146pub const DEFAULT_CODEX_ACP_REASONING_EFFORT: &str = "medium";
147
148/// Default prompt for the vision extraction stage.
149///
150/// Asks the vision model to describe the screenshot as structured JSON
151/// so a text-only rubric model (e.g. DeepSeek V4 via opencode) can score it.
152#[cfg(feature = "vision-api")]
153pub const DEFAULT_VISION_PROMPT: &str = "\
154You are a UI description engine. Given a screenshot, produce a structured JSON \
155description of all visible user interface elements, their text content, layout, \
156and any visual issues (clipping, overlap, blank regions, contrast problems). \
157Output ONLY valid JSON with no additional text.";
158
159/// Reads and base64-encodes a PNG file.
160///
161/// # Errors
162///
163/// Returns [`PoolError::Rpc`] when the PNG cannot be read.
164pub fn encode_png(png_path: &Path) -> Result<String, PoolError> {
165    let bytes = std::fs::read(png_path)
166        .map_err(|e| PoolError::Rpc(format!("read png {}: {e}", png_path.display())))?;
167    Ok(base64::engine::general_purpose::STANDARD.encode(bytes))
168}
169
170/// Evaluates a PNG and returns an error when the verdict is not pass.
171///
172/// # Errors
173///
174/// Returns [`RubricError`] for PNG IO, ACP, JSON parsing, or failed
175/// assertion errors.
176#[cfg(feature = "codex-acp")]
177pub fn assert_image_rubric(png_path: &Path, name: &str, question: &str) -> Result<(), RubricError> {
178    let verdict = evaluate_image_rubric(png_path, question)?;
179    assert_verdict(name, verdict)
180}
181
182/// Evaluates a PNG with default options.
183///
184/// # Errors
185///
186/// Returns [`RubricError`] for PNG IO, ACP, or verdict parsing failures.
187#[cfg(feature = "codex-acp")]
188pub fn evaluate_image_rubric(
189    png_path: &Path,
190    question: &str,
191) -> Result<RubricVerdict, RubricError> {
192    evaluate_image_rubric_with_options(png_path, question, default_options())
193}
194
195/// Evaluates a PNG with caller-provided model options.
196///
197/// # Errors
198///
199/// Returns [`RubricError`] for PNG IO, ACP, or verdict parsing failures.
200#[cfg(feature = "codex-acp")]
201pub fn evaluate_image_rubric_with_options(
202    png_path: &Path,
203    question: &str,
204    opts: RubricOptions,
205) -> Result<RubricVerdict, RubricError> {
206    evaluate_image_rubric_with_config(png_path, question, opts, RubricRunConfig::default())
207}
208
209/// Evaluates a PNG with caller-provided model and runtime configuration.
210///
211/// # Errors
212///
213/// Returns [`RubricError`] for PNG IO, ACP, or verdict parsing failures.
214#[cfg(feature = "codex-acp")]
215pub fn evaluate_image_rubric_with_config(
216    png_path: &Path,
217    question: &str,
218    opts: RubricOptions,
219    config: RubricRunConfig,
220) -> Result<RubricVerdict, RubricError> {
221    let bytes = std::fs::read(png_path).map_err(|source| RubricError::ReadPng {
222        path: png_path.to_path_buf(),
223        source,
224    })?;
225    let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes);
226    let text = run_codex_acp_rubric(
227        &b64,
228        question,
229        opts.model
230            .as_deref()
231            .map_or(DEFAULT_CODEX_ACP_MODEL, |model| model),
232        opts.effort
233            .as_deref()
234            .map_or(DEFAULT_CODEX_ACP_REASONING_EFFORT, |effort| effort),
235        opts.system_prompt
236            .as_deref()
237            .map_or(DEFAULT_SYSTEM_PROMPT, |system_prompt| system_prompt),
238        &config,
239    )?;
240
241    parse_verdict(&text).map_err(|source| RubricError::ParseVerdict { text, source })
242}
243
244/// Evaluates an ordered screenshot sequence as one interaction journey.
245///
246/// The evaluator receives every checkpoint in order, including labels, so it
247/// can verify both the visible state at each point and the before/after
248/// transition implied by the sequence. A single-frame rubric cannot prove
249/// that an interaction actually changed the UI.
250///
251/// # Errors
252///
253/// Returns [`RubricError`] for PNG IO, ACP, or verdict parsing failures.
254#[cfg(feature = "codex-acp")]
255pub fn evaluate_image_sequence_rubric_with_config(
256    frames: &[SequenceFrame],
257    question: &str,
258    opts: RubricOptions,
259    config: RubricRunConfig,
260) -> Result<RubricVerdict, RubricError> {
261    evaluate_image_sequence_rubric_with_options(
262        frames,
263        question,
264        opts,
265        config,
266        SequenceOptions::default(),
267    )
268}
269
270/// Evaluates an ordered screenshot sequence with an explicit checkpoint
271/// policy and ACP runtime configuration.
272#[cfg(feature = "codex-acp")]
273pub fn evaluate_image_sequence_rubric_with_options(
274    frames: &[SequenceFrame],
275    question: &str,
276    opts: RubricOptions,
277    config: RubricRunConfig,
278    sequence_options: SequenceOptions,
279) -> Result<RubricVerdict, RubricError> {
280    sequence_options
281        .validate()
282        .map_err(PoolError::Rpc)
283        .map_err(RubricError::Pool)?;
284    if frames.is_empty() {
285        return Err(RubricError::Pool(PoolError::Rpc(
286            "sequence rubric requires at least one frame".to_owned(),
287        )));
288    }
289    if sequence_options.require_transition && frames.len() < 2 {
290        return Err(RubricError::Pool(PoolError::Rpc(
291            "sequence transition assessment requires at least two frames".to_owned(),
292        )));
293    }
294    if frames.len() > sequence_options.max_frames {
295        return Err(RubricError::Pool(PoolError::Rpc(format!(
296            "sequence contains {} frames, maximum is {}",
297            frames.len(),
298            sequence_options.max_frames
299        ))));
300    }
301    let mut encoded = Vec::with_capacity(frames.len());
302    for frame in frames {
303        let bytes = std::fs::read(&frame.path).map_err(|source| RubricError::ReadPng {
304            path: frame.path.clone(),
305            source,
306        })?;
307        encoded.push((
308            frame.label.clone(),
309            base64::engine::general_purpose::STANDARD.encode(bytes),
310        ));
311    }
312    let transition_instruction = if sequence_options.require_transition {
313        "Check every checkpoint and whether each before/after transition is visible and semantically correct."
314    } else {
315        "Check every checkpoint for visual and semantic correctness. Transition assessment is disabled."
316    };
317    let system_prompt = opts
318        .system_prompt
319        .as_deref()
320        .map_or(DEFAULT_SYSTEM_PROMPT, |prompt| prompt);
321    let prompt = format!(
322        "{system_prompt}\n\nEvaluate this ordered interaction sequence. {transition_instruction}\nQuestion: {question}"
323    );
324    let text = run_codex_acp_sequence(
325        &encoded,
326        &prompt,
327        opts.model
328            .as_deref()
329            .map_or(DEFAULT_CODEX_ACP_MODEL, |model| model),
330        opts.effort
331            .as_deref()
332            .map_or(DEFAULT_CODEX_ACP_REASONING_EFFORT, |effort| effort),
333        &config,
334    )?;
335    parse_verdict(&text).map_err(|source| RubricError::ParseVerdict { text, source })
336}
337
338/// Runs the rubric step: either an HTTP call to a text model or ACP.
339///
340/// When `config.url` is set, posts `rubric_prompt` to an OpenAI-compatible
341/// text model.  Otherwise spawns an ACP child process.
342#[cfg(feature = "pipeline")]
343fn run_rubric_prompt(rubric_prompt: &str, config: &RubricRunConfig) -> Result<String, RubricError> {
344    #[cfg(feature = "http-rubric")]
345    if let Some(ref url) = config.url {
346        let api_config = VisionApiConfig {
347            url: url.clone(),
348            model: config.api_model.clone().unwrap_or_default(),
349            api_key: None,
350        };
351        return vision::call_text_api(rubric_prompt, &api_config).map_err(RubricError::Pool);
352    }
353    #[cfg(feature = "acp")]
354    {
355        let mut acp = AcpClient::spawn(
356            &config.codex_acp_binary,
357            &config.acp_args,
358            &config.extra_env,
359            config.cwd.as_deref(),
360        )
361        .map_err(RubricError::Pool)?;
362        acp.start_session(config.cwd.as_deref(), config.api_model.as_deref(), None)
363            .map_err(RubricError::Pool)?;
364        acp.prompt_text(rubric_prompt).map_err(RubricError::Pool)
365    }
366    #[cfg(not(feature = "acp"))]
367    Err(RubricError::Pool(PoolError::Spawn(
368        "no ACP backend enabled; enable 'acp' feature".to_string(),
369    )))
370}
371
372/// Two-stage pipeline evaluation: vision model → rubric model.
373///
374/// Stage 1: Sends the screenshot to an OpenAI-compatible vision API and
375/// returns a structured JSON description.
376///
377/// Stage 2: When [`RubricRunConfig::url`] is set, sends the structured
378/// description (plus the rubric question) to an OpenAI-compatible text
379/// model.  Otherwise falls back to the configured ACP backend.
380///
381/// # Errors
382///
383/// Returns [`RubricError`] for PNG IO, vision API, ACP, or verdict parsing
384/// failures.
385#[cfg(feature = "pipeline")]
386pub fn evaluate_image_rubric_pipeline(
387    png_path: &Path,
388    question: &str,
389    vision_config: &VisionApiConfig,
390    vision_prompt: &str,
391    rubric_options: &RubricOptions,
392    rubric_config: &RubricRunConfig,
393) -> Result<RubricVerdict, RubricError> {
394    let bytes = std::fs::read(png_path).map_err(|source| RubricError::ReadPng {
395        path: png_path.to_path_buf(),
396        source,
397    })?;
398    let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes);
399
400    let structured =
401        vision::call_vision_api(&b64, vision_prompt, vision_config).map_err(RubricError::Pool)?;
402
403    let system_prompt = rubric_options
404        .system_prompt
405        .as_deref()
406        .map_or(DEFAULT_SYSTEM_PROMPT, |system_prompt| system_prompt);
407    let rubric_prompt =
408        format!("{system_prompt}\n\nUI description:\n{structured}\n\nQuestion: {question}");
409
410    let text = run_rubric_prompt(&rubric_prompt, rubric_config)?;
411
412    parse_verdict(&text).map_err(|source| RubricError::ParseVerdict { text, source })
413}
414
415/// Like [`evaluate_image_rubric_pipeline`] but also returns the vision
416/// model's text description alongside the rubric verdict.
417///
418/// The first element of the tuple is the rubric verdict, the second is
419/// the structured UI description from the vision model.
420///
421/// # Errors
422///
423/// See [`evaluate_image_rubric_pipeline`].
424#[cfg(feature = "pipeline")]
425pub fn evaluate_image_rubric_pipeline_with_vision(
426    png_path: &Path,
427    question: &str,
428    vision_config: &VisionApiConfig,
429    vision_prompt: &str,
430    rubric_options: &RubricOptions,
431    rubric_config: &RubricRunConfig,
432) -> Result<(RubricVerdict, String), RubricError> {
433    let bytes = std::fs::read(png_path).map_err(|source| RubricError::ReadPng {
434        path: png_path.to_path_buf(),
435        source,
436    })?;
437    let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes);
438
439    let structured =
440        vision::call_vision_api(&b64, vision_prompt, vision_config).map_err(RubricError::Pool)?;
441
442    let system_prompt = rubric_options
443        .system_prompt
444        .as_deref()
445        .map_or(DEFAULT_SYSTEM_PROMPT, |system_prompt| system_prompt);
446    let rubric_prompt =
447        format!("{system_prompt}\n\nUI description:\n{structured}\n\nQuestion: {question}");
448
449    let text = run_rubric_prompt(&rubric_prompt, rubric_config)?;
450
451    let verdict =
452        parse_verdict(&text).map_err(|source| RubricError::ParseVerdict { text, source })?;
453    Ok((verdict, structured))
454}
455
456/// Two-stage evaluation for an ordered interaction sequence.
457///
458/// The vision stage receives all labelled frames and the rubric stage receives
459/// the resulting ordered description, preserving transition context in both
460/// supported backends.
461#[cfg(feature = "pipeline")]
462pub fn evaluate_image_sequence_rubric_pipeline(
463    frames: &[SequenceFrame],
464    question: &str,
465    vision_config: &VisionApiConfig,
466    vision_prompt: &str,
467    rubric_options: &RubricOptions,
468    rubric_config: &RubricRunConfig,
469) -> Result<RubricVerdict, RubricError> {
470    evaluate_image_sequence_rubric_pipeline_with_options(
471        frames,
472        question,
473        vision_config,
474        vision_prompt,
475        rubric_options,
476        rubric_config,
477        SequenceOptions::default(),
478    )
479}
480
481/// Two-stage sequence evaluation with an explicit checkpoint policy.
482#[cfg(feature = "pipeline")]
483pub fn evaluate_image_sequence_rubric_pipeline_with_options(
484    frames: &[SequenceFrame],
485    question: &str,
486    vision_config: &VisionApiConfig,
487    vision_prompt: &str,
488    rubric_options: &RubricOptions,
489    rubric_config: &RubricRunConfig,
490    sequence_options: SequenceOptions,
491) -> Result<RubricVerdict, RubricError> {
492    sequence_options
493        .validate()
494        .map_err(PoolError::VisionApi)
495        .map_err(RubricError::Pool)?;
496    if frames.is_empty() {
497        return Err(RubricError::Pool(PoolError::VisionApi(
498            "sequence rubric requires at least one frame".to_owned(),
499        )));
500    }
501    if sequence_options.require_transition && frames.len() < 2 {
502        return Err(RubricError::Pool(PoolError::VisionApi(
503            "sequence transition assessment requires at least two frames".to_owned(),
504        )));
505    }
506    if frames.len() > sequence_options.max_frames {
507        return Err(RubricError::Pool(PoolError::VisionApi(format!(
508            "sequence contains {} frames, maximum is {}",
509            frames.len(),
510            sequence_options.max_frames
511        ))));
512    }
513    let mut encoded = Vec::with_capacity(frames.len());
514    for frame in frames {
515        let bytes = std::fs::read(&frame.path).map_err(|source| RubricError::ReadPng {
516            path: frame.path.clone(),
517            source,
518        })?;
519        encoded.push((
520            frame.label.clone(),
521            base64::engine::general_purpose::STANDARD.encode(bytes),
522        ));
523    }
524    let structured = vision::call_vision_api_sequence(&encoded, vision_prompt, vision_config)
525        .map_err(RubricError::Pool)?;
526    let system_prompt = rubric_options
527        .system_prompt
528        .as_deref()
529        .map_or(DEFAULT_SYSTEM_PROMPT, |prompt| prompt);
530    let transition_instruction = if sequence_options.require_transition {
531        "Assess the semantic before/after transition between adjacent checkpoints."
532    } else {
533        "Assess each checkpoint independently; transition assessment is disabled."
534    };
535    let rubric_prompt = format!(
536        "{system_prompt}\n\nOrdered UI journey description:\n{structured}\n\n{transition_instruction}\nQuestion: {question}"
537    );
538    let text = run_rubric_prompt(&rubric_prompt, rubric_config)?;
539    parse_verdict(&text).map_err(|source| RubricError::ParseVerdict { text, source })
540}
541
542/// Runs the CLI command.
543///
544/// # Errors
545///
546/// Returns command parsing, IO, ACP, or audit failures as [`anyhow::Error`].
547pub fn run(cli: Cli) -> anyhow::Result<()> {
548    cli::run(cli)
549}
550
551#[cfg(feature = "codex-acp")]
552fn run_codex_acp_rubric(
553    b64_png: &str,
554    question: &str,
555    model: &str,
556    effort: &str,
557    system_prompt: &str,
558    config: &RubricRunConfig,
559) -> Result<String, PoolError> {
560    let mut acp = AcpClient::spawn(
561        &config.codex_acp_binary,
562        &config.acp_args,
563        &config.extra_env,
564        config.cwd.as_deref(),
565    )?;
566    acp.start_session(config.cwd.as_deref(), Some(model), Some(effort))?;
567
568    let prompt = format!("{system_prompt}\n\nQuestion: {question}");
569    acp.prompt_image(&prompt, b64_png)
570}
571
572#[cfg(feature = "codex-acp")]
573fn run_codex_acp_sequence(
574    frames: &[(String, String)],
575    prompt: &str,
576    model: &str,
577    effort: &str,
578    config: &RubricRunConfig,
579) -> Result<String, PoolError> {
580    let mut acp = AcpClient::spawn(
581        &config.codex_acp_binary,
582        &config.acp_args,
583        &config.extra_env,
584        config.cwd.as_deref(),
585    )?;
586    acp.start_session(config.cwd.as_deref(), Some(model), Some(effort))?;
587    acp.prompt_images(prompt, frames)
588}
589
590#[cfg(test)]
591mod tests;