#![warn(missing_docs)]
#![allow(
clippy::io_other_error,
clippy::manual_checked_ops,
clippy::single_match
)]
#[cfg(feature = "acp")]
mod acp;
#[cfg(feature = "batch")]
mod batch;
#[cfg(feature = "audit")]
mod browser;
pub mod calibration;
pub mod cli;
mod config;
mod configured_eval;
pub mod coverage;
pub mod deterministic;
mod errors;
pub mod manifest;
#[cfg(feature = "pool")]
mod pool;
pub mod presets;
pub mod qa;
mod report;
mod typed_strings;
mod verdict;
#[cfg(any(feature = "vision-api", feature = "http-rubric"))]
pub mod vision;
use std::path::Path;
#[cfg(any(feature = "codex-acp", feature = "pipeline"))]
use std::path::PathBuf;
#[cfg(feature = "acp")]
use acp::AcpClient;
use base64::Engine as _;
#[cfg(any(feature = "vision-api", feature = "http-rubric"))]
use vision::VisionApiConfig;
#[cfg(feature = "batch")]
pub use batch::{
AggregateStatus, AssetChange, AssetRubricReport, AssetRubricResult, AssetSnapshot,
BatchRubricConfig, BatchRubricReport, BatchRubricRun, IssueClassificationInput,
IssueClassifier, IssueRecommendation, RecommendationSeverity, SelectionMode, diff_snapshots,
select_changed,
};
pub use calibration::{
CALIBRATION_CORPUS_SCHEMA_VERSION, CalibrationCaptureV1, CalibrationCorpusV1,
REQUIRED_CALIBRATION_CATEGORIES,
};
pub use cli::Cli;
pub use config::{
ConfigMode, RubricOptions, RubricRunConfig, TomlConfig, TomlRubric, TomlSequence, TomlVision,
load_config_toml,
};
#[cfg(feature = "codex-acp")]
pub use config::{default_codex_acp_binary, default_options, direct_codex_gpt_config};
pub use configured_eval::{evaluate_configured, evaluate_configured_with_vision};
pub use coverage::{
COVERAGE_CONTRACT_SCHEMA_VERSION, COVERAGE_REPORT_SCHEMA_VERSION, CoverageContractV1,
CoverageExclusionV1, CoverageReportV1, CoverageSurfaceV1, CoverageTransitionV1,
};
pub use deterministic::{
DETERMINISTIC_EVIDENCE_SCHEMA_VERSION, DeterministicEvidenceV1, DeterministicObservationV1,
};
pub use errors::{PoolError, RateLimitEvent, RubricError};
pub use manifest::{
ArtifactDigest, CAPTURE_MANIFEST_SCHEMA_VERSION, CaptureCell, CaptureEnvironment,
CaptureManifest, ManifestError, Viewport,
};
#[cfg(feature = "pool")]
pub use pool::{LogCaptureConfig, LogPathMode, PoolConfig, PoolStats, RubricPool};
pub use qa::{
CALIBRATION_SENTINEL_SCHEMA_VERSION, CalibrationExpectation, CalibrationSentinelV1,
FINDING_SCHEMA_VERSION, FindingV1, OBSERVATION_SCHEMA_VERSION, ObservationV1,
VISUAL_RUN_REPORT_SCHEMA_VERSION, VisualRunReportV1, VisualRunStatus, finding_fingerprint,
validate_calibration_sentinels, validate_visual_run, validate_visual_run_against_manifest,
};
pub use report::{PageResult, RubricReport};
pub use typed_strings::{RubricEffort, RubricVerdictStatus};
pub use verdict::{RubricVerdict, assert_verdict, parse_verdict};
pub const DEFAULT_SYSTEM_PROMPT: &str = presets::UI_REGRESSION_SYSTEM_PROMPT;
#[cfg(any(feature = "codex-acp", feature = "pipeline"))]
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct SequenceFrame {
pub label: String,
pub path: PathBuf,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct SequenceOptions {
pub max_frames: usize,
pub require_transition: bool,
}
impl Default for SequenceOptions {
fn default() -> Self {
Self {
max_frames: 8,
require_transition: true,
}
}
}
impl SequenceOptions {
pub fn validate(self) -> Result<(), String> {
if self.max_frames == 0 {
return Err("sequence max_frames must be greater than zero".to_owned());
}
if self.max_frames > 32 {
return Err("sequence max_frames must not exceed 32".to_owned());
}
Ok(())
}
}
#[cfg(feature = "codex-acp")]
pub const DEFAULT_CODEX_ACP_MODEL: &str = "gpt-5.5";
#[cfg(feature = "codex-acp")]
pub const DEFAULT_CODEX_ACP_REASONING_EFFORT: &str = "medium";
#[cfg(feature = "vision-api")]
pub const DEFAULT_VISION_PROMPT: &str = "\
You are a UI description engine. Given a screenshot, produce a structured JSON \
description of all visible user interface elements, their text content, layout, \
and any visual issues (clipping, overlap, blank regions, contrast problems). \
Output ONLY valid JSON with no additional text.";
pub fn encode_png(png_path: &Path) -> Result<String, PoolError> {
let bytes = std::fs::read(png_path)
.map_err(|e| PoolError::Rpc(format!("read png {}: {e}", png_path.display())))?;
Ok(base64::engine::general_purpose::STANDARD.encode(bytes))
}
#[cfg(feature = "codex-acp")]
pub fn assert_image_rubric(png_path: &Path, name: &str, question: &str) -> Result<(), RubricError> {
let verdict = evaluate_image_rubric(png_path, question)?;
assert_verdict(name, verdict)
}
#[cfg(feature = "codex-acp")]
pub fn evaluate_image_rubric(
png_path: &Path,
question: &str,
) -> Result<RubricVerdict, RubricError> {
evaluate_image_rubric_with_options(png_path, question, default_options())
}
#[cfg(feature = "codex-acp")]
pub fn evaluate_image_rubric_with_options(
png_path: &Path,
question: &str,
opts: RubricOptions,
) -> Result<RubricVerdict, RubricError> {
evaluate_image_rubric_with_config(png_path, question, opts, RubricRunConfig::default())
}
#[cfg(feature = "codex-acp")]
pub fn evaluate_image_rubric_with_config(
png_path: &Path,
question: &str,
opts: RubricOptions,
config: RubricRunConfig,
) -> Result<RubricVerdict, RubricError> {
let bytes = std::fs::read(png_path).map_err(|source| RubricError::ReadPng {
path: png_path.to_path_buf(),
source,
})?;
let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes);
let text = run_codex_acp_rubric(
&b64,
question,
opts.model
.as_deref()
.map_or(DEFAULT_CODEX_ACP_MODEL, |model| model),
opts.effort
.as_deref()
.map_or(DEFAULT_CODEX_ACP_REASONING_EFFORT, |effort| effort),
opts.system_prompt
.as_deref()
.map_or(DEFAULT_SYSTEM_PROMPT, |system_prompt| system_prompt),
&config,
)?;
parse_verdict(&text).map_err(|source| RubricError::ParseVerdict { text, source })
}
#[cfg(feature = "codex-acp")]
pub fn evaluate_image_sequence_rubric_with_config(
frames: &[SequenceFrame],
question: &str,
opts: RubricOptions,
config: RubricRunConfig,
) -> Result<RubricVerdict, RubricError> {
evaluate_image_sequence_rubric_with_options(
frames,
question,
opts,
config,
SequenceOptions::default(),
)
}
#[cfg(feature = "codex-acp")]
pub fn evaluate_image_sequence_rubric_with_options(
frames: &[SequenceFrame],
question: &str,
opts: RubricOptions,
config: RubricRunConfig,
sequence_options: SequenceOptions,
) -> Result<RubricVerdict, RubricError> {
sequence_options
.validate()
.map_err(PoolError::Rpc)
.map_err(RubricError::Pool)?;
if frames.is_empty() {
return Err(RubricError::Pool(PoolError::Rpc(
"sequence rubric requires at least one frame".to_owned(),
)));
}
if sequence_options.require_transition && frames.len() < 2 {
return Err(RubricError::Pool(PoolError::Rpc(
"sequence transition assessment requires at least two frames".to_owned(),
)));
}
if frames.len() > sequence_options.max_frames {
return Err(RubricError::Pool(PoolError::Rpc(format!(
"sequence contains {} frames, maximum is {}",
frames.len(),
sequence_options.max_frames
))));
}
let mut encoded = Vec::with_capacity(frames.len());
for frame in frames {
let bytes = std::fs::read(&frame.path).map_err(|source| RubricError::ReadPng {
path: frame.path.clone(),
source,
})?;
encoded.push((
frame.label.clone(),
base64::engine::general_purpose::STANDARD.encode(bytes),
));
}
let transition_instruction = if sequence_options.require_transition {
"Check every checkpoint and whether each before/after transition is visible and semantically correct."
} else {
"Check every checkpoint for visual and semantic correctness. Transition assessment is disabled."
};
let system_prompt = opts
.system_prompt
.as_deref()
.map_or(DEFAULT_SYSTEM_PROMPT, |prompt| prompt);
let prompt = format!(
"{system_prompt}\n\nEvaluate this ordered interaction sequence. {transition_instruction}\nQuestion: {question}"
);
let text = run_codex_acp_sequence(
&encoded,
&prompt,
opts.model
.as_deref()
.map_or(DEFAULT_CODEX_ACP_MODEL, |model| model),
opts.effort
.as_deref()
.map_or(DEFAULT_CODEX_ACP_REASONING_EFFORT, |effort| effort),
&config,
)?;
parse_verdict(&text).map_err(|source| RubricError::ParseVerdict { text, source })
}
#[cfg(feature = "pipeline")]
fn run_rubric_prompt(rubric_prompt: &str, config: &RubricRunConfig) -> Result<String, RubricError> {
#[cfg(feature = "http-rubric")]
if let Some(ref url) = config.url {
let api_config = VisionApiConfig {
url: url.clone(),
model: config.api_model.clone().unwrap_or_default(),
api_key: None,
};
return vision::call_text_api(rubric_prompt, &api_config).map_err(RubricError::Pool);
}
#[cfg(feature = "acp")]
{
let mut acp = AcpClient::spawn(
&config.codex_acp_binary,
&config.acp_args,
&config.extra_env,
config.cwd.as_deref(),
)
.map_err(RubricError::Pool)?;
acp.start_session(config.cwd.as_deref(), config.api_model.as_deref(), None)
.map_err(RubricError::Pool)?;
acp.prompt_text(rubric_prompt).map_err(RubricError::Pool)
}
#[cfg(not(feature = "acp"))]
Err(RubricError::Pool(PoolError::Spawn(
"no ACP backend enabled; enable 'acp' feature".to_string(),
)))
}
#[cfg(feature = "pipeline")]
pub fn evaluate_image_rubric_pipeline(
png_path: &Path,
question: &str,
vision_config: &VisionApiConfig,
vision_prompt: &str,
rubric_options: &RubricOptions,
rubric_config: &RubricRunConfig,
) -> Result<RubricVerdict, RubricError> {
let bytes = std::fs::read(png_path).map_err(|source| RubricError::ReadPng {
path: png_path.to_path_buf(),
source,
})?;
let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes);
let structured =
vision::call_vision_api(&b64, vision_prompt, vision_config).map_err(RubricError::Pool)?;
let system_prompt = rubric_options
.system_prompt
.as_deref()
.map_or(DEFAULT_SYSTEM_PROMPT, |system_prompt| system_prompt);
let rubric_prompt =
format!("{system_prompt}\n\nUI description:\n{structured}\n\nQuestion: {question}");
let text = run_rubric_prompt(&rubric_prompt, rubric_config)?;
parse_verdict(&text).map_err(|source| RubricError::ParseVerdict { text, source })
}
#[cfg(feature = "pipeline")]
pub fn evaluate_image_rubric_pipeline_with_vision(
png_path: &Path,
question: &str,
vision_config: &VisionApiConfig,
vision_prompt: &str,
rubric_options: &RubricOptions,
rubric_config: &RubricRunConfig,
) -> Result<(RubricVerdict, String), RubricError> {
let bytes = std::fs::read(png_path).map_err(|source| RubricError::ReadPng {
path: png_path.to_path_buf(),
source,
})?;
let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes);
let structured =
vision::call_vision_api(&b64, vision_prompt, vision_config).map_err(RubricError::Pool)?;
let system_prompt = rubric_options
.system_prompt
.as_deref()
.map_or(DEFAULT_SYSTEM_PROMPT, |system_prompt| system_prompt);
let rubric_prompt =
format!("{system_prompt}\n\nUI description:\n{structured}\n\nQuestion: {question}");
let text = run_rubric_prompt(&rubric_prompt, rubric_config)?;
let verdict =
parse_verdict(&text).map_err(|source| RubricError::ParseVerdict { text, source })?;
Ok((verdict, structured))
}
#[cfg(feature = "pipeline")]
pub fn evaluate_image_sequence_rubric_pipeline(
frames: &[SequenceFrame],
question: &str,
vision_config: &VisionApiConfig,
vision_prompt: &str,
rubric_options: &RubricOptions,
rubric_config: &RubricRunConfig,
) -> Result<RubricVerdict, RubricError> {
evaluate_image_sequence_rubric_pipeline_with_options(
frames,
question,
vision_config,
vision_prompt,
rubric_options,
rubric_config,
SequenceOptions::default(),
)
}
#[cfg(feature = "pipeline")]
pub fn evaluate_image_sequence_rubric_pipeline_with_options(
frames: &[SequenceFrame],
question: &str,
vision_config: &VisionApiConfig,
vision_prompt: &str,
rubric_options: &RubricOptions,
rubric_config: &RubricRunConfig,
sequence_options: SequenceOptions,
) -> Result<RubricVerdict, RubricError> {
sequence_options
.validate()
.map_err(PoolError::VisionApi)
.map_err(RubricError::Pool)?;
if frames.is_empty() {
return Err(RubricError::Pool(PoolError::VisionApi(
"sequence rubric requires at least one frame".to_owned(),
)));
}
if sequence_options.require_transition && frames.len() < 2 {
return Err(RubricError::Pool(PoolError::VisionApi(
"sequence transition assessment requires at least two frames".to_owned(),
)));
}
if frames.len() > sequence_options.max_frames {
return Err(RubricError::Pool(PoolError::VisionApi(format!(
"sequence contains {} frames, maximum is {}",
frames.len(),
sequence_options.max_frames
))));
}
let mut encoded = Vec::with_capacity(frames.len());
for frame in frames {
let bytes = std::fs::read(&frame.path).map_err(|source| RubricError::ReadPng {
path: frame.path.clone(),
source,
})?;
encoded.push((
frame.label.clone(),
base64::engine::general_purpose::STANDARD.encode(bytes),
));
}
let structured = vision::call_vision_api_sequence(&encoded, vision_prompt, vision_config)
.map_err(RubricError::Pool)?;
let system_prompt = rubric_options
.system_prompt
.as_deref()
.map_or(DEFAULT_SYSTEM_PROMPT, |prompt| prompt);
let transition_instruction = if sequence_options.require_transition {
"Assess the semantic before/after transition between adjacent checkpoints."
} else {
"Assess each checkpoint independently; transition assessment is disabled."
};
let rubric_prompt = format!(
"{system_prompt}\n\nOrdered UI journey description:\n{structured}\n\n{transition_instruction}\nQuestion: {question}"
);
let text = run_rubric_prompt(&rubric_prompt, rubric_config)?;
parse_verdict(&text).map_err(|source| RubricError::ParseVerdict { text, source })
}
pub fn run(cli: Cli) -> anyhow::Result<()> {
cli::run(cli)
}
#[cfg(feature = "codex-acp")]
fn run_codex_acp_rubric(
b64_png: &str,
question: &str,
model: &str,
effort: &str,
system_prompt: &str,
config: &RubricRunConfig,
) -> Result<String, PoolError> {
let mut acp = AcpClient::spawn(
&config.codex_acp_binary,
&config.acp_args,
&config.extra_env,
config.cwd.as_deref(),
)?;
acp.start_session(config.cwd.as_deref(), Some(model), Some(effort))?;
let prompt = format!("{system_prompt}\n\nQuestion: {question}");
acp.prompt_image(&prompt, b64_png)
}
#[cfg(feature = "codex-acp")]
fn run_codex_acp_sequence(
frames: &[(String, String)],
prompt: &str,
model: &str,
effort: &str,
config: &RubricRunConfig,
) -> Result<String, PoolError> {
let mut acp = AcpClient::spawn(
&config.codex_acp_binary,
&config.acp_args,
&config.extra_env,
config.cwd.as_deref(),
)?;
acp.start_session(config.cwd.as_deref(), Some(model), Some(effort))?;
acp.prompt_images(prompt, frames)
}
#[cfg(test)]
mod tests;