1#![warn(missing_docs)]
11#![allow(
12 clippy::io_other_error,
13 clippy::manual_checked_ops,
14 clippy::single_match
15)]
16
17#[cfg(feature = "acp")]
18mod acp;
19#[cfg(feature = "batch")]
20mod batch;
21#[cfg(feature = "audit")]
22mod browser;
23pub mod calibration;
24pub mod cli;
25mod config;
26mod configured_eval;
27pub mod coverage;
28pub mod deterministic;
29mod errors;
30pub mod manifest;
31#[cfg(feature = "pool")]
32mod pool;
33pub mod presets;
34pub mod qa;
35mod report;
36mod typed_strings;
37mod verdict;
38#[cfg(any(feature = "vision-api", feature = "http-rubric"))]
39pub mod vision;
40
41use std::path::Path;
42#[cfg(any(feature = "codex-acp", feature = "pipeline"))]
43use std::path::PathBuf;
44
45#[cfg(feature = "acp")]
46use acp::AcpClient;
47use base64::Engine as _;
48#[cfg(any(feature = "vision-api", feature = "http-rubric"))]
49use vision::VisionApiConfig;
50
51#[cfg(feature = "batch")]
52pub use batch::{
53 AggregateStatus, AssetChange, AssetRubricReport, AssetRubricResult, AssetSnapshot,
54 BatchRubricConfig, BatchRubricReport, BatchRubricRun, IssueClassificationInput,
55 IssueClassifier, IssueRecommendation, RecommendationSeverity, SelectionMode, diff_snapshots,
56 select_changed,
57};
58pub use calibration::{
59 CALIBRATION_CORPUS_SCHEMA_VERSION, CalibrationCaptureV1, CalibrationCorpusV1,
60 REQUIRED_CALIBRATION_CATEGORIES,
61};
62pub use cli::Cli;
63pub use config::{
64 ConfigMode, RubricOptions, RubricRunConfig, TomlConfig, TomlRubric, TomlSequence, TomlVision,
65 load_config_toml,
66};
67#[cfg(feature = "codex-acp")]
68pub use config::{default_codex_acp_binary, default_options, direct_codex_gpt_config};
69pub use configured_eval::{evaluate_configured, evaluate_configured_with_vision};
70pub use coverage::{
71 COVERAGE_CONTRACT_SCHEMA_VERSION, COVERAGE_REPORT_SCHEMA_VERSION, CoverageContractV1,
72 CoverageExclusionV1, CoverageReportV1, CoverageSurfaceV1, CoverageTransitionV1,
73};
74pub use deterministic::{
75 DETERMINISTIC_EVIDENCE_SCHEMA_VERSION, DeterministicEvidenceV1, DeterministicObservationV1,
76};
77pub use errors::{PoolError, RateLimitEvent, RubricError};
78pub use manifest::{
79 ArtifactDigest, CAPTURE_MANIFEST_SCHEMA_VERSION, CaptureCell, CaptureEnvironment,
80 CaptureManifest, ManifestError, Viewport,
81};
82#[cfg(feature = "pool")]
83pub use pool::{LogCaptureConfig, LogPathMode, PoolConfig, PoolStats, RubricPool};
84pub use qa::{
85 CALIBRATION_SENTINEL_SCHEMA_VERSION, CalibrationExpectation, CalibrationSentinelV1,
86 FINDING_SCHEMA_VERSION, FindingV1, OBSERVATION_SCHEMA_VERSION, ObservationV1,
87 VISUAL_RUN_REPORT_SCHEMA_VERSION, VisualRunReportV1, VisualRunStatus, finding_fingerprint,
88 validate_calibration_sentinels, validate_visual_run, validate_visual_run_against_manifest,
89};
90pub use report::{PageResult, RubricReport};
91pub use typed_strings::{RubricEffort, RubricVerdictStatus};
92pub use verdict::{RubricVerdict, assert_verdict, parse_verdict};
93
94pub const DEFAULT_SYSTEM_PROMPT: &str = presets::UI_REGRESSION_SYSTEM_PROMPT;
98
99#[cfg(any(feature = "codex-acp", feature = "pipeline"))]
101#[derive(Clone, Debug, PartialEq, Eq)]
102pub struct SequenceFrame {
103 pub label: String,
105 pub path: PathBuf,
107}
108
109#[derive(Clone, Copy, Debug, PartialEq, Eq)]
111pub struct SequenceOptions {
112 pub max_frames: usize,
114 pub require_transition: bool,
117}
118
119impl Default for SequenceOptions {
120 fn default() -> Self {
121 Self {
122 max_frames: 8,
123 require_transition: true,
124 }
125 }
126}
127
128impl SequenceOptions {
129 pub fn validate(self) -> Result<(), String> {
131 if self.max_frames == 0 {
132 return Err("sequence max_frames must be greater than zero".to_owned());
133 }
134 if self.max_frames > 32 {
135 return Err("sequence max_frames must not exceed 32".to_owned());
136 }
137 Ok(())
138 }
139}
140
141#[cfg(feature = "codex-acp")]
143pub const DEFAULT_CODEX_ACP_MODEL: &str = "gpt-5.5";
144#[cfg(feature = "codex-acp")]
146pub const DEFAULT_CODEX_ACP_REASONING_EFFORT: &str = "medium";
147
148#[cfg(feature = "vision-api")]
153pub const DEFAULT_VISION_PROMPT: &str = "\
154You are a UI description engine. Given a screenshot, produce a structured JSON \
155description of all visible user interface elements, their text content, layout, \
156and any visual issues (clipping, overlap, blank regions, contrast problems). \
157Output ONLY valid JSON with no additional text.";
158
159pub fn encode_png(png_path: &Path) -> Result<String, PoolError> {
165 let bytes = std::fs::read(png_path)
166 .map_err(|e| PoolError::Rpc(format!("read png {}: {e}", png_path.display())))?;
167 Ok(base64::engine::general_purpose::STANDARD.encode(bytes))
168}
169
170#[cfg(feature = "codex-acp")]
177pub fn assert_image_rubric(png_path: &Path, name: &str, question: &str) -> Result<(), RubricError> {
178 let verdict = evaluate_image_rubric(png_path, question)?;
179 assert_verdict(name, verdict)
180}
181
182#[cfg(feature = "codex-acp")]
188pub fn evaluate_image_rubric(
189 png_path: &Path,
190 question: &str,
191) -> Result<RubricVerdict, RubricError> {
192 evaluate_image_rubric_with_options(png_path, question, default_options())
193}
194
195#[cfg(feature = "codex-acp")]
201pub fn evaluate_image_rubric_with_options(
202 png_path: &Path,
203 question: &str,
204 opts: RubricOptions,
205) -> Result<RubricVerdict, RubricError> {
206 evaluate_image_rubric_with_config(png_path, question, opts, RubricRunConfig::default())
207}
208
209#[cfg(feature = "codex-acp")]
215pub fn evaluate_image_rubric_with_config(
216 png_path: &Path,
217 question: &str,
218 opts: RubricOptions,
219 config: RubricRunConfig,
220) -> Result<RubricVerdict, RubricError> {
221 let bytes = std::fs::read(png_path).map_err(|source| RubricError::ReadPng {
222 path: png_path.to_path_buf(),
223 source,
224 })?;
225 let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes);
226 let text = run_codex_acp_rubric(
227 &b64,
228 question,
229 opts.model
230 .as_deref()
231 .map_or(DEFAULT_CODEX_ACP_MODEL, |model| model),
232 opts.effort
233 .as_deref()
234 .map_or(DEFAULT_CODEX_ACP_REASONING_EFFORT, |effort| effort),
235 opts.system_prompt
236 .as_deref()
237 .map_or(DEFAULT_SYSTEM_PROMPT, |system_prompt| system_prompt),
238 &config,
239 )?;
240
241 parse_verdict(&text).map_err(|source| RubricError::ParseVerdict { text, source })
242}
243
244#[cfg(feature = "codex-acp")]
255pub fn evaluate_image_sequence_rubric_with_config(
256 frames: &[SequenceFrame],
257 question: &str,
258 opts: RubricOptions,
259 config: RubricRunConfig,
260) -> Result<RubricVerdict, RubricError> {
261 evaluate_image_sequence_rubric_with_options(
262 frames,
263 question,
264 opts,
265 config,
266 SequenceOptions::default(),
267 )
268}
269
270#[cfg(feature = "codex-acp")]
273pub fn evaluate_image_sequence_rubric_with_options(
274 frames: &[SequenceFrame],
275 question: &str,
276 opts: RubricOptions,
277 config: RubricRunConfig,
278 sequence_options: SequenceOptions,
279) -> Result<RubricVerdict, RubricError> {
280 sequence_options
281 .validate()
282 .map_err(PoolError::Rpc)
283 .map_err(RubricError::Pool)?;
284 if frames.is_empty() {
285 return Err(RubricError::Pool(PoolError::Rpc(
286 "sequence rubric requires at least one frame".to_owned(),
287 )));
288 }
289 if sequence_options.require_transition && frames.len() < 2 {
290 return Err(RubricError::Pool(PoolError::Rpc(
291 "sequence transition assessment requires at least two frames".to_owned(),
292 )));
293 }
294 if frames.len() > sequence_options.max_frames {
295 return Err(RubricError::Pool(PoolError::Rpc(format!(
296 "sequence contains {} frames, maximum is {}",
297 frames.len(),
298 sequence_options.max_frames
299 ))));
300 }
301 let mut encoded = Vec::with_capacity(frames.len());
302 for frame in frames {
303 let bytes = std::fs::read(&frame.path).map_err(|source| RubricError::ReadPng {
304 path: frame.path.clone(),
305 source,
306 })?;
307 encoded.push((
308 frame.label.clone(),
309 base64::engine::general_purpose::STANDARD.encode(bytes),
310 ));
311 }
312 let transition_instruction = if sequence_options.require_transition {
313 "Check every checkpoint and whether each before/after transition is visible and semantically correct."
314 } else {
315 "Check every checkpoint for visual and semantic correctness. Transition assessment is disabled."
316 };
317 let system_prompt = opts
318 .system_prompt
319 .as_deref()
320 .map_or(DEFAULT_SYSTEM_PROMPT, |prompt| prompt);
321 let prompt = format!(
322 "{system_prompt}\n\nEvaluate this ordered interaction sequence. {transition_instruction}\nQuestion: {question}"
323 );
324 let text = run_codex_acp_sequence(
325 &encoded,
326 &prompt,
327 opts.model
328 .as_deref()
329 .map_or(DEFAULT_CODEX_ACP_MODEL, |model| model),
330 opts.effort
331 .as_deref()
332 .map_or(DEFAULT_CODEX_ACP_REASONING_EFFORT, |effort| effort),
333 &config,
334 )?;
335 parse_verdict(&text).map_err(|source| RubricError::ParseVerdict { text, source })
336}
337
338#[cfg(feature = "pipeline")]
343fn run_rubric_prompt(rubric_prompt: &str, config: &RubricRunConfig) -> Result<String, RubricError> {
344 #[cfg(feature = "http-rubric")]
345 if let Some(ref url) = config.url {
346 let api_config = VisionApiConfig {
347 url: url.clone(),
348 model: config.api_model.clone().unwrap_or_default(),
349 api_key: None,
350 };
351 return vision::call_text_api(rubric_prompt, &api_config).map_err(RubricError::Pool);
352 }
353 #[cfg(feature = "acp")]
354 {
355 let mut acp = AcpClient::spawn(
356 &config.codex_acp_binary,
357 &config.acp_args,
358 &config.extra_env,
359 config.cwd.as_deref(),
360 )
361 .map_err(RubricError::Pool)?;
362 acp.start_session(config.cwd.as_deref(), config.api_model.as_deref(), None)
363 .map_err(RubricError::Pool)?;
364 acp.prompt_text(rubric_prompt).map_err(RubricError::Pool)
365 }
366 #[cfg(not(feature = "acp"))]
367 Err(RubricError::Pool(PoolError::Spawn(
368 "no ACP backend enabled; enable 'acp' feature".to_string(),
369 )))
370}
371
372#[cfg(feature = "pipeline")]
386pub fn evaluate_image_rubric_pipeline(
387 png_path: &Path,
388 question: &str,
389 vision_config: &VisionApiConfig,
390 vision_prompt: &str,
391 rubric_options: &RubricOptions,
392 rubric_config: &RubricRunConfig,
393) -> Result<RubricVerdict, RubricError> {
394 let bytes = std::fs::read(png_path).map_err(|source| RubricError::ReadPng {
395 path: png_path.to_path_buf(),
396 source,
397 })?;
398 let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes);
399
400 let structured =
401 vision::call_vision_api(&b64, vision_prompt, vision_config).map_err(RubricError::Pool)?;
402
403 let system_prompt = rubric_options
404 .system_prompt
405 .as_deref()
406 .map_or(DEFAULT_SYSTEM_PROMPT, |system_prompt| system_prompt);
407 let rubric_prompt =
408 format!("{system_prompt}\n\nUI description:\n{structured}\n\nQuestion: {question}");
409
410 let text = run_rubric_prompt(&rubric_prompt, rubric_config)?;
411
412 parse_verdict(&text).map_err(|source| RubricError::ParseVerdict { text, source })
413}
414
415#[cfg(feature = "pipeline")]
425pub fn evaluate_image_rubric_pipeline_with_vision(
426 png_path: &Path,
427 question: &str,
428 vision_config: &VisionApiConfig,
429 vision_prompt: &str,
430 rubric_options: &RubricOptions,
431 rubric_config: &RubricRunConfig,
432) -> Result<(RubricVerdict, String), RubricError> {
433 let bytes = std::fs::read(png_path).map_err(|source| RubricError::ReadPng {
434 path: png_path.to_path_buf(),
435 source,
436 })?;
437 let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes);
438
439 let structured =
440 vision::call_vision_api(&b64, vision_prompt, vision_config).map_err(RubricError::Pool)?;
441
442 let system_prompt = rubric_options
443 .system_prompt
444 .as_deref()
445 .map_or(DEFAULT_SYSTEM_PROMPT, |system_prompt| system_prompt);
446 let rubric_prompt =
447 format!("{system_prompt}\n\nUI description:\n{structured}\n\nQuestion: {question}");
448
449 let text = run_rubric_prompt(&rubric_prompt, rubric_config)?;
450
451 let verdict =
452 parse_verdict(&text).map_err(|source| RubricError::ParseVerdict { text, source })?;
453 Ok((verdict, structured))
454}
455
456#[cfg(feature = "pipeline")]
462pub fn evaluate_image_sequence_rubric_pipeline(
463 frames: &[SequenceFrame],
464 question: &str,
465 vision_config: &VisionApiConfig,
466 vision_prompt: &str,
467 rubric_options: &RubricOptions,
468 rubric_config: &RubricRunConfig,
469) -> Result<RubricVerdict, RubricError> {
470 evaluate_image_sequence_rubric_pipeline_with_options(
471 frames,
472 question,
473 vision_config,
474 vision_prompt,
475 rubric_options,
476 rubric_config,
477 SequenceOptions::default(),
478 )
479}
480
481#[cfg(feature = "pipeline")]
483pub fn evaluate_image_sequence_rubric_pipeline_with_options(
484 frames: &[SequenceFrame],
485 question: &str,
486 vision_config: &VisionApiConfig,
487 vision_prompt: &str,
488 rubric_options: &RubricOptions,
489 rubric_config: &RubricRunConfig,
490 sequence_options: SequenceOptions,
491) -> Result<RubricVerdict, RubricError> {
492 sequence_options
493 .validate()
494 .map_err(PoolError::VisionApi)
495 .map_err(RubricError::Pool)?;
496 if frames.is_empty() {
497 return Err(RubricError::Pool(PoolError::VisionApi(
498 "sequence rubric requires at least one frame".to_owned(),
499 )));
500 }
501 if sequence_options.require_transition && frames.len() < 2 {
502 return Err(RubricError::Pool(PoolError::VisionApi(
503 "sequence transition assessment requires at least two frames".to_owned(),
504 )));
505 }
506 if frames.len() > sequence_options.max_frames {
507 return Err(RubricError::Pool(PoolError::VisionApi(format!(
508 "sequence contains {} frames, maximum is {}",
509 frames.len(),
510 sequence_options.max_frames
511 ))));
512 }
513 let mut encoded = Vec::with_capacity(frames.len());
514 for frame in frames {
515 let bytes = std::fs::read(&frame.path).map_err(|source| RubricError::ReadPng {
516 path: frame.path.clone(),
517 source,
518 })?;
519 encoded.push((
520 frame.label.clone(),
521 base64::engine::general_purpose::STANDARD.encode(bytes),
522 ));
523 }
524 let structured = vision::call_vision_api_sequence(&encoded, vision_prompt, vision_config)
525 .map_err(RubricError::Pool)?;
526 let system_prompt = rubric_options
527 .system_prompt
528 .as_deref()
529 .map_or(DEFAULT_SYSTEM_PROMPT, |prompt| prompt);
530 let transition_instruction = if sequence_options.require_transition {
531 "Assess the semantic before/after transition between adjacent checkpoints."
532 } else {
533 "Assess each checkpoint independently; transition assessment is disabled."
534 };
535 let rubric_prompt = format!(
536 "{system_prompt}\n\nOrdered UI journey description:\n{structured}\n\n{transition_instruction}\nQuestion: {question}"
537 );
538 let text = run_rubric_prompt(&rubric_prompt, rubric_config)?;
539 parse_verdict(&text).map_err(|source| RubricError::ParseVerdict { text, source })
540}
541
542pub fn run(cli: Cli) -> anyhow::Result<()> {
548 cli::run(cli)
549}
550
551#[cfg(feature = "codex-acp")]
552fn run_codex_acp_rubric(
553 b64_png: &str,
554 question: &str,
555 model: &str,
556 effort: &str,
557 system_prompt: &str,
558 config: &RubricRunConfig,
559) -> Result<String, PoolError> {
560 let mut acp = AcpClient::spawn(
561 &config.codex_acp_binary,
562 &config.acp_args,
563 &config.extra_env,
564 config.cwd.as_deref(),
565 )?;
566 acp.start_session(config.cwd.as_deref(), Some(model), Some(effort))?;
567
568 let prompt = format!("{system_prompt}\n\nQuestion: {question}");
569 acp.prompt_image(&prompt, b64_png)
570}
571
572#[cfg(feature = "codex-acp")]
573fn run_codex_acp_sequence(
574 frames: &[(String, String)],
575 prompt: &str,
576 model: &str,
577 effort: &str,
578 config: &RubricRunConfig,
579) -> Result<String, PoolError> {
580 let mut acp = AcpClient::spawn(
581 &config.codex_acp_binary,
582 &config.acp_args,
583 &config.extra_env,
584 config.cwd.as_deref(),
585 )?;
586 acp.start_session(config.cwd.as_deref(), Some(model), Some(effort))?;
587 acp.prompt_images(prompt, frames)
588}
589
590#[cfg(test)]
591mod tests;