1use std::path::PathBuf;
4
5#[cfg(feature = "codex-acp")]
6use anyhow::Context as _;
7use anyhow::{Result, anyhow};
8use clap::{Args, Parser, Subcommand};
9
10use crate::presets::PresetError;
11
12#[cfg(feature = "audit")]
13mod audit;
14#[cfg(feature = "audit")]
15mod calibration;
16#[cfg(feature = "audit")]
17mod capture;
18pub mod configured;
19#[cfg(feature = "pipeline")]
20pub mod pipeline;
21mod static_server;
22#[cfg(test)]
23mod tests;
24
25#[cfg(all(test, feature = "audit"))]
26use audit::RubricReport;
27#[cfg(feature = "audit")]
28use audit::run_audit;
29#[cfg(feature = "audit")]
30pub use audit::{AuditReport, AuditStatus};
31#[cfg(feature = "audit")]
32use calibration::run_calibration;
33#[cfg(feature = "audit")]
34use capture::run_capture;
35use static_server::StaticServer;
36#[cfg(test)]
37use static_server::{content_type, resolve_static_path};
38
39#[derive(Debug, Parser)]
41#[command(name = "visual-rubric")]
42pub struct Cli {
43 #[command(subcommand)]
44 command: Option<Commands>,
45 #[command(flatten)]
46 image: LegacyImageArgs,
47}
48
49#[derive(Debug, Subcommand)]
50#[allow(clippy::large_enum_variant)]
51enum Commands {
52 #[cfg(feature = "codex-acp")]
54 Image(ImageArgs),
55 #[cfg(feature = "codex-acp")]
57 Sequence(SequenceArgs),
58 #[cfg(feature = "audit")]
60 Audit(AuditArgs),
61 #[cfg(feature = "audit")]
63 Capture(CaptureArgs),
64 #[cfg(feature = "audit")]
66 Calibrate(CalibrateArgs),
67 Serve(ServeArgs),
69 #[cfg(feature = "pipeline")]
76 Pipeline(pipeline::PipelineArgs),
77 Configured(configured::ConfiguredArgs),
83}
84
85#[derive(Clone, Debug, Parser)]
86struct LegacyImageArgs {
87 #[arg(long)]
88 image: Option<PathBuf>,
89 #[arg(long)]
90 question: Option<String>,
91 #[arg(long)]
92 preset: Option<String>,
93 #[arg(long)]
94 system_prompt: Option<String>,
95 #[arg(long)]
96 model: Option<String>,
97 #[arg(long)]
98 effort: Option<String>,
99 #[cfg(feature = "codex-acp")]
100 #[arg(long)]
101 codex_acp: Option<PathBuf>,
102 #[arg(long, default_value = "vnc-screenshot")]
103 name: String,
104 #[arg(long)]
105 json: bool,
106}
107
108#[cfg(feature = "codex-acp")]
109#[derive(Clone, Debug, Parser)]
110struct ImageArgs {
111 #[arg(long)]
112 image: PathBuf,
113 #[command(flatten)]
114 questions: QuestionSource,
115 #[arg(long)]
116 system_prompt: Option<String>,
117 #[arg(long)]
118 model: Option<String>,
119 #[arg(long)]
120 effort: Option<String>,
121 #[arg(long)]
122 codex_acp: Option<PathBuf>,
123 #[arg(long, default_value = "screenshot")]
124 name: String,
125 #[arg(long)]
126 json: bool,
127}
128
129#[cfg(feature = "codex-acp")]
130#[derive(Clone, Debug, Parser)]
131struct SequenceArgs {
132 #[arg(long = "frame", value_parser = parse_sequence_frame, required = true, num_args = 1..)]
134 frames: Vec<SequenceFrameArg>,
135 #[command(flatten)]
136 questions: QuestionSource,
137 #[arg(long)]
138 model: Option<String>,
139 #[arg(long)]
140 effort: Option<String>,
141 #[arg(long)]
142 codex_acp: Option<PathBuf>,
143 #[arg(long, default_value = "sequence")]
144 name: String,
145 #[arg(long)]
146 json: bool,
147 #[arg(long)]
149 max_frames: Option<usize>,
150 #[arg(long)]
152 no_require_transition: bool,
153}
154
155#[cfg(feature = "codex-acp")]
156#[derive(Clone, Debug)]
157struct SequenceFrameArg {
158 label: String,
159 path: PathBuf,
160}
161
162#[cfg(feature = "audit")]
163#[derive(Clone, Debug, Parser)]
164struct AuditArgs {
165 #[arg(long)]
167 root: PathBuf,
168 #[arg(long, default_value = "/")]
170 path: String,
171 #[arg(long, default_value = "target/visual-rubric")]
173 screenshots: PathBuf,
174 #[arg(long, default_value = "target/visual-rubric/report.json")]
176 report: PathBuf,
177 #[arg(long, env = "VISUAL_RUBRIC_BROWSER", default_value = "chromium")]
179 browser: PathBuf,
180 #[arg(long = "browser-arg")]
182 browser_args: Vec<String>,
183 #[arg(long, default_value_t = 0)]
185 wait_ms: u64,
186 #[arg(long)]
188 device_scale_factor: Option<f32>,
189 #[arg(long, default_value_t = 0)]
191 capture_retries: u32,
192 #[arg(long)]
194 fail_on_rubric: bool,
195 #[arg(long = "viewport")]
197 viewports: Vec<ViewportArg>,
198 #[command(flatten)]
199 questions: QuestionSource,
200 #[arg(long)]
201 system_prompt: Option<String>,
202 #[arg(long)]
203 model: Option<String>,
204 #[arg(long)]
205 effort: Option<String>,
206 #[arg(long)]
207 codex_acp: Option<PathBuf>,
208 #[arg(long)]
210 fake_pass: bool,
211 #[arg(long)]
213 skip_ai: bool,
214}
215
216#[cfg(feature = "audit")]
217#[derive(Clone, Debug, Parser)]
218struct CaptureArgs {
219 #[arg(long)]
221 root: PathBuf,
222 #[arg(long)]
224 workspace: Option<PathBuf>,
225 #[arg(long)]
227 base_url: Option<String>,
228 #[arg(long)]
230 job: PathBuf,
231 #[arg(long, default_value = "target/visual-rubric/captures")]
233 output: PathBuf,
234 #[arg(long, default_value = "target/visual-rubric/capture_manifest.json")]
236 manifest: PathBuf,
237 #[arg(long, default_value = "target/visual-rubric/run_report.json")]
239 report: PathBuf,
240 #[arg(long, env = "VISUAL_RUBRIC_BROWSER", default_value = "chromium")]
242 browser: PathBuf,
243 #[arg(long = "browser-arg")]
245 browser_args: Vec<String>,
246 #[arg(long, default_value_t = 4)]
248 rubric_workers: usize,
249 #[arg(long)]
251 cache_dir: Option<PathBuf>,
252 #[command(flatten)]
253 questions: QuestionSource,
254 #[arg(long)]
255 system_prompt: Option<String>,
256 #[arg(long)]
257 model: Option<String>,
258 #[arg(long)]
259 effort: Option<String>,
260 #[arg(long)]
261 codex_acp: Option<PathBuf>,
262 #[arg(long)]
264 fake_pass: bool,
265 #[arg(long)]
267 skip_ai: bool,
268}
269
270#[cfg(feature = "audit")]
271#[derive(Clone, Debug, Parser)]
272struct CalibrateArgs {
273 #[arg(long)]
275 corpus: PathBuf,
276 #[arg(long)]
278 manifest: PathBuf,
279 #[arg(long)]
281 report: PathBuf,
282}
283
284#[derive(Clone, Debug, Parser)]
285struct ServeArgs {
286 #[arg(long)]
287 root: PathBuf,
288 #[arg(long, default_value_t = 1111)]
289 port: u16,
290}
291
292#[cfg(feature = "audit")]
293#[derive(Clone, Debug)]
294struct ViewportArg {
295 name: String,
296 width: u32,
297 height: u32,
298}
299
300#[derive(Clone, Debug, Args)]
303pub struct QuestionSource {
304 #[arg(long, required_unless_present = "preset")]
306 pub question: Option<String>,
307
308 #[arg(long, required_unless_present = "question")]
310 pub preset: Option<String>,
311}
312
313impl QuestionSource {
314 pub fn from_question(question: String) -> Self {
316 Self {
317 question: Some(question),
318 preset: None,
319 }
320 }
321
322 pub fn resolve(&self) -> Result<String, PresetError> {
327 match (&self.preset, &self.question) {
328 (Some(name), _) => {
329 let questions = crate::presets::resolve(name)?;
330 Ok(questions.join("\n"))
331 }
332 (None, Some(q)) => Ok(q.clone()),
333 (None, None) => Err(PresetError::MissingQuestionSource),
334 }
335 }
336
337 pub fn resolve_system_prompt(&self) -> Result<Option<String>, PresetError> {
343 match &self.preset {
344 Some(name) => Ok(crate::presets::find(name)?
345 .system_prompt()
346 .map(str::to_owned)),
347 None => Ok(None),
348 }
349 }
350}
351
352pub fn run(cli: Cli) -> Result<()> {
358 match cli.command {
359 #[cfg(feature = "codex-acp")]
360 Some(Commands::Image(args)) => run_image(args),
361 #[cfg(feature = "codex-acp")]
362 Some(Commands::Sequence(args)) => run_sequence(args),
363 #[cfg(feature = "audit")]
364 Some(Commands::Audit(args)) => run_audit(args),
365 #[cfg(feature = "audit")]
366 Some(Commands::Capture(args)) => run_capture(args),
367 #[cfg(feature = "audit")]
368 Some(Commands::Calibrate(args)) => run_calibration(args),
369 Some(Commands::Serve(args)) => run_serve(args),
370 #[cfg(feature = "pipeline")]
371 Some(Commands::Pipeline(args)) => pipeline::run_pipeline(args),
372 Some(Commands::Configured(args)) => configured::run_configured(args),
373 #[cfg(feature = "codex-acp")]
374 None => run_image(cli.image.try_into()?),
375 #[cfg(not(feature = "codex-acp"))]
376 None => Err(anyhow!(
377 "no subcommand given; use --help for available commands"
378 )),
379 }
380}
381
382#[cfg(feature = "codex-acp")]
383fn run_image(args: ImageArgs) -> Result<()> {
384 let verdict = evaluate_image(&args)?;
385 if args.json {
386 println!("{}", serde_json::to_string(&verdict)?);
387 return Ok(());
388 }
389 crate::assert_verdict(&args.name, verdict)
390 .map(|()| println!("visual rubric passed"))
391 .map_err(|error| anyhow!(error))
392}
393
394#[cfg(feature = "codex-acp")]
395fn run_sequence(args: SequenceArgs) -> Result<()> {
396 let question = args.questions.resolve().map_err(|error| anyhow!(error))?;
397 let frames = args
398 .frames
399 .into_iter()
400 .map(|frame| crate::SequenceFrame {
401 label: frame.label,
402 path: frame.path,
403 })
404 .collect::<Vec<_>>();
405 let options = crate::RubricOptions {
406 model: args.model,
407 effort: args.effort.map(Into::into),
408 system_prompt: args
409 .questions
410 .resolve_system_prompt()
411 .map_err(|e| anyhow!(e))?,
412 };
413 let configured_sequence = crate::SequenceOptions::from_config_toml(None);
414 let sequence_options = crate::SequenceOptions {
415 max_frames: args.max_frames.unwrap_or(configured_sequence.max_frames),
416 require_transition: configured_sequence.require_transition && !args.no_require_transition,
417 };
418 let mut runtime_config = crate::RubricRunConfig::from_config_toml(None);
419 if let Some(binary) = args.codex_acp {
420 runtime_config.codex_acp_binary = binary;
421 }
422 let verdict = crate::evaluate_image_sequence_rubric_with_options(
423 &frames,
424 &question,
425 options,
426 runtime_config,
427 sequence_options,
428 )?;
429 if args.json {
430 println!("{}", serde_json::to_string(&verdict)?);
431 return Ok(());
432 }
433 crate::assert_verdict(&args.name, verdict)
434 .map(|()| println!("visual sequence rubric passed"))
435 .map_err(|error| anyhow!(error))
436}
437
438fn run_serve(args: ServeArgs) -> Result<()> {
439 let server = StaticServer::start(args.root, args.port)?;
440 println!("{}", server.base_url());
441 server.wait_forever()
442}
443
444#[cfg(feature = "codex-acp")]
445fn evaluate_image(args: &ImageArgs) -> Result<crate::RubricVerdict> {
446 let question = args.questions.resolve().map_err(|e| anyhow!(e))?;
447 let system_prompt = match args.system_prompt.clone() {
448 Some(prompt) => Some(prompt),
449 None => args
450 .questions
451 .resolve_system_prompt()
452 .map_err(|e| anyhow!(e))?,
453 };
454 let options = crate::RubricOptions {
455 model: args.model.clone(),
456 effort: args.effort.clone().map(Into::into),
457 system_prompt,
458 };
459 if let Some(codex_acp) = &args.codex_acp {
460 let pool = crate::RubricPool::new(crate::PoolConfig {
461 workers: 1,
462 codex_acp_binary: codex_acp.clone(),
463 default_options: merge_with_defaults(options),
464 ..crate::PoolConfig::default()
465 })?;
466 let verdict = pool.submit(&args.image, &question, crate::RubricOptions::default())?;
467 let _ = pool.shutdown();
468 Ok(verdict)
469 } else {
470 crate::evaluate_image_rubric_with_options(&args.image, &question, options)
471 .map_err(|error| anyhow!(error))
472 }
473}
474
475#[cfg(feature = "codex-acp")]
476fn merge_with_defaults(mut options: crate::RubricOptions) -> crate::RubricOptions {
477 let defaults = crate::default_options();
478 if options.model.is_none() {
479 options.model = defaults.model;
480 }
481 if options.effort.is_none() {
482 options.effort = defaults.effort;
483 }
484 if options.system_prompt.is_none() {
485 options.system_prompt = defaults.system_prompt;
486 }
487 options
488}
489
490#[cfg(feature = "codex-acp")]
491impl TryFrom<LegacyImageArgs> for ImageArgs {
492 type Error = anyhow::Error;
493
494 fn try_from(value: LegacyImageArgs) -> Result<Self> {
495 let questions = match (value.question, value.preset) {
496 (Some(q), _) => QuestionSource::from_question(q),
497 (None, Some(preset)) => QuestionSource {
498 question: None,
499 preset: Some(preset),
500 },
501 (None, None) => {
502 anyhow::bail!("either --question or --preset is required");
503 }
504 };
505 Ok(Self {
506 image: value.image.context("--image is required")?,
507 questions,
508 system_prompt: value.system_prompt,
509 model: value.model,
510 effort: value.effort,
511 codex_acp: value.codex_acp,
512 name: value.name,
513 json: value.json,
514 })
515 }
516}
517
518#[cfg(feature = "codex-acp")]
519fn parse_sequence_frame(value: &str) -> Result<SequenceFrameArg, String> {
520 let (label, path) = value
521 .split_once('=')
522 .ok_or_else(|| "frame must be LABEL=PATH".to_owned())?;
523 if label.trim().is_empty() {
524 return Err("frame label must not be empty".to_owned());
525 }
526 if path.trim().is_empty() {
527 return Err("frame path must not be empty".to_owned());
528 }
529 Ok(SequenceFrameArg {
530 label: label.to_owned(),
531 path: PathBuf::from(path),
532 })
533}
534
535#[cfg(feature = "audit")]
536impl std::str::FromStr for ViewportArg {
537 type Err = anyhow::Error;
538
539 fn from_str(value: &str) -> Result<Self> {
540 let (name, size) = value
541 .split_once('=')
542 .context("viewport must be name=WIDTHxHEIGHT")?;
543 let (width, height) = size
544 .split_once('x')
545 .context("viewport size must be WIDTHxHEIGHT")?;
546 if name.is_empty() {
547 anyhow::bail!("viewport name must not be empty");
548 }
549 if name.contains('/') || name.contains('\\') {
550 anyhow::bail!("viewport name {name:?} must not contain path separators");
551 }
552 let width = width.parse().context("viewport width must be an integer")?;
553 let height = height
554 .parse()
555 .context("viewport height must be an integer")?;
556 if width == 0 || height == 0 {
557 anyhow::bail!("viewport dimensions must be greater than zero");
558 }
559 Ok(Self {
560 name: name.to_string(),
561 width,
562 height,
563 })
564 }
565}