Skip to main content

visual_rubric/
cli.rs

1//! Command-line interface and audit report types.
2
3use std::path::PathBuf;
4
5#[cfg(feature = "codex-acp")]
6use anyhow::Context as _;
7use anyhow::{Result, anyhow};
8use clap::{Args, Parser, Subcommand};
9
10use crate::presets::PresetError;
11
12#[cfg(feature = "audit")]
13mod audit;
14#[cfg(feature = "audit")]
15mod calibration;
16#[cfg(feature = "audit")]
17mod capture;
18pub mod configured;
19#[cfg(feature = "pipeline")]
20pub mod pipeline;
21mod static_server;
22#[cfg(test)]
23mod tests;
24
25#[cfg(all(test, feature = "audit"))]
26use audit::RubricReport;
27#[cfg(feature = "audit")]
28use audit::run_audit;
29#[cfg(feature = "audit")]
30pub use audit::{AuditReport, AuditStatus};
31#[cfg(feature = "audit")]
32use calibration::run_calibration;
33#[cfg(feature = "audit")]
34use capture::run_capture;
35use static_server::StaticServer;
36#[cfg(test)]
37use static_server::{content_type, resolve_static_path};
38
39/// Parsed command-line interface for `visual-rubric`.
40#[derive(Debug, Parser)]
41#[command(name = "visual-rubric")]
42pub struct Cli {
43    #[command(subcommand)]
44    command: Option<Commands>,
45    #[command(flatten)]
46    image: LegacyImageArgs,
47}
48
49#[derive(Debug, Subcommand)]
50#[allow(clippy::large_enum_variant)]
51enum Commands {
52    /// Evaluate one screenshot.
53    #[cfg(feature = "codex-acp")]
54    Image(ImageArgs),
55    /// Evaluate an ordered before/after screenshot journey.
56    #[cfg(feature = "codex-acp")]
57    Sequence(SequenceArgs),
58    /// Host a local static site, capture screenshots, and evaluate them.
59    #[cfg(feature = "audit")]
60    Audit(AuditArgs),
61    /// Capture a complete matrix with one browser session and one rubric batch.
62    #[cfg(feature = "audit")]
63    Capture(CaptureArgs),
64    /// Validate a captured deterministic report against a calibration corpus.
65    #[cfg(feature = "audit")]
66    Calibrate(CalibrateArgs),
67    /// Serve a local static directory for manual browser testing.
68    Serve(ServeArgs),
69    /// Two-stage pipeline: vision model → rubric model.
70    ///
71    /// Stage 1 sends the screenshot to an OpenAI-compatible vision API
72    /// (e.g. Qwen VL via llama-swap) producing structured JSON.
73    /// Stage 2 sends that description to an ACP backend (opencode with
74    /// DeepSeek V4, or codex-acp) for the final rubric verdict.
75    #[cfg(feature = "pipeline")]
76    Pipeline(pipeline::PipelineArgs),
77    /// Pipeline configured via a TOML file written by the HM module.
78    ///
79    /// Reads configuration from `~/.config/visual-rubric/config.toml` so the
80    /// user only needs `--image` and `--question`. CLI flags override
81    /// individual TOML fields for one-off testing.
82    Configured(configured::ConfiguredArgs),
83}
84
85#[derive(Clone, Debug, Parser)]
86struct LegacyImageArgs {
87    #[arg(long)]
88    image: Option<PathBuf>,
89    #[arg(long)]
90    question: Option<String>,
91    #[arg(long)]
92    preset: Option<String>,
93    #[arg(long)]
94    system_prompt: Option<String>,
95    #[arg(long)]
96    model: Option<String>,
97    #[arg(long)]
98    effort: Option<String>,
99    #[cfg(feature = "codex-acp")]
100    #[arg(long)]
101    codex_acp: Option<PathBuf>,
102    #[arg(long, default_value = "vnc-screenshot")]
103    name: String,
104    #[arg(long)]
105    json: bool,
106}
107
108#[cfg(feature = "codex-acp")]
109#[derive(Clone, Debug, Parser)]
110struct ImageArgs {
111    #[arg(long)]
112    image: PathBuf,
113    #[command(flatten)]
114    questions: QuestionSource,
115    #[arg(long)]
116    system_prompt: Option<String>,
117    #[arg(long)]
118    model: Option<String>,
119    #[arg(long)]
120    effort: Option<String>,
121    #[arg(long)]
122    codex_acp: Option<PathBuf>,
123    #[arg(long, default_value = "screenshot")]
124    name: String,
125    #[arg(long)]
126    json: bool,
127}
128
129#[cfg(feature = "codex-acp")]
130#[derive(Clone, Debug, Parser)]
131struct SequenceArgs {
132    /// Ordered checkpoint as LABEL=PATH. May be repeated.
133    #[arg(long = "frame", value_parser = parse_sequence_frame, required = true, num_args = 1..)]
134    frames: Vec<SequenceFrameArg>,
135    #[command(flatten)]
136    questions: QuestionSource,
137    #[arg(long)]
138    model: Option<String>,
139    #[arg(long)]
140    effort: Option<String>,
141    #[arg(long)]
142    codex_acp: Option<PathBuf>,
143    #[arg(long, default_value = "sequence")]
144    name: String,
145    #[arg(long)]
146    json: bool,
147    /// Override the configured maximum number of ordered checkpoints.
148    #[arg(long)]
149    max_frames: Option<usize>,
150    /// Disable semantic before/after transition assessment for this run.
151    #[arg(long)]
152    no_require_transition: bool,
153}
154
155#[cfg(feature = "codex-acp")]
156#[derive(Clone, Debug)]
157struct SequenceFrameArg {
158    label: String,
159    path: PathBuf,
160}
161
162#[cfg(feature = "audit")]
163#[derive(Clone, Debug, Parser)]
164struct AuditArgs {
165    /// Static site root to serve.
166    #[arg(long)]
167    root: PathBuf,
168    /// Path under the hosted root to capture.
169    #[arg(long, default_value = "/")]
170    path: String,
171    /// Output directory for screenshots.
172    #[arg(long, default_value = "target/visual-rubric")]
173    screenshots: PathBuf,
174    /// JSON report path.
175    #[arg(long, default_value = "target/visual-rubric/report.json")]
176    report: PathBuf,
177    /// Browser binary for headless screenshots.
178    #[arg(long, env = "VISUAL_RUBRIC_BROWSER", default_value = "chromium")]
179    browser: PathBuf,
180    /// Extra argument passed to the browser. May be repeated.
181    #[arg(long = "browser-arg")]
182    browser_args: Vec<String>,
183    /// Delay before each browser capture, in milliseconds.
184    #[arg(long, default_value_t = 0)]
185    wait_ms: u64,
186    /// Device scale factor passed to Chromium.
187    #[arg(long)]
188    device_scale_factor: Option<f32>,
189    /// Number of times to retry a failed browser capture.
190    #[arg(long, default_value_t = 0)]
191    capture_retries: u32,
192    /// Return a non-zero exit when any rubric fails or errors.
193    #[arg(long)]
194    fail_on_rubric: bool,
195    /// Viewports as name=WIDTHxHEIGHT. May be repeated.
196    #[arg(long = "viewport")]
197    viewports: Vec<ViewportArg>,
198    #[command(flatten)]
199    questions: QuestionSource,
200    #[arg(long)]
201    system_prompt: Option<String>,
202    #[arg(long)]
203    model: Option<String>,
204    #[arg(long)]
205    effort: Option<String>,
206    #[arg(long)]
207    codex_acp: Option<PathBuf>,
208    /// Generate pass verdicts without starting codex-acp.
209    #[arg(long)]
210    fake_pass: bool,
211    /// Capture screenshots and report deterministic data without model calls.
212    #[arg(long)]
213    skip_ai: bool,
214}
215
216#[cfg(feature = "audit")]
217#[derive(Clone, Debug, Parser)]
218struct CaptureArgs {
219    /// Static site root to serve when `--base-url` is not supplied.
220    #[arg(long)]
221    root: PathBuf,
222    /// Workspace that owns emitted capture artifacts. Defaults to the current directory.
223    #[arg(long)]
224    workspace: Option<PathBuf>,
225    /// Existing HTTP(S) site to capture; skips starting the static server.
226    #[arg(long)]
227    base_url: Option<String>,
228    /// JSON capture-job description.
229    #[arg(long)]
230    job: PathBuf,
231    /// Root directory for emitted screenshots and metadata.
232    #[arg(long, default_value = "target/visual-rubric/captures")]
233    output: PathBuf,
234    /// JSON capture manifest path.
235    #[arg(long, default_value = "target/visual-rubric/capture_manifest.json")]
236    manifest: PathBuf,
237    /// JSON producer report path.
238    #[arg(long, default_value = "target/visual-rubric/run_report.json")]
239    report: PathBuf,
240    /// Browser binary for headless captures.
241    #[arg(long, env = "VISUAL_RUBRIC_BROWSER", default_value = "chromium")]
242    browser: PathBuf,
243    /// Extra argument passed to the browser. May be repeated.
244    #[arg(long = "browser-arg")]
245    browser_args: Vec<String>,
246    /// Number of rubric workers in the shared batch.
247    #[arg(long, default_value_t = 4)]
248    rubric_workers: usize,
249    /// Content-addressed rubric verdict cache directory.
250    #[arg(long)]
251    cache_dir: Option<PathBuf>,
252    #[command(flatten)]
253    questions: QuestionSource,
254    #[arg(long)]
255    system_prompt: Option<String>,
256    #[arg(long)]
257    model: Option<String>,
258    #[arg(long)]
259    effort: Option<String>,
260    #[arg(long)]
261    codex_acp: Option<PathBuf>,
262    /// Generate pass verdicts without starting codex-acp.
263    #[arg(long)]
264    fake_pass: bool,
265    /// Capture deterministic evidence without model calls.
266    #[arg(long)]
267    skip_ai: bool,
268}
269
270#[cfg(feature = "audit")]
271#[derive(Clone, Debug, Parser)]
272struct CalibrateArgs {
273    /// Checked-in calibration corpus JSON.
274    #[arg(long)]
275    corpus: PathBuf,
276    /// Schema-3 capture manifest produced by `capture`.
277    #[arg(long)]
278    manifest: PathBuf,
279    /// Deterministic report produced by `capture`.
280    #[arg(long)]
281    report: PathBuf,
282}
283
284#[derive(Clone, Debug, Parser)]
285struct ServeArgs {
286    #[arg(long)]
287    root: PathBuf,
288    #[arg(long, default_value_t = 1111)]
289    port: u16,
290}
291
292#[cfg(feature = "audit")]
293#[derive(Clone, Debug)]
294struct ViewportArg {
295    name: String,
296    width: u32,
297    height: u32,
298}
299
300/// Source of the rubric question: either an explicit `--question` string
301/// or a named `--preset`.  At least one of the two must be provided.
302#[derive(Clone, Debug, Args)]
303pub struct QuestionSource {
304    /// Explicit rubric question (required if --preset is not set).
305    #[arg(long, required_unless_present = "preset")]
306    pub question: Option<String>,
307
308    /// Named question preset (required if --question is not set).
309    #[arg(long, required_unless_present = "question")]
310    pub preset: Option<String>,
311}
312
313impl QuestionSource {
314    /// Build a source from an already-known question string (no preset).
315    pub fn from_question(question: String) -> Self {
316        Self {
317            question: Some(question),
318            preset: None,
319        }
320    }
321
322    /// Resolves the effective question text.
323    ///
324    /// When `--preset` was provided, delegates to [`crate::presets::resolve`].
325    /// When `--question` was provided, returns it verbatim.
326    pub fn resolve(&self) -> Result<String, PresetError> {
327        match (&self.preset, &self.question) {
328            (Some(name), _) => {
329                let questions = crate::presets::resolve(name)?;
330                Ok(questions.join("\n"))
331            }
332            (None, Some(q)) => Ok(q.clone()),
333            (None, None) => Err(PresetError::MissingQuestionSource),
334        }
335    }
336
337    /// Resolves the preset's standard system prompt.
338    ///
339    /// Returns `None` when no preset was selected or the preset does not
340    /// define a system prompt.  Callers should prefer an explicit
341    /// `--system-prompt` over this value.
342    pub fn resolve_system_prompt(&self) -> Result<Option<String>, PresetError> {
343        match &self.preset {
344            Some(name) => Ok(crate::presets::find(name)?
345                .system_prompt()
346                .map(str::to_owned)),
347            None => Ok(None),
348        }
349    }
350}
351
352/// Runs a parsed CLI command.
353///
354/// # Errors
355///
356/// Returns command, IO, browser, ACP, or rubric audit failures.
357pub fn run(cli: Cli) -> Result<()> {
358    match cli.command {
359        #[cfg(feature = "codex-acp")]
360        Some(Commands::Image(args)) => run_image(args),
361        #[cfg(feature = "codex-acp")]
362        Some(Commands::Sequence(args)) => run_sequence(args),
363        #[cfg(feature = "audit")]
364        Some(Commands::Audit(args)) => run_audit(args),
365        #[cfg(feature = "audit")]
366        Some(Commands::Capture(args)) => run_capture(args),
367        #[cfg(feature = "audit")]
368        Some(Commands::Calibrate(args)) => run_calibration(args),
369        Some(Commands::Serve(args)) => run_serve(args),
370        #[cfg(feature = "pipeline")]
371        Some(Commands::Pipeline(args)) => pipeline::run_pipeline(args),
372        Some(Commands::Configured(args)) => configured::run_configured(args),
373        #[cfg(feature = "codex-acp")]
374        None => run_image(cli.image.try_into()?),
375        #[cfg(not(feature = "codex-acp"))]
376        None => Err(anyhow!(
377            "no subcommand given; use --help for available commands"
378        )),
379    }
380}
381
382#[cfg(feature = "codex-acp")]
383fn run_image(args: ImageArgs) -> Result<()> {
384    let verdict = evaluate_image(&args)?;
385    if args.json {
386        println!("{}", serde_json::to_string(&verdict)?);
387        return Ok(());
388    }
389    crate::assert_verdict(&args.name, verdict)
390        .map(|()| println!("visual rubric passed"))
391        .map_err(|error| anyhow!(error))
392}
393
394#[cfg(feature = "codex-acp")]
395fn run_sequence(args: SequenceArgs) -> Result<()> {
396    let question = args.questions.resolve().map_err(|error| anyhow!(error))?;
397    let frames = args
398        .frames
399        .into_iter()
400        .map(|frame| crate::SequenceFrame {
401            label: frame.label,
402            path: frame.path,
403        })
404        .collect::<Vec<_>>();
405    let options = crate::RubricOptions {
406        model: args.model,
407        effort: args.effort.map(Into::into),
408        system_prompt: args
409            .questions
410            .resolve_system_prompt()
411            .map_err(|e| anyhow!(e))?,
412    };
413    let configured_sequence = crate::SequenceOptions::from_config_toml(None);
414    let sequence_options = crate::SequenceOptions {
415        max_frames: args.max_frames.unwrap_or(configured_sequence.max_frames),
416        require_transition: configured_sequence.require_transition && !args.no_require_transition,
417    };
418    let mut runtime_config = crate::RubricRunConfig::from_config_toml(None);
419    if let Some(binary) = args.codex_acp {
420        runtime_config.codex_acp_binary = binary;
421    }
422    let verdict = crate::evaluate_image_sequence_rubric_with_options(
423        &frames,
424        &question,
425        options,
426        runtime_config,
427        sequence_options,
428    )?;
429    if args.json {
430        println!("{}", serde_json::to_string(&verdict)?);
431        return Ok(());
432    }
433    crate::assert_verdict(&args.name, verdict)
434        .map(|()| println!("visual sequence rubric passed"))
435        .map_err(|error| anyhow!(error))
436}
437
438fn run_serve(args: ServeArgs) -> Result<()> {
439    let server = StaticServer::start(args.root, args.port)?;
440    println!("{}", server.base_url());
441    server.wait_forever()
442}
443
444#[cfg(feature = "codex-acp")]
445fn evaluate_image(args: &ImageArgs) -> Result<crate::RubricVerdict> {
446    let question = args.questions.resolve().map_err(|e| anyhow!(e))?;
447    let system_prompt = match args.system_prompt.clone() {
448        Some(prompt) => Some(prompt),
449        None => args
450            .questions
451            .resolve_system_prompt()
452            .map_err(|e| anyhow!(e))?,
453    };
454    let options = crate::RubricOptions {
455        model: args.model.clone(),
456        effort: args.effort.clone().map(Into::into),
457        system_prompt,
458    };
459    if let Some(codex_acp) = &args.codex_acp {
460        let pool = crate::RubricPool::new(crate::PoolConfig {
461            workers: 1,
462            codex_acp_binary: codex_acp.clone(),
463            default_options: merge_with_defaults(options),
464            ..crate::PoolConfig::default()
465        })?;
466        let verdict = pool.submit(&args.image, &question, crate::RubricOptions::default())?;
467        let _ = pool.shutdown();
468        Ok(verdict)
469    } else {
470        crate::evaluate_image_rubric_with_options(&args.image, &question, options)
471            .map_err(|error| anyhow!(error))
472    }
473}
474
475#[cfg(feature = "codex-acp")]
476fn merge_with_defaults(mut options: crate::RubricOptions) -> crate::RubricOptions {
477    let defaults = crate::default_options();
478    if options.model.is_none() {
479        options.model = defaults.model;
480    }
481    if options.effort.is_none() {
482        options.effort = defaults.effort;
483    }
484    if options.system_prompt.is_none() {
485        options.system_prompt = defaults.system_prompt;
486    }
487    options
488}
489
490#[cfg(feature = "codex-acp")]
491impl TryFrom<LegacyImageArgs> for ImageArgs {
492    type Error = anyhow::Error;
493
494    fn try_from(value: LegacyImageArgs) -> Result<Self> {
495        let questions = match (value.question, value.preset) {
496            (Some(q), _) => QuestionSource::from_question(q),
497            (None, Some(preset)) => QuestionSource {
498                question: None,
499                preset: Some(preset),
500            },
501            (None, None) => {
502                anyhow::bail!("either --question or --preset is required");
503            }
504        };
505        Ok(Self {
506            image: value.image.context("--image is required")?,
507            questions,
508            system_prompt: value.system_prompt,
509            model: value.model,
510            effort: value.effort,
511            codex_acp: value.codex_acp,
512            name: value.name,
513            json: value.json,
514        })
515    }
516}
517
518#[cfg(feature = "codex-acp")]
519fn parse_sequence_frame(value: &str) -> Result<SequenceFrameArg, String> {
520    let (label, path) = value
521        .split_once('=')
522        .ok_or_else(|| "frame must be LABEL=PATH".to_owned())?;
523    if label.trim().is_empty() {
524        return Err("frame label must not be empty".to_owned());
525    }
526    if path.trim().is_empty() {
527        return Err("frame path must not be empty".to_owned());
528    }
529    Ok(SequenceFrameArg {
530        label: label.to_owned(),
531        path: PathBuf::from(path),
532    })
533}
534
535#[cfg(feature = "audit")]
536impl std::str::FromStr for ViewportArg {
537    type Err = anyhow::Error;
538
539    fn from_str(value: &str) -> Result<Self> {
540        let (name, size) = value
541            .split_once('=')
542            .context("viewport must be name=WIDTHxHEIGHT")?;
543        let (width, height) = size
544            .split_once('x')
545            .context("viewport size must be WIDTHxHEIGHT")?;
546        if name.is_empty() {
547            anyhow::bail!("viewport name must not be empty");
548        }
549        if name.contains('/') || name.contains('\\') {
550            anyhow::bail!("viewport name {name:?} must not contain path separators");
551        }
552        let width = width.parse().context("viewport width must be an integer")?;
553        let height = height
554            .parse()
555            .context("viewport height must be an integer")?;
556        if width == 0 || height == 0 {
557            anyhow::bail!("viewport dimensions must be greater than zero");
558        }
559        Ok(Self {
560            name: name.to_string(),
561            width,
562            height,
563        })
564    }
565}