visual-rubric 0.4.2

AI-assisted screenshot rubric runner for local visual UX review
Documentation
#![cfg(feature = "codex-acp")]

mod common;

use std::ffi::OsString;
use std::process::Command;

use visual_rubric::{
    ConfigMode, PoolError, RubricError, RubricOptions, RubricRunConfig, SequenceFrame,
    SequenceOptions, direct_codex_gpt_config, evaluate_image_rubric_with_config,
    evaluate_image_sequence_rubric_with_options,
};

#[test]
fn image_json_outputs_verdict() {
    let Some(fake) = common::fake_codex_acp_binary() else {
        eprintln!("skipping: fake-codex-acp feature is not enabled");
        return;
    };
    let temp = tempfile::TempDir::new().expect("tempdir");
    let image = common::write_fixture_png(&temp);
    let output = Command::new(env!("CARGO_BIN_EXE_visual-rubric"))
        .arg("image")
        .arg("--image")
        .arg(&image)
        .arg("--question")
        .arg("Does it pass?")
        .arg("--codex-acp")
        .arg(fake)
        .arg("--json")
        .env("FAKE_CODEX_ACP_MODE", "pass")
        .output()
        .expect("run visual-rubric");

    assert!(output.status.success(), "{output:?}");
    let json: serde_json::Value = serde_json::from_slice(&output.stdout).expect("json stdout");
    assert_eq!(json["verdict"], "pass");
    assert_eq!(json["reason"], "fake pass");
}

#[test]
fn image_fail_verdict_returns_error_without_json() {
    let Some(fake) = common::fake_codex_acp_binary() else {
        eprintln!("skipping: fake-codex-acp feature is not enabled");
        return;
    };
    let temp = tempfile::TempDir::new().expect("tempdir");
    let image = common::write_fixture_png(&temp);
    let output = Command::new(env!("CARGO_BIN_EXE_visual-rubric"))
        .arg("image")
        .arg("--image")
        .arg(&image)
        .arg("--question")
        .arg("Does it fail?")
        .arg("--name")
        .arg("fixture")
        .arg("--codex-acp")
        .arg(fake)
        .env("FAKE_CODEX_ACP_MODE", "fail")
        .output()
        .expect("run visual-rubric");

    assert!(!output.status.success(), "{output:?}");
    let stderr = String::from_utf8_lossy(&output.stderr);
    assert!(stderr.contains("fixture"), "{stderr}");
    assert!(stderr.contains("fake fail"), "{stderr}");
}

#[test]
fn image_forwards_model_effort_and_system_prompt_to_custom_acp() {
    let Some(fake) = common::fake_codex_acp_binary() else {
        eprintln!("skipping: fake-codex-acp feature is not enabled");
        return;
    };
    let temp = tempfile::TempDir::new().expect("tempdir");
    let image = common::write_fixture_png(&temp);
    let config_log = temp.path().join("config.log");
    let prompt_log = temp.path().join("prompts.log");
    let output = Command::new(env!("CARGO_BIN_EXE_visual-rubric"))
        .arg("image")
        .arg("--image")
        .arg(&image)
        .arg("--question")
        .arg("Does it pass?")
        .arg("--system-prompt")
        .arg("Project rubric")
        .arg("--model")
        .arg("custom-model")
        .arg("--effort")
        .arg("high")
        .arg("--codex-acp")
        .arg(fake)
        .arg("--json")
        .env("FAKE_CODEX_ACP_MODE", "pass")
        .env("FAKE_CODEX_ACP_CONFIG_LOG", &config_log)
        .env("FAKE_CODEX_ACP_PROMPT_LOG", &prompt_log)
        .output()
        .expect("run visual-rubric");

    assert!(output.status.success(), "{output:?}");
    let config = std::fs::read_to_string(config_log).expect("config log");
    assert!(config.contains("\"configId\":\"model\""), "{config}");
    assert!(config.contains("\"value\":\"custom-model\""), "{config}");
    assert!(
        config.contains("\"configId\":\"reasoning_effort\""),
        "{config}"
    );
    assert!(config.contains("\"value\":\"high\""), "{config}");
    assert!(config.contains("\"configId\":\"mode\""), "{config}");
    let prompts = std::fs::read_to_string(prompt_log).expect("prompt log");
    assert!(prompts.contains("Project rubric"), "{prompts}");
    assert!(prompts.contains("Question: Does it pass?"), "{prompts}");
}

#[test]
fn sequence_api_honors_custom_and_default_system_prompts() {
    let Some(fake) = common::fake_codex_acp_binary() else {
        eprintln!("skipping: fake-codex-acp feature is not enabled");
        return;
    };
    let temp = tempfile::TempDir::new().expect("tempdir");
    let image = common::write_fixture_png(&temp);
    let frames = ["before", "after"].map(|label| SequenceFrame {
        label: label.to_owned(),
        path: image.clone(),
    });
    for custom in [Some("Project-specific sequence rubric"), None] {
        let prompt_log = temp.path().join(if custom.is_some() {
            "custom.log"
        } else {
            "default.log"
        });
        let verdict = evaluate_image_sequence_rubric_with_options(
            &frames,
            "Did the menu open?",
            RubricOptions {
                system_prompt: custom.map(str::to_owned),
                ..RubricOptions::default()
            },
            RubricRunConfig {
                codex_acp_binary: fake.clone(),
                extra_env: vec![
                    (
                        OsString::from("FAKE_CODEX_ACP_MODE"),
                        OsString::from("pass"),
                    ),
                    (
                        OsString::from("FAKE_CODEX_ACP_PROMPT_LOG"),
                        prompt_log.as_os_str().to_owned(),
                    ),
                ],
                ..RubricRunConfig::default()
            },
            SequenceOptions::default(),
        )
        .expect("sequence verdict");
        assert_eq!(verdict.verdict, "pass");
        let prompt = std::fs::read_to_string(prompt_log).expect("prompt log");
        assert!(
            prompt.contains(custom.unwrap_or(visual_rubric::DEFAULT_SYSTEM_PROMPT)),
            "{prompt}"
        );
        assert!(prompt.contains("Question: Did the menu open?"), "{prompt}");
        assert!(prompt.contains("before/after transition"), "{prompt}");
        if custom.is_some() {
            assert!(
                !prompt.contains(visual_rubric::DEFAULT_SYSTEM_PROMPT),
                "{prompt}"
            );
        }
    }
}

#[test]
fn sequence_cli_forwards_preset_system_prompt() {
    let Some(fake) = common::fake_codex_acp_binary() else {
        eprintln!("skipping: fake-codex-acp feature is not enabled");
        return;
    };
    let temp = tempfile::TempDir::new().expect("tempdir");
    let image = common::write_fixture_png(&temp);
    let prompt_log = temp.path().join("prompts.log");
    let output = Command::new(env!("CARGO_BIN_EXE_visual-rubric"))
        .arg("sequence")
        .arg("--frame")
        .arg(format!("before={}", image.display()))
        .arg("--frame")
        .arg(format!("after={}", image.display()))
        .args(["--preset", "website-install", "--max-frames", "2"])
        .arg("--codex-acp")
        .arg(fake)
        .arg("--json")
        .env("XDG_CONFIG_HOME", temp.path())
        .env("FAKE_CODEX_ACP_MODE", "pass")
        .env("FAKE_CODEX_ACP_PROMPT_LOG", &prompt_log)
        .output()
        .expect("run sequence CLI");
    assert!(output.status.success(), "{output:?}");
    let prompt = std::fs::read_to_string(prompt_log).expect("prompt log");
    assert!(
        prompt.contains(visual_rubric::presets::WEBSITE_INSTALL_SYSTEM_PROMPT),
        "{prompt}"
    );
}

#[test]
fn sequence_rejects_excess_frames_before_reading_files() {
    let temp = tempfile::TempDir::new().expect("tempdir");
    let frames = ["before", "during", "after"].map(|label| SequenceFrame {
        label: label.to_owned(),
        path: temp.path().join(format!("missing-{label}.png")),
    });
    let result = evaluate_image_sequence_rubric_with_options(
        &frames,
        "Did the menu open?",
        RubricOptions::default(),
        RubricRunConfig::default(),
        SequenceOptions {
            max_frames: 2,
            require_transition: true,
        },
    );
    assert!(
        matches!(result, Err(RubricError::Pool(PoolError::Rpc(ref message)))
        if message == "sequence contains 3 frames, maximum is 2"),
        "{result:?}"
    );
}

#[test]
fn configured_direct_mode_sends_one_multimodal_codex_prompt() {
    let Some(fake) = common::fake_codex_acp_binary() else {
        eprintln!("skipping: fake-codex-acp feature is not enabled");
        return;
    };
    let temp = tempfile::TempDir::new().expect("tempdir");
    let image = common::write_fixture_png(&temp);
    let config = temp.path().join("config.toml");
    let kind_log = temp.path().join("prompt-kind.log");
    std::fs::write(
        &config,
        format!(
            r#"
mode = "direct"

[rubric]
backend = "{}"
model = "gpt-5.5"
effort = "medium"
"#,
            fake.display()
        ),
    )
    .expect("write config");

    let output = Command::new(env!("CARGO_BIN_EXE_visual-rubric"))
        .arg("configured")
        .arg("--config")
        .arg(&config)
        .arg("--image")
        .arg(&image)
        .arg("--question")
        .arg("Does it pass?")
        .arg("--json")
        .env("FAKE_CODEX_ACP_MODE", "pass")
        .env("FAKE_CODEX_ACP_PROMPT_KIND_LOG", &kind_log)
        .output()
        .expect("run visual-rubric configured");

    assert!(output.status.success(), "{output:?}");
    let prompt_kind = std::fs::read_to_string(kind_log).expect("prompt kind log");
    assert!(prompt_kind.contains("has_image=true"), "{prompt_kind}");
}

#[test]
fn direct_codex_gpt_config_uses_subscription_defaults() {
    let config = direct_codex_gpt_config(None, None);

    assert_eq!(config.mode, Some(ConfigMode::Direct));
    assert_eq!(config.rubric.backend.as_deref(), Some("codex-acp"));
    assert_eq!(config.rubric.model.as_deref(), Some("gpt-5.5"));
    assert_eq!(config.rubric.effort.as_deref(), Some("medium"));
    assert!(config.vision.url.is_none());
}

#[test]
fn public_api_accepts_custom_binary_env_and_cwd() {
    let Some(fake) = common::fake_codex_acp_binary() else {
        eprintln!("skipping: fake-codex-acp feature is not enabled");
        return;
    };
    let temp = tempfile::TempDir::new().expect("tempdir");
    let image = common::write_fixture_png(&temp);
    let cwd = temp.path().join("cwd");
    std::fs::create_dir(&cwd).expect("cwd");
    let cwd_log = temp.path().join("cwd.log");
    let env_log = temp.path().join("env.log");

    let verdict = evaluate_image_rubric_with_config(
        &image,
        "Does it pass?",
        RubricOptions::default(),
        RubricRunConfig {
            codex_acp_binary: fake,
            acp_args: Vec::new(),
            url: None,
            api_model: None,
            extra_env: vec![
                (
                    OsString::from("FAKE_CODEX_ACP_MODE"),
                    OsString::from("pass"),
                ),
                (
                    OsString::from("FAKE_CODEX_ACP_CWD_LOG"),
                    cwd_log.as_os_str().to_os_string(),
                ),
                (
                    OsString::from("FAKE_CODEX_ACP_ENV_LOG"),
                    env_log.as_os_str().to_os_string(),
                ),
                (
                    OsString::from("FAKE_CODEX_ACP_CUSTOM_ENV"),
                    OsString::from("custom-value"),
                ),
            ],
            cwd: Some(cwd.clone()),
        },
    )
    .expect("verdict");

    assert_eq!(verdict.verdict, "pass");
    assert_eq!(
        std::fs::read_to_string(cwd_log).unwrap().trim(),
        cwd.to_string_lossy()
    );
    assert_eq!(
        std::fs::read_to_string(env_log).unwrap().trim(),
        "custom-value"
    );
}

#[test]
fn public_api_forwards_custom_acp_args() {
    let Some(fake) = common::fake_codex_acp_binary() else {
        eprintln!("skipping: fake-codex-acp feature is not enabled");
        return;
    };
    let temp = tempfile::TempDir::new().expect("tempdir");
    let image = common::write_fixture_png(&temp);
    let args_log = temp.path().join("args.log");

    let verdict = evaluate_image_rubric_with_config(
        &image,
        "Does it pass?",
        RubricOptions::default(),
        RubricRunConfig {
            codex_acp_binary: fake,
            acp_args: vec!["acp".to_string()],
            url: None,
            api_model: None,
            extra_env: vec![
                (
                    OsString::from("FAKE_CODEX_ACP_MODE"),
                    OsString::from("pass"),
                ),
                (
                    OsString::from("FAKE_CODEX_ACP_ARG_LOG"),
                    args_log.as_os_str().to_os_string(),
                ),
            ],
            cwd: None,
        },
    )
    .expect("verdict");

    assert_eq!(verdict.verdict, "pass");
    let args = std::fs::read_to_string(args_log).expect("args log");
    assert!(args.lines().any(|line| line == "acp"), "{args}");
    assert!(!args.contains("model=\""), "{args}");
}