pointlock_vision/

lib.rs

//! # pointlock-vision
//!
//! The [`VisionVerifier`] plugin interface (verify role only, principle 7:
//! vision never locates or acts) and the default [`StubVisionVerifier`].
//!
//! The verifier is the runtime consumer of the `vision` verify channel
//! (spine §6.3): it answers an author-written prompt against localized
//! screenshot bytes with a three-valued verdict. A `pass`/`fail` answer is
//! a *completed* evaluation (fail is final, spine R5); an `unknown` answer
//! means the channel could not complete and the chain advances — for the
//! vision channel, which is only legal at the chain tail, that exhausts the
//! chain into assertion `unknown` (principle 4).
//!
//! The v0.1 default is the stub: it always answers `unknown` with the
//! reason `"vision verifier not configured"`. The runner treats an absent
//! verifier (`RunOptions::vision == None`) as exactly equivalent.

use async_trait::async_trait;
use pointlock_ir::{RectIR, VerdictStatus};

/// The reason the stub (and an unconfigured runner) yields `unknown`.
pub const STUB_REASON: &str = "vision verifier not configured";

/// One vision verification request: the author-written prompt (never
/// synthesized by the compiler, principle 6), an optional region of
/// interest, and the *localized* screenshot bytes (evidence is localized
/// during `observing`, spine §6.6 — the verifier never reaches back into a
/// provider session).
#[derive(Debug, Clone, PartialEq)]
pub struct VisionRequest<'a> {
    /// The author-written prompt, verbatim (`AssertionIR.visionPrompt` for
    /// `elementState`/`elementText` chain tails; `predicate.prompt` for
    /// `visual` predicates).
    pub prompt: &'a str,
    /// Optional region of interest (`visual` predicates only).
    pub region: Option<&'a RectIR>,
    /// The localized screenshot bytes.
    pub screenshot: &'a [u8],
    /// The screenshot's media type, e.g. `image/png`.
    pub media_type: &'a str,
}

/// Identity of a vision judge implementation (evidence honesty: once the
/// model is swappable, a verdict must say which model answered).
#[derive(Debug, Clone, PartialEq)]
pub struct VisionJudge {
    /// Implementation family, e.g. `anthropic` / `openai-compat`.
    pub provider: String,
    /// The requested model id, when the implementation has one.
    pub model: Option<String>,
}

/// The verifier's three-valued answer with a human-readable reason.
///
/// `pass`/`fail` are completed evaluations; `unknown` means the verifier
/// could not complete (unconfigured, low confidence, unusable image) and
/// carries the why. The verdict never panics its way out — model/transport
/// failures inside an implementation must fold to `unknown` (principle 4).
#[derive(Debug, Clone, PartialEq)]
pub struct VisionVerdict {
    /// The three-valued status.
    pub status: VerdictStatus,
    /// Why the verifier answered this way.
    pub reason: String,
    /// Which judge answered, when a configured implementation did
    /// (`None` from the stub — nothing judged).
    pub judge: Option<VisionJudge>,
    /// The judge's self-reported on-screen facts (the look-then-judge
    /// answer protocol), bounded by [`MAX_OBSERVATIONS`] and
    /// [`MAX_OBSERVATION_CHARS`]; empty when none were reported.
    pub observations: Vec<String>,
}

/// The vision verification plugin interface (verify role only).
#[async_trait]
pub trait VisionVerifier: Send + Sync {
    /// Answers `request.prompt` against the screenshot. Infallible by
    /// construction: anything that prevents an answer is an `unknown`
    /// verdict with a reason, never an error (principle 4).
    async fn verify(&self, request: VisionRequest<'_>) -> VisionVerdict;
}

/// The v0.1 default verifier: always `unknown` with [`STUB_REASON`].
#[derive(Debug, Clone, Copy, Default)]
pub struct StubVisionVerifier;

#[async_trait]
impl VisionVerifier for StubVisionVerifier {
    async fn verify(&self, _request: VisionRequest<'_>) -> VisionVerdict {
        VisionVerdict {
            status: VerdictStatus::Unknown,
            reason: STUB_REASON.to_owned(),
            judge: None,
            observations: Vec::new(),
        }
    }
}

// ─── The Anthropic-backed verifier (M3a-W4: the first usable one) ───────────

/// The default model of [`AnthropicVisionVerifier`].
pub const DEFAULT_VISION_MODEL: &str = "claude-opus-4-8";

/// Total per-request deadline. Without one, a TCP-accepted-but-silent
/// endpoint stalls `verify()` forever — an unbounded await is a failure
/// mode that never folds to `unknown`, breaking the module contract
/// (principle 4). The step's `timeout_ms` governs only provider execute,
/// not the assert phase, so the bound must live here.
const REQUEST_TIMEOUT_SECS: u64 = 60;

/// Connection-establishment deadline (part of the same fail-to-unknown
/// bound; kept tighter so dead endpoints answer fast).
const CONNECT_TIMEOUT_SECS: u64 = 10;

/// The first usable verifier (08 §6.4): asks an Anthropic vision model to
/// answer the author's prompt against the screenshot, over raw HTTP (no
/// official Rust SDK exists). Discipline unchanged from the trait docs:
/// verify-only, chain tail only, and every failure mode — missing key,
/// transport, non-200, unparseable answer, model uncertainty — folds to
/// `unknown` with a reason, never an error and never a guessed pass
/// (principles 4/7).
pub struct AnthropicVisionVerifier {
    api_key: String,
    model: String,
    base_url: String,
    client: reqwest::Client,
}

impl AnthropicVisionVerifier {
    /// Builds a verifier from explicit configuration.
    pub fn new(
        api_key: impl Into<String>,
        model: impl Into<String>,
        base_url: impl Into<String>,
    ) -> Self {
        AnthropicVisionVerifier {
            api_key: api_key.into(),
            model: model.into(),
            base_url: base_url.into(),
            client: http_client(),
        }
    }

    /// Builds from the environment: `ANTHROPIC_API_KEY` (required — `None`
    /// without it), `POINTLOCK_VISION_MODEL` (default
    /// [`DEFAULT_VISION_MODEL`]), `ANTHROPIC_BASE_URL` (default the public
    /// API; overriding it is also how the tests run against a local
    /// canned-response server).
    pub fn from_env() -> Option<Self> {
        Self::from_lookup(|key| std::env::var(key).ok())
    }

    /// The injectable body of [`Self::from_env`] (unit-testable without
    /// process-global env mutation).
    fn from_lookup(get: impl Fn(&str) -> Option<String>) -> Option<Self> {
        let api_key = get("ANTHROPIC_API_KEY").filter(|key| !key.is_empty())?;
        let model = get("POINTLOCK_VISION_MODEL")
            .filter(|model| !model.is_empty())
            .unwrap_or_else(|| DEFAULT_VISION_MODEL.to_owned());
        let base_url = get("ANTHROPIC_BASE_URL")
            .filter(|url| !url.is_empty())
            .unwrap_or_else(|| "https://api.anthropic.com".to_owned());
        Some(Self::new(api_key, model, base_url))
    }

    fn judge(&self) -> VisionJudge {
        VisionJudge {
            provider: "anthropic".to_owned(),
            model: Some(self.model.clone()),
        }
    }

    fn unknown(&self, reason: impl Into<String>) -> VisionVerdict {
        VisionVerdict {
            status: VerdictStatus::Unknown,
            reason: reason.into(),
            judge: Some(self.judge()),
            observations: Vec::new(),
        }
    }
}

/// Cap on self-reported observations kept per answer, and per-observation
/// character bound — both bound ledger growth: observations enter the
/// durable assertion record verbatim.
pub const MAX_OBSERVATIONS: usize = 16;
/// See [`MAX_OBSERVATIONS`].
pub const MAX_OBSERVATION_CHARS: usize = 300;

/// The shared verification instruction (the look-then-judge answer
/// protocol): the judge first lists the on-screen facts it bases its
/// answer on, then gives exactly one verdict line. The observation lines
/// become checkable evidence next to the verdict; the pinned vocabulary
/// keeps parsing fail-closed.
fn verification_instruction(request: &VisionRequest<'_>) -> String {
    let region_note = request.region.map_or(String::new(), |region| {
        format!(
            " Consider ONLY the region at x={}, y={}, width={}, height={} (pixels from the top-left).",
            region.x, region.y, region.width, region.height
        )
    });
    format!(
        "You are a visual verification oracle for a device-automation audit trail. \
         Judge the following claim against the screenshot.{region_note}\n\
         Claim: {}\n\
         Answer in EXACTLY this form and nothing else. First, zero or more lines, each:\n\
         OBSERVED: <one concrete on-screen fact relevant to the claim>\n\
         Then exactly one final line, one of:\n\
         PASS: <what you see that confirms it>\n\
         FAIL: <what you see that contradicts it>\n\
         UNKNOWN: <why it cannot be determined>\n\
         Answer UNKNOWN unless the claim is clearly confirmed or clearly contradicted.",
        request.prompt
    )
}

/// The shared HTTP client of the remote verifiers. Deadlines: see the
/// timeout consts. `no_proxy` keeps egress deterministic (v0.1 makes no
/// proxy promise) and the canned local-endpoint tests hermetic under
/// ambient HTTP(S)_PROXY / ALL_PROXY environments. The `expect` matches
/// `reqwest::Client::new`'s own panic-on-TLS-init semantics.
fn http_client() -> reqwest::Client {
    reqwest::Client::builder()
        .connect_timeout(std::time::Duration::from_secs(CONNECT_TIMEOUT_SECS))
        .timeout(std::time::Duration::from_secs(REQUEST_TIMEOUT_SECS))
        .no_proxy()
        .build()
        .expect("reqwest client construction")
}

/// Parses one answer under the look-then-judge protocol: zero or more
/// leading `OBSERVED:` fact lines, then exactly one verdict line. A
/// missing or malformed verdict line parses to `unknown` (fail-closed —
/// a chatty answer is not a verdict); lines after the verdict are
/// ignored. Observations are bounded before they can enter any durable
/// record, and ride along even on a failed parse (they are still what
/// the judge reported seeing).
fn parse_answer(text: &str, judge: &VisionJudge) -> VisionVerdict {
    let mut observations: Vec<String> = Vec::new();
    let mut verdict_line: Option<&str> = None;
    for line in text.trim().lines().map(str::trim) {
        if line.is_empty() {
            continue;
        }
        if let Some(fact) = line.strip_prefix("OBSERVED:") {
            if observations.len() < MAX_OBSERVATIONS {
                observations.push(bounded_chars(fact.trim(), MAX_OBSERVATION_CHARS));
            }
            continue;
        }
        verdict_line = Some(line);
        break;
    }
    let unknown = |reason: String, observations: Vec<String>| VisionVerdict {
        status: VerdictStatus::Unknown,
        reason,
        judge: Some(judge.clone()),
        observations,
    };
    let Some(first) = verdict_line else {
        return unknown(
            "the verifier answer carried no verdict line".to_owned(),
            observations,
        );
    };
    let (status, rest) = if let Some(rest) = first.strip_prefix("PASS:") {
        (VerdictStatus::Pass, rest)
    } else if let Some(rest) = first.strip_prefix("FAIL:") {
        (VerdictStatus::Fail, rest)
    } else if let Some(rest) = first.strip_prefix("UNKNOWN:") {
        (VerdictStatus::Unknown, rest)
    } else {
        return unknown(
            format!("unparseable verifier answer: {first:.120}"),
            observations,
        );
    };
    VisionVerdict {
        status,
        reason: format!("vision: {}", rest.trim()),
        judge: Some(judge.clone()),
        observations,
    }
}

/// Truncates to a character bound on a char boundary, marking the cut.
fn bounded_chars(value: &str, max_chars: usize) -> String {
    if value.chars().count() <= max_chars {
        return value.to_owned();
    }
    let mut bounded: String = value.chars().take(max_chars).collect();
    bounded.push('…');
    bounded
}

#[async_trait]
impl VisionVerifier for AnthropicVisionVerifier {
    async fn verify(&self, request: VisionRequest<'_>) -> VisionVerdict {
        use base64::Engine as _;
        let data = base64::engine::general_purpose::STANDARD.encode(request.screenshot);
        let instruction = verification_instruction(&request);
        let body = serde_json::json!({
            "model": self.model,
            // Room for the observation lines ahead of the verdict line.
            "max_tokens": 2048,
            "messages": [{
                "role": "user",
                "content": [
                    { "type": "image", "source": {
                        "type": "base64",
                        "media_type": request.media_type,
                        "data": data,
                    }},
                    { "type": "text", "text": instruction },
                ],
            }],
        });

        let response = match self
            .client
            .post(format!("{}/v1/messages", self.base_url))
            .header("x-api-key", &self.api_key)
            .header("anthropic-version", "2023-06-01")
            .json(&body)
            .send()
            .await
        {
            Ok(response) => response,
            Err(err) => return self.unknown(format!("vision transport failed: {err}")),
        };
        if !response.status().is_success() {
            let status = response.status();
            let body = response.text().await.unwrap_or_default();
            return self.unknown(format!(
                "vision API answered {status}: {:.200}",
                body.trim()
            ));
        }
        let parsed: serde_json::Value = match response.json().await {
            Ok(parsed) => parsed,
            Err(err) => return self.unknown(format!("vision response unreadable: {err}")),
        };
        // Concatenate the text blocks (thinking blocks are skipped).
        let text: String = parsed
            .get("content")
            .and_then(|content| content.as_array())
            .map(|blocks| {
                blocks
                    .iter()
                    .filter(|block| block.get("type").and_then(|t| t.as_str()) == Some("text"))
                    .filter_map(|block| block.get("text").and_then(|t| t.as_str()))
                    .collect::<Vec<_>>()
                    .join("")
            })
            .unwrap_or_default();
        if text.trim().is_empty() {
            return self.unknown("vision response carried no text answer");
        }
        parse_answer(&text, &self.judge())
    }
}

// ─── The OpenAI-compatible verifier (graphics-focused open models) ──────────

/// A verifier over the OpenAI-compatible chat-completions wire format —
/// how graphics-focused open models (Qwen-VL, GLM-4V, …) are typically
/// served, both hosted and self-hosted (vLLM). Same discipline as the
/// Anthropic verifier: verify-only, chain tail only, every failure mode
/// folds to `unknown` with a reason (principles 4/7), and the same
/// look-then-judge answer protocol so verdicts stay comparable across
/// providers.
pub struct OpenAiCompatVisionVerifier {
    /// Bearer token; absent for endpoints that need none (local vLLM).
    api_key: Option<String>,
    model: String,
    /// Endpoint base *including* the ecosystem-conventional `/v1`
    /// (e.g. `http://127.0.0.1:8000/v1`); `/chat/completions` is appended.
    base_url: String,
    client: reqwest::Client,
}

impl OpenAiCompatVisionVerifier {
    /// Builds a verifier from explicit configuration.
    pub fn new(
        api_key: Option<String>,
        model: impl Into<String>,
        base_url: impl Into<String>,
    ) -> Self {
        OpenAiCompatVisionVerifier {
            api_key,
            model: model.into(),
            base_url: base_url.into(),
            client: http_client(),
        }
    }

    /// Builds from the environment: `POINTLOCK_VISION_BASE_URL` and
    /// `POINTLOCK_VISION_MODEL` are both required (`None` without them —
    /// there is no canonical public endpoint or model to default to);
    /// `POINTLOCK_VISION_API_KEY` is optional (self-hosted endpoints
    /// commonly need none).
    pub fn from_env() -> Option<Self> {
        Self::from_lookup(|key| std::env::var(key).ok())
    }

    /// The injectable body of [`Self::from_env`] (unit-testable without
    /// process-global env mutation).
    fn from_lookup(get: impl Fn(&str) -> Option<String>) -> Option<Self> {
        let base_url = get("POINTLOCK_VISION_BASE_URL").filter(|url| !url.is_empty())?;
        let model = get("POINTLOCK_VISION_MODEL").filter(|model| !model.is_empty())?;
        let api_key = get("POINTLOCK_VISION_API_KEY").filter(|key| !key.is_empty());
        Some(Self::new(api_key, model, base_url))
    }

    fn judge(&self) -> VisionJudge {
        VisionJudge {
            provider: "openai-compat".to_owned(),
            model: Some(self.model.clone()),
        }
    }

    fn unknown(&self, reason: impl Into<String>) -> VisionVerdict {
        VisionVerdict {
            status: VerdictStatus::Unknown,
            reason: reason.into(),
            judge: Some(self.judge()),
            observations: Vec::new(),
        }
    }
}

#[async_trait]
impl VisionVerifier for OpenAiCompatVisionVerifier {
    async fn verify(&self, request: VisionRequest<'_>) -> VisionVerdict {
        use base64::Engine as _;
        let data = base64::engine::general_purpose::STANDARD.encode(request.screenshot);
        let instruction = verification_instruction(&request);
        let body = serde_json::json!({
            "model": self.model,
            // Room for the observation lines ahead of the verdict line.
            "max_tokens": 2048,
            "messages": [{
                "role": "user",
                "content": [
                    { "type": "image_url", "image_url": {
                        "url": format!("data:{};base64,{data}", request.media_type),
                    }},
                    { "type": "text", "text": instruction },
                ],
            }],
        });

        let mut post = self
            .client
            .post(format!("{}/chat/completions", self.base_url))
            .json(&body);
        if let Some(key) = &self.api_key {
            post = post.bearer_auth(key);
        }
        let response = match post.send().await {
            Ok(response) => response,
            Err(err) => return self.unknown(format!("vision transport failed: {err}")),
        };
        if !response.status().is_success() {
            let status = response.status();
            let body = response.text().await.unwrap_or_default();
            return self.unknown(format!(
                "vision API answered {status}: {:.200}",
                body.trim()
            ));
        }
        let parsed: serde_json::Value = match response.json().await {
            Ok(parsed) => parsed,
            Err(err) => return self.unknown(format!("vision response unreadable: {err}")),
        };
        // `choices[0].message.content` is a string on the chat-completions
        // format; some servers answer an array of typed parts instead —
        // accept both, concatenating the text parts.
        let content = &parsed["choices"][0]["message"]["content"];
        let text: String = match content {
            serde_json::Value::String(text) => text.clone(),
            serde_json::Value::Array(parts) => parts
                .iter()
                .filter(|part| part.get("type").and_then(|t| t.as_str()) == Some("text"))
                .filter_map(|part| part.get("text").and_then(|t| t.as_str()))
                .collect::<Vec<_>>()
                .join(""),
            _ => String::new(),
        };
        if text.trim().is_empty() {
            return self.unknown("vision response carried no text answer");
        }
        parse_answer(&text, &self.judge())
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn from_lookup_requires_a_nonempty_api_key() {
        assert!(AnthropicVisionVerifier::from_lookup(|_| None).is_none());
        assert!(
            AnthropicVisionVerifier::from_lookup(|key| {
                (key == "ANTHROPIC_API_KEY").then(String::new)
            })
            .is_none()
        );
    }

    #[test]
    fn from_lookup_honors_the_model_override_and_defaults_without_it() {
        let with_override = AnthropicVisionVerifier::from_lookup(|key| match key {
            "ANTHROPIC_API_KEY" => Some("k".to_owned()),
            "POINTLOCK_VISION_MODEL" => Some("claude-haiku-4-5".to_owned()),
            _ => None,
        })
        .expect("key present");
        assert_eq!(with_override.model, "claude-haiku-4-5");

        let defaulted = AnthropicVisionVerifier::from_lookup(|key| {
            (key == "ANTHROPIC_API_KEY").then(|| "k".to_owned())
        })
        .expect("key present");
        assert_eq!(defaulted.model, DEFAULT_VISION_MODEL);
        assert_eq!(defaulted.base_url, "https://api.anthropic.com");
    }

    #[tokio::test]
    async fn stub_always_answers_unknown_with_the_fixed_reason() {
        let verdict = StubVisionVerifier
            .verify(VisionRequest {
                prompt: "the Wi-Fi toggle is visible",
                region: None,
                screenshot: b"png-bytes",
                media_type: "image/png",
            })
            .await;
        assert_eq!(verdict.status, VerdictStatus::Unknown);
        assert_eq!(verdict.reason, STUB_REASON);
        // Nothing judged: no judge identity, no observations.
        assert_eq!(verdict.judge, None);
        assert!(verdict.observations.is_empty());
    }

    fn judge() -> VisionJudge {
        VisionJudge {
            provider: "test".to_owned(),
            model: Some("test-model".to_owned()),
        }
    }

    #[test]
    fn parse_collects_observations_ahead_of_the_verdict() {
        let verdict = parse_answer(
            "OBSERVED: the SSID field shows HomeWifi\n\
             OBSERVED: the connect button is enabled\n\
             PASS: the field shows the requested name",
            &judge(),
        );
        assert_eq!(verdict.status, VerdictStatus::Pass);
        assert_eq!(verdict.reason, "vision: the field shows the requested name");
        assert_eq!(verdict.judge, Some(judge()));
        assert_eq!(
            verdict.observations,
            vec![
                "the SSID field shows HomeWifi".to_owned(),
                "the connect button is enabled".to_owned(),
            ]
        );
    }

    #[test]
    fn parse_accepts_a_bare_verdict_without_observed_lines() {
        let verdict = parse_answer("FAIL: the field is empty", &judge());
        assert_eq!(verdict.status, VerdictStatus::Fail);
        assert_eq!(verdict.judge, Some(judge()));
        assert!(verdict.observations.is_empty());
    }

    #[test]
    fn parse_fails_closed_but_keeps_observations_without_a_verdict() {
        let missing = parse_answer("OBSERVED: a dialog covers the screen", &judge());
        assert_eq!(missing.status, VerdictStatus::Unknown);
        assert!(
            missing.reason.contains("no verdict line"),
            "{}",
            missing.reason
        );
        assert_eq!(
            missing.observations,
            vec!["a dialog covers the screen".to_owned()]
        );

        let chatty = parse_answer(
            "OBSERVED: a dialog covers the screen\nSure! I believe it passes.",
            &judge(),
        );
        assert_eq!(chatty.status, VerdictStatus::Unknown);
        assert!(chatty.reason.contains("unparseable"), "{}", chatty.reason);
        assert_eq!(
            chatty.observations,
            vec!["a dialog covers the screen".to_owned()]
        );
    }

    #[test]
    fn observation_bounds_cap_count_and_length_on_char_boundaries() {
        let mut answer = String::new();
        for index in 0..(MAX_OBSERVATIONS + 3) {
            answer.push_str(&format!("OBSERVED: fact {index}\n"));
        }
        answer.push_str("PASS: ok");
        let verdict = parse_answer(&answer, &judge());
        assert_eq!(verdict.observations.len(), MAX_OBSERVATIONS);

        // Multi-byte characters must be cut on a char boundary.
        let long = format!(
            "OBSERVED: {}\nPASS: ok",
            "界".repeat(MAX_OBSERVATION_CHARS + 5)
        );
        let verdict = parse_answer(&long, &judge());
        let kept = &verdict.observations[0];
        assert_eq!(kept.chars().count(), MAX_OBSERVATION_CHARS + 1);
        assert!(kept.ends_with('…'));
    }

    #[test]
    fn openai_from_lookup_requires_base_url_and_model_with_the_key_optional() {
        assert!(OpenAiCompatVisionVerifier::from_lookup(|_| None).is_none());
        assert!(
            OpenAiCompatVisionVerifier::from_lookup(|key| {
                (key == "POINTLOCK_VISION_BASE_URL").then(|| "http://127.0.0.1:1/v1".to_owned())
            })
            .is_none()
        );
        let keyless = OpenAiCompatVisionVerifier::from_lookup(|key| match key {
            "POINTLOCK_VISION_BASE_URL" => Some("http://127.0.0.1:1/v1".to_owned()),
            "POINTLOCK_VISION_MODEL" => Some("qwen2.5-vl".to_owned()),
            _ => None,
        })
        .expect("base url and model present");
        assert_eq!(keyless.api_key, None);
        assert_eq!(keyless.model, "qwen2.5-vl");
        let keyed = OpenAiCompatVisionVerifier::from_lookup(|key| match key {
            "POINTLOCK_VISION_BASE_URL" => Some("http://127.0.0.1:1/v1".to_owned()),
            "POINTLOCK_VISION_MODEL" => Some("qwen2.5-vl".to_owned()),
            "POINTLOCK_VISION_API_KEY" => Some("k".to_owned()),
            _ => None,
        })
        .expect("all present");
        assert_eq!(keyed.api_key.as_deref(), Some("k"));
    }
}