pointlock_vision/
lib.rs

1//! # pointlock-vision
2//!
3//! The [`VisionVerifier`] plugin interface (verify role only, principle 7:
4//! vision never locates or acts) and the default [`StubVisionVerifier`].
5//!
6//! The verifier is the runtime consumer of the `vision` verify channel
7//! (spine §6.3): it answers an author-written prompt against localized
8//! screenshot bytes with a three-valued verdict. A `pass`/`fail` answer is
9//! a *completed* evaluation (fail is final, spine R5); an `unknown` answer
10//! means the channel could not complete and the chain advances — for the
11//! vision channel, which is only legal at the chain tail, that exhausts the
12//! chain into assertion `unknown` (principle 4).
13//!
14//! The v0.1 default is the stub: it always answers `unknown` with the
15//! reason `"vision verifier not configured"`. The runner treats an absent
16//! verifier (`RunOptions::vision == None`) as exactly equivalent.
17
18use async_trait::async_trait;
19use pointlock_ir::{RectIR, VerdictStatus};
20
21/// The reason the stub (and an unconfigured runner) yields `unknown`.
22pub const STUB_REASON: &str = "vision verifier not configured";
23
24/// One vision verification request: the author-written prompt (never
25/// synthesized by the compiler, principle 6), an optional region of
26/// interest, and the *localized* screenshot bytes (evidence is localized
27/// during `observing`, spine §6.6 — the verifier never reaches back into a
28/// provider session).
29#[derive(Debug, Clone, PartialEq)]
30pub struct VisionRequest<'a> {
31    /// The author-written prompt, verbatim (`AssertionIR.visionPrompt` for
32    /// `elementState`/`elementText` chain tails; `predicate.prompt` for
33    /// `visual` predicates).
34    pub prompt: &'a str,
35    /// Optional region of interest (`visual` predicates only).
36    pub region: Option<&'a RectIR>,
37    /// The localized screenshot bytes.
38    pub screenshot: &'a [u8],
39    /// The screenshot's media type, e.g. `image/png`.
40    pub media_type: &'a str,
41}
42
43/// The verifier's three-valued answer with a human-readable reason.
44///
45/// `pass`/`fail` are completed evaluations; `unknown` means the verifier
46/// could not complete (unconfigured, low confidence, unusable image) and
47/// carries the why. The verdict never panics its way out — model/transport
48/// failures inside an implementation must fold to `unknown` (principle 4).
49#[derive(Debug, Clone, PartialEq)]
50pub struct VisionVerdict {
51    /// The three-valued status.
52    pub status: VerdictStatus,
53    /// Why the verifier answered this way.
54    pub reason: String,
55}
56
57/// The vision verification plugin interface (verify role only).
58#[async_trait]
59pub trait VisionVerifier: Send + Sync {
60    /// Answers `request.prompt` against the screenshot. Infallible by
61    /// construction: anything that prevents an answer is an `unknown`
62    /// verdict with a reason, never an error (principle 4).
63    async fn verify(&self, request: VisionRequest<'_>) -> VisionVerdict;
64}
65
66/// The v0.1 default verifier: always `unknown` with [`STUB_REASON`].
67#[derive(Debug, Clone, Copy, Default)]
68pub struct StubVisionVerifier;
69
70#[async_trait]
71impl VisionVerifier for StubVisionVerifier {
72    async fn verify(&self, _request: VisionRequest<'_>) -> VisionVerdict {
73        VisionVerdict {
74            status: VerdictStatus::Unknown,
75            reason: STUB_REASON.to_owned(),
76        }
77    }
78}
79
80// ─── The Anthropic-backed verifier (M3a-W4: the first usable one) ───────────
81
82/// The default model of [`AnthropicVisionVerifier`].
83pub const DEFAULT_VISION_MODEL: &str = "claude-opus-4-8";
84
85/// Total per-request deadline. Without one, a TCP-accepted-but-silent
86/// endpoint stalls `verify()` forever — an unbounded await is a failure
87/// mode that never folds to `unknown`, breaking the module contract
88/// (principle 4). The step's `timeout_ms` governs only provider execute,
89/// not the assert phase, so the bound must live here.
90const REQUEST_TIMEOUT_SECS: u64 = 60;
91
92/// Connection-establishment deadline (part of the same fail-to-unknown
93/// bound; kept tighter so dead endpoints answer fast).
94const CONNECT_TIMEOUT_SECS: u64 = 10;
95
96/// The first usable verifier (08 §6.4): asks an Anthropic vision model to
97/// answer the author's prompt against the screenshot, over raw HTTP (no
98/// official Rust SDK exists). Discipline unchanged from the trait docs:
99/// verify-only, chain tail only, and every failure mode — missing key,
100/// transport, non-200, unparseable answer, model uncertainty — folds to
101/// `unknown` with a reason, never an error and never a guessed pass
102/// (principles 4/7).
103pub struct AnthropicVisionVerifier {
104    api_key: String,
105    model: String,
106    base_url: String,
107    client: reqwest::Client,
108}
109
110impl AnthropicVisionVerifier {
111    /// Builds a verifier from explicit configuration.
112    pub fn new(
113        api_key: impl Into<String>,
114        model: impl Into<String>,
115        base_url: impl Into<String>,
116    ) -> Self {
117        AnthropicVisionVerifier {
118            api_key: api_key.into(),
119            model: model.into(),
120            base_url: base_url.into(),
121            // Deadlines: see the timeout consts. `no_proxy` keeps egress
122            // deterministic (v0.1 makes no proxy promise) and the canned
123            // local-endpoint tests hermetic under ambient HTTP(S)_PROXY /
124            // ALL_PROXY environments. The `expect` matches
125            // `reqwest::Client::new`'s own panic-on-TLS-init semantics.
126            client: reqwest::Client::builder()
127                .connect_timeout(std::time::Duration::from_secs(CONNECT_TIMEOUT_SECS))
128                .timeout(std::time::Duration::from_secs(REQUEST_TIMEOUT_SECS))
129                .no_proxy()
130                .build()
131                .expect("reqwest client construction"),
132        }
133    }
134
135    /// Builds from the environment: `ANTHROPIC_API_KEY` (required — `None`
136    /// without it), `POINTLOCK_VISION_MODEL` (default
137    /// [`DEFAULT_VISION_MODEL`]), `ANTHROPIC_BASE_URL` (default the public
138    /// API; overriding it is also how the tests run against a local
139    /// canned-response server).
140    pub fn from_env() -> Option<Self> {
141        Self::from_lookup(|key| std::env::var(key).ok())
142    }
143
144    /// The injectable body of [`Self::from_env`] (unit-testable without
145    /// process-global env mutation).
146    fn from_lookup(get: impl Fn(&str) -> Option<String>) -> Option<Self> {
147        let api_key = get("ANTHROPIC_API_KEY").filter(|key| !key.is_empty())?;
148        let model = get("POINTLOCK_VISION_MODEL")
149            .filter(|model| !model.is_empty())
150            .unwrap_or_else(|| DEFAULT_VISION_MODEL.to_owned());
151        let base_url = get("ANTHROPIC_BASE_URL")
152            .filter(|url| !url.is_empty())
153            .unwrap_or_else(|| "https://api.anthropic.com".to_owned());
154        Some(Self::new(api_key, model, base_url))
155    }
156
157    fn unknown(reason: impl Into<String>) -> VisionVerdict {
158        VisionVerdict {
159            status: VerdictStatus::Unknown,
160            reason: reason.into(),
161        }
162    }
163}
164
165/// The verifier's answer protocol: exactly one leading verdict line. The
166/// instruction pins the vocabulary; anything else parses to `unknown`
167/// (fail-closed — a chatty answer is not a verdict).
168fn parse_answer(text: &str) -> VisionVerdict {
169    let first = text.trim().lines().next().unwrap_or_default().trim();
170    let (status, rest) = if let Some(rest) = first.strip_prefix("PASS:") {
171        (VerdictStatus::Pass, rest)
172    } else if let Some(rest) = first.strip_prefix("FAIL:") {
173        (VerdictStatus::Fail, rest)
174    } else if let Some(rest) = first.strip_prefix("UNKNOWN:") {
175        (VerdictStatus::Unknown, rest)
176    } else {
177        return AnthropicVisionVerifier::unknown(format!(
178            "unparseable verifier answer: {first:.120}"
179        ));
180    };
181    VisionVerdict {
182        status,
183        reason: format!("vision: {}", rest.trim()),
184    }
185}
186
187#[async_trait]
188impl VisionVerifier for AnthropicVisionVerifier {
189    async fn verify(&self, request: VisionRequest<'_>) -> VisionVerdict {
190        use base64::Engine as _;
191        let data = base64::engine::general_purpose::STANDARD.encode(request.screenshot);
192        let region_note = request.region.map_or(String::new(), |region| {
193            format!(
194                " Consider ONLY the region at x={}, y={}, width={}, height={} (pixels from the top-left).",
195                region.x, region.y, region.width, region.height
196            )
197        });
198        let instruction = format!(
199            "You are a visual verification oracle for a device-automation audit trail. \
200             Judge the following claim against the screenshot.{region_note}\n\
201             Claim: {}\n\
202             Answer with EXACTLY one line and nothing else:\n\
203             PASS: <what you see that confirms it>\n\
204             FAIL: <what you see that contradicts it>\n\
205             UNKNOWN: <why it cannot be determined>\n\
206             Answer UNKNOWN unless the claim is clearly confirmed or clearly contradicted.",
207            request.prompt
208        );
209        let body = serde_json::json!({
210            "model": self.model,
211            "max_tokens": 1024,
212            "messages": [{
213                "role": "user",
214                "content": [
215                    { "type": "image", "source": {
216                        "type": "base64",
217                        "media_type": request.media_type,
218                        "data": data,
219                    }},
220                    { "type": "text", "text": instruction },
221                ],
222            }],
223        });
224
225        let response = match self
226            .client
227            .post(format!("{}/v1/messages", self.base_url))
228            .header("x-api-key", &self.api_key)
229            .header("anthropic-version", "2023-06-01")
230            .json(&body)
231            .send()
232            .await
233        {
234            Ok(response) => response,
235            Err(err) => return Self::unknown(format!("vision transport failed: {err}")),
236        };
237        if !response.status().is_success() {
238            let status = response.status();
239            let body = response.text().await.unwrap_or_default();
240            return Self::unknown(format!(
241                "vision API answered {status}: {:.200}",
242                body.trim()
243            ));
244        }
245        let parsed: serde_json::Value = match response.json().await {
246            Ok(parsed) => parsed,
247            Err(err) => return Self::unknown(format!("vision response unreadable: {err}")),
248        };
249        // Concatenate the text blocks (thinking blocks are skipped).
250        let text: String = parsed
251            .get("content")
252            .and_then(|content| content.as_array())
253            .map(|blocks| {
254                blocks
255                    .iter()
256                    .filter(|block| block.get("type").and_then(|t| t.as_str()) == Some("text"))
257                    .filter_map(|block| block.get("text").and_then(|t| t.as_str()))
258                    .collect::<Vec<_>>()
259                    .join("")
260            })
261            .unwrap_or_default();
262        if text.trim().is_empty() {
263            return Self::unknown("vision response carried no text answer");
264        }
265        parse_answer(&text)
266    }
267}
268
269#[cfg(test)]
270mod tests {
271    use super::*;
272
273    #[test]
274    fn from_lookup_requires_a_nonempty_api_key() {
275        assert!(AnthropicVisionVerifier::from_lookup(|_| None).is_none());
276        assert!(
277            AnthropicVisionVerifier::from_lookup(|key| {
278                (key == "ANTHROPIC_API_KEY").then(String::new)
279            })
280            .is_none()
281        );
282    }
283
284    #[test]
285    fn from_lookup_honors_the_model_override_and_defaults_without_it() {
286        let with_override = AnthropicVisionVerifier::from_lookup(|key| match key {
287            "ANTHROPIC_API_KEY" => Some("k".to_owned()),
288            "POINTLOCK_VISION_MODEL" => Some("claude-haiku-4-5".to_owned()),
289            _ => None,
290        })
291        .expect("key present");
292        assert_eq!(with_override.model, "claude-haiku-4-5");
293
294        let defaulted = AnthropicVisionVerifier::from_lookup(|key| {
295            (key == "ANTHROPIC_API_KEY").then(|| "k".to_owned())
296        })
297        .expect("key present");
298        assert_eq!(defaulted.model, DEFAULT_VISION_MODEL);
299        assert_eq!(defaulted.base_url, "https://api.anthropic.com");
300    }
301
302    #[tokio::test]
303    async fn stub_always_answers_unknown_with_the_fixed_reason() {
304        let verdict = StubVisionVerifier
305            .verify(VisionRequest {
306                prompt: "the Wi-Fi toggle is visible",
307                region: None,
308                screenshot: b"png-bytes",
309                media_type: "image/png",
310            })
311            .await;
312        assert_eq!(verdict.status, VerdictStatus::Unknown);
313        assert_eq!(verdict.reason, STUB_REASON);
314    }
315}
pointlock_vision/lib.rs

pointlock_vision/
lib.rs
