Skip to main content

smix_ai_tier/
lib.rs

1#![doc = include_str!("../README.md")]
2#![deny(missing_docs)]
3#![forbid(unsafe_code)]
4
5use std::path::{Path, PathBuf};
6use std::time::{Duration, SystemTime, UNIX_EPOCH};
7
8use serde::{Deserialize, Serialize};
9use smix_error::{ExpectationFailure, FailureCode, FailureInit};
10use tokio::process::Command;
11
12/// How to reach the judge.
13#[derive(Clone, Debug)]
14pub struct AiTierConfig {
15    /// Path to the `claude` CLI. Defaults to `claude`, i.e. a PATH lookup.
16    pub claude_bin: String,
17    /// How long the judge may take before the flow gives up on it.
18    pub timeout_secs: u64,
19}
20
21impl Default for AiTierConfig {
22    fn default() -> Self {
23        Self {
24            claude_bin: "claude".into(),
25            timeout_secs: 60,
26        }
27    }
28}
29
30/// What the judge decided.
31///
32/// This is a judgement, not a measurement: the same screen and the same
33/// condition may not produce the same verdict twice. Callers surface it
34/// marked as such.
35#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
36pub struct StructuredVerdict {
37    /// Whether the condition holds.
38    pub pass: bool,
39    /// What the judge saw, in its own words. Carried into the failure
40    /// message so a reader knows why.
41    pub reason: String,
42}
43
44/// Ask the judge whether `condition` holds on the screen in `screenshot_png`.
45///
46/// Every error here means the judge never reached a verdict — a missing CLI, a
47/// timeout, unreadable output. None of them collapse into `pass: false`, which
48/// would report a broken app instead of a judge that never ran.
49pub async fn judge(
50    screenshot_png: &[u8],
51    condition: &str,
52    cfg: &AiTierConfig,
53) -> Result<StructuredVerdict, ExpectationFailure> {
54    let reply = with_staged_image(screenshot_png, cfg, |image| {
55        format!(
56            "Read the screenshot at {}.\n\n\
57             Decide whether this condition holds on that screen:\n\
58             {condition}\n\n\
59             Reply with one JSON object and nothing else:\n\
60             {{\"pass\": true|false, \"reason\": \"<one sentence naming what you saw>\"}}",
61            image.display()
62        )
63    })
64    .await?;
65
66    parse_json_object(&reply).ok_or_else(|| {
67        driver_error(
68            format!(
69                "ai-tier: no verdict in the judge's reply — wanted a JSON object, got: {}",
70                reply.trim()
71            ),
72            Some("the judge answered but not in the shape asked for; this is not an assertion failure".into()),
73        )
74    })
75}
76
77/// Read `fields` off the screen.
78///
79/// Values come back as strings. The expression engine's output store is flat
80/// and stringly-typed, and what a judge reads off a screen is text anyway.
81pub async fn extract(
82    screenshot_png: &[u8],
83    fields: &[String],
84    cfg: &AiTierConfig,
85) -> Result<std::collections::BTreeMap<String, String>, ExpectationFailure> {
86    let wanted = fields.join(", ");
87    let reply = with_staged_image(screenshot_png, cfg, |image| {
88        format!(
89            "Read the screenshot at {}.\n\n\
90             Read these fields off that screen: {wanted}\n\n\
91             Reply with one JSON object and nothing else, mapping each field \
92             name to its value as a string. Use an empty string for a field \
93             you cannot find:\n\
94             {{\"<field>\": \"<value>\"}}",
95            image.display()
96        )
97    })
98    .await?;
99
100    parse_json_object(&reply).ok_or_else(|| {
101        driver_error(
102            format!(
103                "ai-tier: no field object in the judge's reply — wanted a JSON object, got: {}",
104                reply.trim()
105            ),
106            Some("the judge answered but not in the shape asked for".into()),
107        )
108    })
109}
110
111/// Stage the screenshot where the CLI can read it, ask, then clean up.
112async fn with_staged_image(
113    screenshot_png: &[u8],
114    cfg: &AiTierConfig,
115    prompt: impl FnOnce(&Path) -> String,
116) -> Result<String, ExpectationFailure> {
117    // The CLI reads the image off disk, so it has to land somewhere first.
118    let image = scratch_png_path();
119    tokio::fs::write(&image, screenshot_png)
120        .await
121        .map_err(|e| {
122            driver_error(
123                format!(
124                    "ai-tier: could not stage the screenshot at {}: {e}",
125                    image.display()
126                ),
127                None,
128            )
129        })?;
130
131    let reply = ask_with_attachments(prompt(&image), &[Attachment::image(&image)], cfg).await;
132    // Best-effort: a leftover temp png must never turn into a flow failure.
133    let _ = tokio::fs::remove_file(&image).await;
134    reply
135}
136
137/// Something the question is about, alongside the words.
138///
139/// The primitive is prompt-and-attachments to text. Stating an image as an
140/// attachment rather than writing its path into the question is what keeps
141/// the primitive independent of who answers it: a local CLI locates it by
142/// path, and something speaking a network protocol would send the bytes.
143/// A caller that writes the path into its own prose has decided, on the
144/// satisfier's behalf, that the reader can open local files.
145#[derive(Debug, Clone, PartialEq, Eq)]
146pub enum Attachment {
147    /// An image on disk.
148    Image {
149        /// Where it is.
150        path: PathBuf,
151    },
152}
153
154impl Attachment {
155    /// An image attachment.
156    #[must_use]
157    pub fn image(path: impl Into<PathBuf>) -> Self {
158        Self::Image { path: path.into() }
159    }
160}
161
162/// The argv for the local-CLI satisfier.
163///
164/// Separated from running it so the decisions are testable: that a
165/// text-only question grants no tools, and that an attachment is what
166/// causes read access rather than it being granted always.
167fn claude_argv(prompt: &str, attachments: &[Attachment], _cfg: &AiTierConfig) -> Vec<String> {
168    let mut argv: Vec<String> = Vec::new();
169
170    // Read access exists to open the attachments and nothing else, so a
171    // question that carries none does not get it. Granting it always made
172    // the text case look as though it needed a filesystem.
173    if !attachments.is_empty() {
174        argv.push("--tools".into());
175        argv.push("Read".into());
176    }
177
178    let mut full = String::from(prompt);
179    for attachment in attachments {
180        match attachment {
181            Attachment::Image { path } => {
182                full.push_str(&format!("\n\nThe screenshot is at {}", path.display()));
183            }
184        }
185    }
186
187    argv.push("-p".into());
188    argv.push(full);
189    argv.push("--output-format".into());
190    argv.push("text".into());
191    argv
192}
193
194/// Ask once, and return what came back.
195///
196/// The whole AI surface is this one call: words in, optional attachments
197/// alongside, text out. Who satisfies it is configuration — today a local
198/// `claude` CLI, which is the default and the only one wired. What smix
199/// does not do is branch on provider or keep a capability matrix; that is
200/// the abstraction §9 forbids, and it is a different thing from being
201/// able to reach a model over a different transport.
202///
203/// Every error means no answer was produced — a missing binary, a
204/// timeout, a non-zero exit — surfaced as a `DriverError`, never folded
205/// into a made-up reply. Other tiers (`smix-authoring-propose`) build on
206/// this rather than spawning their own.
207///
208/// This form carries no attachments; [`ask_with_attachments`] is the same
209/// call for a question that is about a file. Two entry points rather than
210/// one changed signature: callers outside this workspace compile against
211/// the old one, and a minor release is not where that should break.
212pub async fn ask(prompt: String, cfg: &AiTierConfig) -> Result<String, ExpectationFailure> {
213    ask_with_attachments(prompt, &[], cfg).await
214}
215
216/// Ask about something, with the something stated rather than described.
217///
218/// See [`ask`] for what the primitive is. This is the same call with
219/// attachments alongside the words — the form that keeps a question about
220/// an image independent of whether the answerer can open local files.
221pub async fn ask_with_attachments(
222    prompt: String,
223    attachments: &[Attachment],
224    cfg: &AiTierConfig,
225) -> Result<String, ExpectationFailure> {
226    let mut cmd = Command::new(&cfg.claude_bin);
227    cmd.args(claude_argv(&prompt, attachments, cfg))
228        // Timing out only drops the future — without this the judge keeps
229        // running, outliving the step that gave up on it.
230        .kill_on_drop(true);
231
232    let output =
233        match tokio::time::timeout(Duration::from_secs(cfg.timeout_secs), cmd.output()).await {
234            Err(_) => {
235                return Err(driver_error(
236                    format!("ai-tier: the judge timed out after {}s", cfg.timeout_secs),
237                    Some("raise timeout_secs, or narrow the condition".into()),
238                ));
239            }
240            Ok(Err(e)) => {
241                return Err(driver_error(
242                    format!(
243                        "ai-tier: could not run the claude CLI at `{}`: {e}",
244                        cfg.claude_bin
245                    ),
246                    Some(format!(
247                        "install the claude CLI, or point claude_bin at it (currently `{}`)",
248                        cfg.claude_bin
249                    )),
250                ));
251            }
252            Ok(Ok(output)) => output,
253        };
254
255    if !output.status.success() {
256        return Err(driver_error(
257            format!(
258                "ai-tier: the claude CLI exited {}: {}",
259                output.status.code().unwrap_or(-1),
260                why_it_failed(&output.stderr, &output.stdout)
261            ),
262            None,
263        ));
264    }
265
266    Ok(String::from_utf8_lossy(&output.stdout).into_owned())
267}
268
269/// What a failed CLI said about itself, from both streams. The claude CLI
270/// reports a usage limit on stdout, and can put an unrelated warning (a
271/// sign-in about to end) on stderr in the same run: reading stderr alone
272/// kept the warning and lost the reason.
273fn why_it_failed(stderr: &[u8], stdout: &[u8]) -> String {
274    const TAIL: usize = 800;
275    let pick = |b: &[u8]| String::from_utf8_lossy(b).trim().to_string();
276    let said = [pick(stderr), pick(stdout)]
277        .into_iter()
278        .filter(|s| !s.is_empty())
279        .collect::<Vec<_>>()
280        .join("\n");
281    if said.is_empty() {
282        return "no output on stdout or stderr".into();
283    }
284    match said.char_indices().rev().nth(TAIL - 1) {
285        Some((i, _)) => format!("…{}", &said[i..]),
286        None => said,
287    }
288}
289
290fn driver_error(message: String, hint: Option<String>) -> ExpectationFailure {
291    ExpectationFailure::new(FailureInit {
292        code: Some(FailureCode::DriverError),
293        message,
294        hint,
295        ..Default::default()
296    })
297}
298
299fn scratch_png_path() -> PathBuf {
300    let nanos = SystemTime::now()
301        .duration_since(UNIX_EPOCH)
302        .map(|d| d.as_nanos())
303        .unwrap_or(0);
304    std::env::temp_dir().join(format!("smix-ai-tier-{}-{nanos}.png", std::process::id()))
305}
306
307/// Take the first JSON object out of the reply, tolerating prose around it —
308/// models like to introduce themselves, and that is not a reason to fail a
309/// flow. The generic type is the caller's target shape; a reply that carries
310/// no object, or one that does not deserialize into `T`, yields `None`.
311pub fn parse_json_object<T: serde::de::DeserializeOwned>(reply: &str) -> Option<T> {
312    let start = reply.find('{')?;
313    let end = reply.rfind('}')?;
314    if end < start {
315        return None;
316    }
317    serde_json::from_str(&reply[start..=end]).ok()
318}
319
320#[cfg(test)]
321mod transport_tests {
322    use super::*;
323    use std::path::PathBuf;
324
325    fn cfg() -> AiTierConfig {
326        AiTierConfig::default()
327    }
328
329    #[test]
330    fn a_text_only_ask_grants_no_tools() {
331        // The primitive is prompt-to-text. Handing the satisfier file
332        // access for a question that carries no files widens what it can
333        // reach for no reason, and makes the text case look like it needs
334        // a filesystem when it does not.
335        let argv = claude_argv("is the sky blue?", &[], &cfg());
336        assert!(
337            !argv.iter().any(|a| a == "--tools"),
338            "text-only ask should grant nothing: {argv:?}"
339        );
340        assert!(argv.iter().any(|a| a == "-p"));
341    }
342
343    #[test]
344    fn an_attachment_is_named_in_the_prompt_and_readable() {
345        // How an attachment reaches the model is the satisfier's business.
346        // This one is a local CLI, so it passes a path and the permission
347        // to open it; a satisfier speaking HTTP would inline the bytes
348        // instead. What the caller states is *that* there is an image.
349        let png = PathBuf::from("/tmp/shot.png");
350        let argv = claude_argv("what is on screen?", &[Attachment::image(&png)], &cfg());
351        let tools = argv
352            .iter()
353            .position(|a| a == "--tools")
354            .expect("an attachment needs read access");
355        assert_eq!(argv[tools + 1], "Read");
356        let prompt = argv
357            .iter()
358            .position(|a| a == "-p")
359            .map(|i| argv[i + 1].clone())
360            .expect("prompt");
361        assert!(
362            prompt.contains("/tmp/shot.png"),
363            "the CLI satisfier locates an attachment by path: {prompt}"
364        );
365    }
366
367    #[test]
368    fn the_caller_does_not_write_the_path_into_its_own_prose() {
369        // The judge used to embed the path in the question it wrote. That
370        // is what made the primitive untransportable: the prompt itself
371        // assumed the reader could open local files.
372        let png = PathBuf::from("/tmp/a.png");
373        let argv = claude_argv("describe it", &[Attachment::image(&png)], &cfg());
374        let prompt = argv
375            .iter()
376            .position(|a| a == "-p")
377            .map(|i| argv[i + 1].clone())
378            .expect("prompt");
379        let written_by_caller = prompt.split("/tmp/a.png").count() - 1;
380        assert_eq!(
381            written_by_caller, 1,
382            "the path should appear once, added here"
383        );
384    }
385
386    #[test]
387    fn a_failed_cli_reports_both_streams() {
388        let said = why_it_failed(
389            b"Warning: this sign-in can end at any time from now until 2026-11-05.\n",
390            b"You've reached your usage limit. Switch to another model.\n",
391        );
392        assert!(said.contains("sign-in"), "{said}");
393        assert!(said.contains("reached your usage limit"), "{said}");
394    }
395
396    #[test]
397    fn a_failed_cli_with_one_stream_reports_that_one() {
398        assert_eq!(why_it_failed(b"", b"limit\n"), "limit");
399        assert_eq!(why_it_failed(b"boom\n", b""), "boom");
400        assert_eq!(why_it_failed(b"  ", b"\n"), "no output on stdout or stderr");
401    }
402
403    #[test]
404    fn a_long_report_keeps_its_end() {
405        let long = "x".repeat(2000) + " the reason";
406        let said = why_it_failed(b"", long.as_bytes());
407        assert!(
408            said.starts_with('…') && said.ends_with("the reason"),
409            "{said}"
410        );
411    }
412}