speccoder_runtime/
external.rs

1//! External executors: another coding
2//! agent — Claude Code (`claude -p`), Codex (`codex exec`) or any command —
3//! makes the attempt; speccoder keeps everything around it.
4//!
5//! Guarantees are diff-level, not action-level: the agent edits the worktree
6//! with its own tools and its own permission system, so speccoder cannot
7//! classify each call. What it still does: the isolated worktree, frozen and
8//! spec-derived acceptance tests written before the attempt, the integrity
9//! audit of the diff (protected files, deleted or weakened tests), the
10//! checks and the verdict, retries with the verifier's digest, the gaming
11//! monitor, ticking and committing. Where the agent has a permission system,
12//! speccoder's command allowances are passed to it.
13
14use std::path::Path;
15use std::time::Duration;
16
17use serde::Deserialize;
18use serde_json::{Value, json};
19
20/// Which agent.
21#[derive(Debug, Clone, PartialEq, Eq)]
22pub enum Kind {
23    /// Claude Code: `claude -p --output-format json`.
24    Claude,
25    /// Codex: `codex exec --sandbox workspace-write`.
26    Codex,
27    /// Any program: the prompt on stdin, the summary on stdout.
28    Command(Vec<String>),
29}
30
31/// `[executor]` in `speccoder.toml`.
32#[derive(Debug, Clone, PartialEq, Eq)]
33pub struct Executor {
34    /// Which agent.
35    pub kind: Kind,
36    /// The agent's model, if set.
37    pub model: Option<String>,
38    /// How long one attempt may take.
39    pub timeout: Duration,
40}
41
42#[derive(Debug, Deserialize)]
43#[serde(deny_unknown_fields)]
44struct Raw {
45    kind: String,
46    #[serde(default)]
47    argv: Vec<String>,
48    model: Option<String>,
49    timeout_secs: Option<u64>,
50}
51
52/// Reads `[executor]`; `None` (or `kind = "native"`) is speccoder's own loop.
53pub fn parse(text: &str) -> Result<Option<Executor>, String> {
54    let table: toml::Table = toml::from_str(text).map_err(|e| e.to_string())?;
55    let Some(v) = table.get("executor") else {
56        return Ok(None);
57    };
58    let raw: Raw = v
59        .clone()
60        .try_into()
61        .map_err(|e| format!("[executor]: {e}"))?;
62    let kind = match raw.kind.as_str() {
63        "native" => return Ok(None),
64        "claude" => Kind::Claude,
65        "codex" => Kind::Codex,
66        "command" => {
67            if raw.argv.first().is_none_or(String::is_empty) {
68                return Err("[executor]: kind = \"command\" needs `argv`".to_owned());
69            }
70            Kind::Command(raw.argv)
71        }
72        other => {
73            return Err(format!(
74                "[executor]: kind = \"{other}\" (native, claude, codex or command)"
75            ));
76        }
77    };
78    Ok(Some(Executor {
79        kind,
80        model: raw.model,
81        timeout: Duration::from_secs(raw.timeout_secs.unwrap_or(1800)),
82    }))
83}
84
85/// What the external agent is told: the same context a native attempt gets,
86/// the rules speccoder will check afterwards, and how to stop honestly.
87#[must_use]
88pub fn prompt(bundle: &Value, digest: Option<&Value>) -> String {
89    prompt_in(crate::format::reference().wording(), bundle, digest)
90}
91
92/// [`prompt`] for a project whose format is worded `w`.
93#[must_use]
94pub fn prompt_in(w: &crate::format::Wording, bundle: &Value, digest: Option<&Value>) -> String {
95    let s = |k: &str| bundle[k].as_str().unwrap_or_default();
96    let task = &bundle["task"];
97    let mut out = format!(
98        "You are implementing exactly one task of a {product} {unit} in this git worktree.\n\n\
99Rules — speccoder checks them after you finish, from the diff:\n\
100- Change only what the task needs. Do not edit the spec ({intent}), the checks' configuration (`speccoder.toml`, CI files, test-runner config) or anything under `.git`.\n\
101- Test files that exist now are frozen: add new tests in new files; never weaken, skip or delete tests.\n\
102- Do not commit, branch or push; speccoder commits a task that passes.\n\
103- Run the project's tests to see whether your change works; speccoder runs them again and decides.\n\
104- If the task cannot be done as written (it contradicts the spec, a test is wrong, information is missing), stop and make your final message start with `BLOCKED:` and the reason.\n\n\
105# Task {}\n\n{}\n\nPhase: {}\n",
106        task["id"].as_str().unwrap_or_default(),
107        task["title"].as_str().unwrap_or_default(),
108        task["phase"].as_str().unwrap_or_default(),
109        product = w.product,
110        unit = w.unit,
111        intent = w.intent_code,
112    );
113    for (title, key) in [
114        (w.story, "story"),
115        (w.plan_short, "plan"),
116        (w.constitution, "constitution"),
117    ] {
118        if !s(key).trim().is_empty() {
119            out.push_str(&format!("\n# {title}\n\n{}\n", s(key)));
120        }
121    }
122    if let Some(d) = digest.filter(|d| !d.is_null()) {
123        out.push_str(&format!(
124            "\n# Previous attempt (failed; its changes were undone)\n\nThe verifier's facts:\n\n```json\n{}\n```\n",
125            serde_json::to_string_pretty(d).unwrap_or_default()
126        ));
127    }
128    out
129}
130
131/// What an external attempt produced, in the shape of a native attempt's output.
132#[derive(Debug, Clone, PartialEq)]
133pub struct Outcome {
134    /// An id for reports (`external_<agent>_<unix time>`); external runs are
135    /// not journaled.
136    pub run_id: String,
137    /// `{status, report?, reason?, tool_calls, usage}` like a native attempt.
138    pub output: Value,
139}
140
141/// The argv for an executor; `allow` are the commands speccoder allows
142/// (checks and `[commands] allow`), passed to agents with a permission system;
143/// `withheld` is a directory the agent is told not to read (the held-out
144/// tests) — withheld, not sealed: only Claude Code takes such a rule.
145#[must_use]
146pub fn argv(
147    exec: &Executor,
148    worktree: &Path,
149    last_message: &Path,
150    allow: &[Vec<String>],
151    withheld: Option<&Path>,
152) -> Vec<String> {
153    match &exec.kind {
154        Kind::Claude => {
155            let mut tools = vec!["Read", "Edit", "Write", "MultiEdit", "Glob", "Grep", "LS"]
156                .into_iter()
157                .map(str::to_owned)
158                .collect::<Vec<_>>();
159            tools.extend(allow.iter().map(|p| format!("Bash({}:*)", p.join(" "))));
160            let mut a = vec![
161                "claude".to_owned(),
162                "-p".to_owned(),
163                "--output-format".to_owned(),
164                "json".to_owned(),
165                "--permission-mode".to_owned(),
166                "acceptEdits".to_owned(),
167                "--allowedTools".to_owned(),
168                tools.join(","),
169                "--disallowedTools".to_owned(),
170                match withheld {
171                    Some(dir) => format!("WebFetch,WebSearch,Read({}/**)", dir.display()),
172                    None => "WebFetch,WebSearch".to_owned(),
173                },
174            ];
175            if let Some(m) = &exec.model {
176                a.extend(["--model".to_owned(), m.clone()]);
177            }
178            a
179        }
180        Kind::Codex => {
181            let mut a = vec![
182                "codex".to_owned(),
183                "exec".to_owned(),
184                "--sandbox".to_owned(),
185                "workspace-write".to_owned(),
186                "--skip-git-repo-check".to_owned(),
187                "-C".to_owned(),
188                worktree.display().to_string(),
189                "-o".to_owned(),
190                last_message.display().to_string(),
191            ];
192            if let Some(m) = &exec.model {
193                a.extend(["-m".to_owned(), m.clone()]);
194            }
195            a.push("-".to_owned());
196            a
197        }
198        Kind::Command(argv) => argv.clone(),
199    }
200}
201
202/// Reads an agent's final message and usage from its stdout (and, for
203/// Codex, its last-message file).
204#[must_use]
205pub fn interpret(
206    exec: &Executor,
207    exit: Option<i32>,
208    stdout: &str,
209    last_message: Option<&str>,
210) -> Value {
211    let (message, usage) = match &exec.kind {
212        Kind::Claude => match serde_json::from_str::<Value>(stdout.trim()) {
213            Ok(v) => {
214                let u = &v["usage"];
215                let n = |k: &str| u[k].as_u64().unwrap_or(0);
216                let usage = json!({
217                    "input_tokens": n("input_tokens") + n("cache_read_input_tokens") + n("cache_creation_input_tokens"),
218                    "output_tokens": n("output_tokens"),
219                    "cache_read_tokens": n("cache_read_input_tokens"),
220                    "cache_write_tokens": n("cache_creation_input_tokens"),
221                    "cost_usd": v["total_cost_usd"],
222                });
223                let failed = v["is_error"] == true;
224                (
225                    if failed {
226                        format!(
227                            "FAILED: {}",
228                            v["result"]
229                                .as_str()
230                                .unwrap_or("the agent reported an error")
231                        )
232                    } else {
233                        v["result"].as_str().unwrap_or_default().to_owned()
234                    },
235                    Some(usage),
236                )
237            }
238            Err(_) => (stdout.to_owned(), None),
239        },
240        Kind::Codex => (last_message.unwrap_or(stdout).to_owned(), None),
241        Kind::Command(_) => (stdout.to_owned(), None),
242    };
243    let message = message.trim().to_owned();
244    let summary: String = message.chars().take(2000).collect();
245    let (status, extra) = if exit != Some(0) {
246        (
247            "failed",
248            json!({ "reason": format!("the agent exited {}: {}", exit.map_or("by signal".to_owned(), |c| c.to_string()), summary.chars().take(500).collect::<String>()) }),
249        )
250    } else if let Some(reason) = message.strip_prefix("BLOCKED:") {
251        ("blocked", json!({ "reason": reason.trim() }))
252    } else if message.starts_with("FAILED:") {
253        ("failed", json!({ "reason": summary }))
254    } else {
255        (
256            "completed",
257            json!({ "report": { "summary": summary, "status": "done", "files_changed": [], "checks_run": [], "notes": ["external executor"] } }),
258        )
259    };
260    let mut out =
261        json!({ "status": status, "tool_calls": [], "usage": usage.unwrap_or(Value::Null) });
262    if let (Some(o), Some(e)) = (out.as_object_mut(), extra.as_object()) {
263        o.extend(e.clone());
264    }
265    out
266}
267
268/// Runs one attempt with `exec` in `worktree`.
269#[must_use]
270pub fn run(
271    exec: &Executor,
272    worktree: &Path,
273    prompt: &str,
274    allow: &[Vec<String>],
275    tmp: &Path,
276    withheld: Option<&Path>,
277) -> Outcome {
278    use crate::verify::process::{CommandRunner as _, Invocation, LocalRunner, OUTPUT_CAP};
279    let _ = std::fs::create_dir_all(tmp);
280    let stamp = std::time::SystemTime::now()
281        .duration_since(std::time::UNIX_EPOCH)
282        .map_or(0, |d| d.as_secs());
283    let last = tmp.join(format!("external-{stamp}.txt"));
284    let prompt_file = tmp.join(format!("external-{stamp}.prompt"));
285    let _ = std::fs::write(&prompt_file, prompt);
286    let agent = match &exec.kind {
287        Kind::Claude => "claude",
288        Kind::Codex => "codex",
289        Kind::Command(_) => "command",
290    };
291    // The prompt arrives on stdin (a shell redirect keeps the runner's
292    // closed-stdin rule for everything else); the agent keeps the person's
293    // environment, because it needs its own credentials.
294    let mut inner = argv(exec, worktree, &last, allow, withheld);
295    let quoted: Vec<String> = inner
296        .drain(..)
297        .map(|a| format!("'{}'", a.replace('\'', "'\\''")))
298        .collect();
299    let script = format!(
300        "exec {} < '{}'",
301        quoted.join(" "),
302        prompt_file.display().to_string().replace('\'', "'\\''")
303    );
304    let out = LocalRunner.run(&Invocation {
305        argv: vec!["sh".to_owned(), "-c".to_owned(), script],
306        cwd: worktree.to_path_buf(),
307        env: std::env::vars().collect(),
308        timeout: exec.timeout,
309        output_cap: OUTPUT_CAP,
310    });
311    let last_message = std::fs::read_to_string(&last).ok();
312    let stdout = String::from_utf8_lossy(&out.stdout.kept);
313    let mut output = interpret(exec, out.exit_code, &stdout, last_message.as_deref());
314    // A failing agent often explains itself only on stderr.
315    if out.exit_code != Some(0) && stdout.trim().is_empty() {
316        let stderr = String::from_utf8_lossy(&out.stderr.kept);
317        let lines: Vec<&str> = stderr.lines().filter(|l| !l.trim().is_empty()).collect();
318        let tail = lines[lines.len().saturating_sub(5)..].join(" | ");
319        output["reason"] = json!(format!(
320            "the agent exited {}: {}",
321            out.exit_code
322                .map_or("by signal".to_owned(), |c| c.to_string()),
323            tail.chars().take(500).collect::<String>()
324        ));
325    }
326    if let Some(e) = &out.spawn_error {
327        output["status"] = json!("failed");
328        output["reason"] = json!(format!("the agent could not be started: {e}"));
329    }
330    if out.timed_out {
331        output["status"] = json!("failed");
332        output["reason"] = json!(format!(
333            "the agent did not finish within {} s",
334            exec.timeout.as_secs()
335        ));
336    }
337    let _ = std::fs::remove_file(&prompt_file);
338    let _ = std::fs::remove_file(&last);
339    Outcome {
340        run_id: format!("external_{agent}_{stamp}"),
341        output,
342    }
343}
344
345#[cfg(test)]
346mod tests {
347    use super::*;
348
349    #[test]
350    fn executors_parse_and_claude_gets_the_allowances() {
351        assert!(parse("").unwrap_or_default().is_none());
352        assert!(
353            parse("[executor]\nkind = \"native\"\n")
354                .unwrap_or_default()
355                .is_none()
356        );
357        let c = parse("[executor]\nkind = \"claude\"\nmodel = \"sonnet\"\n")
358            .unwrap_or_default()
359            .unwrap_or_else(|| panic!("claude"));
360        let a = argv(
361            &c,
362            Path::new("/w"),
363            Path::new("/t/last"),
364            &[vec!["cargo".to_owned(), "test".to_owned()]],
365            None,
366        );
367        assert!(
368            a.contains(&"Read,Edit,Write,MultiEdit,Glob,Grep,LS,Bash(cargo test:*)".to_owned()),
369            "{a:?}"
370        );
371        assert!(a.ends_with(&["--model".to_owned(), "sonnet".to_owned()]));
372        let held = argv(
373            &c,
374            Path::new("/w"),
375            Path::new("/t/last"),
376            &[],
377            Some(Path::new("/s/held-out/f/T1")),
378        );
379        assert!(
380            held.contains(&"WebFetch,WebSearch,Read(/s/held-out/f/T1/**)".to_owned()),
381            "{held:?}"
382        );
383        let x = parse("[executor]\nkind = \"codex\"\n")
384            .unwrap_or_default()
385            .unwrap_or_else(|| panic!("codex"));
386        let a = argv(&x, Path::new("/w"), Path::new("/t/last"), &[], None);
387        assert_eq!(&a[..4], ["codex", "exec", "--sandbox", "workspace-write"]);
388        assert!(parse("[executor]\nkind = \"command\"\n").is_err());
389        assert!(parse("[executor]\nkind = \"gpt\"\n").is_err());
390    }
391
392    #[test]
393    fn final_messages_become_statuses() {
394        let c = Executor {
395            kind: Kind::Claude,
396            model: None,
397            timeout: Duration::from_secs(1),
398        };
399        let ok = interpret(
400            &c,
401            Some(0),
402            r#"{"type":"result","is_error":false,"result":"Implemented add.","total_cost_usd":0.01,"usage":{"input_tokens":10,"cache_read_input_tokens":90,"output_tokens":5}}"#,
403            None,
404        );
405        assert_eq!(ok["status"], "completed");
406        assert_eq!(ok["usage"]["input_tokens"], 100);
407        let blocked = interpret(
408            &c,
409            Some(0),
410            r#"{"is_error":false,"result":"BLOCKED: the test asserts 2+2=5"}"#,
411            None,
412        );
413        assert_eq!(
414            (blocked["status"].as_str(), blocked["reason"].as_str()),
415            (Some("blocked"), Some("the test asserts 2+2=5"))
416        );
417        let cmd = Executor {
418            kind: Kind::Command(vec!["x".to_owned()]),
419            model: None,
420            timeout: Duration::from_secs(1),
421        };
422        assert_eq!(interpret(&cmd, Some(2), "boom", None)["status"], "failed");
423    }
424}
speccoder_runtime/external.rs

speccoder_runtime/
external.rs
