speccoder_runtime/
external.rs

1//! External executors: another coding
2//! agent — Claude Code (`claude -p`), Codex (`codex exec`) or any command —
3//! makes the attempt; speccoder keeps everything around it.
4//!
5//! Guarantees are diff-level, not action-level: the agent edits the worktree
6//! with its own tools and its own permission system, so speccoder cannot
7//! classify each call. What it still does: the isolated worktree, frozen and
8//! spec-derived acceptance tests written before the attempt, the integrity
9//! audit of the diff (protected files, deleted or weakened tests), the
10//! checks and the verdict, retries with the verifier's digest, the gaming
11//! monitor, ticking and committing. Where the agent has a permission system,
12//! speccoder's command allowances are passed to it.
13
14use std::path::Path;
15use std::time::Duration;
16
17use serde::Deserialize;
18use serde_json::{Value, json};
19
20/// Which agent.
21#[derive(Debug, Clone, PartialEq, Eq)]
22pub enum Kind {
23    /// Claude Code: `claude -p --output-format json`.
24    Claude,
25    /// Codex: `codex exec --sandbox workspace-write`.
26    Codex,
27    /// Any program: the prompt on stdin, the summary on stdout.
28    Command(Vec<String>),
29}
30
31/// `[executor]` in `speccoder.toml`.
32#[derive(Debug, Clone, PartialEq, Eq)]
33pub struct Executor {
34    /// Which agent.
35    pub kind: Kind,
36    /// The agent's model, if set.
37    pub model: Option<String>,
38    /// How long one attempt may take.
39    pub timeout: Duration,
40}
41
42#[derive(Debug, Deserialize)]
43#[serde(deny_unknown_fields)]
44struct Raw {
45    kind: String,
46    #[serde(default)]
47    argv: Vec<String>,
48    model: Option<String>,
49    timeout_secs: Option<u64>,
50}
51
52/// Reads `[executor]`; `None` (or `kind = "native"`) is speccoder's own loop.
53pub fn parse(text: &str) -> Result<Option<Executor>, String> {
54    let table: toml::Table = toml::from_str(text).map_err(|e| e.to_string())?;
55    let Some(v) = table.get("executor") else {
56        return Ok(None);
57    };
58    let raw: Raw = v
59        .clone()
60        .try_into()
61        .map_err(|e| format!("[executor]: {e}"))?;
62    let kind = match raw.kind.as_str() {
63        "native" => return Ok(None),
64        "claude" => Kind::Claude,
65        "codex" => Kind::Codex,
66        "command" => {
67            if raw.argv.first().is_none_or(String::is_empty) {
68                return Err("[executor]: kind = \"command\" needs `argv`".to_owned());
69            }
70            Kind::Command(raw.argv)
71        }
72        other => {
73            return Err(format!(
74                "[executor]: kind = \"{other}\" (native, claude, codex or command)"
75            ));
76        }
77    };
78    Ok(Some(Executor {
79        kind,
80        model: raw.model,
81        timeout: Duration::from_secs(raw.timeout_secs.unwrap_or(1800)),
82    }))
83}
84
85/// What the external agent is told: the same context a native attempt gets,
86/// the rules speccoder will check afterwards, and how to stop honestly.
87#[must_use]
88pub fn prompt(bundle: &Value, digest: Option<&Value>) -> String {
89    prompt_in(crate::format::reference().wording(), bundle, digest)
90}
91
92/// [`prompt`] for a project whose format is worded `w`.
93#[must_use]
94pub fn prompt_in(w: &crate::format::Wording, bundle: &Value, digest: Option<&Value>) -> String {
95    let s = |k: &str| bundle[k].as_str().unwrap_or_default();
96    let task = &bundle["task"];
97    let mut out = format!(
98        "You are implementing exactly one task of a {product} {unit} in this git worktree.\n\n\
99Rules — speccoder checks them after you finish, from the diff:\n\
100- Change only what the task needs. Do not edit the spec ({intent}), the checks' configuration (`speccoder.toml`, CI files, test-runner config) or anything under `.git`.\n\
101- Test files that exist now are frozen: add new tests in new files; never weaken, skip or delete tests.\n\
102- Do not commit, branch or push; speccoder commits a task that passes.\n\
103- Run the project's tests to see whether your change works; speccoder runs them again and decides.\n\
104- If the task cannot be done as written (it contradicts the spec, a test is wrong, information is missing), stop and make your final message start with `BLOCKED:` and the reason.\n\n\
105# Task {}\n\n{}\n\nPhase: {}\n",
106        task["id"].as_str().unwrap_or_default(),
107        task["title"].as_str().unwrap_or_default(),
108        task["phase"].as_str().unwrap_or_default(),
109        product = w.product,
110        unit = w.unit,
111        intent = w.intent_code,
112    );
113    for (title, key) in [
114        (w.story, "story"),
115        (w.plan_short, "plan"),
116        (w.constitution, "constitution"),
117    ] {
118        if !s(key).trim().is_empty() {
119            out.push_str(&format!("\n# {title}\n\n{}\n", s(key)));
120        }
121    }
122    if let Some(d) = digest.filter(|d| !d.is_null()) {
123        out.push_str(&format!(
124            "\n# Previous attempt (failed; its changes were undone)\n\nThe verifier's facts:\n\n```json\n{}\n```\n",
125            serde_json::to_string_pretty(d).unwrap_or_default()
126        ));
127    }
128    out
129}
130
131/// What an external attempt produced, in the shape of a native attempt's output.
132#[derive(Debug, Clone, PartialEq)]
133pub struct Outcome {
134    /// An id for reports (`external_<agent>_<unix time>`); external runs are
135    /// not journaled.
136    pub run_id: String,
137    /// `{status, report?, reason?, tool_calls, usage}` like a native attempt.
138    pub output: Value,
139}
140
141/// The argv for an executor; `allow` are the commands speccoder allows
142/// (checks and `[commands] allow`), passed to agents with a permission system;
143/// `withheld` is a directory the agent is told not to read (the held-out
144/// tests) — withheld, not sealed: only Claude Code takes such a rule.
145#[must_use]
146pub fn argv(
147    exec: &Executor,
148    worktree: &Path,
149    last_message: &Path,
150    allow: &[Vec<String>],
151    withheld: Option<&Path>,
152) -> Vec<String> {
153    match &exec.kind {
154        Kind::Claude => {
155            let mut tools = vec!["Read", "Edit", "Write", "MultiEdit", "Glob", "Grep", "LS"]
156                .into_iter()
157                .map(str::to_owned)
158                .collect::<Vec<_>>();
159            // Exact commands, as speccoder allows them: Claude Code reads
160            // `Bash(cmd:*)` as a prefix, `Bash(cmd)` as the command alone.
161            tools.extend(allow.iter().map(|p| format!("Bash({})", p.join(" "))));
162            let mut a = vec![
163                "claude".to_owned(),
164                "-p".to_owned(),
165                "--output-format".to_owned(),
166                "json".to_owned(),
167                "--permission-mode".to_owned(),
168                "acceptEdits".to_owned(),
169                "--allowedTools".to_owned(),
170                tools.join(","),
171                "--disallowedTools".to_owned(),
172                match withheld {
173                    Some(dir) => format!("WebFetch,WebSearch,Read({}/**)", dir.display()),
174                    None => "WebFetch,WebSearch".to_owned(),
175                },
176            ];
177            if let Some(m) = &exec.model {
178                a.extend(["--model".to_owned(), m.clone()]);
179            }
180            a
181        }
182        Kind::Codex => {
183            let mut a = vec![
184                "codex".to_owned(),
185                "exec".to_owned(),
186                "--sandbox".to_owned(),
187                "workspace-write".to_owned(),
188                "--skip-git-repo-check".to_owned(),
189                "-C".to_owned(),
190                worktree.display().to_string(),
191                "-o".to_owned(),
192                last_message.display().to_string(),
193            ];
194            if let Some(m) = &exec.model {
195                a.extend(["-m".to_owned(), m.clone()]);
196            }
197            a.push("-".to_owned());
198            a
199        }
200        Kind::Command(argv) => argv.clone(),
201    }
202}
203
204/// Reads an agent's final message and usage from its stdout (and, for
205/// Codex, its last-message file).
206#[must_use]
207pub fn interpret(
208    exec: &Executor,
209    exit: Option<i32>,
210    stdout: &str,
211    last_message: Option<&str>,
212) -> Value {
213    let (message, usage) = match &exec.kind {
214        Kind::Claude => match serde_json::from_str::<Value>(stdout.trim()) {
215            Ok(v) => {
216                let u = &v["usage"];
217                let n = |k: &str| u[k].as_u64().unwrap_or(0);
218                let usage = json!({
219                    "input_tokens": n("input_tokens") + n("cache_read_input_tokens") + n("cache_creation_input_tokens"),
220                    "output_tokens": n("output_tokens"),
221                    "cache_read_tokens": n("cache_read_input_tokens"),
222                    "cache_write_tokens": n("cache_creation_input_tokens"),
223                    "cost_usd": v["total_cost_usd"],
224                });
225                let failed = v["is_error"] == true;
226                (
227                    if failed {
228                        format!(
229                            "FAILED: {}",
230                            v["result"]
231                                .as_str()
232                                .unwrap_or("the agent reported an error")
233                        )
234                    } else {
235                        v["result"].as_str().unwrap_or_default().to_owned()
236                    },
237                    Some(usage),
238                )
239            }
240            Err(_) => (stdout.to_owned(), None),
241        },
242        Kind::Codex => (last_message.unwrap_or(stdout).to_owned(), None),
243        Kind::Command(_) => (stdout.to_owned(), None),
244    };
245    let message = message.trim().to_owned();
246    let summary: String = message.chars().take(2000).collect();
247    let (status, extra) = if exit != Some(0) {
248        (
249            "failed",
250            json!({ "reason": format!("the agent exited {}: {}", exit.map_or("by signal".to_owned(), |c| c.to_string()), summary.chars().take(500).collect::<String>()) }),
251        )
252    } else if let Some(reason) = message.strip_prefix("BLOCKED:") {
253        ("blocked", json!({ "reason": reason.trim() }))
254    } else if message.starts_with("FAILED:") {
255        ("failed", json!({ "reason": summary }))
256    } else {
257        (
258            "completed",
259            json!({ "report": { "summary": summary, "status": "done", "files_changed": [], "checks_run": [], "notes": ["external executor"] } }),
260        )
261    };
262    let mut out =
263        json!({ "status": status, "tool_calls": [], "usage": usage.unwrap_or(Value::Null) });
264    if let (Some(o), Some(e)) = (out.as_object_mut(), extra.as_object()) {
265        o.extend(e.clone());
266    }
267    out
268}
269
270/// Runs one attempt with `exec` in `worktree`.
271#[must_use]
272pub fn run(
273    exec: &Executor,
274    worktree: &Path,
275    prompt: &str,
276    allow: &[Vec<String>],
277    tmp: &Path,
278    withheld: Option<&Path>,
279) -> Outcome {
280    use crate::verify::process::{CommandRunner as _, Invocation, LocalRunner, OUTPUT_CAP};
281    let _ = std::fs::create_dir_all(tmp);
282    let stamp = std::time::SystemTime::now()
283        .duration_since(std::time::UNIX_EPOCH)
284        .map_or(0, |d| d.as_secs());
285    let last = tmp.join(format!("external-{stamp}.txt"));
286    let prompt_file = tmp.join(format!("external-{stamp}.prompt"));
287    let _ = std::fs::write(&prompt_file, prompt);
288    let agent = match &exec.kind {
289        Kind::Claude => "claude",
290        Kind::Codex => "codex",
291        Kind::Command(_) => "command",
292    };
293    // The prompt arrives on stdin (a shell redirect keeps the runner's
294    // closed-stdin rule for everything else); the agent keeps the person's
295    // environment, because it needs its own credentials.
296    let mut inner = argv(exec, worktree, &last, allow, withheld);
297    let quoted: Vec<String> = inner
298        .drain(..)
299        .map(|a| format!("'{}'", a.replace('\'', "'\\''")))
300        .collect();
301    let script = format!(
302        "exec {} < '{}'",
303        quoted.join(" "),
304        prompt_file.display().to_string().replace('\'', "'\\''")
305    );
306    let out = LocalRunner.run(&Invocation {
307        argv: vec!["sh".to_owned(), "-c".to_owned(), script],
308        cwd: worktree.to_path_buf(),
309        env: std::env::vars().collect(),
310        timeout: exec.timeout,
311        output_cap: OUTPUT_CAP,
312    });
313    let last_message = std::fs::read_to_string(&last).ok();
314    let stdout = String::from_utf8_lossy(&out.stdout.kept);
315    let mut output = interpret(exec, out.exit_code, &stdout, last_message.as_deref());
316    // A failing agent often explains itself only on stderr.
317    if out.exit_code != Some(0) && stdout.trim().is_empty() {
318        let stderr = String::from_utf8_lossy(&out.stderr.kept);
319        let lines: Vec<&str> = stderr.lines().filter(|l| !l.trim().is_empty()).collect();
320        let tail = lines[lines.len().saturating_sub(5)..].join(" | ");
321        output["reason"] = json!(format!(
322            "the agent exited {}: {}",
323            out.exit_code
324                .map_or("by signal".to_owned(), |c| c.to_string()),
325            tail.chars().take(500).collect::<String>()
326        ));
327    }
328    if let Some(e) = &out.spawn_error {
329        output["status"] = json!("failed");
330        output["reason"] = json!(format!("the agent could not be started: {e}"));
331    }
332    if out.timed_out {
333        output["status"] = json!("failed");
334        output["reason"] = json!(format!(
335            "the agent did not finish within {} s",
336            exec.timeout.as_secs()
337        ));
338    }
339    let _ = std::fs::remove_file(&prompt_file);
340    let _ = std::fs::remove_file(&last);
341    Outcome {
342        run_id: format!("external_{agent}_{stamp}"),
343        output,
344    }
345}
346
347#[cfg(test)]
348mod tests {
349    use super::*;
350
351    #[test]
352    fn executors_parse_and_claude_gets_the_allowances() {
353        assert!(parse("").unwrap_or_default().is_none());
354        assert!(
355            parse("[executor]\nkind = \"native\"\n")
356                .unwrap_or_default()
357                .is_none()
358        );
359        let c = parse("[executor]\nkind = \"claude\"\nmodel = \"sonnet\"\n")
360            .unwrap_or_default()
361            .unwrap_or_else(|| panic!("claude"));
362        let a = argv(
363            &c,
364            Path::new("/w"),
365            Path::new("/t/last"),
366            &[vec!["cargo".to_owned(), "test".to_owned()]],
367            None,
368        );
369        assert!(
370            a.contains(&"Read,Edit,Write,MultiEdit,Glob,Grep,LS,Bash(cargo test)".to_owned()),
371            "{a:?}"
372        );
373        assert!(a.ends_with(&["--model".to_owned(), "sonnet".to_owned()]));
374        let held = argv(
375            &c,
376            Path::new("/w"),
377            Path::new("/t/last"),
378            &[],
379            Some(Path::new("/s/held-out/f/T1")),
380        );
381        assert!(
382            held.contains(&"WebFetch,WebSearch,Read(/s/held-out/f/T1/**)".to_owned()),
383            "{held:?}"
384        );
385        let x = parse("[executor]\nkind = \"codex\"\n")
386            .unwrap_or_default()
387            .unwrap_or_else(|| panic!("codex"));
388        let a = argv(&x, Path::new("/w"), Path::new("/t/last"), &[], None);
389        assert_eq!(&a[..4], ["codex", "exec", "--sandbox", "workspace-write"]);
390        assert!(parse("[executor]\nkind = \"command\"\n").is_err());
391        assert!(parse("[executor]\nkind = \"gpt\"\n").is_err());
392    }
393
394    #[test]
395    fn final_messages_become_statuses() {
396        let c = Executor {
397            kind: Kind::Claude,
398            model: None,
399            timeout: Duration::from_secs(1),
400        };
401        let ok = interpret(
402            &c,
403            Some(0),
404            r#"{"type":"result","is_error":false,"result":"Implemented add.","total_cost_usd":0.01,"usage":{"input_tokens":10,"cache_read_input_tokens":90,"output_tokens":5}}"#,
405            None,
406        );
407        assert_eq!(ok["status"], "completed");
408        assert_eq!(ok["usage"]["input_tokens"], 100);
409        let blocked = interpret(
410            &c,
411            Some(0),
412            r#"{"is_error":false,"result":"BLOCKED: the test asserts 2+2=5"}"#,
413            None,
414        );
415        assert_eq!(
416            (blocked["status"].as_str(), blocked["reason"].as_str()),
417            (Some("blocked"), Some("the test asserts 2+2=5"))
418        );
419        let cmd = Executor {
420            kind: Kind::Command(vec!["x".to_owned()]),
421            model: None,
422            timeout: Duration::from_secs(1),
423        };
424        assert_eq!(interpret(&cmd, Some(2), "boom", None)["status"], "failed");
425    }
426}
speccoder_runtime/external.rs

speccoder_runtime/
external.rs
