io_harness/run.rs
1//! The orchestration loop: observe, reason, act, verify, stop — bounded by
2//! budgets, resilient to transient step failures, and resumable.
3//!
4//! v0.2 adds three budgets (step, time, cost-in-tokens) each with its own stop
5//! outcome, per-step retry with escalation, a full trace written to the store,
6//! and [`resume`], which continues an interrupted run under its original id
7//! instead of restarting.
8
9use std::cell::Cell;
10use std::future::Future;
11use std::path::{Path, PathBuf};
12use std::pin::Pin;
13use std::sync::Arc;
14use std::time::Duration;
15
16use serde_json::json;
17use tracing::info;
18
19use crate::approve::{ApproveAll, Approver, Decision, Request};
20use crate::containment::{Containment, Draw, Ledger};
21use crate::context::{
22 assemble, bound, entry_cap_chars, Assembly, Ledger as ContextLedger, ObsKind, Observation,
23};
24use crate::contract::TaskContract;
25use crate::error::{Error, Result};
26use crate::mcp::McpSession;
27use crate::net::{self, NetGuard};
28use crate::observe::{EventKind, Ignore, Observer, RunEvent};
29use crate::policy::{Act, Effect, Policy, Rule};
30use crate::provider::{CompletionRequest, CompletionResponse, Provider, ToolCall, ToolSpec};
31use crate::resilience::{Progress, Progressing};
32use crate::skills::Skills;
33use crate::state::PolicyEvent;
34use crate::state::{AgentEvent, ContextEvent, RunStatus, StepRecord, Store};
35use crate::toolchain::Toolchain;
36use crate::tools::exec::{Exec, ExecOutcome};
37use crate::tools::git::{Git, GitCmd, GitOutcome};
38#[cfg(feature = "barcode")]
39use crate::tools::BARCODE_DECODE_TOOL;
40#[cfg(feature = "pptx")]
41use crate::tools::PPTX_READ_TOOL;
42
43/// The path a git built-in names when it asks the policy about the repository
44/// itself. Reading history reads it; committing writes it. A run under a narrow
45/// write policy must allow it explicitly, which is stated where the tools are
46/// documented rather than left to be discovered by a refusal.
47const GIT_DIR: &str = ".git";
48#[cfg(feature = "media")]
49use crate::tools::VIEW_IMAGE_TOOL;
50use crate::tools::{
51 FsTool, Toolbox, Workspace, EDIT_FILE_TOOL, EXEC_TOOL, FIND_TOOL, GREP_TOOL, READ_FILE_TOOL,
52 READ_SKILL_TOOL, REMEMBER_TOOL, WRITE_FILE_TOOL,
53};
54#[cfg(feature = "docx")]
55use crate::tools::{DOCX_READ_TOOL, DOCX_WRITE_TOOL};
56use crate::tools::{GIT_ADD_TOOL, GIT_COMMIT_TOOL, GIT_DIFF_TOOL, GIT_LOG_TOOL, GIT_STATUS_TOOL};
57#[cfg(feature = "pdf")]
58use crate::tools::{PDF_FILL_FORM_TOOL, PDF_READ_TOOL, PDF_WATERMARK_TOOL, PDF_WRITE_TOOL};
59#[cfg(feature = "xlsx")]
60use crate::tools::{XLSX_READ_TOOL, XLSX_SET_CELL_TOOL, XLSX_SHEETS_TOOL, XLSX_WRITE_TOOL};
61use crate::verify::{ExecGuard, Verification};
62
63/// The tool a parent agent calls to spawn a contained sub-agent.
64///
65/// It is the name the model sees, and the name that appears in the trace and in
66/// [`EventKind::ToolCall`](crate::EventKind::ToolCall) when an agent fans out —
67/// which is the reason to know it. A consumer watching a tree matches on it to
68/// tell composition apart from ordinary work:
69///
70/// ```
71/// use io_harness::{EventKind, Flow, Observer, RunEvent, SPAWN_TOOL};
72///
73/// struct TreeShape;
74///
75/// impl Observer for TreeShape {
76/// fn event(&self, event: &RunEvent) -> Flow {
77/// match &event.kind {
78/// // A parent asking for a child, before the child exists.
79/// EventKind::ToolCall { name, target } if name == SPAWN_TOOL => {
80/// println!("{:indent$}spawning: {target}", "", indent = event.depth as usize * 2);
81/// }
82/// // The child that resulted, with its own run id to route on.
83/// EventKind::Spawned { child_run_id, goal } => {
84/// println!("{:indent$}run {child_run_id}: {goal}", "",
85/// indent = (event.depth as usize + 1) * 2);
86/// }
87/// _ => {}
88/// }
89/// Flow::Continue
90/// }
91/// }
92/// ```
93///
94/// Only [`run_tree`] offers it. [`run`] and [`run_with`] never put it in the
95/// tool list, so a contract cannot opt into sub-agents by accident.
96///
97/// It is deliberately *not* governed by the exec policy the way a registered
98/// tool's name is — a spawn is intercepted by the tree loop before dispatch, and
99/// its ceilings are [`Containment`]'s: total agents, concurrency, depth, and the
100/// shared token ledger. To forbid composition, use [`run_with`]; to bound it,
101/// lower those caps.
102pub const SPAWN_TOOL: &str = "spawn_agent";
103
104/// How many grep hits are folded into one observation. A relevance ceiling, not a
105/// size one — the size ceiling is the budget-derived per-entry cap on top of it.
106const OBS_GREP_CAP: usize = 50;
107
108/// Why a run stopped.
109///
110/// A run stopping is not a run failing — only one of these variants is success,
111/// and lumping the rest together as "it didn't work" loses the difference
112/// between a run that needs more budget, one that needs a human, and one that
113/// needs a different task. The match is what a caller writes around every entry
114/// point:
115///
116/// ```
117/// use io_harness::RunOutcome;
118///
119/// # fn next_step(outcome: RunOutcome) -> &'static str {
120/// match outcome {
121/// RunOutcome::Success { .. } => "verification passed; ship it",
122///
123/// // Paused, not finished. The pending action is persisted under
124/// // `request_id` and this process may exit; whoever decides later calls
125/// // `resume_with_decision` with that id. Never retry the run from scratch.
126/// RunOutcome::AwaitingApproval { request_id: _, .. } => "ask a human, then resume",
127///
128/// // Ceilings, and they mean different things. More steps or more time is a
129/// // knob; a token ceiling that keeps being hit is usually a task too big
130/// // for one contract.
131/// RunOutcome::StepCapReached { .. } | RunOutcome::TimeBudgetExceeded { .. } => "raise the bound and resume",
132/// RunOutcome::CostBudgetExceeded { .. } | RunOutcome::BudgetCeilingReached { .. } => "split the task",
133///
134/// // The agent is going in circles and was already told once. Resuming
135/// // spends the rest of the budget proving it again — change the goal.
136/// RunOutcome::Stalled { .. } => "rewrite the contract",
137///
138/// // Both are reported *after the fact*: the run that escalated or was
139/// // refused returned the `Err` itself, and a later `resume` reports this
140/// // instead of re-driving the loop.
141/// RunOutcome::Escalated { retryable: true, .. } => "transient provider failure; resume",
142/// RunOutcome::Escalated { .. } => "wrong key or bad request; fix it first",
143/// RunOutcome::Refused { .. } => "the provider's host was denied; widen the net policy",
144///
145/// RunOutcome::Denied { .. } => "a human said no; the action never happened",
146/// // An observer returned `Flow::Cancel`. Finished cleanly, and still resumable.
147/// RunOutcome::Cancelled { .. } => "resume when you want it to continue",
148///
149/// // Only a `Verification::None` run reaches this: it stopped because the
150/// // agent stopped, not because a ceiling did. Nothing checked the work —
151/// // read it, rather than shipping it the way a `Success` may be shipped.
152/// RunOutcome::Finished { .. } => "the agent is done; nothing verified it",
153/// }
154/// # }
155/// ```
156///
157/// Every variant carries `steps`, which is how many steps *completed* — so a
158/// `StepCapReached { steps: 12 }` and a `Success { steps: 12 }` cost the same
159/// and only one of them produced anything. For what the run actually spent, use
160/// [`RunResult::summary`].
161#[derive(Debug, Clone, PartialEq, Eq)]
162pub enum RunOutcome {
163 /// Verification passed. `steps` is the step it passed on.
164 Success { steps: u32 },
165 /// The step budget was reached before verification passed.
166 StepCapReached { steps: u32 },
167 /// The time budget was exceeded. `steps` is how many steps completed.
168 TimeBudgetExceeded { steps: u32 },
169 /// The cost (token) budget was exceeded. `steps` is how many steps completed.
170 CostBudgetExceeded { steps: u32 },
171 /// A human denied a deferred action on resume, so the run stopped without
172 /// performing it. `steps` is how many steps completed.
173 Denied { steps: u32 },
174 /// An approver deferred a decision. The run is paused, not finished: the
175 /// pending action is persisted under `request_id` and survives this
176 /// process, so [`resume_with_decision`] can continue it once a human
177 /// decides. `steps` is how many steps completed.
178 AwaitingApproval { request_id: i64, steps: u32 },
179 /// The agent stopped making progress: for `StallPolicy::window` consecutive
180 /// steps it changed nothing in the workspace while repeating a tool call it had
181 /// already made, and it had already been told once. The run stops here rather
182 /// than spending the rest of its step budget proving it is stuck. `steps` is
183 /// how many steps completed.
184 Stalled { steps: u32 },
185 /// A provider failure exhausted its retries and the run was escalated to the
186 /// caller. `retryable` is whether the failure was one another attempt could have
187 /// survived — a rate limit or a 503 — as opposed to a wrong key or an
188 /// unacceptable request. Reached through [`resume`] after the fact: the run that
189 /// escalated returned the `Err` itself.
190 Escalated { steps: u32, retryable: bool },
191 /// (sub-agent trees) The tree's aggregate spend ceiling was crossed, so the
192 /// whole tree halts — not this one agent hitting its own budget. `steps` is
193 /// how many steps this agent completed before the tree-wide halt.
194 BudgetCeilingReached { steps: u32 },
195 /// The run never started, because reaching the provider needed network access
196 /// the policy asked about and a human denied. The authorization happens before
197 /// the run's first step, so `steps` is normally 0.
198 ///
199 /// Reached through [`resume`] after the fact: the run that was refused
200 /// returned the `Err` itself, exactly as an escalation does. Added in 0.12.0 —
201 /// `"refused"` was written to the store from 0.8.0 onward with no variant and
202 /// no mapping, so resuming a refused run fell back into the loop and asked
203 /// the human again.
204 Refused { steps: u32 },
205 /// An [`Observer`] asked the run to stop, and it stopped — at the next step
206 /// boundary rather than where the request landed, so no step was abandoned
207 /// half-done. `steps` is how many steps completed before it stopped.
208 ///
209 /// Added in 0.12.0 with [`Flow::Cancel`](crate::Flow::Cancel), which is the
210 /// first supported way to stop a run in flight: dropping the run's future
211 /// abandons it mid-step and leaves `runs.status` as `running` forever, which
212 /// nothing can tell apart from a process that crashed. A cancelled run is
213 /// finished rather than abandoned, and stays resumable — a resume reports this
214 /// outcome instead of re-driving the loop.
215 Cancelled { steps: u32 },
216 /// The agent finished. Only a [`Verification::None`] run reaches this: with
217 /// no criterion to pass, an assistant turn that calls no tool is the run
218 /// saying it is done, and the loop stops there.
219 ///
220 /// Distinct from every ceiling on purpose. An unattended run that completed
221 /// its work and one that ran out of steps both stop, and treating them alike
222 /// is how a fleet operator ends up re-driving finished work — or worse,
223 /// shipping the output of a run that never got there. `steps` is the step it
224 /// finished on.
225 ///
226 /// It is **not** a claim the work is correct. Nothing checked it; that is
227 /// what choosing [`Verification::None`] means. A run with a criterion reports
228 /// [`RunOutcome::Success`], and that one *is* a claim — bounded by what the
229 /// criterion checked and no wider.
230 ///
231 /// Added in 0.17.0 with [`Verification::None`].
232 ///
233 /// [`Verification::None`]: crate::Verification::None
234 Finished { steps: u32 },
235}
236
237/// The result of a run, including the persisted run id for audit.
238///
239/// Three things, and the `run_id` is the one that outlives the process. Every
240/// step, refusal, spawn and budget draw is in the store under it, so a run is
241/// still readable long after the program that drove it exited — and it is the
242/// handle every `resume*` entry point takes:
243///
244/// ```no_run
245/// use io_harness::{run_with, ApproveAll, OpenRouter, Policy, RunOutcome, Store, TaskContract};
246///
247/// # async fn demo(contract: &TaskContract, policy: &Policy) -> io_harness::Result<()> {
248/// let store = Store::open("runs.db")?;
249/// let result = run_with(contract, &OpenRouter::from_env()?, &store, policy, &ApproveAll).await?;
250///
251/// // What it cost and how long it took, read back from the same row an auditor
252/// // would read — so the caller and the audit cannot disagree. `None` while a
253/// // run is paused awaiting a human: there is no ending to summarise yet.
254/// if let Some(summary) = result.summary(&store)? {
255/// println!("{} tokens", summary.tokens);
256/// }
257///
258/// // Rules an approver asked to remember. The crate applied them for the rest of
259/// // this run and hands them back here; persisting them across runs is the
260/// // caller's decision, because config files are the application's to own.
261/// for rule in &result.remembered {
262/// println!("remember: {:?} {} {:?}", rule.act, rule.pattern, rule.effect);
263/// }
264///
265/// // Keep the id if the run is not finished — it is all a later process needs.
266/// if !matches!(result.outcome, RunOutcome::Success { .. }) {
267/// println!("resume run {}", result.run_id);
268/// }
269/// # Ok(()) }
270/// ```
271#[derive(Debug, Clone)]
272pub struct RunResult {
273 /// Why the run stopped.
274 pub outcome: RunOutcome,
275 /// The run's id in the [`Store`], for reading its trace back.
276 pub run_id: i64,
277 /// Rules an approver asked to remember during this run. The crate applies
278 /// them for the rest of the run and hands them back here; persisting them
279 /// across runs is the caller's decision, since config files are app-owned.
280 pub remembered: Vec<Rule>,
281}
282
283impl RunResult {
284 /// What this run cost and whether it worked, read back from the store.
285 ///
286 /// Added in 0.12.0. Before it, a caller holding a `RunResult` had an outcome
287 /// discriminant and a `run_id`: spend needed a follow-up query, and latency
288 /// was not recorded anywhere at all.
289 ///
290 /// `None` when the run has not finished — a run paused awaiting a human has
291 /// no ending to summarise yet.
292 ///
293 /// A method rather than a field, deliberately. A field would have to be
294 /// filled at every one of the entry points' return sites, including the ones
295 /// that return `Err` and never build a `RunResult`, so the two could drift.
296 /// Reading it from the store means the caller and an auditor are looking at
297 /// the same row by construction. It also keeps this struct's existing
298 /// exhaustive-pattern compatibility intact: no new field, no break.
299 pub fn summary(&self, store: &Store) -> Result<Option<crate::RunSummary>> {
300 store.run_summary(self.run_id)
301 }
302}
303
304impl RunResult {
305 fn new(outcome: RunOutcome, run_id: i64) -> Self {
306 Self {
307 outcome,
308 run_id,
309 remembered: Vec::new(),
310 }
311 }
312
313 fn with_remembered(mut self, remembered: Vec<Rule>) -> Self {
314 self.remembered = remembered;
315 self
316 }
317}
318
319/// Run a task contract to a verified result using `provider` and `store`.
320///
321/// Each iteration: read the file into context, ask the model (offering the
322/// `write_file` tool, retrying transient failures), apply any write, record the
323/// trace, then verify. Stops on the first passing verify, or when any budget —
324/// steps, time, or tokens — is reached.
325///
326/// The smallest thing that works, and the right entry point when the boundary is
327/// the *task* rather than a policy: one file, one tool, and a criterion the model
328/// cannot talk its way past.
329///
330/// ```no_run
331/// use io_harness::{run, OpenRouter, RunOutcome, Store, TaskContract, Verification};
332///
333/// # async fn demo() -> io_harness::Result<()> {
334/// let contract = TaskContract::new(
335/// "add a `hello` function returning 42",
336/// "src/hello.rs",
337/// // Execution-based: the project's own build has to succeed, so `fn hello`
338/// // written as a literal string fails — which is exactly what a model did to
339/// // the cheaper `FileContains` in the 0.1.0 live run.
340/// Verification::Command { argv: vec!["cargo".into(), "build".into()], expect_exit: 0 },
341/// )
342/// .with_max_steps(6);
343///
344/// let result = run(&contract, &OpenRouter::from_env()?, &Store::open("runs.db")?).await?;
345/// match result.outcome {
346/// RunOutcome::Success { steps } => println!("verified in {steps} steps"),
347/// // Keep the id: the file on disk is the run's state, so a resume continues
348/// // from it rather than starting over.
349/// other => println!("{other:?} — resume run {}", result.run_id),
350/// }
351/// # Ok(()) }
352/// ```
353///
354/// It applies [`Policy::permissive`] and approves everything, because there is no
355/// policy-aware tool layer in single-file mode — passing a real policy to
356/// [`run_with`] with a single-file contract is refused with
357/// [`Error::Config`](crate::Error::Config) rather than silently ignored. Use
358/// [`TaskContract::workspace`] and [`run_with`] as soon as a boundary matters.
359pub async fn run<P: Provider>(
360 contract: &TaskContract,
361 provider: &P,
362 store: &Store,
363) -> Result<RunResult> {
364 run_observed(contract, provider, store, &Ignore).await
365}
366
367/// [`run`], reporting to `observer` as it happens.
368///
369/// The observed twin of every entry point takes the [`Observer`] last and does
370/// exactly what its unobserved original does — the originals *are* these
371/// functions, called with [`Ignore`]. Adding a parameter to the seven existing
372/// signatures would have broken every caller of a 0.11.0 API to add something
373/// opt-in; a builder would have added a second way to start a run for the same
374/// reason.
375///
376/// Reach for it when a run is long enough that silence is a problem. Without an
377/// observer the only thing between "started" and "finished" is the SQLite trace,
378/// which nobody is watching while it happens:
379///
380/// ```no_run
381/// use io_harness::{run_observed, EventKind, Flow, Observer, OpenRouter, RunEvent,
382/// Store, TaskContract};
383///
384/// /// A progress line per committed step — the boundary at which work is durable.
385/// struct Progress;
386///
387/// impl Observer for Progress {
388/// fn event(&self, event: &RunEvent) -> Flow {
389/// if let EventKind::Step { decision, tokens, changed, .. } = &event.kind {
390/// let mark = if *changed { "*" } else { " " };
391/// println!("{mark} step {} ({tokens} tokens): {decision}", event.step);
392/// }
393/// Flow::Continue
394/// }
395/// }
396///
397/// # async fn demo(contract: &TaskContract) -> io_harness::Result<()> {
398/// run_observed(contract, &OpenRouter::from_env()?, &Store::memory()?, &Progress).await?;
399/// # Ok(()) }
400/// ```
401///
402/// Events are delivered in order, on the run's own task, so an observer that
403/// blocks holds the run up. Anything slow belongs on a channel.
404pub async fn run_observed<P: Provider>(
405 contract: &TaskContract,
406 provider: &P,
407 store: &Store,
408 observer: &dyn Observer,
409) -> Result<RunResult> {
410 run_with_observed(
411 contract,
412 provider,
413 store,
414 &Policy::permissive(),
415 &ApproveAll,
416 observer,
417 )
418 .await
419}
420
421/// Run a task contract under a permission `policy`, routing anything the policy
422/// marks [`Effect::Ask`] to `approver` before it happens.
423///
424/// An action the policy *denies* is refused without consulting the approver and
425/// reported to the model as a tool result it can adapt to; the refusal consumes
426/// the step, so a model that keeps retrying a denied action reaches the step cap
427/// rather than looping forever.
428///
429/// This is the entry point most callers want: a workspace, a boundary, and a
430/// human only for the grey tier.
431///
432/// ```no_run
433/// use io_harness::{run_with, OpenRouter, Policy, StdinApprover, Store, TaskContract,
434/// Verification};
435///
436/// # async fn demo() -> io_harness::Result<()> {
437/// let contract = TaskContract::workspace(
438/// "make the failing test in tests/parse.rs pass",
439/// "/path/to/repo",
440/// Verification::Command { argv: vec!["cargo".into(), "test".into()], expect_exit: 0 },
441/// );
442///
443/// // Three tiers, and the middle one is the only one anybody is asked about.
444/// // `Policy::default()` already denies `.env`, `*.pem` and the other secret
445/// // paths outright, so those never become a question at 3am.
446/// let policy = Policy::default()
447/// .layer("app")
448/// .allow_read("*")
449/// .allow_write("src/*") // routine, proceeds silently
450/// .deny_write("src/main.rs"); // never, and the approver is not consulted
451///
452/// // Everything else the policy marks `Ask` — a write outside src/, say —
453/// // stops here and waits, for as long as it takes.
454/// let result = run_with(
455/// &contract, &OpenRouter::from_env()?, &Store::open("runs.db")?, &policy, &StdinApprover,
456/// )
457/// .await?;
458/// println!("{:?}", result.outcome);
459/// # Ok(()) }
460/// ```
461///
462/// The policy is recorded against the run, so a later [`resume_from_stored_policy`]
463/// can recover the boundary this run executed under without the caller having to
464/// reconstruct it.
465pub async fn run_with<P: Provider>(
466 contract: &TaskContract,
467 provider: &P,
468 store: &Store,
469 policy: &Policy,
470 approver: &dyn Approver,
471) -> Result<RunResult> {
472 run_with_observed(contract, provider, store, policy, approver, &Ignore).await
473}
474
475/// [`run_with`], reporting to `observer` as it happens. See [`run_observed`].
476///
477/// The events a *policed* run adds are the ones worth watching: a refusal names
478/// the rule and layer that made it, which turns "the agent kept failing" into
479/// "one line of the ops baseline is too tight".
480///
481/// ```no_run
482/// use io_harness::{run_with_observed, ApproveAll, EventKind, Flow, Observer, OpenRouter,
483/// Policy, RunEvent, Store, TaskContract};
484/// use std::sync::Mutex;
485///
486/// /// Collects every refusal so the operator sees which rules the task ran into,
487/// /// rather than only that it ran out of steps.
488/// #[derive(Default)]
489/// struct Friction(Mutex<Vec<String>>);
490///
491/// impl Observer for Friction {
492/// fn event(&self, event: &RunEvent) -> Flow {
493/// if let EventKind::Refused { act, target, rule, layer } = &event.kind {
494/// self.0.lock().unwrap().push(format!(
495/// "{act} {target} <- {} in {}",
496/// rule.as_deref().unwrap_or("tier default"),
497/// layer.as_deref().unwrap_or("-"),
498/// ));
499/// }
500/// Flow::Continue
501/// }
502/// }
503///
504/// # async fn demo(contract: &TaskContract, policy: &Policy) -> io_harness::Result<()> {
505/// let friction = Friction::default();
506/// run_with_observed(
507/// contract, &OpenRouter::from_env()?, &Store::memory()?, policy, &ApproveAll, &friction,
508/// )
509/// .await?;
510/// for line in friction.0.lock().unwrap().iter() {
511/// println!("refused: {line}");
512/// }
513/// # Ok(()) }
514/// ```
515///
516/// `&self`, not `&mut self`: one observer serves a whole tree, so state goes
517/// behind a `Mutex` as above.
518pub async fn run_with_observed<P: Provider>(
519 contract: &TaskContract,
520 provider: &P,
521 store: &Store,
522 policy: &Policy,
523 approver: &dyn Approver,
524 observer: &dyn Observer,
525) -> Result<RunResult> {
526 // Arbitration before anything else: a toolbox that cannot be dispatched
527 // unambiguously is a configuration mistake, and the caller should hear about
528 // it before a run row exists and before the provider is billed for a turn.
529 contract.tools.validate()?;
530 // Same reason, same point: a skills directory that cannot be read is a
531 // configuration mistake, and the caller hears the path before a run row
532 // exists rather than getting a silently empty catalogue mid-run.
533 let skills = contract.discover_skills()?;
534 let file_str = contract.file.display().to_string();
535 let run_id = store.start_run(&contract.goal, &file_str)?;
536 store.set_provider(run_id, provider.name())?;
537 // The caller's policy, before the provider layer below is merged into it —
538 // what the caller asked for, not what the harness added. Recorded so a later
539 // `resume` can tell a run that had a boundary from one that never did; see
540 // [`resume`], which refuses the first rather than resuming it permissively.
541 store.record_run_policy(run_id, policy)?;
542 // The run row exists and the provider is set, which is what `Started` reports:
543 // emitted before the network authorization below, so an observer watching a run
544 // that is refused before its first step still saw it begin.
545 let watch = &Watch::new(observer);
546 watch.emit(RunEvent::new(
547 run_id,
548 0,
549 EventKind::Started {
550 goal: contract.goal.clone(),
551 provider: provider.name().to_string(),
552 },
553 ));
554 // Decided against the *caller's* policy, before the provider layer is merged
555 // in: the harness adding a network layer of its own must not turn a
556 // permissive caller into a policy-bearing one and push it off the
557 // single-file path.
558 let caller_enforces = !policy.is_permissive();
559 let policy = &match authorize_provider(provider, policy, store, run_id, approver, watch).await?
560 {
561 ProviderAccess::Granted(p) => p,
562 ProviderAccess::Pending(request_id) => {
563 return Ok(RunResult::new(
564 RunOutcome::AwaitingApproval {
565 request_id,
566 steps: 0,
567 },
568 run_id,
569 ))
570 }
571 };
572 match contract.root.clone() {
573 Some(root) => {
574 let mcp = McpSession::connect(&contract.mcp, policy, store, run_id, watch).await?;
575 let result = run_workspace_from(
576 contract, provider, store, run_id, &root, 1, policy, approver, &mcp, &skills, watch,
577 )
578 .await;
579 mcp.shutdown(store, run_id, watch).await;
580 result
581 }
582 // Single-file mode has no policy-aware tool layer in 0.4.0. Silently
583 // ignoring a policy here would be worse than not supporting it: the
584 // caller would believe a boundary was enforced when nothing was
585 // checking. Refuse loudly instead.
586 None if caller_enforces => Err(crate::error::Error::Config(
587 "a permission policy requires workspace mode — build the contract \
588 with TaskContract::workspace(goal, root, verify). Single-file \
589 contracts are not policy-enforced in 0.4.0."
590 .into(),
591 )),
592 None => run_from(contract, provider, store, run_id, 1, watch).await,
593 }
594}
595
596/// Resume an interrupted run under its original `run_id`. Continues from the
597/// step after the last one recorded, reusing the file on disk as the current
598/// state — it does not restart from step one.
599///
600/// This is the resume for a run that had **no** permission boundary. It drives
601/// the loop permissively, and a run that *was* started under a policy is refused
602/// with [`Error::Resume`] rather than resumed without it — use [`resume_with`]
603/// and supply the policy. Through 0.12.0 this function substituted
604/// [`Policy::permissive`] for every workspace run it resumed, so a caller who
605/// ran under a deny-by-default policy and crashed came back with no boundary and
606/// nothing said so. Refusing is the only behaviour that cannot silently widen
607/// what an agent may do.
608///
609/// What it preserves: the run id, the step it reached, its token and wall-clock
610/// budgets, and — since 0.13.0 — the observation ledger it had assembled, so the
611/// resumed run asks the model what the interrupted one would have. What it does
612/// not: a permission policy, which it refuses to guess at rather than
613/// substituting one. A run with no recorded policy, which is every run
614/// checkpointed before 0.13.0, resumes exactly as it did then.
615///
616/// The shape a supervisor process wants: the run id is the only thing that has
617/// to survive the crash, and resuming twice is a no-op rather than a second run.
618///
619/// ```no_run
620/// use io_harness::{run, resume, OpenRouter, RunOutcome, Store, TaskContract};
621///
622/// # async fn demo(contract: &TaskContract) -> io_harness::Result<()> {
623/// let store = Store::open("runs.db")?;
624/// let provider = OpenRouter::from_env()?;
625///
626/// let first = run(contract, &provider, &store).await?;
627/// // ... the process is killed here, mid-step ...
628///
629/// // A crash leaves either a whole step or none of it — the trace, the budget
630/// // draw and the checkpoint commit in one transaction — so this continues from
631/// // the last committed step rather than restarting. Completed steps are
632/// // skipped, spend is restored from durable totals rather than reset, and the
633/// // time budget counts the downtime as elapsed.
634/// let again = resume(contract, &provider, &store, first.run_id).await?;
635///
636/// // Idempotent: a finished run reports its outcome instead of re-driving.
637/// if let RunOutcome::Success { steps } = again.outcome {
638/// println!("done at step {steps}");
639/// }
640/// # Ok(()) }
641/// ```
642///
643/// It refuses a run that had a boundary — that is the point of it. Use
644/// [`resume_with`] when you hold the policy, and [`resume_from_stored_policy`]
645/// when you do not and want the one the run actually executed under.
646pub async fn resume<P: Provider>(
647 contract: &TaskContract,
648 provider: &P,
649 store: &Store,
650 run_id: i64,
651) -> Result<RunResult> {
652 resume_observed(contract, provider, store, run_id, &Ignore).await
653}
654
655/// [`resume`], reporting to `observer` as it happens. See [`run_observed`].
656///
657/// A resume of an already-finished run is a no-op and reports no events: it drives
658/// nothing, so there is nothing to watch.
659///
660/// That silence is the useful signal, and it is the reason to observe a resume at
661/// all: an observer that hears nothing but `Started` and `Finished` is looking at
662/// a run that had already completed before the crash, not one that did work.
663///
664/// ```no_run
665/// use io_harness::{resume_observed, EventKind, Flow, Observer, OpenRouter, RunEvent,
666/// Store, TaskContract};
667/// use std::sync::atomic::{AtomicU32, Ordering};
668///
669/// /// Counts only what *this* process drove. Steps committed before the crash
670/// /// are replayed from the store, not re-run, so they emit nothing.
671/// #[derive(Default)]
672/// struct DrivenHere(AtomicU32);
673///
674/// impl Observer for DrivenHere {
675/// fn event(&self, event: &RunEvent) -> Flow {
676/// if matches!(event.kind, EventKind::Step { .. }) {
677/// self.0.fetch_add(1, Ordering::Relaxed);
678/// }
679/// Flow::Continue
680/// }
681/// }
682///
683/// # async fn demo(contract: &TaskContract, run_id: i64) -> io_harness::Result<()> {
684/// let driven = DrivenHere::default();
685/// resume_observed(contract, &OpenRouter::from_env()?, &Store::open("runs.db")?, run_id, &driven)
686/// .await?;
687/// println!("{} new steps after the restart", driven.0.load(Ordering::Relaxed));
688/// # Ok(()) }
689/// ```
690pub async fn resume_observed<P: Provider>(
691 contract: &TaskContract,
692 provider: &P,
693 store: &Store,
694 run_id: i64,
695 observer: &dyn Observer,
696) -> Result<RunResult> {
697 // Refuse a store from a newer checkpoint format or a missing run with a
698 // typed error, rather than misreading it or panicking. Before the policy
699 // gate below, so an unknown run still reports as an unknown run.
700 store.check_resumable(run_id)?;
701 // Before the gate, because a finished run is a read and not a resume: it
702 // drives no loop, performs no action, and asks no provider, so there is no
703 // boundary to drop and refusing it would break the "report, don't re-drive"
704 // contract a refused or escalated run relies on.
705 if let Some(o) = finished_outcome(store, run_id)? {
706 return Ok(RunResult::new(o, run_id));
707 }
708 // The gate. `None` means nothing recorded a policy for this run — a run
709 // written by 0.12.0 or earlier — and is deliberately not read as "the caller
710 // chose permissive": it resumes as it always did, which is what 0.7.0's
711 // resume contract promised those runs. A recorded permissive policy is the
712 // same case, said explicitly. Anything else had a boundary, and this
713 // function cannot honour it.
714 if let Some(recorded) = store.run_policy(run_id)? {
715 if !recorded.is_permissive() {
716 return Err(crate::error::Error::Resume {
717 reason: format!(
718 "run {run_id} was started under a permission policy; resume it with \
719 resume_with (or resume_with_observed), supplying that policy — resuming \
720 here would drop the boundary the run was executing under"
721 ),
722 });
723 }
724 }
725 resume_with_observed(
726 contract,
727 provider,
728 store,
729 run_id,
730 &Policy::permissive(),
731 &ApproveAll,
732 observer,
733 )
734 .await
735}
736
737/// Resume an interrupted run under `policy`, routing anything the policy marks
738/// [`Effect::Ask`] to `approver` — the resume twin of [`run_with`].
739///
740/// The policy given here is the one that governs the resumed run; it is recorded
741/// against the run, so the store answers what rules the run actually executed
742/// under rather than only what it started under. Supplying
743/// [`Policy::permissive`] deliberately downgrades a run that had a boundary,
744/// which is a caller's decision to make explicitly and is exactly what [`resume`]
745/// will not do on its behalf.
746///
747/// Use it when the policy is something your program *builds* — from config it
748/// still holds, or from config that has since changed and should now apply:
749///
750/// ```no_run
751/// use io_harness::{resume_with, OpenRouter, Policy, StdinApprover, Store, TaskContract};
752///
753/// # async fn demo(contract: &TaskContract, run_id: i64) -> io_harness::Result<()> {
754/// // The same policy the run started under, rebuilt from the same config — plus
755/// // one deny added since, which now applies to the rest of the run. A resume is
756/// // the natural place to tighten: nothing forces the resumed run to inherit a
757/// // boundary the operator has since decided was too wide.
758/// let policy = Policy::default()
759/// .layer("app")
760/// .allow_read("*")
761/// .allow_write("src/*")
762/// .layer("incident-2026-07")
763/// .deny_write("src/billing/*");
764///
765/// resume_with(
766/// contract, &OpenRouter::from_env()?, &Store::open("runs.db")?, run_id, &policy,
767/// &StdinApprover,
768/// )
769/// .await?;
770/// # Ok(()) }
771/// ```
772///
773/// The policy given here is recorded against the run, so the store keeps
774/// answering what the run actually executed under. If you cannot reconstruct one,
775/// do not pass [`Policy::permissive`] to get moving — use
776/// [`resume_from_stored_policy`], which reads back the real one.
777pub async fn resume_with<P: Provider>(
778 contract: &TaskContract,
779 provider: &P,
780 store: &Store,
781 run_id: i64,
782 policy: &Policy,
783 approver: &dyn Approver,
784) -> Result<RunResult> {
785 resume_with_observed(contract, provider, store, run_id, policy, approver, &Ignore).await
786}
787
788/// Resume a run under the policy it was started with, read back from the store.
789///
790/// [`resume_with`] takes a policy because a resumed run must not silently lose
791/// its boundary — that was 0.13.0's subject. But the caller still had to
792/// reconstruct one, and a caller resuming after a crash in another process may
793/// have nothing to reconstruct it from. The policy has been durable since
794/// 0.13.0; this is the entry point that uses it.
795///
796/// It matters more from 0.15.0 on than it did when it was first noticed: this is
797/// the first release in which a crashed run may already have taken an
798/// irreversible action — a commit — under a policy the resuming caller cannot
799/// name.
800///
801/// Fails with [`Error::Resume`] when the store holds no policy for the run,
802/// rather than substituting a permissive one. A run whose boundary cannot be
803/// recovered is not resumed under no boundary; that substitution is exactly the
804/// defect 0.13.0 closed.
805///
806/// The entry point for a restart supervisor, which knows a run id and nothing
807/// else — it did not build the policy and has no config to rebuild it from:
808///
809/// ```no_run
810/// use io_harness::{resume_from_stored_policy, DenyAll, Error, OpenRouter, RunStatus,
811/// Store, TaskContract};
812///
813/// # async fn sweep(contract: &TaskContract) -> io_harness::Result<()> {
814/// let store = Store::open("runs.db")?;
815/// let provider = OpenRouter::from_env()?;
816///
817/// for run_id in [17_i64, 18, 19] {
818/// if store.run_status(run_id)? != Some(RunStatus::Running) {
819/// continue; // already finished, or paused on a human
820/// }
821/// // No policy argument, and that is the point: the boundary comes back from
822/// // the store, so a supervisor cannot silently widen what an agent may do by
823/// // being the process that happened to restart it.
824/// match resume_from_stored_policy(contract, &provider, &store, run_id, &DenyAll).await {
825/// Ok(result) => println!("run {run_id}: {:?}", result.outcome),
826/// // Refused rather than resumed unbounded. A run whose boundary cannot
827/// // be recovered stays stopped until a human names one.
828/// Err(Error::Resume { reason }) => eprintln!("run {run_id} needs a human: {reason}"),
829/// Err(e) => return Err(e),
830/// }
831/// }
832/// # Ok(()) }
833/// ```
834///
835/// Prefer it to [`resume_with`] whenever the resuming process is not the one that
836/// wrote the policy — from 0.15.0 a crashed run may already have committed, so
837/// the boundary it was working under is not a detail that can be approximated.
838pub async fn resume_from_stored_policy<P: Provider>(
839 contract: &TaskContract,
840 provider: &P,
841 store: &Store,
842 run_id: i64,
843 approver: &dyn Approver,
844) -> Result<RunResult> {
845 resume_from_stored_policy_observed(contract, provider, store, run_id, approver, &Ignore).await
846}
847
848/// [`resume_from_stored_policy`], reporting to `observer` as it happens. See
849/// [`run_observed`].
850///
851/// The one thing this combination shows that no other does: the recovered
852/// boundary in action. The caller never names the policy, so the refusals the
853/// observer reports — each attributed to its rule and layer — are the only
854/// visible evidence of which boundary came back from the store.
855///
856/// ```no_run
857/// use io_harness::{resume_from_stored_policy_observed, DenyAll, EventKind, Flow, Observer,
858/// OpenRouter, RunEvent, Store, TaskContract};
859///
860/// /// Logs the layers the recovered policy is actually enforcing.
861/// struct RecoveredBoundary;
862///
863/// impl Observer for RecoveredBoundary {
864/// fn event(&self, event: &RunEvent) -> Flow {
865/// if let EventKind::Refused { act, target, layer, .. } = &event.kind {
866/// println!("still enforcing {}: refused {act} {target}",
867/// layer.as_deref().unwrap_or("tier default"));
868/// }
869/// Flow::Continue
870/// }
871/// }
872///
873/// # async fn demo(contract: &TaskContract, run_id: i64) -> io_harness::Result<()> {
874/// resume_from_stored_policy_observed(
875/// contract, &OpenRouter::from_env()?, &Store::open("runs.db")?, run_id, &DenyAll,
876/// &RecoveredBoundary,
877/// )
878/// .await?;
879/// # Ok(()) }
880/// ```
881pub async fn resume_from_stored_policy_observed<P: Provider>(
882 contract: &TaskContract,
883 provider: &P,
884 store: &Store,
885 run_id: i64,
886 approver: &dyn Approver,
887 observer: &dyn Observer,
888) -> Result<RunResult> {
889 let Some(policy) = store.run_policy(run_id)? else {
890 return Err(Error::Resume {
891 reason: format!(
892 "run {run_id} has no recorded policy, so the boundary it ran under cannot be \
893 recovered; pass one explicitly with `resume_with` if you know what it was"
894 ),
895 });
896 };
897 resume_with_observed(
898 contract, provider, store, run_id, &policy, approver, observer,
899 )
900 .await
901}
902
903/// [`resume_with`], reporting to `observer` as it happens. See [`run_observed`].
904///
905/// An observer can also *stop* a run, which is worth pairing with a resume
906/// because the two are the same mechanism seen from both ends: cancelling
907/// finishes a run cleanly at the next step boundary and leaves it resumable, so
908/// "stop it for now" and "carry on later" are one loop.
909///
910/// ```no_run
911/// use io_harness::{resume_with_observed, ApproveAll, EventKind, Flow, Observer, OpenRouter,
912/// Policy, RunEvent, Store, TaskContract};
913/// use std::sync::atomic::{AtomicU64, Ordering};
914///
915/// /// Stops the run once it has spent more than the operator is willing to.
916/// struct SpendCeiling { limit: u64, spent: AtomicU64 }
917///
918/// impl Observer for SpendCeiling {
919/// fn event(&self, event: &RunEvent) -> Flow {
920/// if let EventKind::Step { tokens, .. } = &event.kind {
921/// if self.spent.fetch_add(*tokens, Ordering::Relaxed) + tokens > self.limit {
922/// // Honoured at the next step boundary, never mid-step: no tool
923/// // call is abandoned in flight and no file is left half-written.
924/// // The run records `cancelled` and stays resumable.
925/// return Flow::Cancel;
926/// }
927/// }
928/// Flow::Continue
929/// }
930/// }
931///
932/// # async fn demo(contract: &TaskContract, policy: &Policy, run_id: i64)
933/// # -> io_harness::Result<()> {
934/// let ceiling = SpendCeiling { limit: 50_000, spent: AtomicU64::new(0) };
935/// let result = resume_with_observed(
936/// contract, &OpenRouter::from_env()?, &Store::open("runs.db")?, run_id, policy,
937/// &ApproveAll, &ceiling,
938/// )
939/// .await?;
940/// // `RunOutcome::Cancelled` — finished, not abandoned. Dropping the future
941/// // instead would leave `runs.status` as `running` forever, indistinguishable
942/// // from a crashed process.
943/// println!("{:?}", result.outcome);
944/// # Ok(()) }
945/// ```
946#[allow(clippy::too_many_arguments)]
947pub async fn resume_with_observed<P: Provider>(
948 contract: &TaskContract,
949 provider: &P,
950 store: &Store,
951 run_id: i64,
952 policy: &Policy,
953 approver: &dyn Approver,
954 observer: &dyn Observer,
955) -> Result<RunResult> {
956 contract.tools.validate()?;
957 let skills = contract.discover_skills()?;
958 store.check_resumable(run_id)?;
959
960 if let Some(o) = finished_outcome(store, run_id)? {
961 return Ok(RunResult::new(o, run_id));
962 }
963
964 let caller_enforces = !policy.is_permissive();
965 let start_step = record_resume_markers(store, run_id)?;
966 store.set_provider(run_id, provider.name())?;
967 store.record_run_policy(run_id, policy)?;
968 let watch = &Watch::new(observer);
969 watch.emit(RunEvent::new(
970 run_id,
971 start_step.saturating_sub(1),
972 EventKind::Started {
973 goal: contract.goal.clone(),
974 provider: provider.name().to_string(),
975 },
976 ));
977 match contract.root.clone() {
978 Some(root) => {
979 // Re-authorized on resume rather than trusted from the interrupted
980 // run, for the reason [`resume_tree_observed`] gives: the policy
981 // handed to the resume is the one that governs it, and a host allowed
982 // before a crash may not be allowed after.
983 let policy = &match authorize_provider(provider, policy, store, run_id, approver, watch)
984 .await?
985 {
986 ProviderAccess::Granted(p) => p,
987 ProviderAccess::Pending(request_id) => {
988 return Ok(RunResult::new(
989 RunOutcome::AwaitingApproval {
990 request_id,
991 steps: start_step.saturating_sub(1),
992 },
993 run_id,
994 ))
995 }
996 };
997 let mcp = McpSession::connect(&contract.mcp, policy, store, run_id, watch).await?;
998 let result = run_workspace_from(
999 contract, provider, store, run_id, &root, start_step, policy, approver, &mcp,
1000 &skills, watch,
1001 )
1002 .await;
1003 mcp.shutdown(store, run_id, watch).await;
1004 result
1005 }
1006 // The same refusal [`run_with_observed`] makes, for the same reason:
1007 // single-file mode has no policy-aware tool layer, and silently ignoring
1008 // a policy would leave the caller believing a boundary was enforced when
1009 // nothing was checking.
1010 None if caller_enforces => Err(crate::error::Error::Config(
1011 "a permission policy requires workspace mode — build the contract \
1012 with TaskContract::workspace(goal, root, verify). Single-file \
1013 contracts are not policy-enforced."
1014 .into(),
1015 )),
1016 None => run_from(contract, provider, store, run_id, start_step, watch).await,
1017 }
1018}
1019
1020/// Continue a run that stopped at [`RunOutcome::AwaitingApproval`], once a
1021/// human has decided about the pending action.
1022///
1023/// An approval performs exactly the action that was persisted — the same target
1024/// and the same content the human was shown — and then continues the run under
1025/// its original `run_id`. The decision is re-checked against the policy first,
1026/// so a deny that landed after the pause still holds. A denial closes the run
1027/// without performing the action.
1028///
1029/// Preserves the policy — it is an argument — and, since 0.13.0, the run's
1030/// observation ledger. It is for a run that *paused*, though: a run that crashed
1031/// has no `request_id` and no pending decision to supply, and wants
1032/// [`resume_with`].
1033///
1034/// This is the other half of [`Decision::Defer`], and it is what makes an
1035/// approval able to outlive the process that asked for it — a web app can show
1036/// the pending action, close the request, and continue the run when someone
1037/// clicks approve tomorrow:
1038///
1039/// ```no_run
1040/// use io_harness::{resume_with_decision, ApproveAll, Act, Decision, OpenRouter, Policy,
1041/// Request, RunOutcome, Store, TaskContract};
1042///
1043/// # async fn on_click(contract: &TaskContract, policy: &Policy, request_id: i64, approved: bool)
1044/// # -> io_harness::Result<()> {
1045/// let store = Store::open("runs.db")?;
1046/// let pending = store.pending(request_id)?.expect("a pending request");
1047///
1048/// let decision = if approved {
1049/// // Approving performs exactly what was persisted — the same target, the
1050/// // same bytes the human was shown. Hand back a `modified` request to
1051/// // perform something else instead; it is re-checked against the policy
1052/// // first, so an approver cannot rewrite an action across a deny.
1053/// Decision::Approve {
1054/// modified: Some(Request::new(Act::Write, "docs/NOTES.md")
1055/// .with_content(pending.content.clone().unwrap_or_default())),
1056/// remember: Vec::new(),
1057/// }
1058/// } else {
1059/// // The action never happens and the run closes as `RunOutcome::Denied`.
1060/// Decision::deny("rejected in review")
1061/// };
1062///
1063/// let result = resume_with_decision(
1064/// contract, &OpenRouter::from_env()?, &store, pending.run_id, request_id, decision,
1065/// policy, &ApproveAll,
1066/// )
1067/// .await?;
1068///
1069/// // Deferring again is legal and leaves it pending — the run stays paused.
1070/// if let RunOutcome::AwaitingApproval { .. } = result.outcome {
1071/// println!("still waiting");
1072/// }
1073/// # Ok(()) }
1074/// ```
1075///
1076/// For a paused *tree*, use [`resume_tree_with_decision`]: the pending action
1077/// often belongs to a child rather than the root, and only that function
1078/// validates the request against the whole tree.
1079#[allow(clippy::too_many_arguments)]
1080pub async fn resume_with_decision<P: Provider>(
1081 contract: &TaskContract,
1082 provider: &P,
1083 store: &Store,
1084 run_id: i64,
1085 request_id: i64,
1086 decision: Decision,
1087 policy: &Policy,
1088 approver: &dyn Approver,
1089) -> Result<RunResult> {
1090 resume_with_decision_observed(
1091 contract, provider, store, run_id, request_id, decision, policy, approver, &Ignore,
1092 )
1093 .await
1094}
1095
1096/// [`resume_with_decision`], reporting to `observer` as it happens. See
1097/// [`run_observed`].
1098///
1099/// The event worth listening for here is
1100/// [`EventKind::ApprovalDecided`](crate::EventKind::ApprovalDecided): it is the
1101/// audit record that the decision a human made was the decision the run
1102/// performed, which is the one claim an approval flow has to be able to prove.
1103///
1104/// ```no_run
1105/// use io_harness::{resume_with_decision_observed, ApproveAll, Decision, EventKind, Flow,
1106/// Observer, OpenRouter, Policy, RunEvent, Store, TaskContract};
1107///
1108/// struct AuditTrail;
1109///
1110/// impl Observer for AuditTrail {
1111/// fn event(&self, event: &RunEvent) -> Flow {
1112/// if let EventKind::ApprovalDecided { act, target, decision } = &event.kind {
1113/// println!("run {} step {}: {decision} {act} {target}", event.run_id, event.step);
1114/// }
1115/// Flow::Continue
1116/// }
1117/// }
1118///
1119/// # async fn demo(contract: &TaskContract, policy: &Policy, run_id: i64, request_id: i64)
1120/// # -> io_harness::Result<()> {
1121/// resume_with_decision_observed(
1122/// contract, &OpenRouter::from_env()?, &Store::open("runs.db")?, run_id, request_id,
1123/// Decision::approve(), policy, &ApproveAll, &AuditTrail,
1124/// )
1125/// .await?;
1126/// # Ok(()) }
1127/// ```
1128///
1129/// A re-check against the policy happens before the action, so a deny added
1130/// while the run was paused still wins — in which case the observer sees a
1131/// refusal rather than an approval, and the run closes as
1132/// [`RunOutcome::Denied`].
1133#[allow(clippy::too_many_arguments)]
1134pub async fn resume_with_decision_observed<P: Provider>(
1135 contract: &TaskContract,
1136 provider: &P,
1137 store: &Store,
1138 run_id: i64,
1139 request_id: i64,
1140 decision: Decision,
1141 policy: &Policy,
1142 approver: &dyn Approver,
1143 observer: &dyn Observer,
1144) -> Result<RunResult> {
1145 contract.tools.validate()?;
1146 let skills = contract.discover_skills()?;
1147 let pending = store
1148 .pending(request_id)?
1149 .ok_or_else(|| crate::error::Error::Config(format!("no pending request {request_id}")))?;
1150 if pending.run_id != run_id {
1151 return Err(crate::error::Error::Config(format!(
1152 "request {request_id} belongs to run {}, not {run_id}",
1153 pending.run_id
1154 )));
1155 }
1156
1157 let root = contract.root.clone().ok_or_else(|| {
1158 crate::error::Error::Config("resume_with_decision needs a workspace".into())
1159 })?;
1160 let step = pending.step;
1161 // The run row and its provider have existed since the run that paused, so
1162 // `Started` here says "this process is now driving it" — one per entry point,
1163 // never zero, so a `Finished` below is never the first thing an observer hears.
1164 let watch = &Watch::new(observer);
1165 watch.emit(RunEvent::new(
1166 run_id,
1167 step,
1168 EventKind::Started {
1169 goal: contract.goal.clone(),
1170 provider: provider.name().to_string(),
1171 },
1172 ));
1173
1174 match decision {
1175 // Deferring again leaves it pending and the run paused.
1176 Decision::Defer => Ok(RunResult::new(
1177 RunOutcome::AwaitingApproval {
1178 request_id,
1179 steps: step,
1180 },
1181 run_id,
1182 )),
1183 Decision::Deny { reason } => {
1184 store.resolve_pending(request_id, "deny")?;
1185 store.record_event(
1186 run_id,
1187 &PolicyEvent::decision(
1188 step,
1189 &pending.act,
1190 &pending.target,
1191 "deny",
1192 format!("resumed:{request_id}"),
1193 ),
1194 )?;
1195 info!(run_id, request_id, %reason, "deferred action denied");
1196 finish(store, watch, run_id, 0, step, "denied")?;
1197 Ok(RunResult::new(RunOutcome::Denied { steps: step }, run_id))
1198 }
1199 // A deferred *network* action has no filesystem effect to replay: the
1200 // run paused before its first step, so approving it grants the host and
1201 // starts the loop. Routing it through the write path below would check
1202 // a host against the path policy and then try to create a file named
1203 // after it.
1204 Decision::Approve { ref remember, .. } if pending.act == "net" => {
1205 let effective = policy
1206 .clone()
1207 .merge(net::provider_layer(&pending.target))
1208 .merge(remembered_layer(remember));
1209 store.resolve_pending(request_id, "approve")?;
1210 store.record_event(
1211 run_id,
1212 &PolicyEvent::decision(
1213 step,
1214 "net",
1215 &pending.target,
1216 "approve",
1217 format!("resumed:{request_id}"),
1218 ),
1219 )?;
1220 let remember = remember.clone();
1221 let mcp = McpSession::connect(&contract.mcp, &effective, store, run_id, watch).await?;
1222 let result = run_workspace_from(
1223 contract,
1224 provider,
1225 store,
1226 run_id,
1227 &root,
1228 step + 1,
1229 &effective,
1230 approver,
1231 &mcp,
1232 &skills,
1233 watch,
1234 )
1235 .await;
1236 mcp.shutdown(store, run_id, watch).await;
1237 result.map(|r| r.with_remembered(remember))
1238 }
1239 Decision::Approve { modified, remember } => {
1240 let target = modified
1241 .as_ref()
1242 .map(|m| m.target.clone())
1243 .unwrap_or_else(|| pending.target.clone());
1244 let content = modified
1245 .as_ref()
1246 .and_then(|m| m.content.clone())
1247 .or_else(|| pending.content.clone());
1248
1249 let mut effective = policy.clone();
1250 if !remember.is_empty() {
1251 let mut layer = Policy::permissive().layer("remembered");
1252 for r in &remember {
1253 layer = layer.rule(r.act, r.effect, r.pattern.clone());
1254 }
1255 effective = effective.merge(layer);
1256 }
1257 let ws = Workspace::with_policy(&root, effective.clone());
1258
1259 // The pause does not grant immunity: the policy still decides.
1260 let act = if pending.act == "read" {
1261 Act::Read
1262 } else {
1263 Act::Write
1264 };
1265 let recheck = ws.check_path(act, &target);
1266 if recheck.effect == Effect::Deny {
1267 let mut ev = PolicyEvent::refusal(step, &pending.act, &target);
1268 ev.rule = recheck.rule.clone();
1269 ev.layer = recheck.layer.clone();
1270 store.record_event(run_id, &ev)?;
1271 refused(watch, run_id, 0, &ev);
1272 store.resolve_pending(request_id, "deny")?;
1273 finish(store, watch, run_id, 0, step, "denied")?;
1274 return Ok(RunResult::new(RunOutcome::Denied { steps: step }, run_id));
1275 }
1276
1277 if act == Act::Write {
1278 ws.write_file(&target, content.as_deref().unwrap_or_default())?;
1279 }
1280 store.resolve_pending(request_id, "approve")?;
1281 let mut ev = PolicyEvent::decision(
1282 step,
1283 &pending.act,
1284 &pending.target,
1285 "approve",
1286 format!("resumed:{request_id}"),
1287 );
1288 if target != pending.target {
1289 ev = ev.with_performed(&target);
1290 }
1291 store.record_event(run_id, &ev)?;
1292
1293 // Continue the run under its original id, from the next step.
1294 let mcp = McpSession::connect(&contract.mcp, &effective, store, run_id, watch).await?;
1295 let result = run_workspace_from(
1296 contract,
1297 provider,
1298 store,
1299 run_id,
1300 &root,
1301 step + 1,
1302 &effective,
1303 approver,
1304 &mcp,
1305 &skills,
1306 watch,
1307 )
1308 .await;
1309 mcp.shutdown(store, run_id, watch).await;
1310 result.map(|r| r.with_remembered(remember))
1311 }
1312 }
1313}
1314
1315/// Continue an agent *tree* that paused at [`RunOutcome::AwaitingApproval`],
1316/// once a human has decided — the tree counterpart of [`resume_with_decision`].
1317///
1318/// The pending action belongs to whichever agent in the tree deferred (often a
1319/// child, not the root), so the decision is validated against the whole tree,
1320/// not just the root run id. On approve the deferred action is performed once,
1321/// the pending is resolved, and the tree is resumed from the store exactly as
1322/// [`resume_tree`] does: the root replays its (deliberately uncommitted) pause
1323/// step, re-adopts the paused child, and the child continues past the
1324/// now-applied action. A denial stops the tree.
1325///
1326/// The trap this exists to avoid: the `run_id` you pass is the tree's **root**,
1327/// while `pending.run_id` is whichever agent actually asked — often three levels
1328/// down. Passing the child's id to [`resume_with_decision`] resumes that child
1329/// alone and orphans the tree around it.
1330///
1331/// ```no_run
1332/// use io_harness::{resume_tree_with_decision, ApproveAll, Containment, Decision, OpenRouter,
1333/// Policy, RunOutcome, Store, TaskContract};
1334///
1335/// # async fn decide(contract: &TaskContract, policy: &Policy, paused: RunOutcome, root_run_id: i64)
1336/// # -> io_harness::Result<()> {
1337/// let store = Store::open("runs.db")?;
1338/// let RunOutcome::AwaitingApproval { request_id, .. } = paused else { return Ok(()) };
1339///
1340/// // Show the human which agent in the tree is asking, not just what for.
1341/// let pending = store.pending(request_id)?.expect("a pending request");
1342/// println!("agent {} wants to {} {}", pending.run_id, pending.act, pending.target);
1343///
1344/// // The ROOT id, and the request id from anywhere in the tree. Containment is
1345/// // supplied again because the resumed tree draws against one continuous
1346/// // ceiling — it is restored from durable totals, never reset.
1347/// resume_tree_with_decision(
1348/// contract, &OpenRouter::from_env()?, &store, root_run_id, request_id,
1349/// Decision::approve(), policy, &ApproveAll, &Containment::new(8, 3, 2, 400_000),
1350/// )
1351/// .await?;
1352/// # Ok(()) }
1353/// ```
1354#[allow(clippy::too_many_arguments)]
1355pub async fn resume_tree_with_decision<P: Provider>(
1356 contract: &TaskContract,
1357 provider: &P,
1358 store: &Store,
1359 run_id: i64,
1360 request_id: i64,
1361 decision: Decision,
1362 policy: &Policy,
1363 approver: &dyn Approver,
1364 containment: &Containment,
1365) -> Result<RunResult> {
1366 resume_tree_with_decision_observed(
1367 contract,
1368 provider,
1369 store,
1370 run_id,
1371 request_id,
1372 decision,
1373 policy,
1374 approver,
1375 containment,
1376 &Ignore,
1377 )
1378 .await
1379}
1380
1381/// [`resume_tree_with_decision`], reporting to `observer` as it happens. See
1382/// [`run_observed`].
1383///
1384/// One observer watches the whole tree: every agent's events carry that agent's
1385/// own `run_id` and `depth`, so a consumer routes on those rather than being
1386/// handed one observer per child.
1387///
1388/// Which is what makes it usable here: after a tree-wide pause you want to see
1389/// the deferred action land and then watch the *right* agent carry on, out of
1390/// the several that resume at once.
1391///
1392/// ```no_run
1393/// use io_harness::{resume_tree_with_decision_observed, ApproveAll, Containment, Decision,
1394/// EventKind, Flow, Observer, OpenRouter, Policy, RunEvent, Store, TaskContract};
1395///
1396/// /// Follows one agent out of a whole tree resuming around it.
1397/// struct FollowAgent { run_id: i64 }
1398///
1399/// impl Observer for FollowAgent {
1400/// fn event(&self, event: &RunEvent) -> Flow {
1401/// if event.run_id != self.run_id {
1402/// return Flow::Continue; // some other agent in the tree
1403/// }
1404/// match &event.kind {
1405/// EventKind::ApprovalDecided { decision, target, .. } => {
1406/// println!("resumed on: {decision} {target}");
1407/// }
1408/// EventKind::Step { decision, .. } => println!(" step {}: {decision}", event.step),
1409/// _ => {}
1410/// }
1411/// Flow::Continue
1412/// }
1413/// }
1414///
1415/// # async fn demo(contract: &TaskContract, policy: &Policy, root_run_id: i64, request_id: i64)
1416/// # -> io_harness::Result<()> {
1417/// let store = Store::open("runs.db")?;
1418/// let pending = store.pending(request_id)?.expect("a pending request");
1419/// let follow = FollowAgent { run_id: pending.run_id };
1420///
1421/// resume_tree_with_decision_observed(
1422/// contract, &OpenRouter::from_env()?, &store, root_run_id, request_id,
1423/// Decision::approve(), policy, &ApproveAll, &Containment::new(8, 3, 2, 400_000), &follow,
1424/// )
1425/// .await?;
1426/// # Ok(()) }
1427/// ```
1428///
1429/// A [`Flow::Cancel`](crate::Flow::Cancel) from any agent's event stops the whole
1430/// tree at the next boundary, not only the agent that emitted it — there is one
1431/// cancellation flag per tree, as there is one approver and one ledger.
1432#[allow(clippy::too_many_arguments)]
1433pub async fn resume_tree_with_decision_observed<P: Provider>(
1434 contract: &TaskContract,
1435 provider: &P,
1436 store: &Store,
1437 run_id: i64,
1438 request_id: i64,
1439 decision: Decision,
1440 policy: &Policy,
1441 approver: &dyn Approver,
1442 containment: &Containment,
1443 observer: &dyn Observer,
1444) -> Result<RunResult> {
1445 store.check_resumable(run_id)?;
1446 contract.tools.validate()?;
1447 let skills = contract.discover_skills()?;
1448 let pending = store
1449 .pending(request_id)?
1450 .ok_or_else(|| crate::error::Error::Config(format!("no pending request {request_id}")))?;
1451 // The pending may belong to any agent in this tree, not only the root.
1452 if !store.tree_run_ids(run_id)?.contains(&pending.run_id) {
1453 return Err(crate::error::Error::Config(format!(
1454 "request {request_id} belongs to run {}, which is not in the tree rooted at {run_id}",
1455 pending.run_id
1456 )));
1457 }
1458 let root = contract.root.clone().ok_or_else(|| {
1459 crate::error::Error::Config("resume_tree_with_decision needs a workspace".into())
1460 })?;
1461 let step = pending.step;
1462 let watch = &Watch::new(observer);
1463 watch.emit(RunEvent::new(
1464 run_id,
1465 step,
1466 EventKind::Started {
1467 goal: contract.goal.clone(),
1468 provider: provider.name().to_string(),
1469 },
1470 ));
1471
1472 match decision {
1473 Decision::Defer => Ok(RunResult::new(
1474 RunOutcome::AwaitingApproval {
1475 request_id,
1476 steps: step,
1477 },
1478 run_id,
1479 )),
1480 Decision::Deny { reason } => {
1481 store.resolve_pending(request_id, "deny")?;
1482 store.record_event(
1483 pending.run_id,
1484 &PolicyEvent::decision(
1485 step,
1486 &pending.act,
1487 &pending.target,
1488 "deny",
1489 format!("resumed:{request_id}"),
1490 ),
1491 )?;
1492 info!(run_id, request_id, %reason, "deferred tree action denied; tree stops");
1493 finish(store, watch, run_id, 0, step, "denied")?;
1494 Ok(RunResult::new(RunOutcome::Denied { steps: step }, run_id))
1495 }
1496 // As in `resume_with_decision`: an approved network action grants the
1497 // host and starts the tree, with no filesystem effect to replay.
1498 Decision::Approve { ref remember, .. } if pending.act == "net" => {
1499 let effective = policy
1500 .clone()
1501 .merge(net::provider_layer(&pending.target))
1502 .merge(remembered_layer(remember));
1503 store.resolve_pending(request_id, "approve")?;
1504 store.record_event(
1505 pending.run_id,
1506 &PolicyEvent::decision(
1507 step,
1508 "net",
1509 &pending.target,
1510 "approve",
1511 format!("resumed:{request_id}"),
1512 ),
1513 )?;
1514 let ledger = Arc::new(Ledger::from_state(
1515 containment,
1516 store.spent_tokens_tree(run_id)?,
1517 store.agent_count_tree(run_id)?,
1518 ));
1519 let start_step = record_resume_markers(store, run_id)?;
1520 store.set_provider(run_id, provider.name())?;
1521 let mcp = McpSession::connect(&contract.mcp, &effective, store, run_id, watch).await?;
1522 let tree = Tree {
1523 mcp: &mcp,
1524 tools: &contract.tools,
1525 skills: &skills,
1526 provider,
1527 store,
1528 approver,
1529 watch,
1530 ledger,
1531 containment,
1532 root,
1533 root_run_id: run_id,
1534 };
1535 let outcome = run_agent(&tree, contract, run_id, 0, &effective, start_step).await;
1536 mcp.shutdown(store, run_id, watch).await;
1537 Ok(RunResult::new(outcome?, run_id).with_remembered(remember.clone()))
1538 }
1539 Decision::Approve { modified, remember } => {
1540 let target = modified
1541 .as_ref()
1542 .map(|m| m.target.clone())
1543 .unwrap_or_else(|| pending.target.clone());
1544 let content = modified
1545 .as_ref()
1546 .and_then(|m| m.content.clone())
1547 .or_else(|| pending.content.clone());
1548
1549 // ponytail: the deferred write is re-checked against the tree policy,
1550 // not the child's narrowed policy (not reconstructed here). A denial
1551 // beneath still holds; a narrower child allow is not re-enforced on
1552 // this one performed action. Tighten if child-specific deny of an
1553 // approved action becomes a requirement.
1554 let ws = Workspace::with_policy(&root, policy.clone());
1555 let act = if pending.act == "read" {
1556 Act::Read
1557 } else {
1558 Act::Write
1559 };
1560 if ws.check_path(act, &target).effect == Effect::Deny {
1561 store.resolve_pending(request_id, "deny")?;
1562 finish(store, watch, run_id, 0, step, "denied")?;
1563 return Ok(RunResult::new(RunOutcome::Denied { steps: step }, run_id));
1564 }
1565 if act == Act::Write {
1566 ws.write_file(&target, content.as_deref().unwrap_or_default())?;
1567 }
1568 store.resolve_pending(request_id, "approve")?;
1569 store.record_event(
1570 pending.run_id,
1571 &PolicyEvent::decision(
1572 step,
1573 &pending.act,
1574 &pending.target,
1575 "approve",
1576 format!("resumed:{request_id}"),
1577 ),
1578 )?;
1579
1580 // Resume the whole tree; the root replays its uncommitted pause step
1581 // and re-adopts the (now-unblocked) child.
1582 let mut effective = policy.clone();
1583 if !remember.is_empty() {
1584 let mut layer = Policy::permissive().layer("remembered");
1585 for r in &remember {
1586 layer = layer.rule(r.act, r.effect, r.pattern.clone());
1587 }
1588 effective = effective.merge(layer);
1589 }
1590 let ledger = Arc::new(Ledger::from_state(
1591 containment,
1592 store.spent_tokens_tree(run_id)?,
1593 store.agent_count_tree(run_id)?,
1594 ));
1595 let start_step = record_resume_markers(store, run_id)?;
1596 store.set_provider(run_id, provider.name())?;
1597 let mcp = McpSession::connect(&contract.mcp, &effective, store, run_id, watch).await?;
1598 let tree = Tree {
1599 mcp: &mcp,
1600 tools: &contract.tools,
1601 skills: &skills,
1602 provider,
1603 store,
1604 approver,
1605 watch,
1606 ledger,
1607 containment,
1608 root,
1609 root_run_id: run_id,
1610 };
1611 let outcome = run_agent(&tree, contract, run_id, 0, &effective, start_step).await;
1612 mcp.shutdown(store, run_id, watch).await;
1613 Ok(RunResult::new(outcome?, run_id).with_remembered(remember))
1614 }
1615 }
1616}
1617
1618/// Reconstruct the *final* [`RunOutcome`] of a run that cannot be meaningfully
1619/// re-driven, so a resume of such a run is a faithful no-op. Only genuinely
1620/// final outcomes are returned: `success`, `denied` (a human's no), and a tree
1621/// `budget_ceiling_reached`. A run that merely ran out of step / token / time
1622/// budget is deliberately NOT final — a caller resumes it with a larger budget
1623/// to continue — so those return `None` and resume re-drives the loop (which is
1624/// itself idempotent: re-running with the same budget skips the exhausted loop
1625/// and reports the same outcome without spending anything). `awaiting_approval`
1626/// is `Paused`, not `Completed`, and resumes via [`resume_with_decision`].
1627/// Record the resume marker and one skipped marker per already-committed step,
1628/// so a multi-crash run's full history is reconstructable from the store alone.
1629/// Returns the step to resume from (last committed + 1).
1630/// Did this step end the run, because there is no gate and the agent stopped
1631/// acting?
1632///
1633/// Both halves are required. Without [`Verification::None`] an assistant turn
1634/// with no tool call is an ordinary unproductive step — the agent thinking aloud,
1635/// or asking a question the loop cannot answer — and ending the run there would
1636/// silently cap every existing contract at its first quiet turn. Without the
1637/// empty tool-call list there is no signal at all: no `done` tool is added, so an
1638/// unverified run has exactly the tool surface a verified one has, and a model
1639/// that has finished says so by saying something.
1640fn finished(contract: &TaskContract, response: &CompletionResponse) -> bool {
1641 matches!(contract.verify, Verification::None) && response.tool_calls.is_empty()
1642}
1643
1644fn record_resume_markers(store: &Store, run_id: i64) -> Result<u32> {
1645 let last = store.last_step(run_id)?;
1646 let start_step = last + 1;
1647 store.record_checkpoint_event(&crate::state::CheckpointEvent::resume(
1648 run_id,
1649 start_step,
1650 format!("resuming at step {start_step}, {last} committed step(s) skipped"),
1651 ))?;
1652 for s in 1..=last {
1653 store.record_checkpoint_event(&crate::state::CheckpointEvent::skipped(run_id, s))?;
1654 }
1655 Ok(start_step)
1656}
1657
1658/// A run's ledger as the store has it, with the count already durable.
1659///
1660/// The count is the watermark [`persist_ledger`] appends from: everything below
1661/// it is on disk, everything above it was observed since the last committed
1662/// step.
1663fn restore_ledger(store: &Store, run_id: i64) -> Result<(ContextLedger, usize)> {
1664 let mut ledger = ContextLedger::new();
1665 for obs in store.observations(run_id)? {
1666 ledger.push(obs);
1667 }
1668 let written = ledger.len();
1669 Ok((ledger, written))
1670}
1671
1672/// Append everything observed since the last committed step, and return the new
1673/// watermark.
1674///
1675/// Called at the step boundary that commits, so an observation belonging to a
1676/// step that never committed does not outlive it — the ledger stays consistent
1677/// with the trace rather than running ahead of it.
1678fn persist_ledger(
1679 store: &Store,
1680 run_id: i64,
1681 ledger: &ContextLedger,
1682 written: usize,
1683) -> Result<usize> {
1684 store.record_observations(run_id, &ledger.entries()[written..])?;
1685 Ok(ledger.len())
1686}
1687
1688/// The outcome of a run that is already over, if it is over.
1689///
1690/// Idempotence for every resume entry point: a run that already finished is
1691/// returned as-is, so resuming twice does not re-drive the loop or re-charge the
1692/// budget. A run still `Running` — its process died mid-loop — reads as `None`
1693/// here and is resumed from its last committed step.
1694fn finished_outcome(store: &Store, run_id: i64) -> Result<Option<RunOutcome>> {
1695 if store.run_status(run_id)? != Some(RunStatus::Completed) {
1696 return Ok(None);
1697 }
1698 terminal_outcome(store, run_id)
1699}
1700
1701fn terminal_outcome(store: &Store, run_id: i64) -> Result<Option<RunOutcome>> {
1702 let last = store.last_step(run_id)?;
1703 Ok(store.outcome(run_id)?.and_then(|o| match o.as_str() {
1704 "success" => Some(RunOutcome::Success { steps: last }),
1705 "denied" => Some(RunOutcome::Denied { steps: last }),
1706 "budget_ceiling_reached" => Some(RunOutcome::BudgetCeilingReached { steps: last }),
1707 "stalled" => Some(RunOutcome::Stalled { steps: last }),
1708 // Before 0.11.0 `"escalated"` was unmapped and `finish_run` reported it as a
1709 // plain completion, so resuming an escalated run fell straight back into the
1710 // loop and re-ran it. An unattended run that escalated at 3am was silently
1711 // restarted by the next resume.
1712 "escalated_retryable" => Some(RunOutcome::Escalated {
1713 steps: last,
1714 retryable: true,
1715 }),
1716 "escalated_terminal" | "escalated" => Some(RunOutcome::Escalated {
1717 steps: last,
1718 retryable: false,
1719 }),
1720 // 0.12.0: the same defect, found by the same kind of audit. `"refused"` is
1721 // written when a human denies the network access the provider needs
1722 // (`authorize_provider`), and it was unmapped — so resuming a refused run
1723 // re-entered the loop and asked the human the same question again. A human's
1724 // no is as final as a policy's, which is why `denied` above is final too.
1725 "refused" => Some(RunOutcome::Refused { steps: last }),
1726 // Also 0.12.0: a run its observer stopped. Final for the same reason a
1727 // human's `denied` is — the caller asked for it — so a resume reports it
1728 // rather than quietly starting the run up again.
1729 "cancelled" => Some(RunOutcome::Cancelled { steps: last }),
1730 // 0.17.0: a `Verification::None` run that ended on its own terms. Final —
1731 // there is no criterion left to re-check, so a resume reports it rather
1732 // than driving the loop again to watch the agent say nothing twice.
1733 "finished" => Some(RunOutcome::Finished { steps: last }),
1734 _ => None,
1735 }))
1736}
1737
1738/// The [`Observer`] a run reports to, plus the one bit of state it can set.
1739///
1740/// A wrapper rather than a bare `&dyn Observer` because a cancellation has to
1741/// outlive the `event()` call that asked for it: [`Flow::Cancel`](crate::Flow::Cancel)
1742/// is honoured at the next step boundary, not where it was returned, so the
1743/// request is remembered here — and one `Watch` shared by a whole tree means a
1744/// child's observer can stop the tree, not only itself.
1745///
1746/// A [`Cell`] is enough: [`Store`] is `!Sync` and `run_agent` returns a
1747/// non-`Send` future, so a run and every agent in its tree are driven on one
1748/// task. Nothing here needs a lock, and adding one would be the only `Sync`
1749/// requirement in the loop.
1750pub(crate) struct Watch<'a> {
1751 observer: &'a dyn Observer,
1752 cancelled: Cell<bool>,
1753}
1754
1755impl<'a> Watch<'a> {
1756 fn new(observer: &'a dyn Observer) -> Self {
1757 Self {
1758 observer,
1759 cancelled: Cell::new(false),
1760 }
1761 }
1762
1763 /// Report one event, remembering a cancellation for the next step boundary.
1764 pub(crate) fn emit(&self, event: RunEvent) {
1765 if self.observer.event(&event).is_cancel() {
1766 self.cancelled.set(true);
1767 }
1768 }
1769
1770 /// Whether stopping has been asked for. Read at a step boundary only.
1771 fn cancelled(&self) -> bool {
1772 self.cancelled.get()
1773 }
1774}
1775
1776/// Announce a refusal, reading the event straight off the row that records it.
1777///
1778/// Every `Refused` in the crate goes through here, and takes the `PolicyEvent`
1779/// rather than the four fields, so an event cannot carry a rule or a layer the
1780/// `policy_events` row does not. The two surfaces agree by construction instead
1781/// of by four call sites remembering to keep in step.
1782pub(crate) fn refused(watch: &Watch<'_>, run_id: i64, depth: u32, ev: &PolicyEvent) {
1783 watch.emit(RunEvent::at_depth(
1784 run_id,
1785 ev.step,
1786 depth,
1787 EventKind::Refused {
1788 act: ev.act.clone(),
1789 target: ev.target.clone(),
1790 rule: ev.rule.clone(),
1791 layer: ev.layer.clone(),
1792 },
1793 ));
1794}
1795
1796/// Announce a human's answer, from the row that records it. As [`refused`]: the
1797/// event's `decision` is the row's, never a second literal beside it.
1798fn decided(watch: &Watch<'_>, run_id: i64, depth: u32, ev: &PolicyEvent) {
1799 watch.emit(RunEvent::at_depth(
1800 run_id,
1801 ev.step,
1802 depth,
1803 EventKind::ApprovalDecided {
1804 act: ev.act.clone(),
1805 target: ev.target.clone(),
1806 decision: ev.decision.clone().unwrap_or_default(),
1807 },
1808 ));
1809}
1810
1811/// The one step boundary: commit the step, log it, and tell the observer.
1812///
1813/// Before 0.12.0 the single-file loop, the workspace loop and the sub-agent loop
1814/// each had their own copy of this — their own inline [`StepRecord`], their own
1815/// `checkpoint_step`, and their own differently-named `info!` ("loop step" /
1816/// "workspace step" / "agent step"). One boundary is what stops the three
1817/// drifting, and what makes [`EventKind::Step`] one fact about a committed step
1818/// rather than three approximations of one.
1819///
1820/// `commit` is `false` for exactly one case: the sub-agent loop's step that
1821/// paused because one of its CHILDREN deferred. That step is deliberately left
1822/// uncommitted so a resume replays it and re-adopts the paused child — only the
1823/// parent re-entering `spawn_child` can wait on that child again — and committing
1824/// it would skip the replay, which is the double-execution defect 0.7.0's
1825/// checkpointing exists to prevent. Nothing is committed and no
1826/// [`EventKind::Step`] is emitted, because there is no committed step to report:
1827/// what the caller hears about is the pause, through
1828/// [`RunOutcome::AwaitingApproval`].
1829fn commit_step(
1830 store: &Store,
1831 watch: &Watch<'_>,
1832 run_id: i64,
1833 depth: u32,
1834 record: StepRecord,
1835 changed: bool,
1836 commit: bool,
1837) -> Result<()> {
1838 if !commit {
1839 info!(
1840 run_id,
1841 depth,
1842 step = record.step,
1843 "tree paused for a child's approval (step left uncommitted for replay)"
1844 );
1845 return Ok(());
1846 }
1847 store.checkpoint_step(run_id, &record)?;
1848 info!(
1849 run_id,
1850 depth,
1851 step = record.step,
1852 decision = %record.decision,
1853 tokens = record.tokens,
1854 changed,
1855 "step"
1856 );
1857 // The record's own fields, moved rather than cloned: an event must report
1858 // exactly what was committed, and the unobserved path must not pay an
1859 // allocation per step for the privilege of being ignored.
1860 watch.emit(RunEvent::at_depth(
1861 run_id,
1862 record.step,
1863 depth,
1864 EventKind::Step {
1865 decision: record.decision,
1866 tool_call: record.tool_call,
1867 tokens: record.tokens,
1868 changed,
1869 },
1870 ));
1871 Ok(())
1872}
1873
1874/// Stop the run if the observer asked it to, recording `"cancelled"` as the
1875/// outcome. `None` means carry on.
1876///
1877/// Call this at a step boundary and nowhere else — that is the contract
1878/// [`Flow::Cancel`](crate::Flow::Cancel) states, and the whole reason the request
1879/// is remembered in [`Watch`] rather than acted on where it was returned. The run
1880/// is *finished*, not abandoned: `runs.status` stops being `running`, a summary is
1881/// written, and `terminal_outcome` maps the string back so a resume reports the
1882/// cancellation instead of re-driving the loop.
1883fn cancelled(
1884 store: &Store,
1885 watch: &Watch<'_>,
1886 run_id: i64,
1887 depth: u32,
1888 steps: u32,
1889) -> Result<Option<RunOutcome>> {
1890 if !watch.cancelled() {
1891 return Ok(None);
1892 }
1893 finish(store, watch, run_id, depth, steps, "cancelled")?;
1894 info!(run_id, depth, steps, "run cancelled by its observer");
1895 Ok(Some(RunOutcome::Cancelled { steps }))
1896}
1897
1898/// End a run: write the outcome and tell the observer, so no terminal path can do
1899/// one without the other. Every `finish_run` in this file goes through here.
1900///
1901/// `steps` is what the outcome reports, which is not always the step the loop was
1902/// on — a time-budget stop reports the last step that completed.
1903fn finish(
1904 store: &Store,
1905 watch: &Watch<'_>,
1906 run_id: i64,
1907 depth: u32,
1908 steps: u32,
1909 outcome: &str,
1910) -> Result<()> {
1911 store.finish_run(run_id, outcome)?;
1912 watch.emit(RunEvent::at_depth(
1913 run_id,
1914 steps,
1915 depth,
1916 EventKind::Finished {
1917 outcome: outcome.to_string(),
1918 steps,
1919 // Read back from the store rather than carried in a local: the store
1920 // is what an audit will read, and the two must agree.
1921 tokens: store.spent_tokens(run_id)?,
1922 },
1923 ));
1924 Ok(())
1925}
1926
1927async fn run_from<P: Provider>(
1928 contract: &TaskContract,
1929 provider: &P,
1930 store: &Store,
1931 run_id: i64,
1932 start_step: u32,
1933 watch: &Watch<'_>,
1934) -> Result<RunResult> {
1935 let fs = FsTool::new(&contract.file);
1936 let system = system_prompt();
1937 let tool = write_file_tool();
1938 // Durable budget: spend and elapsed time are restored from the store, so a
1939 // resume continues one continuous budget instead of restarting it at zero.
1940 let mut tokens_used: u64 = store.spent_tokens(run_id)?;
1941 // Single-file mode is not policy-enforced (0.4.0), but the verify gate is
1942 // still sandboxed (0.6.0). A permissive guard carries the trace so the
1943 // sandbox lifecycle is recorded for single-file runs too.
1944 let permissive = Policy::permissive();
1945
1946 for step in start_step..=contract.max_steps {
1947 // A cancellation is acted on here, at the boundary between two steps, and
1948 // nowhere else: the points inside a step are not safe to stop at — a tool
1949 // call is in flight, a file may be half-written — and stopping there is
1950 // what dropping the future already does badly. Checked before the budgets
1951 // because the caller asking to stop outranks a budget saying so.
1952 if let Some(o) = cancelled(store, watch, run_id, 0, step - 1)? {
1953 return Ok(RunResult::new(o, run_id));
1954 }
1955 // Time budget: checked before doing the step's work, against real
1956 // wall-clock elapsed since the run started (durable across a restart).
1957 if let Some(max) = contract.max_duration {
1958 if store.elapsed_secs(run_id)? > max.as_secs_f64() {
1959 finish(store, watch, run_id, 0, step - 1, "time_budget_exceeded")?;
1960 return Ok(RunResult::new(
1961 RunOutcome::TimeBudgetExceeded { steps: step - 1 },
1962 run_id,
1963 ));
1964 }
1965 }
1966
1967 let current = fs.read().await?;
1968 // The whole file goes back every turn, so it is bounded on the same terms
1969 // as any observation: one large file must not exhaust the request. The tail
1970 // is kept, because the end of a file is what a writer needs.
1971 let user =
1972 user_prompt(
1973 contract,
1974 &bound(
1975 ¤t,
1976 entry_cap_chars(contract.context.effective_tokens(
1977 contract.max_tokens.map(|m| m.saturating_sub(tokens_used)),
1978 )),
1979 ObsKind::Read,
1980 ),
1981 );
1982 #[allow(clippy::needless_update)] // `media` is cfg'd out in the default build
1983 let request = CompletionRequest {
1984 system: system.clone(),
1985 user: user.clone(),
1986 tools: vec![tool.clone()],
1987 // Single-file mode has no `view_image` tool, so only the caller's
1988 // images are in play here.
1989 #[cfg(feature = "media")]
1990 media: attach_media(contract, &mut PendingMedia::default())?,
1991 ..Default::default()
1992 };
1993
1994 let response =
1995 complete_with_retry(provider, &request, contract, store, run_id, step, watch, 0)
1996 .await?;
1997
1998 // Which provider answered, when that is not a foregone conclusion. A
1999 // `Fallback` that fell over served this step from its secondary, and a trace
2000 // reader has no other way to know.
2001 if let Some(served) = provider.last_served() {
2002 store.record_context_event(run_id, &ContextEvent::served(step, served.clone()))?;
2003 watch.emit(RunEvent::new(
2004 run_id,
2005 step,
2006 EventKind::FellBackTo { provider: served },
2007 ));
2008 }
2009 let step_tokens = response.usage.map(|u| u.total_tokens).unwrap_or(0);
2010 tokens_used += step_tokens;
2011
2012 let call = response
2013 .tool_calls
2014 .iter()
2015 .find(|c| c.name == WRITE_FILE_TOOL);
2016 let tool_call_json = call.map(|c| c.arguments.to_string()).unwrap_or_default();
2017 let write = call.and_then(|c| c.arguments.get("content").and_then(|v| v.as_str()));
2018
2019 let (decision, result_text) = match write {
2020 Some(content) => {
2021 fs.write(content).await?;
2022 ("wrote file", content.to_string())
2023 }
2024 None => ("no tool call", response.text.clone().unwrap_or_default()),
2025 };
2026 // The file write (if any) is already applied above, before this commit:
2027 // a crash between the write and the commit replays this step, and the
2028 // model re-observes the already-written file, so the edit lands exactly
2029 // once. The committed checkpoint is the step's completion marker.
2030 //
2031 // Single-file mode has no workspace-change signal of its own, so `changed`
2032 // is whether this step wrote the file at all — the nearest true statement
2033 // the mode can make.
2034 commit_step(
2035 store,
2036 watch,
2037 run_id,
2038 0,
2039 StepRecord::new(step, decision, result_text).with_trace(
2040 user,
2041 tool_call_json,
2042 step_tokens,
2043 ),
2044 write.is_some(),
2045 true,
2046 )?;
2047
2048 // Cost budget: checked after this step's tokens are counted.
2049 if let Some(max) = contract.max_tokens {
2050 if tokens_used > max {
2051 finish(store, watch, run_id, 0, step, "cost_budget_exceeded")?;
2052 return Ok(RunResult::new(
2053 RunOutcome::CostBudgetExceeded { steps: step },
2054 run_id,
2055 ));
2056 }
2057 }
2058
2059 // A run with no criterion ends when the agent stops calling tools. After
2060 // the budget checks, because a step that also crossed a ceiling crossed
2061 // it — and before the gate, which for this variant can never pass.
2062 if finished(contract, &response) {
2063 finish(store, watch, run_id, 0, step, "finished")?;
2064 return Ok(RunResult::new(RunOutcome::Finished { steps: step }, run_id));
2065 }
2066
2067 let contents = fs.read().await?;
2068 let guard = ExecGuard::new(&permissive)
2069 .tracing(store, run_id, step)
2070 .watching(watch, 0);
2071 if contract
2072 .verify
2073 .passes_guarded(&contract.file, &contents, &guard)
2074 .await?
2075 {
2076 finish(store, watch, run_id, 0, step, "success")?;
2077 return Ok(RunResult::new(RunOutcome::Success { steps: step }, run_id));
2078 }
2079 }
2080
2081 finish(
2082 store,
2083 watch,
2084 run_id,
2085 0,
2086 contract.max_steps,
2087 "step_cap_reached",
2088 )?;
2089 Ok(RunResult::new(
2090 RunOutcome::StepCapReached {
2091 steps: contract.max_steps,
2092 },
2093 run_id,
2094 ))
2095}
2096
2097/// The workspace loop (0.3 multi-file mode): the agent greps, finds, reads, and
2098/// writes several files under `root`, carrying its own working memory as an
2099/// observation log folded into each turn's prompt. Budgets, retry, trace, and
2100/// resume behave as in single-file mode; verification is multi-file
2101/// ([`Verification::passes_in`]).
2102#[allow(clippy::too_many_arguments)]
2103async fn run_workspace_from<P: Provider>(
2104 contract: &TaskContract,
2105 provider: &P,
2106 store: &Store,
2107 run_id: i64,
2108 root: &Path,
2109 start_step: u32,
2110 policy: &Policy,
2111 approver: &dyn Approver,
2112 mcp: &McpSession,
2113 skills: &Skills,
2114 watch: &Watch<'_>,
2115) -> Result<RunResult> {
2116 // The effective policy grows as approvers remember rules; it is rebuilt as a
2117 // merge so a remembered allow can still never defeat a deny beneath it.
2118 let mut effective = policy.clone();
2119 let mut remembered: Vec<Rule> = Vec::new();
2120 let mut ws = Workspace::with_policy(root, effective.clone());
2121 // MCP tools sit beside the built-ins under their namespaced names, so the
2122 // model chooses between them the same way it chooses between grep and find.
2123 // Registered in-process tools and MCP tools sit beside the built-ins under
2124 // their own names, so the model chooses between them the same way it chooses
2125 // between grep and find.
2126 let mut extra = contract.tools.specs();
2127 extra.extend(mcp.tool_specs());
2128 extra.extend(skill_tool(skills));
2129 let system = with_skill_catalog(with_extra_tools(workspace_system_prompt(), &extra), skills);
2130 let mut tools = workspace_tools();
2131 tools.extend(extra);
2132 // Durable budget: restored from the store so a resume continues the same
2133 // token and wall-clock budget rather than restarting it at zero.
2134 let mut tokens_used: u64 = store.spent_tokens(run_id)?;
2135 // History, append-only. What the model sees of it is decided per turn by
2136 // `assemble`, under the contract's context budget — the log itself is never
2137 // trimmed, so the trace keeps everything.
2138 //
2139 // Restored from the store, so a resumed run continues with the context it had
2140 // rather than re-deriving one from the workspace and asking the model a
2141 // different question than the process before it would have. Empty for a fresh
2142 // run, and empty for a run checkpointed before 0.13.0, which is the same
2143 // re-derivation that binary did.
2144 let (mut ledger, mut written) = restore_ledger(store, run_id)?;
2145 // Is the agent getting anywhere? Restored from nothing on resume by design: a
2146 // resumed run has just been given a fresh chance, and condemning it for the
2147 // window it stalled in before the crash would be a poor welcome.
2148 let mut progress = Progress::new();
2149 // Detected once, before the first turn. The marker files do not change under
2150 // a run often enough to be worth a filesystem walk every step, and a run that
2151 // creates its own `package.json` is creating a project rather than working in
2152 // one.
2153 let toolchain = crate::toolchain::detect(root);
2154 let mem_key = memory_key(root);
2155 // Images the agent looked at last step, carried into this one's request and
2156 // dropped once shown. A viewed image is a tool result, not a permanent part
2157 // of the conversation: the model that wants it again asks again, and the
2158 // request stays bounded by what one step actually needed.
2159 let pending_media = &mut PendingMedia::default();
2160
2161 for step in start_step..=contract.max_steps {
2162 // The step boundary, where a cancellation is honoured (see `cancelled`).
2163 if let Some(o) = cancelled(store, watch, run_id, 0, step - 1)? {
2164 return Ok(RunResult::new(o, run_id).with_remembered(remembered));
2165 }
2166 if let Some(max) = contract.max_duration {
2167 if store.elapsed_secs(run_id)? > max.as_secs_f64() {
2168 finish(store, watch, run_id, 0, step - 1, "time_budget_exceeded")?;
2169 return Ok(RunResult::new(
2170 RunOutcome::TimeBudgetExceeded { steps: step - 1 },
2171 run_id,
2172 )
2173 .with_remembered(remembered));
2174 }
2175 }
2176
2177 // One budget, derived once per turn: it sets both this request's ceiling
2178 // and the per-observation cap the results of this step enter under.
2179 let budget_tokens = contract
2180 .context
2181 .effective_tokens(contract.max_tokens.map(|m| m.saturating_sub(tokens_used)));
2182 let entry_cap = entry_cap_chars(budget_tokens);
2183 // Re-read each turn rather than once at the start, so the notes the model
2184 // sees are the notes the store holds — including one written this run, and
2185 // not one the operator has since cleared.
2186 let notes = store.memory_list(&mem_key)?;
2187 let assembled = assemble(
2188 &ledger,
2189 budget_tokens,
2190 ¬es,
2191 Assembly {
2192 ws: Some(&ws),
2193 policy: &effective,
2194 store,
2195 run_id,
2196 step,
2197 },
2198 )
2199 .await?;
2200 let user = workspace_user_prompt(contract, &assembled.text, toolchain.as_ref());
2201 #[allow(clippy::needless_update)] // `media` is cfg'd out in the default build
2202 let request = CompletionRequest {
2203 system: system.clone(),
2204 user: user.clone(),
2205 tools: tools.clone(),
2206 #[cfg(feature = "media")]
2207 media: attach_media(contract, pending_media)?,
2208 ..Default::default()
2209 };
2210
2211 let response =
2212 complete_with_retry(provider, &request, contract, store, run_id, step, watch, 0)
2213 .await?;
2214
2215 // Which provider answered, when that is not a foregone conclusion. A
2216 // `Fallback` that fell over served this step from its secondary, and a trace
2217 // reader has no other way to know.
2218 if let Some(served) = provider.last_served() {
2219 store.record_context_event(run_id, &ContextEvent::served(step, served.clone()))?;
2220 watch.emit(RunEvent::new(
2221 run_id,
2222 step,
2223 EventKind::FellBackTo { provider: served },
2224 ));
2225 }
2226 let step_tokens = response.usage.map(|u| u.total_tokens).unwrap_or(0);
2227 tokens_used += step_tokens;
2228 // The provider's own number for the request `assemble` just built, beside
2229 // the estimate: the pair is what makes the estimator's drift auditable. A
2230 // silent provider leaves it null rather than recording a zero.
2231 if step_tokens > 0 {
2232 store.record_context_reported(run_id, step, step_tokens)?;
2233 }
2234
2235 // Dispatch every tool call the model made this step, in order, folding
2236 // each result into the observation log the next turn will see.
2237 let mut decisions: Vec<String> = Vec::new();
2238 let mut calls_json: Vec<String> = Vec::new();
2239 // Did this step move the workspace? Only a write that wrote something
2240 // different can, and it is the half of the stall signal that says the agent
2241 // is not merely repeating itself but achieving nothing.
2242 let mut step_changed = false;
2243 if response.tool_calls.is_empty() {
2244 let said = response.text.clone().unwrap_or_default();
2245 ledger.push(Observation::new(
2246 step,
2247 ObsKind::Message,
2248 None,
2249 bound(
2250 &format!("\n[step {step}] (no tool call) {said}\n"),
2251 entry_cap,
2252 ObsKind::Message,
2253 ),
2254 ));
2255 decisions.push("no tool call".into());
2256 }
2257 let mut paused: Option<i64> = None;
2258 let mut new_rules: Vec<Rule> = Vec::new();
2259 for call in &response.tool_calls {
2260 calls_json.push(format!("{}:{}", call.name, call.arguments));
2261 match dispatch(
2262 &ws,
2263 call,
2264 approver,
2265 store,
2266 run_id,
2267 step,
2268 mcp,
2269 &contract.tools,
2270 skills,
2271 entry_cap,
2272 &mem_key,
2273 watch,
2274 0,
2275 pending_media,
2276 &contract.commit_identity,
2277 contract.exec_timeout,
2278 )
2279 .await?
2280 {
2281 Dispatched::Continue {
2282 decision,
2283 obs,
2284 kind,
2285 target,
2286 changed,
2287 remember,
2288 } => {
2289 step_changed |= changed;
2290 ledger.push(Observation::new(step, kind, target, obs));
2291 decisions.push(decision);
2292 new_rules.extend(remember);
2293 }
2294 Dispatched::Pause { request_id } => {
2295 decisions.push(format!("awaiting approval (request {request_id})"));
2296 paused = Some(request_id);
2297 break;
2298 }
2299 }
2300 }
2301
2302 // The trace gets this step's observations unelided, so concatenating the
2303 // rows in step order reproduces the whole log: bounding what the model
2304 // sees must not bound what an operator can audit. A delta rather than the
2305 // whole log per row, so the trace is linear in the step count and a
2306 // 24-hour run does not write the same text hundreds of times.
2307 // ponytail: each row repeats the whole log, so the column grows with the
2308 // square of the step count. Bounded in practice by the step budget times
2309 // the entry cap; write per-step deltas if a long run's store size matters.
2310 //
2311 // The assembly stats this line used to log (`carried`, `stubbed`,
2312 // `est_tokens`) are not lost with the loop's own `info!`: `assemble`
2313 // records them as the step's `"assembled"` context event, which is where a
2314 // reader could already find them.
2315 commit_step(
2316 store,
2317 watch,
2318 run_id,
2319 0,
2320 StepRecord::new(step, decisions.join("; "), ledger.text_for_step(step)).with_trace(
2321 user,
2322 calls_json.join(" | "),
2323 step_tokens,
2324 ),
2325 step_changed,
2326 true,
2327 )?;
2328 // The step is committed, so the observations behind it are safe to make
2329 // durable. After the commit rather than before: a ledger that ran ahead of
2330 // the trace would restore observations for a step the run never took.
2331 written = persist_ledger(store, run_id, &ledger, written)?;
2332
2333 // Did that step get anywhere? A stall needs both halves — nothing changed
2334 // in the workspace AND a tool call this window already saw — because a
2335 // legitimate exploration phase changes nothing either, and flagging that
2336 // would degrade healthy runs to add resilience.
2337 let signature = calls_json.join(" | ");
2338 match progress.step(contract.stall, step_changed, &signature) {
2339 Progressing::Fine => {}
2340 Progressing::Replan => {
2341 store.record_context_event(
2342 run_id,
2343 &ContextEvent::replan(
2344 step,
2345 format!(
2346 "{} steps without progress; replanning",
2347 contract.stall.window
2348 ),
2349 ),
2350 )?;
2351 // The directive is an observation like any other, so it is bounded
2352 // by the same budget and, carrying no target, can never be
2353 // superseded away.
2354 ledger.push(Observation::new(
2355 step,
2356 ObsKind::Message,
2357 None,
2358 bound(
2359 &progress.replan_directive(contract.stall.window, &decisions),
2360 entry_cap,
2361 ObsKind::Message,
2362 ),
2363 ));
2364 info!(run_id, step, "agent told to change approach");
2365 watch.emit(RunEvent::new(
2366 run_id,
2367 step,
2368 EventKind::Replan {
2369 window: contract.stall.window,
2370 },
2371 ));
2372 }
2373 Progressing::Stalled => {
2374 store.record_context_event(
2375 run_id,
2376 &ContextEvent::stalled(step, "still no progress after replanning"),
2377 )?;
2378 info!(run_id, step, "run stopped: stalled");
2379 watch.emit(RunEvent::new(run_id, step, EventKind::Stalled));
2380 finish(store, watch, run_id, 0, step, "stalled")?;
2381 return Ok(RunResult::new(RunOutcome::Stalled { steps: step }, run_id)
2382 .with_remembered(remembered));
2383 }
2384 }
2385
2386 // An approver deferred: persist nothing further, stop, and let the
2387 // caller resume once a human has decided.
2388 if let Some(request_id) = paused {
2389 finish(store, watch, run_id, 0, step, "awaiting_approval")?;
2390 return Ok(RunResult::new(
2391 RunOutcome::AwaitingApproval {
2392 request_id,
2393 steps: step,
2394 },
2395 run_id,
2396 )
2397 .with_remembered(remembered));
2398 }
2399
2400 // Rules an approver asked to remember apply as a top layer for the rest
2401 // of the run. Merging (rather than editing) is what keeps a remembered
2402 // allow from overriding a deny beneath it.
2403 if !new_rules.is_empty() {
2404 let mut layer = Policy::permissive().layer("remembered");
2405 for r in &new_rules {
2406 layer = layer.rule(r.act, r.effect, r.pattern.clone());
2407 }
2408 effective = effective.merge(layer);
2409 ws = Workspace::with_policy(root, effective.clone());
2410 remembered.extend(new_rules);
2411 }
2412
2413 if let Some(max) = contract.max_tokens {
2414 if tokens_used > max {
2415 finish(store, watch, run_id, 0, step, "cost_budget_exceeded")?;
2416 return Ok(
2417 RunResult::new(RunOutcome::CostBudgetExceeded { steps: step }, run_id)
2418 .with_remembered(remembered),
2419 );
2420 }
2421 }
2422
2423 // A run with no criterion ends when the agent stops calling tools. Checked
2424 // after the budgets, so a step that also crossed a ceiling reports the
2425 // ceiling, and before the gate, which for this variant can never pass.
2426 if finished(contract, &response) {
2427 finish(store, watch, run_id, 0, step, "finished")?;
2428 return Ok(RunResult::new(RunOutcome::Finished { steps: step }, run_id)
2429 .with_remembered(remembered));
2430 }
2431
2432 if contract
2433 .verify
2434 .passes_in_guarded(
2435 root,
2436 &ExecGuard::new(&effective)
2437 .tracing(store, run_id, step)
2438 .watching(watch, 0),
2439 )
2440 .await?
2441 {
2442 finish(store, watch, run_id, 0, step, "success")?;
2443 return Ok(RunResult::new(RunOutcome::Success { steps: step }, run_id)
2444 .with_remembered(remembered));
2445 }
2446 }
2447
2448 finish(
2449 store,
2450 watch,
2451 run_id,
2452 0,
2453 contract.max_steps,
2454 "step_cap_reached",
2455 )?;
2456 Ok(RunResult::new(
2457 RunOutcome::StepCapReached {
2458 steps: contract.max_steps,
2459 },
2460 run_id,
2461 ))
2462}
2463
2464/// Shared context for one agent tree: everything every agent in the tree
2465/// draws on — the provider, the store, the one approver, the shared spend
2466/// ledger, the containment caps, and the workspace root.
2467struct Tree<'a, P: Provider> {
2468 /// One MCP session for the whole tree. A server is a stateful process, so
2469 /// 100 concurrent agents get 100 views of one connection, not 100 of their
2470 /// own — the same reason the ledger and the store are shared here.
2471 mcp: &'a McpSession,
2472 /// The caller's registered tools, shared by the whole tree. A child is
2473 /// offered exactly what its parent was: inheritance grants the tool, and the
2474 /// child's own narrowed policy still decides each call. Carried here rather
2475 /// than read from each agent's contract so a spawned child — whose contract
2476 /// the *model* writes — cannot register a tool its parent never had.
2477 tools: &'a Toolbox,
2478 /// The catalogue discovered from the ROOT contract, shared by the whole tree
2479 /// for the same reason `tools` is: a child contract the *model* wrote must
2480 /// not be able to conjure skills its parent was never offered.
2481 skills: &'a Skills,
2482 provider: &'a P,
2483 store: &'a Store,
2484 approver: &'a dyn Approver,
2485 /// One observer for the whole tree, exactly as there is one approver: every
2486 /// event carries the agent's own `run_id` and `depth`, so a consumer routes on
2487 /// those rather than being handed an observer per child. It also carries the
2488 /// tree's single cancellation flag, so a `Flow::Cancel` from any agent's event
2489 /// stops the tree at the next boundary rather than only that agent.
2490 watch: &'a Watch<'a>,
2491 ledger: Arc<Ledger>,
2492 containment: &'a Containment,
2493 root: PathBuf,
2494 /// The tree root's run id, so `Containment::max_total_duration` can be
2495 /// measured against when the TREE started rather than when this agent did.
2496 /// A child spawned twenty hours into a run has its own young `started_at`;
2497 /// the ceiling is about the whole tree, so the root's stamp is the only
2498 /// correct clock. Held here because [`Ledger`] has no store access.
2499 root_run_id: i64,
2500}
2501
2502/// Run a workspace contract as the root of an agent tree under `containment`.
2503///
2504/// The root agent runs the workspace loop with one extra tool, [`SPAWN_TOOL`],
2505/// which launches a contained sub-agent. A child inherits the parent policy and
2506/// can only narrow it ([`Policy::contain`]); the whole tree draws its token
2507/// spend from one shared ledger no child contract can raise; and every spawn,
2508/// refusal, and budget draw is recorded so the tree is a reconstructable graph.
2509///
2510/// Sub-agents are opt-in: this is the only entry point that offers the spawn
2511/// tool. [`run_with`] and [`run`] are unchanged and never expose it.
2512///
2513/// Reach for it when a task decomposes into parts that do not have to share one
2514/// agent's context — and note that the [`Containment`], not the contract, is
2515/// what actually bounds the result:
2516///
2517/// ```no_run
2518/// use io_harness::{run_tree, Containment, OpenRouter, Policy, StdinApprover, Store,
2519/// TaskContract, Verification};
2520/// use std::time::Duration;
2521///
2522/// # async fn demo() -> io_harness::Result<()> {
2523/// let contract = TaskContract::workspace(
2524/// "document every public module under docs/, one file per module",
2525/// "/path/to/repo",
2526/// Verification::WorkspaceFileContains { file: "docs/index.md".into(), needle: "##".into() },
2527/// );
2528///
2529/// // The root's boundary, and therefore the ceiling for the entire tree: a child
2530/// // inherits it through `Policy::contain` and may only narrow it, so no
2531/// // descendant at any depth can write outside docs/ however its goal is worded.
2532/// let policy = Policy::default()
2533/// .layer("app")
2534/// .allow_read("*")
2535/// .allow_write("docs/*");
2536///
2537/// // The spend ceiling belongs here rather than on the contract, because a
2538/// // spawned child's contract is written by the *model* — anything it could set
2539/// // is something it could raise.
2540/// let containment = Containment {
2541/// max_total_agents: 12,
2542/// max_concurrent: 4, // the fan-out bound
2543/// max_depth: 2,
2544/// max_total_tokens: 500_000, // drawn down by the whole tree together
2545/// max_total_cost: None, // reserved and inert; bound money in tokens
2546/// max_total_duration: Some(Duration::from_secs(3600)),
2547/// };
2548///
2549/// let result = run_tree(
2550/// &contract, &OpenRouter::from_env()?, &Store::open("runs.db")?, &policy,
2551/// &StdinApprover, &containment,
2552/// )
2553/// .await?;
2554/// # Ok(()) }
2555/// ```
2556///
2557/// Every spawn, refusal and budget draw is recorded against the tree, so
2558/// `Store::agent_events` reconstructs who spawned whom and what each drew long
2559/// after the process exited.
2560pub async fn run_tree<P: Provider>(
2561 contract: &TaskContract,
2562 provider: &P,
2563 store: &Store,
2564 policy: &Policy,
2565 approver: &dyn Approver,
2566 containment: &Containment,
2567) -> Result<RunResult> {
2568 run_tree_observed(
2569 contract,
2570 provider,
2571 store,
2572 policy,
2573 approver,
2574 containment,
2575 &Ignore,
2576 )
2577 .await
2578}
2579
2580/// [`run_tree`], reporting to `observer` as it happens. See [`run_observed`].
2581///
2582/// One observer watches the whole tree: a child's events carry that child's own
2583/// `run_id` and its non-zero `depth`.
2584///
2585/// A tree is where an observer stops being a nicety. Children run concurrently
2586/// and their output interleaves, so `depth` and `run_id` are what turn a stream
2587/// of events back into a shape a person can read:
2588///
2589/// ```no_run
2590/// use io_harness::{run_tree_observed, Containment, EventKind, Flow, Observer, OpenRouter,
2591/// Policy, RunEvent, StdinApprover, Store, TaskContract};
2592///
2593/// /// Indents by depth, so concurrent children are legible rather than interleaved.
2594/// struct TreeLog;
2595///
2596/// impl Observer for TreeLog {
2597/// fn event(&self, event: &RunEvent) -> Flow {
2598/// let pad = " ".repeat(event.depth as usize);
2599/// match &event.kind {
2600/// EventKind::Spawned { child_run_id, goal } => {
2601/// println!("{pad}+ run {child_run_id}: {goal}");
2602/// }
2603/// // Containment refused the spawn — the tree hit `max_total_agents`,
2604/// // `max_depth` or `max_concurrent`. The parent adapts; nothing fails.
2605/// EventKind::SpawnRefused { cap } => println!("{pad}! spawn refused: {cap} cap"),
2606/// // What the tree has left of its ONE shared ceiling, after this draw.
2607/// EventKind::SpendDraw { remaining, .. } => {
2608/// println!("{pad} budget left: {remaining:?}");
2609/// }
2610/// EventKind::Step { decision, .. } => println!("{pad} {decision}"),
2611/// _ => {}
2612/// }
2613/// Flow::Continue
2614/// }
2615/// }
2616///
2617/// # async fn demo(contract: &TaskContract, policy: &Policy) -> io_harness::Result<()> {
2618/// run_tree_observed(
2619/// contract, &OpenRouter::from_env()?, &Store::memory()?, policy, &StdinApprover,
2620/// &Containment::new(12, 4, 2, 500_000), &TreeLog,
2621/// )
2622/// .await?;
2623/// # Ok(()) }
2624/// ```
2625///
2626/// Events arrive on the run's own task and children share it, so a slow observer
2627/// slows every agent in the tree, not just one.
2628#[allow(clippy::too_many_arguments)]
2629pub async fn run_tree_observed<P: Provider>(
2630 contract: &TaskContract,
2631 provider: &P,
2632 store: &Store,
2633 policy: &Policy,
2634 approver: &dyn Approver,
2635 containment: &Containment,
2636 observer: &dyn Observer,
2637) -> Result<RunResult> {
2638 contract.tools.validate()?;
2639 let skills = contract.discover_skills()?;
2640 let root = contract.root.clone().ok_or_else(|| {
2641 crate::error::Error::Config(
2642 "run_tree needs a workspace contract — build it with TaskContract::workspace".into(),
2643 )
2644 })?;
2645 let ledger = Arc::new(Ledger::new(containment));
2646 let run_id = store.start_run(&contract.goal, &root.display().to_string())?;
2647 store.set_provider(run_id, provider.name())?;
2648 // As in [`run_with_observed`]: the caller's policy, recorded before the
2649 // provider layer is merged in. A tree's own resume already takes a policy,
2650 // so this is for the audit rather than for a gate.
2651 store.record_run_policy(run_id, policy)?;
2652 let watch = &Watch::new(observer);
2653 watch.emit(RunEvent::new(
2654 run_id,
2655 0,
2656 EventKind::Started {
2657 goal: contract.goal.clone(),
2658 provider: provider.name().to_string(),
2659 },
2660 ));
2661 // Authorized once at the root. Children inherit the root's policy through
2662 // `Policy::contain`, so the provider layer flows down the tree and no child
2663 // needs (or gets) its own chance to widen network access.
2664 let policy = &match authorize_provider(provider, policy, store, run_id, approver, watch).await?
2665 {
2666 ProviderAccess::Granted(p) => p,
2667 ProviderAccess::Pending(request_id) => {
2668 return Ok(RunResult::new(
2669 RunOutcome::AwaitingApproval {
2670 request_id,
2671 steps: 0,
2672 },
2673 run_id,
2674 ))
2675 }
2676 };
2677 let mcp = McpSession::connect(&contract.mcp, policy, store, run_id, watch).await?;
2678 let tree = Tree {
2679 mcp: &mcp,
2680 tools: &contract.tools,
2681 skills: &skills,
2682 provider,
2683 store,
2684 approver,
2685 watch,
2686 ledger,
2687 containment,
2688 root,
2689 root_run_id: run_id,
2690 };
2691 let outcome = run_agent(&tree, contract, run_id, 0, policy, 1).await;
2692 mcp.shutdown(store, run_id, watch).await;
2693 Ok(RunResult::new(outcome?, run_id))
2694}
2695
2696/// Resume a crashed agent tree under its original root `run_id`. Reconstructs
2697/// the whole 0.5.0 tree from the store: the shared spend ledger is restored from
2698/// the tree's durable total spend and agent count (so the resumed tree draws
2699/// against one continuous ceiling, never a reset one), and the root agent
2700/// resumes from its last committed step. As it replays its crashed step it
2701/// adopts the children it had already spawned and resumes each from that child's
2702/// own checkpoint (see `spawn_child`), so every agent in the tree continues
2703/// where it stopped rather than restarting.
2704///
2705/// Additive to [`run_tree`], mirroring how [`resume_with`] complements
2706/// [`run_with`]. Takes the policy and the approver, so a tree's boundary was
2707/// never at risk of being dropped across a resume the way the flat workspace
2708/// loop's was; since 0.13.0 every agent in the tree also restores its own
2709/// observation ledger.
2710///
2711/// One call, whole tree — you never resume a child yourself, and the containment
2712/// you pass is what the restored ledger is measured against:
2713///
2714/// ```no_run
2715/// use io_harness::{resume_tree, Containment, OpenRouter, Policy, StdinApprover, Store,
2716/// TaskContract};
2717///
2718/// # async fn after_crash(contract: &TaskContract, policy: &Policy, root_run_id: i64)
2719/// # -> io_harness::Result<()> {
2720/// // The SAME ceiling the tree started under. The ledger is rebuilt from the
2721/// // tree's durable total spend, so a tree that had already used 400k of 500k
2722/// // resumes with 100k left — pass a fresh, larger number and you have raised the
2723/// // ceiling, not restored it.
2724/// let containment = Containment::new(12, 4, 2, 500_000);
2725///
2726/// // The root's run id. Children are re-adopted from the store as the root
2727/// // replays its crashed step, each continuing from its own checkpoint.
2728/// let result = resume_tree(
2729/// contract, &OpenRouter::from_env()?, &Store::open("runs.db")?, root_run_id, policy,
2730/// &StdinApprover, &containment,
2731/// )
2732/// .await?;
2733/// println!("{:?}", result.outcome);
2734/// # Ok(()) }
2735/// ```
2736///
2737/// One caveat worth knowing before you choose this over
2738/// [`resume_tree_from_stored_policy`]: `run_tree` and the two flat loops record
2739/// the caller's policy against the run, and this function does not — so a tree
2740/// resumed here under a widened policy leaves an audit that understates what was
2741/// permitted.
2742#[allow(clippy::too_many_arguments)]
2743pub async fn resume_tree<P: Provider>(
2744 contract: &TaskContract,
2745 provider: &P,
2746 store: &Store,
2747 run_id: i64,
2748 policy: &Policy,
2749 approver: &dyn Approver,
2750 containment: &Containment,
2751) -> Result<RunResult> {
2752 resume_tree_observed(
2753 contract,
2754 provider,
2755 store,
2756 run_id,
2757 policy,
2758 approver,
2759 containment,
2760 &Ignore,
2761 )
2762 .await
2763}
2764
2765/// [`resume_tree`], reporting to `observer` as it happens. See [`run_observed`].
2766///
2767/// What this shows that a fresh [`run_tree_observed`] cannot: how much of the
2768/// tree was already done. Adopted children emit nothing for the steps they
2769/// committed before the crash, so the events that do arrive are exactly the work
2770/// this process is driving.
2771///
2772/// ```no_run
2773/// use io_harness::{resume_tree_observed, ApproveAll, Containment, EventKind, Flow, Observer,
2774/// OpenRouter, Policy, RunEvent, Store, TaskContract};
2775/// use std::collections::BTreeSet;
2776/// use std::sync::Mutex;
2777///
2778/// /// Which agents in the tree still had work left after the restart.
2779/// #[derive(Default)]
2780/// struct StillWorking(Mutex<BTreeSet<i64>>);
2781///
2782/// impl Observer for StillWorking {
2783/// fn event(&self, event: &RunEvent) -> Flow {
2784/// if matches!(event.kind, EventKind::Step { .. }) {
2785/// self.0.lock().unwrap().insert(event.run_id);
2786/// }
2787/// Flow::Continue
2788/// }
2789/// }
2790///
2791/// # async fn demo(contract: &TaskContract, policy: &Policy, root_run_id: i64)
2792/// # -> io_harness::Result<()> {
2793/// let working = StillWorking::default();
2794/// resume_tree_observed(
2795/// contract, &OpenRouter::from_env()?, &Store::open("runs.db")?, root_run_id, policy,
2796/// &ApproveAll, &Containment::new(12, 4, 2, 500_000), &working,
2797/// )
2798/// .await?;
2799/// println!("{:?} had steps left", working.0.lock().unwrap());
2800/// # Ok(()) }
2801/// ```
2802#[allow(clippy::too_many_arguments)]
2803pub async fn resume_tree_observed<P: Provider>(
2804 contract: &TaskContract,
2805 provider: &P,
2806 store: &Store,
2807 run_id: i64,
2808 policy: &Policy,
2809 approver: &dyn Approver,
2810 containment: &Containment,
2811 observer: &dyn Observer,
2812) -> Result<RunResult> {
2813 contract.tools.validate()?;
2814 let skills = contract.discover_skills()?;
2815 store.check_resumable(run_id)?;
2816
2817 // A finished tree is returned as-is — resume is idempotent for the whole tree.
2818 if store.run_status(run_id)? == Some(RunStatus::Completed) {
2819 if let Some(o) = terminal_outcome(store, run_id)? {
2820 return Ok(RunResult::new(o, run_id));
2821 }
2822 }
2823
2824 let root = contract.root.clone().ok_or_else(|| {
2825 crate::error::Error::Config(
2826 "resume_tree needs a workspace contract — build it with TaskContract::workspace".into(),
2827 )
2828 })?;
2829
2830 // Restore the shared ledger from durable tree-wide totals, so the budget is
2831 // continuous across the crash rather than reset to zero.
2832 let ledger = Arc::new(Ledger::from_state(
2833 containment,
2834 store.spent_tokens_tree(run_id)?,
2835 store.agent_count_tree(run_id)?,
2836 ));
2837 let start_step = record_resume_markers(store, run_id)?;
2838 store.set_provider(run_id, provider.name())?;
2839 let watch = &Watch::new(observer);
2840 watch.emit(RunEvent::new(
2841 run_id,
2842 start_step.saturating_sub(1),
2843 EventKind::Started {
2844 goal: contract.goal.clone(),
2845 provider: provider.name().to_string(),
2846 },
2847 ));
2848 // Re-authorized on resume rather than trusted from the crashed run: the
2849 // policy handed to the resume is the one that governs it, and a host allowed
2850 // before a crash may not be allowed after.
2851 let policy = &match authorize_provider(provider, policy, store, run_id, approver, watch).await?
2852 {
2853 ProviderAccess::Granted(p) => p,
2854 ProviderAccess::Pending(request_id) => {
2855 return Ok(RunResult::new(
2856 RunOutcome::AwaitingApproval {
2857 request_id,
2858 steps: start_step.saturating_sub(1),
2859 },
2860 run_id,
2861 ))
2862 }
2863 };
2864 let mcp = McpSession::connect(&contract.mcp, policy, store, run_id, watch).await?;
2865 let tree = Tree {
2866 mcp: &mcp,
2867 tools: &contract.tools,
2868 skills: &skills,
2869 provider,
2870 store,
2871 approver,
2872 watch,
2873 ledger,
2874 containment,
2875 root,
2876 root_run_id: run_id,
2877 };
2878 let outcome = run_agent(&tree, contract, run_id, 0, policy, start_step).await;
2879 mcp.shutdown(store, run_id, watch).await;
2880 Ok(RunResult::new(outcome?, run_id))
2881}
2882
2883/// Resume a crashed agent tree under the policy it was started with, read back
2884/// from the store — [`resume_from_stored_policy`] for the 0.5.0 tree loop.
2885///
2886/// [`resume_tree`] takes a policy, so a tree's boundary was never at risk of
2887/// being silently dropped the way the flat workspace loop's was. But the caller
2888/// still had to have one to hand, and a process that comes up after a crash in
2889/// another process may have nothing to reconstruct it from. The policy has been
2890/// durable since 0.13.0 and the single-file and workspace loops have resumed from
2891/// it since then; the tree loop had no such entry point until 0.16.0, so the
2892/// three resume paths disagreed about whether a restart preserves the boundary —
2893/// a contradiction a release documenting the public contract would otherwise have
2894/// had to write down as a caveat.
2895///
2896/// Fails with [`Error::Resume`] when the store holds no policy for the run,
2897/// rather than substituting a permissive one. That substitution is the exact
2898/// defect 0.13.0 closed in the other two loops, and it is sharper for a tree:
2899/// every child inherits the root's policy through [`Policy::contain`], so a
2900/// guessed-at root boundary is guessed at for the whole tree, which may already
2901/// have taken an irreversible action under the real one.
2902///
2903/// Prefer it to [`resume_tree`] whenever the boundary matters, and not only
2904/// because you might get the policy wrong: it is also the only tree resume that
2905/// leaves an accurate audit. `resume_tree` does not call `record_run_policy`, so
2906/// a tree resumed there under a different policy keeps reporting the one it
2907/// started with; this one reads that row back rather than writing over it.
2908///
2909/// ```no_run
2910/// use io_harness::{resume_tree_from_stored_policy, Containment, DenyAll, Error, OpenRouter,
2911/// Store, TaskContract};
2912///
2913/// # async fn supervisor(contract: &TaskContract, root_run_id: i64) -> io_harness::Result<()> {
2914/// // No policy argument. A process that comes up after a crash in another
2915/// // process has nothing to reconstruct one from, and guessing here would guess
2916/// // for every agent in the tree at once.
2917/// let resumed = resume_tree_from_stored_policy(
2918/// contract, &OpenRouter::from_env()?, &Store::open("runs.db")?, root_run_id, &DenyAll,
2919/// &Containment::new(12, 4, 2, 500_000),
2920/// )
2921/// .await;
2922///
2923/// match resumed {
2924/// Ok(result) => println!("{:?}", result.outcome),
2925/// // No recorded policy — a tree checkpointed by 0.12.0 or earlier. It stays
2926/// // stopped rather than resuming unbounded; a human names the boundary and
2927/// // uses `resume_tree`.
2928/// Err(Error::Resume { reason }) => eprintln!("cannot recover the boundary: {reason}"),
2929/// Err(e) => return Err(e),
2930/// }
2931/// # Ok(()) }
2932/// ```
2933pub async fn resume_tree_from_stored_policy<P: Provider>(
2934 contract: &TaskContract,
2935 provider: &P,
2936 store: &Store,
2937 run_id: i64,
2938 approver: &dyn Approver,
2939 containment: &Containment,
2940) -> Result<RunResult> {
2941 resume_tree_from_stored_policy_observed(
2942 contract,
2943 provider,
2944 store,
2945 run_id,
2946 approver,
2947 containment,
2948 &Ignore,
2949 )
2950 .await
2951}
2952
2953/// [`resume_tree_from_stored_policy`], reporting to `observer` as it happens. See
2954/// [`run_observed`].
2955///
2956/// The combination an unattended supervisor actually wants: recover the boundary
2957/// from the store, resume the whole tree, and keep a live handle on it — because
2958/// a tree resumed by a process nobody is watching should still be stoppable.
2959///
2960/// ```no_run
2961/// use io_harness::{resume_tree_from_stored_policy_observed, Containment, DenyAll, EventKind,
2962/// Flow, Observer, OpenRouter, RunEvent, Store, TaskContract};
2963/// use std::sync::atomic::{AtomicBool, Ordering};
2964///
2965/// /// Logs the recovered boundary doing its job, and stops the tree on request.
2966/// struct Supervised { stop: AtomicBool }
2967///
2968/// impl Observer for Supervised {
2969/// fn event(&self, event: &RunEvent) -> Flow {
2970/// if let EventKind::Refused { act, target, layer, .. } = &event.kind {
2971/// println!("agent {} refused {act} {target} ({})",
2972/// event.run_id, layer.as_deref().unwrap_or("tier default"));
2973/// }
2974/// // One flag for the whole tree: cancelling from any agent's event stops
2975/// // every agent at its next step boundary, and the tree stays resumable.
2976/// if self.stop.load(Ordering::Relaxed) { Flow::Cancel } else { Flow::Continue }
2977/// }
2978/// }
2979///
2980/// # async fn demo(contract: &TaskContract, root_run_id: i64) -> io_harness::Result<()> {
2981/// let supervised = Supervised { stop: AtomicBool::new(false) };
2982/// resume_tree_from_stored_policy_observed(
2983/// contract, &OpenRouter::from_env()?, &Store::open("runs.db")?, root_run_id, &DenyAll,
2984/// &Containment::new(12, 4, 2, 500_000), &supervised,
2985/// )
2986/// .await?;
2987/// # Ok(()) }
2988/// ```
2989#[allow(clippy::too_many_arguments)]
2990pub async fn resume_tree_from_stored_policy_observed<P: Provider>(
2991 contract: &TaskContract,
2992 provider: &P,
2993 store: &Store,
2994 run_id: i64,
2995 approver: &dyn Approver,
2996 containment: &Containment,
2997 observer: &dyn Observer,
2998) -> Result<RunResult> {
2999 let Some(policy) = store.run_policy(run_id)? else {
3000 return Err(Error::Resume {
3001 reason: format!(
3002 "tree {run_id} has no recorded policy, so the boundary it ran under cannot be \
3003 recovered; pass one explicitly with `resume_tree` if you know what it was"
3004 ),
3005 });
3006 };
3007 resume_tree_observed(
3008 contract,
3009 provider,
3010 store,
3011 run_id,
3012 &policy,
3013 approver,
3014 containment,
3015 observer,
3016 )
3017 .await
3018}
3019
3020/// One agent's loop, reused for the root and every child. Identical to the
3021/// workspace loop, plus: it may spawn children (recursively, via [`SPAWN_TOOL`]),
3022/// and its token spend is drawn from the tree's shared ledger rather than only
3023/// its own contract budget.
3024///
3025/// `depth` is 0 at the root; a child's depth is its parent's + 1. Returns the
3026/// agent's [`RunOutcome`]; a tree-wide budget halt propagates up as
3027/// [`RunOutcome::BudgetCeilingReached`].
3028fn run_agent<'f, P: Provider>(
3029 tree: &'f Tree<'_, P>,
3030 contract: &'f TaskContract,
3031 run_id: i64,
3032 depth: u32,
3033 policy: &'f Policy,
3034 start_step: u32,
3035) -> Pin<Box<dyn Future<Output = Result<RunOutcome>> + 'f>> {
3036 // Boxed so the loop can recurse into itself when an agent spawns a child.
3037 Box::pin(async move {
3038 let ws = Workspace::with_policy(&tree.root, policy.clone());
3039 // The tree shares one MCP session, so every agent in it — root or child —
3040 // is offered the same server tools beside its built-ins. Connecting a
3041 // session and then not offering its tools would leave the model unable to
3042 // call something the run had already paid to set up.
3043 let mut extra = tree.tools.specs();
3044 extra.extend(tree.mcp.tool_specs());
3045 extra.extend(skill_tool(tree.skills));
3046 let system =
3047 with_skill_catalog(with_extra_tools(tree_system_prompt(), &extra), tree.skills);
3048 let mut tools = tree_tools();
3049 tools.extend(extra);
3050 // The budget this agent runs under is the smaller of what its contract
3051 // asked for and what the tree has left — a contract cannot raise it.
3052 let token_cap = tree.ledger.effective_token_budget(contract.max_tokens);
3053 // Durable per-agent budget, restored across a restart.
3054 let mut tokens_used: u64 = tree.store.spent_tokens(run_id)?;
3055 // Same ledger and same per-turn assembly as the workspace loop: a tree of
3056 // 100 children each re-sending its own unbounded log is the multiplied
3057 // version of the problem 0.10.0 exists to fix — and, since 0.13.0, the
3058 // same restore, keyed on this agent's own run id. A child that is resumed
3059 // is the same child, at whatever depth it sits.
3060 let (mut ledger, mut written) = restore_ledger(tree.store, run_id)?;
3061 let mut progress = Progress::new();
3062 // Children share their parent's workspace, so they share its detection too.
3063 let toolchain = crate::toolchain::detect(&tree.root);
3064 // Children share their parent's workspace, so they share its memory: one
3065 // note store per workspace, every entry attributed to the run that wrote it.
3066 let mem_key = memory_key(&tree.root);
3067 // See the workspace loop: viewed images ride one step and are dropped.
3068 let pending_media = &mut PendingMedia::default();
3069
3070 for step in start_step..=contract.max_steps {
3071 // The step boundary, where a cancellation is honoured (see `cancelled`).
3072 // One flag for the whole tree, so a cancel asked for while a sibling was
3073 // mid-flight stops this agent too.
3074 if let Some(o) = cancelled(tree.store, tree.watch, run_id, depth, step - 1)? {
3075 return Ok(o);
3076 }
3077 if let Some(max) = contract.max_duration {
3078 if tree.store.elapsed_secs(run_id)? > max.as_secs_f64() {
3079 finish(
3080 tree.store,
3081 tree.watch,
3082 run_id,
3083 depth,
3084 step - 1,
3085 "time_budget_exceeded",
3086 )?;
3087 return Ok(RunOutcome::TimeBudgetExceeded { steps: step - 1 });
3088 }
3089 }
3090
3091 let budget_tokens = contract
3092 .context
3093 .effective_tokens(Some(token_cap.saturating_sub(tokens_used)));
3094 let entry_cap = entry_cap_chars(budget_tokens);
3095 let notes = tree.store.memory_list(&mem_key)?;
3096 let assembled = assemble(
3097 &ledger,
3098 budget_tokens,
3099 ¬es,
3100 Assembly {
3101 ws: Some(&ws),
3102 policy,
3103 store: tree.store,
3104 run_id,
3105 step,
3106 },
3107 )
3108 .await?;
3109 let user = workspace_user_prompt(contract, &assembled.text, toolchain.as_ref());
3110 #[allow(clippy::needless_update)] // `media` is cfg'd out in the default build
3111 let request = CompletionRequest {
3112 system: system.clone(),
3113 user: user.clone(),
3114 tools: tools.clone(),
3115 #[cfg(feature = "media")]
3116 media: attach_media(contract, pending_media)?,
3117 ..Default::default()
3118 };
3119 let response = complete_with_retry(
3120 tree.provider,
3121 &request,
3122 contract,
3123 tree.store,
3124 run_id,
3125 step,
3126 tree.watch,
3127 depth,
3128 )
3129 .await?;
3130
3131 // Which provider answered, when that is not a foregone conclusion. A
3132 // `Fallback` that fell over served this step from its secondary, and a
3133 // trace reader has no other way to know.
3134 if let Some(served) = tree.provider.last_served() {
3135 tree.store
3136 .record_context_event(run_id, &ContextEvent::served(step, served.clone()))?;
3137 tree.watch.emit(RunEvent::at_depth(
3138 run_id,
3139 step,
3140 depth,
3141 EventKind::FellBackTo { provider: served },
3142 ));
3143 }
3144 let step_tokens = response.usage.map(|u| u.total_tokens).unwrap_or(0);
3145 tokens_used += step_tokens;
3146 if step_tokens > 0 {
3147 tree.store
3148 .record_context_reported(run_id, step, step_tokens)?;
3149 }
3150
3151 let mut decisions: Vec<String> = Vec::new();
3152 let mut calls_json: Vec<String> = Vec::new();
3153 let mut step_changed = false;
3154 if response.tool_calls.is_empty() {
3155 let said = response.text.clone().unwrap_or_default();
3156 ledger.push(Observation::new(
3157 step,
3158 ObsKind::Message,
3159 None,
3160 bound(
3161 &format!("\n[step {step}] (no tool call) {said}\n"),
3162 entry_cap,
3163 ObsKind::Message,
3164 ),
3165 ));
3166 decisions.push("no tool call".into());
3167 }
3168 // Non-spawn tools mutate the workspace and the observation log, so
3169 // they run in order. Spawn calls are independent sub-agents, so they
3170 // fan out concurrently, bounded by the tree's `max_concurrent`.
3171 let mut paused: Option<i64> = None;
3172 let mut paused_by_child = false;
3173 let mut spawn_calls: Vec<&ToolCall> = Vec::new();
3174 for call in &response.tool_calls {
3175 calls_json.push(format!("{}:{}", call.name, call.arguments));
3176 if call.name == SPAWN_TOOL {
3177 spawn_calls.push(call);
3178 continue;
3179 }
3180 match dispatch(
3181 &ws,
3182 call,
3183 tree.approver,
3184 tree.store,
3185 run_id,
3186 step,
3187 tree.mcp,
3188 tree.tools,
3189 tree.skills,
3190 entry_cap,
3191 &mem_key,
3192 tree.watch,
3193 depth,
3194 pending_media,
3195 &contract.commit_identity,
3196 contract.exec_timeout,
3197 )
3198 .await?
3199 {
3200 Dispatched::Continue {
3201 decision,
3202 obs,
3203 kind,
3204 target,
3205 changed,
3206 ..
3207 } => {
3208 step_changed |= changed;
3209 ledger.push(Observation::new(step, kind, target, obs));
3210 decisions.push(decision);
3211 }
3212 Dispatched::Pause { request_id } => {
3213 decisions.push(format!("awaiting approval (request {request_id})"));
3214 paused = Some(request_id);
3215 break;
3216 }
3217 }
3218 }
3219 if paused.is_none() && !spawn_calls.is_empty() {
3220 use futures_util::stream::{self, StreamExt};
3221 let max_c = tree.containment.max_concurrent.max(1) as usize;
3222 // `buffered`, not `buffer_unordered`: up to `max_c` children still
3223 // run at once, but their results are collected in the order the
3224 // model asked for them rather than the order they happen to finish.
3225 //
3226 // Until 0.12.0 this was `buffer_unordered`, which made a tree run
3227 // non-reproducible: the composed child observations and the
3228 // `decisions` list — both of which become the `steps.result` and
3229 // `steps.decision` columns — came back in completion order, so the
3230 // same task over the same workspace produced a different trace and
3231 // a different next prompt depending on which child won a race.
3232 // Deterministic replay cannot be built on that.
3233 //
3234 // The cost is that a child which finishes early has its result held
3235 // until the children before it are done. That is bounded by `max_c`
3236 // and changes when a result is *read*, never when the work runs.
3237 let results: Vec<Result<SpawnResult>> = stream::iter(
3238 spawn_calls
3239 .into_iter()
3240 .map(|c| spawn_child(tree, c, run_id, depth, policy, step)),
3241 )
3242 .buffered(max_c)
3243 .collect()
3244 .await;
3245 for r in results {
3246 match r? {
3247 SpawnResult::Composed { decision, obs } => {
3248 // A child's composed result is an observation like any
3249 // other, and is bounded like any other.
3250 ledger.push(Observation::new(
3251 step,
3252 ObsKind::Child,
3253 None,
3254 bound(&obs, entry_cap, ObsKind::Child),
3255 ));
3256 decisions.push(decision);
3257 // A child that ran did work the parent did not have to.
3258 // Whether it changed the workspace is the child's own
3259 // stall problem, tracked in the child's own loop.
3260 step_changed = true;
3261 }
3262 // A child deferred; pause the tree with its request_id.
3263 SpawnResult::Paused { request_id } => {
3264 decisions
3265 .push(format!("child awaiting approval (request {request_id})"));
3266 paused = Some(request_id);
3267 paused_by_child = true;
3268 }
3269 }
3270 }
3271 }
3272
3273 // An agent paused because one of its CHILDREN deferred does NOT commit
3274 // this step: on resume it must replay it to re-adopt and resume that
3275 // paused child (only the parent re-entering `spawn_child` can wait on
3276 // the child again). An agent paused by its OWN gate commits normally —
3277 // it resumes from the step after, past the now-approved action.
3278 //
3279 // The condition is passed to the one boundary rather than branching
3280 // around it, so the uncommitted case is a stated argument at the single
3281 // commit point instead of a second, quieter commit path that a later
3282 // change could forget about. `commit_step` emits no `EventKind::Step`
3283 // for it either: there is no committed step to report, and a resume is
3284 // going to run this step again.
3285 let committed = !(paused.is_some() && paused_by_child);
3286 commit_step(
3287 tree.store,
3288 tree.watch,
3289 run_id,
3290 depth,
3291 StepRecord::new(step, decisions.join("; "), ledger.text_for_step(step)).with_trace(
3292 user,
3293 calls_json.join(" | "),
3294 step_tokens,
3295 ),
3296 step_changed,
3297 committed,
3298 )?;
3299 // Only when the step actually committed. A step paused by a child is
3300 // deliberately left uncommitted so the resume replays it (0.7.0's
3301 // fix for double execution); persisting its observations would mean
3302 // the replay observed everything twice.
3303 if committed {
3304 written = persist_ledger(tree.store, run_id, &ledger, written)?;
3305 }
3306
3307 // 0.5.0 spawns up to a hundred of these, so an agent burning its whole
3308 // step budget going nowhere is the multiplied version of the problem.
3309 let signature = calls_json.join(" | ");
3310 match progress.step(contract.stall, step_changed, &signature) {
3311 Progressing::Fine => {}
3312 Progressing::Replan => {
3313 tree.store.record_context_event(
3314 run_id,
3315 &ContextEvent::replan(
3316 step,
3317 format!(
3318 "{} steps without progress; replanning",
3319 contract.stall.window
3320 ),
3321 ),
3322 )?;
3323 ledger.push(Observation::new(
3324 step,
3325 ObsKind::Message,
3326 None,
3327 bound(
3328 &progress.replan_directive(contract.stall.window, &decisions),
3329 entry_cap,
3330 ObsKind::Message,
3331 ),
3332 ));
3333 info!(run_id, depth, step, "agent told to change approach");
3334 tree.watch.emit(RunEvent::at_depth(
3335 run_id,
3336 step,
3337 depth,
3338 EventKind::Replan {
3339 window: contract.stall.window,
3340 },
3341 ));
3342 }
3343 Progressing::Stalled => {
3344 tree.store.record_context_event(
3345 run_id,
3346 &ContextEvent::stalled(step, "still no progress after replanning"),
3347 )?;
3348 info!(run_id, depth, step, "agent stopped: stalled");
3349 tree.watch
3350 .emit(RunEvent::at_depth(run_id, step, depth, EventKind::Stalled));
3351 finish(tree.store, tree.watch, run_id, depth, step, "stalled")?;
3352 return Ok(RunOutcome::Stalled { steps: step });
3353 }
3354 }
3355
3356 if let Some(request_id) = paused {
3357 finish(
3358 tree.store,
3359 tree.watch,
3360 run_id,
3361 depth,
3362 step,
3363 "awaiting_approval",
3364 )?;
3365 return Ok(RunOutcome::AwaitingApproval {
3366 request_id,
3367 steps: step,
3368 });
3369 }
3370
3371 // Draw this step's tokens against the tree. The draw is recorded even
3372 // when it crosses the ceiling — the tokens were already spent — and a
3373 // crossing halts the whole tree, not just this agent.
3374 let draw = tree.ledger.draw_tokens(step_tokens);
3375 let remaining = tree.ledger.remaining_tokens();
3376 tree.store.record_agent_event(&AgentEvent::budget_draw(
3377 run_id,
3378 step,
3379 step_tokens,
3380 remaining,
3381 ))?;
3382 tree.watch.emit(RunEvent::at_depth(
3383 run_id,
3384 step,
3385 depth,
3386 EventKind::SpendDraw {
3387 tokens: step_tokens,
3388 // A tree always has a ceiling, so there is always a number here;
3389 // the field is optional for a future draw against no ceiling.
3390 remaining: Some(remaining),
3391 },
3392 ));
3393 if draw == Draw::Halted {
3394 finish(
3395 tree.store,
3396 tree.watch,
3397 run_id,
3398 depth,
3399 step,
3400 "budget_ceiling_reached",
3401 )?;
3402 return Ok(RunOutcome::BudgetCeilingReached { steps: step });
3403 }
3404 // The tree's wall-clock ceiling, measured from the ROOT's `started_at`
3405 // rather than this agent's: a child spawned twenty hours in has a young
3406 // stamp of its own, and the ceiling is about the whole tree. Checked
3407 // beside the token draw because both are the same kind of limit — one a
3408 // contract cannot raise, crossing which halts the tree rather than the
3409 // agent that noticed.
3410 //
3411 // `max_total_duration` has existed on `Containment` since 0.5.0 and was
3412 // never read, so a caller could set a ceiling on a 24-hour tree and have
3413 // it silently ignored. Enforced in 0.12.0. Its sibling `max_total_cost`
3414 // still cannot be: there is no price telemetry to compare against.
3415 if let Some(max) = tree.containment.max_total_duration {
3416 if tree.store.elapsed_secs(tree.root_run_id)? > max.as_secs_f64() {
3417 finish(
3418 tree.store,
3419 tree.watch,
3420 run_id,
3421 depth,
3422 step,
3423 "budget_ceiling_reached",
3424 )?;
3425 info!(run_id, depth, step, "tree stopped: duration ceiling");
3426 return Ok(RunOutcome::BudgetCeilingReached { steps: step });
3427 }
3428 }
3429 // This agent's own contract budget (never looser than the tree's).
3430 if tokens_used > token_cap {
3431 finish(
3432 tree.store,
3433 tree.watch,
3434 run_id,
3435 depth,
3436 step,
3437 "cost_budget_exceeded",
3438 )?;
3439 return Ok(RunOutcome::CostBudgetExceeded { steps: step });
3440 }
3441
3442 // As in the workspace loop: no criterion means the agent's own quiet
3443 // turn ends it. A child composes back into its parent carrying this
3444 // outcome, so a parent can tell a child that finished from one that
3445 // ran out of steps.
3446 if finished(contract, &response) {
3447 finish(tree.store, tree.watch, run_id, depth, step, "finished")?;
3448 return Ok(RunOutcome::Finished { steps: step });
3449 }
3450
3451 if contract
3452 .verify
3453 .passes_in_guarded(
3454 &tree.root,
3455 &ExecGuard::new(policy)
3456 .tracing(tree.store, run_id, step)
3457 .watching(tree.watch, depth),
3458 )
3459 .await?
3460 {
3461 finish(tree.store, tree.watch, run_id, depth, step, "success")?;
3462 return Ok(RunOutcome::Success { steps: step });
3463 }
3464 }
3465
3466 finish(
3467 tree.store,
3468 tree.watch,
3469 run_id,
3470 depth,
3471 contract.max_steps,
3472 "step_cap_reached",
3473 )?;
3474 Ok(RunOutcome::StepCapReached {
3475 steps: contract.max_steps,
3476 })
3477 })
3478}
3479
3480/// The result of one [`SPAWN_TOOL`] call.
3481enum SpawnResult {
3482 /// The child finished; fold its composed result into the parent's log.
3483 Composed { decision: String, obs: String },
3484 /// The child deferred a sensitive action to a human. The pending action is
3485 /// persisted under `request_id`; the whole tree pauses so the caller can
3486 /// resume it with [`resume_with_decision`], exactly as a single run does.
3487 Paused { request_id: i64 },
3488}
3489
3490/// Handle one [`SPAWN_TOOL`] call: enforce the containment caps, derive the
3491/// child's narrowed policy, run it, and compose its result back for the parent's
3492/// next turn. A refused spawn is a typed observation the parent can adapt to,
3493/// never a failure of the parent run; a child that defers propagates the pause
3494/// up so the caller can resume the child once a human decides.
3495async fn spawn_child<P: Provider>(
3496 tree: &Tree<'_, P>,
3497 call: &ToolCall,
3498 parent_run_id: i64,
3499 depth: u32,
3500 parent_policy: &Policy,
3501 step: u32,
3502) -> Result<SpawnResult> {
3503 let a = &call.arguments;
3504 let goal = a.get("goal").and_then(|v| v.as_str()).unwrap_or_default();
3505 let file = a
3506 .get("verify_file")
3507 .and_then(|v| v.as_str())
3508 .unwrap_or_default();
3509 let needle = a
3510 .get("verify_contains")
3511 .and_then(|v| v.as_str())
3512 .unwrap_or_default();
3513 if goal.is_empty() || file.is_empty() {
3514 return Ok(SpawnResult::Composed {
3515 decision: "spawn missing fields".into(),
3516 obs: "\n[spawn error] spawn_agent needs \"goal\" and \"verify_file\"\n".into(),
3517 });
3518 }
3519
3520 let child_depth = depth + 1;
3521
3522 // A child inherits the parent policy and may only narrow it. Optional
3523 // `deny_write` globs let the parent tighten the child further.
3524 let mut overlay = Policy::permissive().layer("child");
3525 if let Some(denies) = a.get("deny_write").and_then(|v| v.as_array()) {
3526 for d in denies.iter().filter_map(|v| v.as_str()) {
3527 overlay = overlay.deny_write(d);
3528 }
3529 }
3530 if let Some(denies) = a.get("deny_net").and_then(|v| v.as_array()) {
3531 for d in denies.iter().filter_map(|v| v.as_str()) {
3532 overlay = overlay.deny_net(d);
3533 }
3534 }
3535 let child_policy = parent_policy.contain(&overlay);
3536
3537 let verify = Verification::WorkspaceFileContains {
3538 file: file.into(),
3539 needle: needle.into(),
3540 };
3541 let mut child_contract = TaskContract::workspace(goal, &tree.root, verify);
3542 if let Some(n) = a.get("max_steps").and_then(|v| v.as_u64()) {
3543 child_contract = child_contract.with_max_steps(n as u32);
3544 }
3545
3546 // Spawn-or-adopt. On a fresh run this spawn has no persisted record, so a new
3547 // child is created. On a tree resume the parent replays the same spawn step
3548 // and finds the child it already spawned (keyed by parent+step+goal): it
3549 // adopts that child and resumes it from its OWN last committed step instead
3550 // of creating a duplicate or restarting it. This is what lets every agent in
3551 // a crashed tree continue from its own checkpoint.
3552 let (child_run, child_start) = match tree.store.find_spawn(parent_run_id, step, goal)? {
3553 Some(row) => {
3554 // Adopted: already counted in the reconstructed ledger, so do NOT
3555 // register it again. A finished child is composed from its recorded
3556 // outcome without re-running; a mid-flight child resumes from its
3557 // next step.
3558 if let Some(o) = terminal_outcome(tree.store, row.child_run_id)? {
3559 return Ok(compose_child(row.child_run_id, goal, o));
3560 }
3561 (
3562 row.child_run_id,
3563 tree.store.last_step(row.child_run_id)? + 1,
3564 )
3565 }
3566 None => {
3567 // Fresh: the containment boundary decides whether it may exist, and
3568 // its contract is persisted so a later resume can adopt it.
3569 if let Err(refusal) = tree.ledger.register_agent(child_depth) {
3570 tree.store.record_agent_event(&AgentEvent::spawn_refused(
3571 parent_run_id,
3572 step,
3573 refusal.cap(),
3574 ))?;
3575 // The parent's event, at the parent's depth: no child exists to
3576 // attribute it to, which is the point of the refusal.
3577 tree.watch.emit(RunEvent::at_depth(
3578 parent_run_id,
3579 step,
3580 depth,
3581 EventKind::SpawnRefused {
3582 cap: refusal.cap().to_string(),
3583 },
3584 ));
3585 return Ok(SpawnResult::Composed {
3586 decision: format!("spawn refused ({})", refusal.cap()),
3587 obs: format!(
3588 "\n[spawn refused] {refusal} — adapt or finish with what you have\n"
3589 ),
3590 });
3591 }
3592 let child_run = tree.store.start_child_run(
3593 goal,
3594 &tree.root.display().to_string(),
3595 parent_run_id,
3596 child_depth,
3597 )?;
3598 tree.store.record_agent_event(&AgentEvent::spawn(
3599 parent_run_id,
3600 step,
3601 child_run,
3602 goal,
3603 ))?;
3604 // Attributed to the PARENT's run and depth: the parent is what spawned
3605 // it, and the child's own events (which carry `child_depth`) start
3606 // arriving next.
3607 tree.watch.emit(RunEvent::at_depth(
3608 parent_run_id,
3609 step,
3610 depth,
3611 EventKind::Spawned {
3612 child_run_id: child_run,
3613 goal: goal.to_string(),
3614 },
3615 ));
3616 let deny_json = a
3617 .get("deny_write")
3618 .map(|v| v.to_string())
3619 .unwrap_or_else(|| "[]".into());
3620 tree.store.record_spawn(
3621 parent_run_id,
3622 step,
3623 child_run,
3624 goal,
3625 file,
3626 needle,
3627 a.get("max_steps")
3628 .and_then(|v| v.as_u64())
3629 .map(|n| n as u32),
3630 &deny_json,
3631 )?;
3632 (child_run, 1)
3633 }
3634 };
3635
3636 let outcome = run_agent(
3637 tree,
3638 &child_contract,
3639 child_run,
3640 child_depth,
3641 &child_policy,
3642 child_start,
3643 )
3644 .await?;
3645
3646 // A child that deferred pauses the whole tree, surfacing its request_id so
3647 // the caller can resume that child once a human decides.
3648 if let RunOutcome::AwaitingApproval { request_id, .. } = outcome {
3649 return Ok(SpawnResult::Paused { request_id });
3650 }
3651
3652 Ok(compose_child(child_run, goal, outcome))
3653}
3654
3655/// Fold one child's finished result back into the parent's observation log.
3656fn compose_child(child_run: i64, goal: &str, outcome: RunOutcome) -> SpawnResult {
3657 SpawnResult::Composed {
3658 decision: format!("spawned child {child_run}: {outcome:?}"),
3659 obs: format!("\n[child {child_run} \"{goal}\" -> {outcome:?}]\n"),
3660 }
3661}
3662
3663/// The rules an approver asked to remember, as a mergeable top layer.
3664///
3665/// A layer rather than an edit: merging is what keeps a remembered allow from
3666/// defeating a deny beneath it, since deny is absolute across the stack.
3667fn remembered_layer(rules: &[Rule]) -> Policy {
3668 let mut layer = Policy::permissive().layer("remembered");
3669 for r in rules {
3670 layer = layer.rule(r.act, r.effect, r.pattern.clone());
3671 }
3672 layer
3673}
3674
3675/// The outcome of authorizing the provider's own endpoint, before a run makes
3676/// its first outbound call.
3677enum ProviderAccess {
3678 /// Cleared to run, under the policy the provider layer has been merged into.
3679 Granted(Policy),
3680 /// An approver deferred; the pending decision is persisted under this id and
3681 /// the run stops before it ever dials.
3682 Pending(i64),
3683}
3684
3685/// Authorize the provider's endpoint once, before the first completion.
3686///
3687/// Once per run rather than once per step: a provider's endpoint is fixed for
3688/// the life of the provider, so re-asking each step would be the same question
3689/// with the same answer — and asking a human it repeatedly would train them to
3690/// wave it through.
3691///
3692/// The provider layer is merged *before* the check, not consulted after it. That
3693/// ordering is what makes a network-deny base usable: the `net` default denies,
3694/// the provider layer's allow rule beats a default, and a caller's explicit
3695/// `deny_net` still beats the allow because deny is absolute across layers. So
3696/// "deny everything but the model" needs no host list from the caller, while
3697/// "deny even the model" remains expressible — and fails fast as a refusal
3698/// rather than hanging on a call that is never made.
3699async fn authorize_provider<P: Provider>(
3700 provider: &P,
3701 policy: &Policy,
3702 store: &Store,
3703 run_id: i64,
3704 approver: &dyn Approver,
3705 watch: &Watch<'_>,
3706) -> Result<ProviderAccess> {
3707 // A provider that opens no connection (the mock providers tests drive the
3708 // loop with) has no endpoint to authorize.
3709 // Every host in the chain, not just the first: a `Fallback` can dial its
3710 // secondary, and a host the policy never checked would be a hole in an egress
3711 // model that is deny-by-default everywhere else.
3712 let urls = provider.endpoints();
3713 if urls.is_empty() {
3714 // A provider that opens no connection (the mock providers the tests drive
3715 // the loop with) has no endpoint to authorize.
3716 return Ok(ProviderAccess::Granted(policy.clone()));
3717 }
3718
3719 let mut effective = policy.clone();
3720 let mut ask: Option<String> = None;
3721 for url in urls {
3722 let Some(target) = net::target(url) else {
3723 return Err(crate::error::Error::Refused {
3724 act: "net".into(),
3725 target: url.to_string(),
3726 rule: None,
3727 layer: None,
3728 });
3729 };
3730 effective = effective.merge(net::provider_layer(&target));
3731 // Step 0: the authorization happens before the run's first step.
3732 let verdict = NetGuard::new(&effective)
3733 .tracing(store, run_id, 0)
3734 .watching(watch, 0)
3735 .check_target(&target)?;
3736 if verdict.effect == Effect::Ask {
3737 // One human decision covers the run; the first host that needs asking
3738 // is the one asked about.
3739 ask = Some(target.clone());
3740 }
3741 }
3742
3743 let Some(target) = ask else {
3744 return Ok(ProviderAccess::Granted(effective));
3745 };
3746
3747 // The run is now waiting on a human, before its first step. Step 0 for the
3748 // same reason the rows below are: the authorization precedes step 1.
3749 watch.emit(RunEvent::new(
3750 run_id,
3751 0,
3752 EventKind::ApprovalRequested {
3753 act: "net".into(),
3754 target: target.clone(),
3755 },
3756 ));
3757 match approver.decide(&Request::new(Act::Net, &target)).await {
3758 Decision::Approve { .. } => {
3759 let ev = PolicyEvent::decision(0, "net", &target, "approve", "approver");
3760 store.record_event(run_id, &ev)?;
3761 decided(watch, run_id, 0, &ev);
3762 Ok(ProviderAccess::Granted(effective))
3763 }
3764 Decision::Deny { reason } => {
3765 let ev = PolicyEvent::decision(0, "net", &target, "deny", "approver");
3766 store.record_event(run_id, &ev)?;
3767 decided(watch, run_id, 0, &ev);
3768 // Step 0: the run never started, so it finished having taken no steps.
3769 finish(store, watch, run_id, 0, 0, "refused")?;
3770 Err(crate::error::Error::Refused {
3771 act: "net".into(),
3772 target: format!("{target} — {reason}"),
3773 // The approver denied it, so the refusal is the human's, not a
3774 // rule's: there is no rule to name.
3775 rule: None,
3776 layer: None,
3777 })
3778 }
3779 Decision::Defer => {
3780 let ev = PolicyEvent::decision(0, "net", &target, "defer", "approver");
3781 store.record_event(run_id, &ev)?;
3782 decided(watch, run_id, 0, &ev);
3783 let request_id = store.put_pending(run_id, 0, "net", &target, None)?;
3784 finish(store, watch, run_id, 0, 0, "awaiting_approval")?;
3785 Ok(ProviderAccess::Pending(request_id))
3786 }
3787 }
3788}
3789
3790/// The result of dispatching one tool call.
3791enum Dispatched {
3792 /// The call resolved; fold `obs` into the observation log and carry any
3793 /// rules an approver asked to remember.
3794 ///
3795 /// `kind` and `target` travel with `obs` because assembly reasons about them:
3796 /// what a later observation supersedes, and which read a write invalidates,
3797 /// is a question about the tool and its subject — not something to recover by
3798 /// re-parsing the text afterwards.
3799 Continue {
3800 decision: String,
3801 obs: String,
3802 kind: ObsKind,
3803 target: Option<String>,
3804 /// Whether this call moved the workspace. Only a write can, and only a
3805 /// write that wrote something different — the signal stall detection reads.
3806 changed: bool,
3807 remember: Vec<Rule>,
3808 },
3809 /// An approver deferred; the action is persisted under `request_id` and the
3810 /// run stops until a human decides.
3811 Pause { request_id: i64 },
3812}
3813
3814impl Dispatched {
3815 /// A tool result: what it was, and the subject it names (if any).
3816 fn seen(
3817 decision: impl Into<String>,
3818 obs: impl Into<String>,
3819 kind: ObsKind,
3820 target: Option<String>,
3821 ) -> Self {
3822 Dispatched::Continue {
3823 decision: decision.into(),
3824 obs: obs.into(),
3825 kind,
3826 target,
3827 changed: false,
3828 remember: Vec::new(),
3829 }
3830 }
3831
3832 /// A failure or a refusal. Kept subject-less on purpose: an error about a
3833 /// path is not an observation *of* that path, so it must never supersede one.
3834 fn go(decision: impl Into<String>, obs: impl Into<String>) -> Self {
3835 Self::seen(decision, obs, ObsKind::Error, None)
3836 }
3837}
3838
3839/// Execute one tool call against the workspace, enforcing the policy and
3840/// consulting `approver` for anything it marks [`Effect::Ask`].
3841///
3842/// Tool-level failures (bad regex, path escape, a policy refusal) become
3843/// The images one request carries: the caller's, which are the task's subject and
3844/// ride every step, plus whatever the agent looked at last step, which rides one.
3845///
3846/// Bounded here rather than at either source, because neither can see the total.
3847/// Over the bound the oldest viewed images are dropped first and the model is not
3848/// told a lie about it — the drop is reported in the trace by the caller. The
3849/// caller's own images are never dropped: a task about an image that silently
3850/// stops carrying it is the failure this whole boundary exists to prevent, so an
3851/// over-budget contract is an error at the first step instead.
3852#[cfg(feature = "media")]
3853fn attach_media(
3854 contract: &TaskContract,
3855 pending: &mut PendingMedia,
3856) -> Result<Vec<crate::provider::Media>> {
3857 use crate::provider::MAX_REQUEST_IMAGE_BYTES;
3858 let fixed: usize = contract.images.iter().map(|m| m.byte_len()).sum();
3859 if fixed > MAX_REQUEST_IMAGE_BYTES {
3860 return Err(Error::Config(format!(
3861 "the contract's images total {fixed} bytes, over the \
3862 {MAX_REQUEST_IMAGE_BYTES}-byte per-request bound"
3863 )));
3864 }
3865 let mut out = contract.images.clone();
3866 let mut used = fixed;
3867 for m in pending.drain(..) {
3868 if used + m.byte_len() > MAX_REQUEST_IMAGE_BYTES {
3869 continue;
3870 }
3871 used += m.byte_len();
3872 out.push(m);
3873 }
3874 Ok(out)
3875}
3876
3877/// Images the agent looked at this step, waiting to be attached to the next
3878/// request.
3879///
3880/// An alias rather than a `cfg` on the parameter itself, so the two call sites
3881/// and the signature read the same in both feature states. Without the feature
3882/// it is `()`: there is nothing to carry and nothing to bound.
3883#[cfg(feature = "media")]
3884pub(crate) type PendingMedia = Vec<crate::provider::Media>;
3885/// See the `media` form above.
3886#[cfg(not(feature = "media"))]
3887pub(crate) type PendingMedia = ();
3888
3889/// observations the agent can recover from rather than failing the run — only
3890/// the model can decide what to do about them.
3891#[allow(clippy::too_many_arguments)]
3892// `pending_media` is `()` without the feature, and nothing reads it there.
3893#[cfg_attr(not(feature = "media"), allow(unused_variables))]
3894async fn dispatch(
3895 ws: &Workspace,
3896 call: &ToolCall,
3897 approver: &dyn Approver,
3898 store: &Store,
3899 run_id: i64,
3900 step: u32,
3901 mcp: &McpSession,
3902 custom: &Toolbox,
3903 skills: &Skills,
3904 cap: usize,
3905 memory_key: &str,
3906 watch: &Watch<'_>,
3907 depth: u32,
3908 pending_media: &mut PendingMedia,
3909 identity: &crate::tools::git::Identity,
3910 exec_timeout: Duration,
3911) -> Result<Dispatched> {
3912 let a = &call.arguments;
3913 let s = |k: &str| a.get(k).and_then(|v| v.as_str());
3914 // Announced before the call is made, so a watcher sees what the run is about
3915 // to do rather than only what it did. The subject is whichever of the
3916 // conventional argument names this tool uses; a tool that names none of them
3917 // is its own subject, which is what an MCP or registered tool call is.
3918 watch.emit(RunEvent::at_depth(
3919 run_id,
3920 step,
3921 depth,
3922 EventKind::ToolCall {
3923 name: call.name.clone(),
3924 target: ["path", "pattern", "name_glob", "glob", "key", "name"]
3925 .into_iter()
3926 .find_map(s)
3927 .unwrap_or(&call.name)
3928 .to_string(),
3929 },
3930 ));
3931 let name = call.name.as_str();
3932 Ok(match name {
3933 GREP_TOOL => {
3934 let pattern = s("pattern").unwrap_or_default();
3935 match ws.grep(pattern, s("path_glob")) {
3936 Ok(hits) => {
3937 let shown: Vec<String> = hits
3938 .iter()
3939 .take(OBS_GREP_CAP)
3940 .map(|m| format!("{}:{}: {}", m.path, m.line, m.text))
3941 .collect();
3942 Dispatched::seen(
3943 format!("grep {pattern:?} ({} hits)", hits.len()),
3944 bound(
3945 &format!("\n[grep {pattern:?}]\n{}\n", shown.join("\n")),
3946 cap,
3947 ObsKind::Grep,
3948 ),
3949 ObsKind::Grep,
3950 Some(pattern.to_string()),
3951 )
3952 }
3953 Err(e) => Dispatched::go("grep error", format!("\n[grep error] {e}\n")),
3954 }
3955 }
3956 FIND_TOOL => {
3957 let glob = s("name_glob").or_else(|| s("glob")).unwrap_or_default();
3958 match ws.find(glob) {
3959 Ok(paths) => Dispatched::seen(
3960 format!("find {glob:?} ({} paths)", paths.len()),
3961 bound(
3962 &format!("\n[find {glob:?}]\n{}\n", paths.join("\n")),
3963 cap,
3964 ObsKind::Find,
3965 ),
3966 ObsKind::Find,
3967 Some(glob.to_string()),
3968 ),
3969 Err(e) => Dispatched::go("find error", format!("\n[find error] {e}\n")),
3970 }
3971 }
3972 REMEMBER_TOOL => {
3973 let key = s("key").unwrap_or_default();
3974 let value = s("value").unwrap_or_default();
3975 if key.is_empty() || value.is_empty() {
3976 return Ok(Dispatched::go(
3977 "remember error",
3978 "\n[remember error] both key and value are required\n",
3979 ));
3980 }
3981 // The store bounds the entry and evicts oldest-first to hold the caps;
3982 // it writes no trace rows of its own, so the write and every eviction
3983 // are recorded here, where the run_id and step are known.
3984 let evicted = store.memory_put(memory_key, key, value, run_id, step)?;
3985 store.record_context_event(
3986 run_id,
3987 &ContextEvent::memory_write(
3988 step,
3989 format!("{key} ({} chars)", value.chars().count()),
3990 ),
3991 )?;
3992 // The key only: the row's detail carries a character count too, but
3993 // that is prose about the write, not the note's identity.
3994 watch.emit(RunEvent::at_depth(
3995 run_id,
3996 step,
3997 depth,
3998 EventKind::MemoryWrote {
3999 key: key.to_string(),
4000 },
4001 ));
4002 for gone in &evicted {
4003 store.record_context_event(
4004 run_id,
4005 &ContextEvent::memory_evict(step, format!("{gone} (evicted to hold the cap)")),
4006 )?;
4007 }
4008 info!(run_id, step, key, evicted = evicted.len(), "remembered");
4009 // No target: two notes under one key are the store's business, and a
4010 // remember is not an observation OF anything that could go stale.
4011 Dispatched::seen(
4012 format!("remembered {key}"),
4013 format!("\n[remember {key}]\n"),
4014 ObsKind::Tool,
4015 None,
4016 )
4017 }
4018 READ_FILE_TOOL => {
4019 let path = s("path").unwrap_or_default();
4020 match gate(
4021 ws,
4022 approver,
4023 store,
4024 run_id,
4025 step,
4026 Act::Read,
4027 path,
4028 None,
4029 watch,
4030 depth,
4031 )
4032 .await?
4033 {
4034 Gated::Refused { decision, obs } => Dispatched::go(decision, obs),
4035 Gated::Paused { request_id } => Dispatched::Pause { request_id },
4036 Gated::Go {
4037 target, remember, ..
4038 } => match ws.read_file(&target) {
4039 Ok(c) => Dispatched::Continue {
4040 decision: format!("read {target}"),
4041 obs: format!("\n[read {target}]\n{}\n", bound(&c, cap, ObsKind::Read)),
4042 kind: ObsKind::Read,
4043 target: Some(target.clone()),
4044 changed: false,
4045 remember,
4046 },
4047 Err(e) => Dispatched::go("read error", format!("\n[read error] {e}\n")),
4048 },
4049 }
4050 }
4051 #[cfg(feature = "media")]
4052 VIEW_IMAGE_TOOL => {
4053 let path = s("path").unwrap_or_default();
4054 // The extension decides the media type, and an unknown one is
4055 // reported rather than guessed. Checked before the gate only because
4056 // it costs nothing: the gate still runs for every path that could
4057 // actually be read, so this cannot be used to probe for a file's
4058 // existence outside the policy.
4059 let Some(media_type) = crate::provider::Media::media_type_for(path) else {
4060 return Ok(Dispatched::go(
4061 "view_image unsupported type",
4062 format!(
4063 "\n[view_image error] {path} is not an image this crate can send. \
4064 Supported: {}\n",
4065 crate::provider::IMAGE_MEDIA_TYPES.join(", ")
4066 ),
4067 ));
4068 };
4069 match gate(
4070 ws,
4071 approver,
4072 store,
4073 run_id,
4074 step,
4075 Act::Read,
4076 path,
4077 None,
4078 watch,
4079 depth,
4080 )
4081 .await?
4082 {
4083 Gated::Refused { decision, obs } => Dispatched::go(decision, obs),
4084 Gated::Paused { request_id } => Dispatched::Pause { request_id },
4085 Gated::Go {
4086 target, remember, ..
4087 } => match ws
4088 .read_bytes(&target)
4089 .map_err(|e| e.to_string())
4090 .and_then(|bytes| {
4091 crate::provider::Media::image(media_type, &bytes).map_err(|e| e.to_string())
4092 }) {
4093 Ok(media) => {
4094 // The observation records what was sent, not the image:
4095 // a digest, a size and a type. A trace that held the
4096 // bytes would grow by megabytes a step in exactly the
4097 // long unattended runs this crate exists for.
4098 let obs = format!(
4099 "\n[view_image {target}] attached to the next request \
4100 ({media_type}, {} bytes, digest {})\n",
4101 media.byte_len(),
4102 media.digest()
4103 );
4104 pending_media.push(media);
4105 Dispatched::Continue {
4106 decision: format!("viewed {target}"),
4107 obs,
4108 kind: ObsKind::Read,
4109 target: Some(target.clone()),
4110 changed: false,
4111 remember,
4112 }
4113 }
4114 Err(e) => {
4115 Dispatched::go("view_image error", format!("\n[view_image error] {e}\n"))
4116 }
4117 },
4118 }
4119 }
4120 WRITE_FILE_TOOL => {
4121 let path = s("path").unwrap_or_default();
4122 let content = s("content").unwrap_or_default();
4123 if path.is_empty() {
4124 return Ok(Dispatched::go(
4125 "write missing path",
4126 "\n[write error] write_file needs a \"path\" in workspace mode\n",
4127 ));
4128 }
4129 match gate(
4130 ws,
4131 approver,
4132 store,
4133 run_id,
4134 step,
4135 Act::Write,
4136 path,
4137 Some(content),
4138 watch,
4139 depth,
4140 )
4141 .await?
4142 {
4143 Gated::Refused { decision, obs } => Dispatched::go(decision, obs),
4144 Gated::Paused { request_id } => Dispatched::Pause { request_id },
4145 Gated::Go {
4146 target,
4147 content,
4148 remember,
4149 } => {
4150 let body = content.unwrap_or_default();
4151 // What is there now, read before the write so the line counts
4152 // below have something to compare against. The write gate has
4153 // already passed at this point; this is measurement, and the
4154 // content never reaches the model.
4155 let before = ws
4156 .resolve(&target)
4157 .ok()
4158 .and_then(|abs| std::fs::read_to_string(abs).ok())
4159 .unwrap_or_default();
4160 match ws.write_file(&target, &body) {
4161 Ok(wrote) => {
4162 record_edit(
4163 store,
4164 run_id,
4165 step,
4166 WRITE_FILE_TOOL,
4167 &target,
4168 &before,
4169 &body,
4170 );
4171 Dispatched::Continue {
4172 decision: format!("wrote {target}"),
4173 // A write that changed nothing says so, to the model as
4174 // well as to the trace: an agent rewriting a file with
4175 // what it already held is the shape of a stall, and it
4176 // cannot correct for what it is not told.
4177 obs: bound(
4178 &format!(
4179 "\n[wrote {target}] ({} chars{})\n",
4180 body.chars().count(),
4181 if wrote.moved_the_workspace() {
4182 ""
4183 } else {
4184 ", identical to what was already there — the \
4185 workspace did not change"
4186 }
4187 ),
4188 cap,
4189 ObsKind::Write,
4190 ),
4191 kind: ObsKind::Write,
4192 target: Some(target.clone()),
4193 changed: wrote.moved_the_workspace(),
4194 remember,
4195 }
4196 }
4197 Err(e) => Dispatched::go("write error", format!("\n[write error] {e}\n")),
4198 }
4199 }
4200 }
4201 }
4202 EDIT_FILE_TOOL => {
4203 let path = s("path").unwrap_or_default();
4204 let search = s("search").unwrap_or_default();
4205 let replacement = s("replace").unwrap_or_default();
4206 if path.is_empty() || search.is_empty() {
4207 return Ok(Dispatched::go(
4208 "edit missing arguments",
4209 "\n[edit error] edit_file needs a \"path\" and a non-empty \"search\"\n",
4210 ));
4211 }
4212 // The same act as `write_file`, so the same gate on the same path — a
4213 // partial edit is not a lesser write, and a policy that refuses one
4214 // refuses the other. The replacement text is offered as the content so
4215 // a human deciding an `Ask` sees what is going in rather than only
4216 // where.
4217 match gate(
4218 ws,
4219 approver,
4220 store,
4221 run_id,
4222 step,
4223 Act::Write,
4224 path,
4225 Some(replacement),
4226 watch,
4227 depth,
4228 )
4229 .await?
4230 {
4231 Gated::Refused { decision, obs } => Dispatched::go(decision, obs),
4232 Gated::Paused { request_id } => Dispatched::Pause { request_id },
4233 Gated::Go {
4234 target,
4235 content,
4236 remember,
4237 } => {
4238 let replacement = content.unwrap_or_default();
4239 match ws.edit_file(&target, search, &replacement) {
4240 Ok(wrote) => {
4241 // The replaced text against the text that replaced it.
4242 // Everything outside the match is byte-identical by
4243 // construction, so this is the same answer comparing the
4244 // whole file would give, without reading it twice.
4245 record_edit(
4246 store,
4247 run_id,
4248 step,
4249 EDIT_FILE_TOOL,
4250 &target,
4251 search,
4252 &replacement,
4253 );
4254 Dispatched::Continue {
4255 decision: format!("edited {target}"),
4256 obs: bound(
4257 &format!(
4258 "\n[edited {target}] replaced {} chars with {}{}\n",
4259 search.chars().count(),
4260 replacement.chars().count(),
4261 if wrote.moved_the_workspace() {
4262 ""
4263 } else {
4264 " — the replacement is identical to what was there, so \
4265 the workspace did not change"
4266 }
4267 ),
4268 cap,
4269 ObsKind::Write,
4270 ),
4271 kind: ObsKind::Write,
4272 target: Some(target.clone()),
4273 changed: wrote.moved_the_workspace(),
4274 remember,
4275 }
4276 }
4277 // A miss or an ambiguity is the model's to fix and says how:
4278 // an edit that guessed which of three occurrences was meant
4279 // is the failure this tool exists to make impossible.
4280 Err(e) => Dispatched::go("edit error", format!("\n[edit error] {e}\n")),
4281 }
4282 }
4283 }
4284 }
4285 EXEC_TOOL => {
4286 let argv: Vec<String> = a
4287 .get("argv")
4288 .and_then(|v| v.as_array())
4289 .map(|v| {
4290 v.iter()
4291 .filter_map(|x| x.as_str().map(str::to_string))
4292 .collect()
4293 })
4294 .unwrap_or_default();
4295 let Some(program) = argv.first().filter(|p| !p.trim().is_empty()).cloned() else {
4296 return Ok(Dispatched::go(
4297 "exec missing argv",
4298 "\n[exec error] exec needs a non-empty \"argv\" array whose first element is \
4299 the program\n",
4300 ));
4301 };
4302 // Two checks, and the second is the one that makes a useful rule
4303 // writable. The program alone is what `deny_exec(\"rm\")` names, and it
4304 // holds whatever the arguments are. The whole argv is what
4305 // `allow_exec(\"git log*\")` and `deny_exec(\"git push*\")` name — a
4306 // check on the program could not tell those two apart, which is the
4307 // weakness the git built-ins were built to route around and the reason
4308 // they still exist.
4309 let joined = argv.join(" ");
4310 let mut remembered: Vec<Rule> = Vec::new();
4311 let mut targets = vec![program.clone()];
4312 if joined != program {
4313 targets.push(joined.clone());
4314 }
4315 for target in targets {
4316 match gate(
4317 ws,
4318 approver,
4319 store,
4320 run_id,
4321 step,
4322 Act::Exec,
4323 &target,
4324 None,
4325 watch,
4326 depth,
4327 )
4328 .await?
4329 {
4330 Gated::Refused { decision, obs } => return Ok(Dispatched::go(decision, obs)),
4331 Gated::Paused { request_id } => return Ok(Dispatched::Pause { request_id }),
4332 Gated::Go { remember, .. } => remembered.extend(remember),
4333 }
4334 }
4335
4336 let outcome = Exec::new(ws.root(), exec_timeout, cap).run(&argv).await?;
4337 let (decision, obs) = match &outcome {
4338 ExecOutcome::Unavailable { reason } => (
4339 format!("{program} unavailable"),
4340 format!(
4341 "\n[exec unavailable] {reason}. This machine cannot run that command; \
4342 try another, or carry on without it.\n"
4343 ),
4344 ),
4345 ExecOutcome::TimedOut { after } => (
4346 format!("{program} timed out"),
4347 format!(
4348 "\n[exec timed out] `{joined}` was killed after {}s without finishing. \
4349 Nothing it printed was captured. Run something narrower, or expect this \
4350 command to need longer than this run allows.\n",
4351 after.as_secs()
4352 ),
4353 ),
4354 ExecOutcome::Ran {
4355 code,
4356 stdout,
4357 stderr,
4358 elided,
4359 } => {
4360 let body = crate::verify::joined_streams(stdout, stderr);
4361 (
4362 format!(
4363 "exec {program} {}",
4364 code.map_or("(signal)".to_string(), |c| format!("exit {c}"))
4365 ),
4366 format!(
4367 "\n[exec `{joined}` {}]{}\n{}\n",
4368 code.map_or("killed by a signal".to_string(), |c| format!("exit {c}")),
4369 if *elided > 0 {
4370 format!(
4371 " ({elided} characters of output elided from the middle; the \
4372 start and the end are both here)"
4373 )
4374 } else {
4375 String::new()
4376 },
4377 if body.trim().is_empty() {
4378 "(no output)"
4379 } else {
4380 body.trim_end()
4381 }
4382 ),
4383 )
4384 }
4385 };
4386 Dispatched::Continue {
4387 decision,
4388 obs,
4389 kind: ObsKind::Tool,
4390 target: None,
4391 // Deliberately not `changed`, even for a command that plainly
4392 // wrote files. The stall signal asks whether the agent is getting
4393 // anywhere, and running the same build a fourth time without
4394 // editing anything in between is the shape of an agent that is
4395 // not — the argv is part of the call signature, so a *different*
4396 // command is never mistaken for a repeat.
4397 changed: false,
4398 remember: remembered,
4399 }
4400 }
4401 // Loading one skill's body. Offered only when skills are configured, so
4402 // the name is not special otherwise: a run without skills falls through
4403 // to the unknown-tool arm like any other name.
4404 //
4405 // The body is read through the policy at the moment it is asked for,
4406 // against the skill file's ABSOLUTE path — a skills directory usually
4407 // sits outside the workspace root, so this is a policy check, not a
4408 // workspace-relative one (see `gate`).
4409 READ_SKILL_TOOL if !skills.is_empty() => {
4410 let name = s("name").unwrap_or_default();
4411 let Some(skill) = skills.get(name) else {
4412 // Not an error and not a failed run: the model asked for
4413 // something that does not exist, so it is told what does.
4414 return Ok(Dispatched::go(
4415 format!("unknown skill {name}"),
4416 format!(
4417 "\n[read_skill] there is no skill named {name:?}. Available: {}\n",
4418 skills.names().join(", ")
4419 ),
4420 ));
4421 };
4422 let path = skill.path.display().to_string();
4423 match gate(
4424 ws,
4425 approver,
4426 store,
4427 run_id,
4428 step,
4429 Act::Read,
4430 &path,
4431 None,
4432 watch,
4433 depth,
4434 )
4435 .await?
4436 {
4437 Gated::Refused { decision, obs } => Dispatched::go(decision, obs),
4438 Gated::Paused { request_id } => Dispatched::Pause { request_id },
4439 Gated::Go {
4440 target, remember, ..
4441 } => match std::fs::read_to_string(&target) {
4442 Ok(body) => {
4443 // Capped where it enters the context, like every other
4444 // tool result, under the same budget-derived cap.
4445 let (body, truncated) = crate::tools::cap_result(body, cap);
4446 info!(run_id, step, skill = name, truncated, "skill read");
4447 Dispatched::Continue {
4448 decision: format!("read skill {name}"),
4449 obs: format!("\n[skill {name}]\n{body}\n"),
4450 kind: ObsKind::Skill,
4451 target: Some(name.to_string()),
4452 changed: false,
4453 remember,
4454 }
4455 }
4456 Err(e) => Dispatched::go(
4457 format!("skill {name} read error"),
4458 format!("\n[skill {name} error] {e}\n"),
4459 ),
4460 },
4461 }
4462 }
4463 // Spreadsheet built-ins (0.14.0). Each one gates on the path the model
4464 // named, with `Act::Read` or `Act::Write`, through the same `gate` the
4465 // file built-ins use — so a refusal names the workbook rather than the
4466 // tool, a child's narrowed policy applies to documents exactly as it
4467 // applies to source, and the underlying module reaches the file only
4468 // through `Workspace`'s policy-checked byte IO.
4469 //
4470 // This is why they are built-ins and not registered `Tool`s: a registered
4471 // tool is authorised once by an exec check on its name and then does
4472 // whatever it likes to the filesystem, which for a capability whose whole
4473 // job is reading and writing the user's files is the wrong boundary.
4474 #[cfg(feature = "xlsx")]
4475 XLSX_SHEETS_TOOL | XLSX_READ_TOOL => {
4476 let path = s("path").unwrap_or_default();
4477 let sheet = s("sheet");
4478 let listing = name == XLSX_SHEETS_TOOL;
4479 match gate(
4480 ws,
4481 approver,
4482 store,
4483 run_id,
4484 step,
4485 Act::Read,
4486 path,
4487 None,
4488 watch,
4489 depth,
4490 )
4491 .await?
4492 {
4493 Gated::Refused { decision, obs } => Dispatched::go(decision, obs),
4494 Gated::Paused { request_id } => Dispatched::Pause { request_id },
4495 Gated::Go {
4496 target, remember, ..
4497 } => {
4498 let read = if listing {
4499 crate::tools::documents::xlsx::sheet_names(ws, &target)
4500 .map(|names| names.join("\n"))
4501 } else {
4502 crate::tools::documents::xlsx::read_sheet(ws, &target, sheet)
4503 };
4504 match read {
4505 Ok(text) => Dispatched::Continue {
4506 decision: format!("read {target}"),
4507 obs: format!(
4508 "\n[{name} {target}]\n{}\n",
4509 bound(&text, cap, ObsKind::Read)
4510 ),
4511 kind: ObsKind::Read,
4512 target: Some(target.clone()),
4513 changed: false,
4514 remember,
4515 },
4516 // A corrupt or non-xlsx file is the model's problem to
4517 // route around, not the run's to die on.
4518 Err(e) => Dispatched::go(
4519 "spreadsheet read error",
4520 format!("\n[{name} error] {e}\n"),
4521 ),
4522 }
4523 }
4524 }
4525 }
4526 #[cfg(feature = "xlsx")]
4527 XLSX_WRITE_TOOL | XLSX_SET_CELL_TOOL => {
4528 let path = s("path").unwrap_or_default();
4529 if path.is_empty() {
4530 return Ok(Dispatched::go(
4531 "spreadsheet missing path",
4532 format!("\n[{name} error] needs a \"path\" relative to the workspace root\n"),
4533 ));
4534 }
4535 let sheet = s("sheet").unwrap_or_default().to_string();
4536 let cell = s("cell").unwrap_or_default().to_string();
4537 let value = s("value").unwrap_or_default().to_string();
4538 let rows: Vec<Vec<String>> = call
4539 .arguments
4540 .get("rows")
4541 .and_then(|r| serde_json::from_value(r.clone()).ok())
4542 .unwrap_or_default();
4543 let creating = name == XLSX_WRITE_TOOL;
4544 // The approval preview is the change being asked for, not the file's
4545 // bytes: a human deciding on a spreadsheet write needs to see what it
4546 // does, and a workbook's raw bytes tell them nothing.
4547 let preview = if creating {
4548 format!(
4549 "create workbook {path} sheet {sheet} with {} row(s)",
4550 rows.len()
4551 )
4552 } else {
4553 format!("set {sheet}!{cell} to {value:?} in {path}")
4554 };
4555 match gate(
4556 ws,
4557 approver,
4558 store,
4559 run_id,
4560 step,
4561 Act::Write,
4562 path,
4563 Some(&preview),
4564 watch,
4565 depth,
4566 )
4567 .await?
4568 {
4569 Gated::Refused { decision, obs } => Dispatched::go(decision, obs),
4570 Gated::Paused { request_id } => Dispatched::Pause { request_id },
4571 Gated::Go {
4572 target, remember, ..
4573 } => {
4574 let wrote = if creating {
4575 crate::tools::documents::xlsx::write_new(ws, &target, &sheet, &rows)
4576 } else {
4577 crate::tools::documents::xlsx::set_cell(ws, &target, &sheet, &cell, &value)
4578 };
4579 match wrote {
4580 Ok(w) => Dispatched::Continue {
4581 decision: format!("wrote {target}"),
4582 obs: format!(
4583 "\n[{name} {target}] {}{}\n",
4584 preview,
4585 if w.moved_the_workspace() {
4586 ""
4587 } else {
4588 " — identical to what was already there, the \
4589 workspace did not change"
4590 }
4591 ),
4592 kind: ObsKind::Write,
4593 target: Some(target.clone()),
4594 changed: w.moved_the_workspace(),
4595 remember,
4596 },
4597 Err(e) => Dispatched::go(
4598 "spreadsheet write error",
4599 format!("\n[{name} error] {e}\n"),
4600 ),
4601 }
4602 }
4603 }
4604 }
4605 // The remaining document readers. Same gate, same reason as the
4606 // spreadsheet arms above: `Act::Read` on the path the model named.
4607 #[cfg(any(
4608 feature = "docx",
4609 feature = "pptx",
4610 feature = "pdf",
4611 feature = "barcode"
4612 ))]
4613 n if is_document_read(n) => {
4614 let path = s("path").unwrap_or_default();
4615 match gate(
4616 ws,
4617 approver,
4618 store,
4619 run_id,
4620 step,
4621 Act::Read,
4622 path,
4623 None,
4624 watch,
4625 depth,
4626 )
4627 .await?
4628 {
4629 Gated::Refused { decision, obs } => Dispatched::go(decision, obs),
4630 Gated::Paused { request_id } => Dispatched::Pause { request_id },
4631 Gated::Go {
4632 target, remember, ..
4633 } => match read_document(ws, name, &target) {
4634 Ok(text) => Dispatched::Continue {
4635 decision: format!("read {target}"),
4636 obs: format!(
4637 "\n[{name} {target}]\n{}\n",
4638 bound(&text, cap, ObsKind::Read)
4639 ),
4640 kind: ObsKind::Read,
4641 target: Some(target.clone()),
4642 changed: false,
4643 remember,
4644 },
4645 Err(e) => {
4646 Dispatched::go("document read error", format!("\n[{name} error] {e}\n"))
4647 }
4648 },
4649 }
4650 }
4651 // The remaining document writers.
4652 #[cfg(any(feature = "docx", feature = "pdf"))]
4653 n if is_document_write(n) => {
4654 let path = s("path").unwrap_or_default();
4655 if path.is_empty() {
4656 return Ok(Dispatched::go(
4657 "document missing path",
4658 format!("\n[{name} error] needs a \"path\" relative to the workspace root\n"),
4659 ));
4660 }
4661 let preview = describe_document_write(name, &call.arguments);
4662 match gate(
4663 ws,
4664 approver,
4665 store,
4666 run_id,
4667 step,
4668 Act::Write,
4669 path,
4670 Some(&preview),
4671 watch,
4672 depth,
4673 )
4674 .await?
4675 {
4676 Gated::Refused { decision, obs } => Dispatched::go(decision, obs),
4677 Gated::Paused { request_id } => Dispatched::Pause { request_id },
4678 Gated::Go {
4679 target, remember, ..
4680 } => match write_document(ws, name, &target, &call.arguments) {
4681 Ok(w) => Dispatched::Continue {
4682 decision: format!("wrote {target}"),
4683 obs: format!(
4684 "\n[{name} {target}] {preview}{}\n",
4685 if w.moved_the_workspace() {
4686 ""
4687 } else {
4688 " — identical to what was already there, the workspace \
4689 did not change"
4690 }
4691 ),
4692 kind: ObsKind::Write,
4693 target: Some(target.clone()),
4694 changed: w.moved_the_workspace(),
4695 remember,
4696 },
4697 Err(e) => {
4698 Dispatched::go("document write error", format!("\n[{name} error] {e}\n"))
4699 }
4700 },
4701 }
4702 }
4703 // A tool the embedding program registered. Registration made it
4704 // available; this check is what authorizes the call, on exactly the terms
4705 // an MCP tool gets — an exec check on the name the model used. Deciding
4706 // it here rather than at registration is what lets one policy layer hand
4707 // over a toolbox and another refuse a single tool in it.
4708 //
4709 // Registered tools are matched before the MCP arm and after the
4710 // built-ins, and `Toolbox::validate` has already guaranteed the three
4711 // sets are disjoint, so the order is documentation rather than a
4712 // tie-break.
4713 GIT_LOG_TOOL | GIT_STATUS_TOOL | GIT_DIFF_TOOL | GIT_ADD_TOOL | GIT_COMMIT_TOOL => {
4714 // Paths the model named, if any. Every one of them is data: `argv`
4715 // puts them after `--` and refuses a leading `-`.
4716 let paths: Vec<String> = a
4717 .get("paths")
4718 .and_then(|v| v.as_array())
4719 .map(|v| {
4720 v.iter()
4721 .filter_map(|x| x.as_str().map(str::to_string))
4722 .collect()
4723 })
4724 .unwrap_or_default();
4725
4726 // What the policy is asked, per tool. Reading history reads `.git`.
4727 // Staging copies a file's bytes into the object store, so it needs
4728 // `Act::Read` on that file — which is what stops a path the policy
4729 // denies from reaching a commit. Committing writes `.git`.
4730 let (repo_act, path_act) = match name {
4731 GIT_ADD_TOOL => (Act::Write, Some(Act::Read)),
4732 GIT_COMMIT_TOOL => (Act::Write, None),
4733 _ => (Act::Read, Some(Act::Read)),
4734 };
4735
4736 let mut remembered: Vec<Rule> = Vec::new();
4737 let mut targets: Vec<(Act, String)> = vec![(repo_act, GIT_DIR.to_string())];
4738 if let Some(act) = path_act {
4739 targets.extend(paths.iter().map(|p| (act, p.clone())));
4740 }
4741 let mut refused: Option<Dispatched> = None;
4742 for (act, target) in targets {
4743 match gate(
4744 ws, approver, store, run_id, step, act, &target, None, watch, depth,
4745 )
4746 .await?
4747 {
4748 Gated::Refused { decision, obs } => {
4749 refused = Some(Dispatched::go(decision, obs));
4750 break;
4751 }
4752 Gated::Paused { request_id } => {
4753 refused = Some(Dispatched::Pause { request_id });
4754 break;
4755 }
4756 Gated::Go { remember, .. } => remembered.extend(remember),
4757 }
4758 }
4759 if let Some(d) = refused {
4760 return Ok(d);
4761 }
4762
4763 let cmd = match name {
4764 GIT_STATUS_TOOL => GitCmd::Status { paths },
4765 GIT_DIFF_TOOL => GitCmd::Diff {
4766 staged: a.get("staged").and_then(serde_json::Value::as_bool) == Some(true),
4767 paths,
4768 },
4769 GIT_LOG_TOOL => GitCmd::Log {
4770 // Clamped rather than trusted: a model asking for the whole
4771 // history of a large repository would blow the observation
4772 // cap and learn nothing the first twenty commits do not say.
4773 max_count: a
4774 .get("max_count")
4775 .and_then(serde_json::Value::as_u64)
4776 .unwrap_or(20)
4777 .clamp(1, 200) as u32,
4778 paths,
4779 },
4780 GIT_ADD_TOOL => {
4781 if paths.is_empty() {
4782 return Ok(Dispatched::go(
4783 "git_add missing paths",
4784 "\n[git error] git_add needs a non-empty \"paths\" array\n".to_string(),
4785 ));
4786 }
4787 GitCmd::Add { paths }
4788 }
4789 _ => {
4790 let Some(message) = s("message").filter(|m| !m.trim().is_empty()) else {
4791 return Ok(Dispatched::go(
4792 "git_commit missing message",
4793 "\n[git error] git_commit needs a non-empty \"message\"\n".to_string(),
4794 ));
4795 };
4796 GitCmd::Commit {
4797 message: message.to_string(),
4798 identity: identity.clone(),
4799 }
4800 }
4801 };
4802
4803 let git = Git::new(ws.policy(), ws.root(), cap);
4804 match git.run(&cmd).await? {
4805 GitOutcome::Unavailable { reason } => Dispatched::go(
4806 "git unavailable",
4807 format!(
4808 "\n[git unavailable] {reason}. This workspace cannot be worked as a git \
4809 repository; carry on without it.\n"
4810 ),
4811 ),
4812 out @ GitOutcome::Ran { .. } => {
4813 let GitOutcome::Ran {
4814 code,
4815 stdout,
4816 stderr,
4817 } = &out
4818 else {
4819 unreachable!()
4820 };
4821 let ok = out.ok();
4822 let body = if stdout.trim().is_empty() && !ok {
4823 stderr.clone()
4824 } else {
4825 stdout.clone()
4826 };
4827 // A git that ran and failed is an observation, not a run
4828 // failure — the same treatment a malformed regex gets from
4829 // `grep`. The model reads the message and adapts.
4830 Dispatched::Continue {
4831 decision: format!(
4832 "{name} {}",
4833 if ok {
4834 "ok".to_string()
4835 } else {
4836 format!("exit {}", code.map_or("signal".into(), |c| c.to_string()))
4837 }
4838 ),
4839 obs: format!(
4840 "\n[{name}{}]\n{}\n",
4841 if ok {
4842 String::new()
4843 } else {
4844 " failed".to_string()
4845 },
4846 if body.trim().is_empty() {
4847 "(no output)"
4848 } else {
4849 body.trim_end()
4850 }
4851 ),
4852 kind: ObsKind::Tool,
4853 target: None,
4854 changed: matches!(cmd, GitCmd::Add { .. } | GitCmd::Commit { .. }) && ok,
4855 remember: remembered,
4856 }
4857 }
4858 }
4859 }
4860 name if custom.owns(name) => {
4861 match gate(
4862 ws,
4863 approver,
4864 store,
4865 run_id,
4866 step,
4867 Act::Exec,
4868 name,
4869 None,
4870 watch,
4871 depth,
4872 )
4873 .await?
4874 {
4875 Gated::Refused { decision, obs } => Dispatched::go(decision, obs),
4876 Gated::Paused { request_id } => Dispatched::Pause { request_id },
4877 Gated::Go { remember, .. } => {
4878 // `validate` ran at run start, so the lookup cannot miss.
4879 let tool = custom.get(name).expect("owns() and get() agree");
4880 match tool.invoke(&call.arguments).await {
4881 Ok(out) => {
4882 let (out, truncated) = crate::tools::cap_result(out, cap);
4883 info!(run_id, step, tool = name, truncated, "registered tool call");
4884 Dispatched::Continue {
4885 decision: format!("called {name}"),
4886 obs: format!("\n[{name}]\n{out}\n"),
4887 kind: ObsKind::Tool,
4888 target: Some(name.to_string()),
4889 changed: false,
4890 remember,
4891 }
4892 }
4893 // A tool's own failure is the model's problem to route
4894 // around, not the run's to die on — the same treatment a
4895 // bad regex gets from grep.
4896 Err(e) => {
4897 info!(run_id, step, tool = name, error = %e, "registered tool failed");
4898 Dispatched::Continue {
4899 decision: format!("{name} failed"),
4900 obs: format!("\n[{name} error] {e}\n"),
4901 kind: ObsKind::Error,
4902 target: None,
4903 changed: false,
4904 remember,
4905 }
4906 }
4907 }
4908 }
4909 }
4910 }
4911 // An MCP tool. Invoking it is an exec check on its namespaced name, so a
4912 // policy can allow a server generally and still deny one of its tools.
4913 name if mcp.owns(name) => {
4914 let verdict = ws.policy().check(Act::Exec, name);
4915 if verdict.effect != Effect::Allow {
4916 let mut ev = PolicyEvent::refusal(step, "exec", name);
4917 ev.rule = verdict.rule.clone();
4918 ev.layer = verdict.layer.clone();
4919 store.record_event(run_id, &ev)?;
4920 refused(watch, run_id, depth, &ev);
4921 let why = verdict
4922 .rule
4923 .as_deref()
4924 .map(|r| format!(" (rule {r})"))
4925 .unwrap_or_default();
4926 return Ok(Dispatched::go(
4927 format!("{name} refused"),
4928 format!("\n[{name} refused]{why} — the policy forbids calling this tool\n"),
4929 ));
4930 }
4931 let out = mcp
4932 .call_media(
4933 name,
4934 &call.arguments,
4935 store,
4936 run_id,
4937 step,
4938 cap,
4939 watch,
4940 depth,
4941 pending_media,
4942 )
4943 .await?;
4944 Dispatched::seen(
4945 format!("called {name}"),
4946 format!("\n[{name}]\n{out}\n"),
4947 ObsKind::Mcp,
4948 Some(name.to_string()),
4949 )
4950 }
4951 other => Dispatched::go(
4952 format!("unknown tool {other}"),
4953 format!("\n[unknown tool {other}]\n"),
4954 ),
4955 })
4956}
4957
4958/// What the policy and the approver together decided about one action.
4959enum Gated {
4960 /// Perform it, possibly in the form an approver rewrote it into.
4961 Go {
4962 target: String,
4963 content: Option<String>,
4964 remember: Vec<Rule>,
4965 },
4966 /// Do not perform it; `obs` is what the model is told.
4967 Refused { decision: String, obs: String },
4968 /// An approver deferred the decision.
4969 Paused { request_id: i64 },
4970}
4971
4972/// Evaluate one action against the policy, consulting `approver` only for the
4973/// sensitive-but-permitted tier.
4974///
4975/// A denied action never reaches the approver — refusal and approval are
4976/// different things. An approver that rewrites the action has the rewritten
4977/// form re-evaluated here, so it can narrow or redirect within the policy but
4978/// cannot move an action across a deny.
4979#[allow(clippy::too_many_arguments)]
4980async fn gate(
4981 ws: &Workspace,
4982 approver: &dyn Approver,
4983 store: &Store,
4984 run_id: i64,
4985 step: u32,
4986 act: Act,
4987 target: &str,
4988 content: Option<&str>,
4989 watch: &Watch<'_>,
4990 depth: u32,
4991) -> Result<Gated> {
4992 let kind = format!("{act:?}").to_lowercase();
4993 // Read and write targets are workspace paths, and are resolved so a symlink
4994 // cannot smuggle one outside the root. Exec and net targets are *names* — a
4995 // binary, an MCP tool, a registered tool, a host — and must not be resolved
4996 // against the root, or a file that happens to share a tool's name would
4997 // change what the policy said about calling it.
4998 //
4999 // An ABSOLUTE read/write target is not a workspace path at all — a skill
5000 // file normally lives outside the root — so it is decided by the policy
5001 // directly. `check_path` would resolve it against the root and deny it
5002 // unconditionally, which would make `read_skill` refusable only by accident.
5003 // This relaxes what the *gate* says, not what the workspace does:
5004 // `Workspace::resolve` rejects absolute paths outright and both `read_file`
5005 // and `write_file` go through it, so an absolute path still cannot leave the
5006 // root (asserted in tests/skills.rs).
5007 let check = |act: Act, target: &str| match act {
5008 Act::Exec | Act::Net => ws.policy().check(act, target),
5009 Act::Read | Act::Write if Path::new(target).is_absolute() => ws.policy().check(act, target),
5010 Act::Read | Act::Write => ws.check_path(act, target),
5011 };
5012 let verdict = check(act, target);
5013
5014 match verdict.effect {
5015 Effect::Deny => {
5016 let mut ev = PolicyEvent::refusal(step, &kind, target);
5017 if let (Some(rule), layer) = (verdict.rule.clone(), verdict.layer.clone()) {
5018 ev.rule = Some(rule);
5019 ev.layer = layer;
5020 }
5021 store.record_event(run_id, &ev)?;
5022 refused(watch, run_id, depth, &ev);
5023 let why = verdict
5024 .rule
5025 .as_deref()
5026 .map(|r| format!(" (rule {r})"))
5027 .unwrap_or_default();
5028 Ok(Gated::Refused {
5029 decision: format!("{kind} refused"),
5030 obs: format!("\n[{kind} refused] {target}{why} — the policy forbids this; try another path\n"),
5031 })
5032 }
5033 Effect::Allow => Ok(Gated::Go {
5034 target: target.to_string(),
5035 content: content.map(str::to_string),
5036 remember: Vec::new(),
5037 }),
5038 Effect::Ask => {
5039 let mut request = Request::new(act, target);
5040 if let Some(c) = content {
5041 request = request.with_content(c);
5042 }
5043 watch.emit(RunEvent::at_depth(
5044 run_id,
5045 step,
5046 depth,
5047 EventKind::ApprovalRequested {
5048 act: kind.clone(),
5049 target: target.to_string(),
5050 },
5051 ));
5052 match approver.decide(&request).await {
5053 Decision::Approve { modified, remember } => {
5054 let performed = modified.unwrap_or_else(|| request.clone());
5055 // The rewritten action gets the same scrutiny as the original.
5056 let recheck = check(act, &performed.target);
5057 if recheck.effect == Effect::Deny {
5058 let mut ev = PolicyEvent::refusal(step, &kind, &performed.target);
5059 ev.rule = recheck.rule.clone();
5060 ev.layer = recheck.layer.clone();
5061 store.record_event(run_id, &ev)?;
5062 // A refusal, not a decision: the row is a refusal too, and
5063 // the approval it overrode never took effect.
5064 refused(watch, run_id, depth, &ev);
5065 return Ok(Gated::Refused {
5066 decision: format!("{kind} refused after approval"),
5067 obs: format!(
5068 "\n[{kind} refused] {} — an approved change may not cross a deny\n",
5069 performed.target
5070 ),
5071 });
5072 }
5073 let mut ev = PolicyEvent::decision(step, &kind, target, "approve", "approver");
5074 if performed.target != target {
5075 ev = ev.with_performed(&performed.target);
5076 }
5077 store.record_event(run_id, &ev)?;
5078 decided(watch, run_id, depth, &ev);
5079 Ok(Gated::Go {
5080 target: performed.target,
5081 content: performed.content,
5082 remember,
5083 })
5084 }
5085 Decision::Deny { reason } => {
5086 let ev = PolicyEvent::decision(step, &kind, target, "deny", "approver");
5087 store.record_event(run_id, &ev)?;
5088 decided(watch, run_id, depth, &ev);
5089 Ok(Gated::Refused {
5090 decision: format!("{kind} denied"),
5091 obs: format!("\n[{kind} denied] {target} — {reason}\n"),
5092 })
5093 }
5094 Decision::Defer => {
5095 let ev = PolicyEvent::decision(step, &kind, target, "defer", "approver");
5096 store.record_event(run_id, &ev)?;
5097 decided(watch, run_id, depth, &ev);
5098 let request_id = store.put_pending(run_id, step, &kind, target, content)?;
5099 Ok(Gated::Paused { request_id })
5100 }
5101 }
5102 }
5103 }
5104}
5105
5106/// The key one workspace's durable memory is stored under.
5107///
5108/// Canonicalised, so the same directory reached by two different paths is one
5109/// workspace rather than two. The path as given is the fallback: a root that cannot
5110/// be canonicalised yet should still have memory rather than none.
5111fn memory_key(root: &Path) -> String {
5112 std::fs::canonicalize(root)
5113 .unwrap_or_else(|_| root.to_path_buf())
5114 .to_string_lossy()
5115 .into_owned()
5116}
5117
5118/// Call the provider, retrying a failing call up to `max_retries` times. Each
5119/// failed attempt is recorded in the trace. After the limit the error is
5120/// escalated (recorded, the run marked `escalated`, and returned).
5121#[allow(clippy::too_many_arguments)]
5122async fn complete_with_retry<P: Provider>(
5123 provider: &P,
5124 request: &CompletionRequest,
5125 contract: &TaskContract,
5126 store: &Store,
5127 run_id: i64,
5128 step: u32,
5129 watch: &Watch<'_>,
5130 depth: u32,
5131) -> Result<CompletionResponse> {
5132 // The general media boundary. Every completion in every loop goes through
5133 // here, so this covers an out-of-tree `Provider` as well as the three built
5134 // in — and it runs before the first attempt, so a refused request costs no
5135 // retry, no token and no wall clock. The built-in providers check again
5136 // inside their own `complete`, which is what stops a caller reaching one
5137 // directly from bypassing it.
5138 #[cfg(feature = "media")]
5139 crate::provider::ensure_media_accepted(provider.name(), provider.accepts_images(), request)?;
5140 let max_retries = contract.max_retries;
5141 let retry = contract.retry;
5142 let max_duration = contract.max_duration;
5143 let mut attempt = 0;
5144 loop {
5145 // 0.18.0: one `provider_calls` row per attempt, written here because this
5146 // is the only place that knows an attempt happened — a `Fallback` is one
5147 // `complete` call from the outside, and `steps.tokens` collapses a
5148 // retried step into a single integer attributed to nothing.
5149 //
5150 // The clock is the harness's own and brackets `complete`, so it includes
5151 // this crate's request building and stream consumption as well as the
5152 // provider's part. `CONTRACT.md` says so rather than implying the figure
5153 // is the vendor's.
5154 let started = std::time::Instant::now();
5155 let outcome = provider.complete(request.clone()).await;
5156 let latency_ms = started.elapsed().as_millis() as u64;
5157 record_provider_call(
5158 store,
5159 run_id,
5160 step,
5161 attempt,
5162 provider.name(),
5163 latency_ms,
5164 &outcome,
5165 );
5166 match outcome {
5167 Ok(response) => return Ok(response),
5168 // Only ask again if asking again could answer differently. Before
5169 // 0.11.0 every error was retried identically — including a 401 and a
5170 // missing API key, which cost three calls each to learn nothing, while
5171 // the one failure worth waiting for got no wait at all.
5172 Err(e) if attempt < max_retries && retryable(&e) => {
5173 attempt += 1;
5174 let wait = retry.wait(attempt, retry_after(&e));
5175 // A wait is not a way to escape the time budget: if the run's
5176 // deadline falls inside this sleep, stop now rather than after it.
5177 if let Some(max) = max_duration {
5178 let elapsed = store.elapsed_secs(run_id)?;
5179 if elapsed + wait.as_secs_f64() > max.as_secs_f64() {
5180 store.record(
5181 run_id,
5182 &StepRecord::new(
5183 step,
5184 // Same "escalated after <kind>" prefix as every
5185 // other escalation, so a trace reader grepping for
5186 // escalations does not miss this class of them.
5187 format!(
5188 "escalated after {} (a retry would outlast the time budget)",
5189 kind_of(&e)
5190 ),
5191 e.to_string(),
5192 ),
5193 )?;
5194 // The step that failed never completed, so the count comes from the
5195 // trace itself — which is also what `terminal_outcome` reports
5196 // when this run is later resumed, so the two cannot disagree.
5197 let steps = store.last_step(run_id)?;
5198 finish(store, watch, run_id, depth, steps, escalation_outcome(&e))?;
5199 return Err(e);
5200 }
5201 }
5202 store.record(
5203 run_id,
5204 &StepRecord::new(
5205 step,
5206 format!("retry {attempt} after {} in {:?}", kind_of(&e), wait),
5207 e.to_string(),
5208 ),
5209 )?;
5210 // The same `kind_of` string the row above records, so the event and
5211 // the trace name the failure identically rather than nearly so.
5212 watch.emit(RunEvent::at_depth(
5213 run_id,
5214 step,
5215 depth,
5216 EventKind::Retry {
5217 kind: kind_of(&e),
5218 attempt,
5219 delay_ms: wait.as_millis() as u64,
5220 },
5221 ));
5222 if !wait.is_zero() {
5223 tokio::time::sleep(wait).await;
5224 }
5225 }
5226 Err(e) => {
5227 store.record(
5228 run_id,
5229 &StepRecord::new(
5230 step,
5231 format!("escalated after {}", kind_of(&e)),
5232 e.to_string(),
5233 ),
5234 )?;
5235 // The step that failed never completed, so the count comes from the
5236 // trace itself — which is also what `terminal_outcome` reports
5237 // when this run is later resumed, so the two cannot disagree.
5238 let steps = store.last_step(run_id)?;
5239 finish(store, watch, run_id, depth, steps, escalation_outcome(&e))?;
5240 return Err(e);
5241 }
5242 }
5243 }
5244}
5245
5246/// Record one file change and the lines it added and removed (0.18.0).
5247///
5248/// Swallowed on a store failure for the same reason a provider call is: an edit
5249/// that reached the disk is not undone by failing to write its bookkeeping row,
5250/// and turning the run into an error here would lose the work as well as the
5251/// row.
5252fn record_edit(
5253 store: &Store,
5254 run_id: i64,
5255 step: u32,
5256 tool: &str,
5257 path: &str,
5258 before: &str,
5259 after: &str,
5260) {
5261 let edit = crate::state::Edit::measure(step, tool, path, before, after);
5262 if let Err(e) = store.record_edit(run_id, &edit) {
5263 tracing::warn!("could not record the edit to {path} at step {step}: {e}");
5264 }
5265}
5266
5267/// Record one provider call, answered or failed (0.18.0).
5268///
5269/// A failed attempt is recorded too, and deliberately: a model that produced
5270/// tokens and then hit a broken connection was still billed for them, so a trace
5271/// that kept only the winning attempt would understate the money.
5272///
5273/// A store that cannot take the row is logged and swallowed. The alternative is
5274/// failing a run that the provider answered because the accounting could not be
5275/// written, which trades a real answer for a bookkeeping entry.
5276fn record_provider_call(
5277 store: &Store,
5278 run_id: i64,
5279 step: u32,
5280 attempt: u32,
5281 provider: &str,
5282 latency_ms: u64,
5283 outcome: &Result<CompletionResponse>,
5284) {
5285 let call = crate::state::ProviderCall {
5286 step,
5287 attempt,
5288 provider: provider.to_string(),
5289 model: outcome.as_ref().ok().and_then(|r| r.model.clone()),
5290 usage: outcome.as_ref().ok().and_then(|r| r.usage),
5291 latency_ms,
5292 ttft_ms: outcome.as_ref().ok().and_then(|r| r.ttft_ms),
5293 finish_reason: outcome.as_ref().ok().and_then(|r| r.finish_reason.clone()),
5294 // The same short name the retry and escalation rows use, so the two
5295 // surfaces name one failure identically rather than nearly so.
5296 failure: outcome.as_ref().err().map(kind_of),
5297 };
5298 if let Err(e) = store.record_provider_call(run_id, &call) {
5299 tracing::warn!("could not record the provider call for step {step}: {e}");
5300 }
5301}
5302
5303/// Whether this failure is worth another attempt. A non-provider error — a bad
5304/// configuration, an IO failure — is not: it will fail the same way next time.
5305fn retryable(e: &Error) -> bool {
5306 matches!(e, Error::Provider { kind, .. } if kind.is_retryable())
5307}
5308
5309/// What the server asked us to wait, if it asked.
5310fn retry_after(e: &Error) -> Option<std::time::Duration> {
5311 match e {
5312 Error::Provider { retry_after, .. } => *retry_after,
5313 _ => None,
5314 }
5315}
5316
5317/// A short name for the trace row, so a reader can tell a wait from a hammer.
5318fn kind_of(e: &Error) -> String {
5319 match e {
5320 Error::Provider { kind, status, .. } => match status {
5321 Some(s) => format!("{kind:?} (HTTP {s})"),
5322 None => format!("{kind:?}"),
5323 },
5324 other => format!("{other}"),
5325 }
5326}
5327
5328/// The outcome string an escalation records, carrying whether the failure was one
5329/// another attempt could have survived.
5330///
5331/// Two strings rather than one because a resumed run and a trace reader have to
5332/// reach the same conclusion the caller did, and the caller's `Error` does not
5333/// survive into the store.
5334fn escalation_outcome(e: &Error) -> &'static str {
5335 if retryable(e) {
5336 "escalated_retryable"
5337 } else {
5338 "escalated_terminal"
5339 }
5340}
5341
5342fn system_prompt() -> String {
5343 "You are an agent that edits exactly one file to meet a stated specification. \
5344 Call the `write_file` tool with the file's full new contents. Do not explain; \
5345 make the edit. The file will be checked against the success criterion after \
5346 each write."
5347 .to_string()
5348}
5349
5350fn user_prompt(contract: &TaskContract, current: &str) -> String {
5351 let constraints = if contract.constraints.is_empty() {
5352 "(none)".to_string()
5353 } else {
5354 contract.constraints.join("; ")
5355 };
5356 format!(
5357 "Goal: {goal}\nConstraints: {constraints}\nSuccess criterion: {criterion}\n\n\
5358 Current file contents:\n---\n{current}\n---\n\n\
5359 Call write_file with the full new contents that satisfy the success criterion.",
5360 goal = contract.goal,
5361 criterion = contract.verify.describe(),
5362 )
5363}
5364
5365fn write_file_tool() -> ToolSpec {
5366 ToolSpec {
5367 name: WRITE_FILE_TOOL.to_string(),
5368 description: "Write the full new contents of the target file.".to_string(),
5369 parameters: json!({
5370 "type": "object",
5371 "properties": {
5372 "content": { "type": "string", "description": "Full new file contents." }
5373 },
5374 "required": ["content"]
5375 }),
5376 }
5377}
5378
5379fn workspace_system_prompt() -> String {
5380 "You are an agent working across a repository to meet a stated specification. \
5381 Use `grep` to search file contents and `find` to locate files by name, then \
5382 `read_file` to inspect a file before changing it, and `write_file` with the \
5383 file's path and full new contents to edit it. You may edit several files. \
5384 Work in small steps; after each of your steps the whole set is checked \
5385 against the success criterion. Do not explain; call tools."
5386 .to_string()
5387}
5388
5389/// Tell the model about tools the built-in prompt does not enumerate.
5390///
5391/// Without this the system prompt describes a world of exactly four tools while
5392/// the request carries more, and a model that trusts the prose over the schema
5393/// either ignores an MCP tool or, worse, calls one repeatedly without noticing
5394/// it already answered. Naming them — and saying plainly that a result lands in
5395/// the observations — is what turns a discovered tool into a usable one.
5396fn with_extra_tools(base: String, extra: &[ToolSpec]) -> String {
5397 if extra.is_empty() {
5398 return base;
5399 }
5400 let names: Vec<&str> = extra.iter().map(|t| t.name.as_str()).collect();
5401 format!(
5402 "{base} These extra tools are also available and work the same way: {}. \
5403 Each tool's result appears in the observations below; once a tool has \
5404 returned what you asked for, move on rather than calling it again.",
5405 names.join(", ")
5406 )
5407}
5408
5409/// [`READ_SKILL_TOOL`], offered only when the contract configures skills — a
5410/// tool that could do nothing but fail would cost a slot in every request of
5411/// every other run. Same rule MCP tools get: they appear when servers do.
5412fn skill_tool(skills: &Skills) -> Option<ToolSpec> {
5413 if skills.is_empty() {
5414 return None;
5415 }
5416 Some(ToolSpec {
5417 name: READ_SKILL_TOOL.to_string(),
5418 description: "Load one skill's full instructions into your observations, by the name it \
5419 is listed under. Read a skill when its description says it covers what you \
5420 are about to do."
5421 .to_string(),
5422 parameters: json!({
5423 "type": "object",
5424 "properties": {
5425 "name": { "type": "string", "description": "The skill's name, as listed in the system prompt." }
5426 },
5427 "required": ["name"]
5428 }),
5429 })
5430}
5431
5432/// Name the available skills in the system prompt: one line each, name and
5433/// description. A body is never here — that is what [`READ_SKILL_TOOL`] loads,
5434/// once, on demand, so a caller with twenty skills does not pay for twenty
5435/// bodies on every turn.
5436fn with_skill_catalog(base: String, skills: &Skills) -> String {
5437 if skills.is_empty() {
5438 return base;
5439 }
5440 format!(
5441 "{base}\n\nSkills available to you — instructions written for this repository. Only each \
5442 skill's name and description is shown; call `{READ_SKILL_TOOL}` with a name to read that \
5443 skill's full text when its description matches what you are doing.\n{}",
5444 skills.catalog()
5445 )
5446}
5447
5448fn workspace_user_prompt(
5449 contract: &TaskContract,
5450 observations: &str,
5451 toolchain: Option<&Toolchain>,
5452) -> String {
5453 let constraints = if contract.constraints.is_empty() {
5454 "(none)".to_string()
5455 } else {
5456 contract.constraints.join("; ")
5457 };
5458 let obs = if observations.is_empty() {
5459 "(nothing yet — start by grepping or finding)".to_string()
5460 } else {
5461 observations.to_string()
5462 };
5463 // Every turn, not only the first. An agent forty steps into a run has had the
5464 // first turn compacted out from under it by `ContextBudget`, and the project's
5465 // build command is exactly the fact it would then have to rediscover — which
5466 // is what this exists to stop it paying for twice.
5467 let project = match toolchain {
5468 Some(t) => format!("Project: {}\n", t.describe()),
5469 None => String::new(),
5470 };
5471 format!(
5472 "Goal: {goal}\nConstraints: {constraints}\nSuccess criterion: {criterion}\n\
5473 {project}\n\
5474 Observations so far (results of your tool calls):\n{obs}\n\n\
5475 Call a tool to make progress toward the success criterion.",
5476 goal = contract.goal,
5477 criterion = contract.verify.describe(),
5478 )
5479}
5480
5481fn tree_system_prompt() -> String {
5482 "You are an agent working across a repository to meet a stated specification. \
5483 Use `grep`, `find`, `read_file`, and `write_file` as in a normal run. You may \
5484 also decompose the work: call `spawn_agent` to launch a sub-agent that pursues \
5485 a smaller goal over the same workspace, and its result is reported back to you. \
5486 A sub-agent inherits your permissions and can only be more restricted, never \
5487 less. Prefer spawning when parts of the task are independent. Work in small \
5488 steps; the whole set is checked against the success criterion after each. Do \
5489 not explain; call tools."
5490 .to_string()
5491}
5492
5493/// Workspace tools plus [`SPAWN_TOOL`] — offered only inside an agent tree.
5494fn tree_tools() -> Vec<ToolSpec> {
5495 let mut tools = workspace_tools();
5496 tools.push(ToolSpec {
5497 name: SPAWN_TOOL.to_string(),
5498 description: "Spawn a contained sub-agent to pursue a smaller goal over the same \
5499 workspace. The sub-agent inherits your permissions (it can only be \
5500 further restricted) and its outcome is reported back to you."
5501 .to_string(),
5502 parameters: json!({
5503 "type": "object",
5504 "properties": {
5505 "goal": { "type": "string", "description": "The sub-agent's goal." },
5506 "verify_file": { "type": "string", "description": "File (relative to the workspace root) whose contents decide the sub-agent's success." },
5507 "verify_contains": { "type": "string", "description": "Text that file must contain for the sub-agent to succeed." },
5508 "deny_write": { "type": "array", "items": { "type": "string" }, "description": "Optional globs the sub-agent must not write — tightens its inherited policy." },
5509 "deny_net": { "type": "array", "items": { "type": "string" }, "description": "Optional host globs (host or host:port) the sub-agent must not reach — tightens its inherited policy." },
5510 "max_steps": { "type": "integer", "description": "Optional step budget for the sub-agent." }
5511 },
5512 "required": ["goal", "verify_file", "verify_contains"]
5513 }),
5514 });
5515 tools
5516}
5517
5518/// Whether `name` is a document tool that only reads.
5519///
5520/// A free function rather than a match arm per format: the arms differ only in
5521/// which module they call, and four near-identical arms would drift.
5522#[cfg(any(
5523 feature = "docx",
5524 feature = "pptx",
5525 feature = "pdf",
5526 feature = "barcode"
5527))]
5528fn is_document_read(name: &str) -> bool {
5529 #[cfg(feature = "docx")]
5530 if name == DOCX_READ_TOOL {
5531 return true;
5532 }
5533 #[cfg(feature = "pptx")]
5534 if name == PPTX_READ_TOOL {
5535 return true;
5536 }
5537 #[cfg(feature = "pdf")]
5538 if name == PDF_READ_TOOL {
5539 return true;
5540 }
5541 #[cfg(feature = "barcode")]
5542 if name == BARCODE_DECODE_TOOL {
5543 return true;
5544 }
5545 false
5546}
5547
5548/// Whether `name` is a document tool that writes.
5549#[cfg(any(feature = "docx", feature = "pdf"))]
5550fn is_document_write(name: &str) -> bool {
5551 #[cfg(feature = "docx")]
5552 if name == DOCX_WRITE_TOOL {
5553 return true;
5554 }
5555 #[cfg(feature = "pdf")]
5556 if name == PDF_WRITE_TOOL || name == PDF_WATERMARK_TOOL || name == PDF_FILL_FORM_TOOL {
5557 return true;
5558 }
5559 false
5560}
5561
5562/// Read one document, choosing the reader by the tool the model called rather
5563/// than by the file's extension: the model named the format it believes it is
5564/// dealing with, and letting the extension decide would silently read something
5565/// else than what was asked for.
5566#[cfg(any(
5567 feature = "docx",
5568 feature = "pptx",
5569 feature = "pdf",
5570 feature = "barcode"
5571))]
5572fn read_document(ws: &Workspace, name: &str, target: &str) -> Result<String> {
5573 use crate::tools::documents;
5574 match name {
5575 #[cfg(feature = "docx")]
5576 n if n == DOCX_READ_TOOL => documents::docx::read_text(ws, target),
5577 #[cfg(feature = "pptx")]
5578 n if n == PPTX_READ_TOOL => documents::pptx::read_text(ws, target),
5579 #[cfg(feature = "pdf")]
5580 n if n == PDF_READ_TOOL => documents::pdf::read_text(ws, target),
5581 #[cfg(feature = "barcode")]
5582 n if n == BARCODE_DECODE_TOOL => documents::barcode::decode(ws, target).map(|found| {
5583 if found.is_empty() {
5584 // Not an error: "I looked and there was nothing there" is a fact
5585 // the model can act on, and the run continues.
5586 "no barcode or QR code found in this image".to_string()
5587 } else {
5588 found
5589 .iter()
5590 .map(|d| format!("{}: {}", d.format, d.text))
5591 .collect::<Vec<_>>()
5592 .join("\n")
5593 }
5594 }),
5595 other => Err(crate::error::Error::Config(format!(
5596 "not a document read tool: {other}"
5597 ))),
5598 }
5599}
5600
5601/// What a document write is about to do, for the approval preview and the trace.
5602/// The change, never the bytes — a human deciding on a document write cannot
5603/// decide on a blob.
5604#[cfg(any(feature = "docx", feature = "pdf"))]
5605fn describe_document_write(name: &str, args: &serde_json::Value) -> String {
5606 let count = |key: &str| {
5607 args.get(key)
5608 .and_then(|v| v.as_array())
5609 .map(|a| a.len())
5610 .unwrap_or(0)
5611 };
5612 match name {
5613 #[cfg(feature = "docx")]
5614 n if n == DOCX_WRITE_TOOL => {
5615 format!("create a document of {} paragraph(s)", count("paragraphs"))
5616 }
5617 #[cfg(feature = "pdf")]
5618 n if n == PDF_WRITE_TOOL => format!("create a PDF of {} page(s)", count("pages")),
5619 #[cfg(feature = "pdf")]
5620 n if n == PDF_WATERMARK_TOOL => format!(
5621 "watermark every page with {:?}",
5622 args.get("text")
5623 .and_then(|v| v.as_str())
5624 .unwrap_or_default()
5625 ),
5626 #[cfg(feature = "pdf")]
5627 n if n == PDF_FILL_FORM_TOOL => format!("fill {} form field(s)", count("fields")),
5628 other => format!("write via {other}"),
5629 }
5630}
5631
5632/// Perform one document write, chosen by the tool the model called.
5633#[cfg(any(feature = "docx", feature = "pdf"))]
5634fn write_document(
5635 ws: &Workspace,
5636 name: &str,
5637 target: &str,
5638 args: &serde_json::Value,
5639) -> Result<crate::tools::workspace::Wrote> {
5640 use crate::tools::documents;
5641 #[allow(unused_variables)]
5642 let strings = |key: &str| -> Vec<String> {
5643 args.get(key)
5644 .and_then(|v| serde_json::from_value(v.clone()).ok())
5645 .unwrap_or_default()
5646 };
5647 match name {
5648 #[cfg(feature = "docx")]
5649 n if n == DOCX_WRITE_TOOL => documents::docx::write_new(ws, target, &strings("paragraphs")),
5650 #[cfg(feature = "pdf")]
5651 n if n == PDF_WRITE_TOOL => documents::pdf::write_new(ws, target, &strings("pages")),
5652 #[cfg(feature = "pdf")]
5653 n if n == PDF_WATERMARK_TOOL => documents::pdf::watermark(
5654 ws,
5655 target,
5656 args.get("text")
5657 .and_then(|v| v.as_str())
5658 .unwrap_or_default(),
5659 ),
5660 #[cfg(feature = "pdf")]
5661 n if n == PDF_FILL_FORM_TOOL => {
5662 let fields: Vec<(String, String)> = args
5663 .get("fields")
5664 .and_then(|v| v.as_object().cloned())
5665 .map(|m| {
5666 m.into_iter()
5667 .map(|(k, v)| (k, v.as_str().unwrap_or_default().to_string()))
5668 .collect()
5669 })
5670 .unwrap_or_default();
5671 documents::pdf::fill_form(ws, target, &fields)
5672 }
5673 other => Err(crate::error::Error::Config(format!(
5674 "not a document write tool: {other}"
5675 ))),
5676 }
5677}
5678
5679fn workspace_tools() -> Vec<ToolSpec> {
5680 #[allow(unused_mut)]
5681 let mut v = vec![
5682 ToolSpec {
5683 name: GREP_TOOL.to_string(),
5684 description: "Search file contents by regex (a plain substring is valid). Returns file:line: matches.".to_string(),
5685 parameters: json!({
5686 "type": "object",
5687 "properties": {
5688 "pattern": { "type": "string", "description": "Regex or substring to search for." },
5689 "path_glob": { "type": "string", "description": "Optional glob limiting which files are searched, e.g. src/*.rs." }
5690 },
5691 "required": ["pattern"]
5692 }),
5693 },
5694 ToolSpec {
5695 name: FIND_TOOL.to_string(),
5696 description: "List files whose name or relative path matches a glob (* and ?).".to_string(),
5697 parameters: json!({
5698 "type": "object",
5699 "properties": {
5700 "name_glob": { "type": "string", "description": "Glob to match, e.g. *.rs or src/*.rs." }
5701 },
5702 "required": ["name_glob"]
5703 }),
5704 },
5705 ToolSpec {
5706 name: READ_FILE_TOOL.to_string(),
5707 description: "Read a file (path relative to the workspace root) into context.".to_string(),
5708 parameters: json!({
5709 "type": "object",
5710 "properties": {
5711 "path": { "type": "string", "description": "File path relative to the workspace root." }
5712 },
5713 "required": ["path"]
5714 }),
5715 },
5716 ToolSpec {
5717 name: GIT_STATUS_TOOL.to_string(),
5718 description: "Show what has changed in the git repository at the workspace root: \
5719 modified, staged and untracked files."
5720 .to_string(),
5721 parameters: json!({
5722 "type": "object",
5723 "properties": {
5724 "paths": { "type": "array", "items": { "type": "string" }, "description": "Optional paths to limit the report to." }
5725 }
5726 }),
5727 },
5728 ToolSpec {
5729 name: GIT_DIFF_TOOL.to_string(),
5730 description: "Show the diff of the working tree, or of what is staged.".to_string(),
5731 parameters: json!({
5732 "type": "object",
5733 "properties": {
5734 "staged": { "type": "boolean", "description": "Diff what is staged instead of the working tree." },
5735 "paths": { "type": "array", "items": { "type": "string" }, "description": "Optional paths to limit the diff to." }
5736 }
5737 }),
5738 },
5739 ToolSpec {
5740 name: GIT_LOG_TOOL.to_string(),
5741 description: "Read the repository's recent commit history, newest first.".to_string(),
5742 parameters: json!({
5743 "type": "object",
5744 "properties": {
5745 "max_count": { "type": "integer", "description": "How many commits to show (1-200, default 20)." },
5746 "paths": { "type": "array", "items": { "type": "string" }, "description": "Optional paths to limit the history to." }
5747 }
5748 }),
5749 },
5750 ToolSpec {
5751 name: GIT_ADD_TOOL.to_string(),
5752 description: "Stage the named files for the next commit. Honours .gitignore; an \
5753 ignored file is reported rather than staged."
5754 .to_string(),
5755 parameters: json!({
5756 "type": "object",
5757 "properties": {
5758 "paths": { "type": "array", "items": { "type": "string" }, "description": "Files to stage, relative to the workspace root." }
5759 },
5760 "required": ["paths"]
5761 }),
5762 },
5763 ToolSpec {
5764 name: GIT_COMMIT_TOOL.to_string(),
5765 description: "Commit what you have staged, on the branch that is checked out. There \
5766 is no push, no branch switching and no history rewriting: your work \
5767 stays local for a human to review."
5768 .to_string(),
5769 parameters: json!({
5770 "type": "object",
5771 "properties": {
5772 "message": { "type": "string", "description": "The commit message." }
5773 },
5774 "required": ["message"]
5775 }),
5776 },
5777 #[cfg(feature = "media")]
5778 ToolSpec {
5779 name: VIEW_IMAGE_TOOL.to_string(),
5780 description: "Look at an image in the workspace. The image is attached to your next \
5781 message, so you see it on the following step rather than in this tool's \
5782 result. Costs a step; the file must be a jpeg, png, gif or webp."
5783 .to_string(),
5784 parameters: json!({
5785 "type": "object",
5786 "properties": {
5787 "path": { "type": "string", "description": "Image path relative to the workspace root." }
5788 },
5789 "required": ["path"]
5790 }),
5791 },
5792 ToolSpec {
5793 name: REMEMBER_TOOL.to_string(),
5794 description: "Record a short fact or decision worth keeping for a later run over this \
5795 workspace — a build command, a layout you had to discover, a decision and \
5796 why. Notes are yours, not instructions, and are recalled at the start of \
5797 later runs so you do not rediscover the same thing twice."
5798 .to_string(),
5799 parameters: json!({
5800 "type": "object",
5801 "properties": {
5802 "key": { "type": "string", "description": "Short name to recall it by; writing the same key again replaces it." },
5803 "value": { "type": "string", "description": "The fact, in one or two sentences." }
5804 },
5805 "required": ["key", "value"]
5806 }),
5807 },
5808 ToolSpec {
5809 name: WRITE_FILE_TOOL.to_string(),
5810 description: "Write the full new contents of a file (path relative to the workspace root); creates it if absent.".to_string(),
5811 parameters: json!({
5812 "type": "object",
5813 "properties": {
5814 "path": { "type": "string", "description": "File path relative to the workspace root." },
5815 "content": { "type": "string", "description": "Full new file contents." }
5816 },
5817 "required": ["path", "content"]
5818 }),
5819 },
5820 ToolSpec {
5821 name: EDIT_FILE_TOOL.to_string(),
5822 description: "Change part of an existing file, leaving the rest of it exactly as it \
5823 was. Prefer this to write_file for anything but a new file. The search \
5824 text must appear EXACTLY ONCE: if it appears zero times or more than \
5825 once the edit is refused and nothing changes, so include enough \
5826 surrounding lines to make it unique."
5827 .to_string(),
5828 parameters: json!({
5829 "type": "object",
5830 "properties": {
5831 "path": { "type": "string", "description": "File path relative to the workspace root." },
5832 "search": { "type": "string", "description": "The exact text to replace, copied from the file including its whitespace. Must occur exactly once." },
5833 "replace": { "type": "string", "description": "What to put in its place. An empty string deletes the searched text." }
5834 },
5835 "required": ["path", "search", "replace"]
5836 }),
5837 },
5838 ToolSpec {
5839 name: EXEC_TOOL.to_string(),
5840 description: "Run a command in the workspace root and read back its exit status and \
5841 its output — the project's own build, tests, linter, formatter or \
5842 package manager. There is NO shell: give the command as an array of \
5843 strings, one element per argument. Pipes, redirection, `&&`, `;` and \
5844 `$(...)` are not interpreted; they are ordinary characters inside \
5845 whichever argument contains them. Run one command per call. A command \
5846 that runs too long is killed and reported as a timeout, and very long \
5847 output keeps its start and its end with the middle elided."
5848 .to_string(),
5849 parameters: json!({
5850 "type": "object",
5851 "properties": {
5852 "argv": {
5853 "type": "array",
5854 "description": "The command, one array element per argument, program first — e.g. [\"npm\", \"test\"] or [\"cargo\", \"test\", \"--all-features\"].",
5855 "items": { "type": "string" }
5856 }
5857 },
5858 "required": ["argv"]
5859 }),
5860 },
5861 ];
5862 #[cfg(feature = "xlsx")]
5863 // Offered only when the feature is on. A model is told about a tool it can
5864 // actually call, never about one the build does not contain.
5865 v.extend([
5866 ToolSpec {
5867 name: XLSX_SHEETS_TOOL.to_string(),
5868 description: "List the sheet names of an .xlsx workbook in the workspace.".to_string(),
5869 parameters: json!({
5870 "type": "object",
5871 "properties": {
5872 "path": { "type": "string", "description": "Workbook path relative to the workspace root." }
5873 },
5874 "required": ["path"]
5875 }),
5876 },
5877 ToolSpec {
5878 name: XLSX_READ_TOOL.to_string(),
5879 description: "Read one sheet of an .xlsx workbook as text. Omit \"sheet\" for the first sheet.".to_string(),
5880 parameters: json!({
5881 "type": "object",
5882 "properties": {
5883 "path": { "type": "string", "description": "Workbook path relative to the workspace root." },
5884 "sheet": { "type": "string", "description": "Sheet name; the first sheet if omitted." }
5885 },
5886 "required": ["path"]
5887 }),
5888 },
5889 ToolSpec {
5890 name: XLSX_WRITE_TOOL.to_string(),
5891 description: "Create a NEW .xlsx workbook with one sheet of rows. Replaces the file if it exists; to change one cell of an existing workbook use xlsx_set_cell instead, which keeps the rest of it.".to_string(),
5892 parameters: json!({
5893 "type": "object",
5894 "properties": {
5895 "path": { "type": "string", "description": "Workbook path relative to the workspace root." },
5896 "sheet": { "type": "string", "description": "Name for the sheet." },
5897 "rows": {
5898 "type": "array",
5899 "description": "Rows, each an array of cell values as strings.",
5900 "items": { "type": "array", "items": { "type": "string" } }
5901 }
5902 },
5903 "required": ["path", "sheet", "rows"]
5904 }),
5905 },
5906 ToolSpec {
5907 name: XLSX_SET_CELL_TOOL.to_string(),
5908 description: "Set one cell of an EXISTING .xlsx workbook, keeping every other sheet, cell and format as it was.".to_string(),
5909 parameters: json!({
5910 "type": "object",
5911 "properties": {
5912 "path": { "type": "string", "description": "Workbook path relative to the workspace root." },
5913 "sheet": { "type": "string", "description": "Sheet name." },
5914 "cell": { "type": "string", "description": "A1-style cell reference, e.g. B7." },
5915 "value": { "type": "string", "description": "New cell value." }
5916 },
5917 "required": ["path", "sheet", "cell", "value"]
5918 }),
5919 },
5920 ]);
5921 #[cfg(feature = "docx")]
5922 v.extend([
5923 ToolSpec {
5924 name: DOCX_READ_TOOL.to_string(),
5925 description: "Read the text of a .docx Word document in the workspace.".to_string(),
5926 parameters: json!({
5927 "type": "object",
5928 "properties": { "path": { "type": "string", "description": "Document path relative to the workspace root." } },
5929 "required": ["path"]
5930 }),
5931 },
5932 ToolSpec {
5933 name: DOCX_WRITE_TOOL.to_string(),
5934 description: "Create a NEW .docx Word document from paragraphs. There is no in-place edit for Word: to change an existing document, read it and write a new one, accepting that formatting this crate does not model is not carried over.".to_string(),
5935 parameters: json!({
5936 "type": "object",
5937 "properties": {
5938 "path": { "type": "string", "description": "Document path relative to the workspace root." },
5939 "paragraphs": { "type": "array", "description": "Paragraphs, in order.", "items": { "type": "string" } }
5940 },
5941 "required": ["path", "paragraphs"]
5942 }),
5943 },
5944 ]);
5945 #[cfg(feature = "pptx")]
5946 v.push(ToolSpec {
5947 name: PPTX_READ_TOOL.to_string(),
5948 description: "Read the text of a .pptx slide deck, slide by slide. Reading only — this crate cannot write PowerPoint.".to_string(),
5949 parameters: json!({
5950 "type": "object",
5951 "properties": { "path": { "type": "string", "description": "Deck path relative to the workspace root." } },
5952 "required": ["path"]
5953 }),
5954 });
5955 #[cfg(feature = "pdf")]
5956 v.extend([
5957 ToolSpec {
5958 name: PDF_READ_TOOL.to_string(),
5959 description: "Extract the text of a PDF. Best effort: reading order across columns and tables is not guaranteed.".to_string(),
5960 parameters: json!({
5961 "type": "object",
5962 "properties": { "path": { "type": "string", "description": "PDF path relative to the workspace root." } },
5963 "required": ["path"]
5964 }),
5965 },
5966 ToolSpec {
5967 name: PDF_WRITE_TOOL.to_string(),
5968 description: "Create a NEW PDF, one page per string.".to_string(),
5969 parameters: json!({
5970 "type": "object",
5971 "properties": {
5972 "path": { "type": "string", "description": "PDF path relative to the workspace root." },
5973 "pages": { "type": "array", "description": "Page text, one entry per page.", "items": { "type": "string" } }
5974 },
5975 "required": ["path", "pages"]
5976 }),
5977 },
5978 ToolSpec {
5979 name: PDF_WATERMARK_TOOL.to_string(),
5980 description: "Stamp text across every page of an existing PDF, keeping its content.".to_string(),
5981 parameters: json!({
5982 "type": "object",
5983 "properties": {
5984 "path": { "type": "string", "description": "PDF path relative to the workspace root." },
5985 "text": { "type": "string", "description": "Watermark text." }
5986 },
5987 "required": ["path", "text"]
5988 }),
5989 },
5990 ToolSpec {
5991 name: PDF_FILL_FORM_TOOL.to_string(),
5992 description: "Fill the form fields of an existing PDF, by field name.".to_string(),
5993 parameters: json!({
5994 "type": "object",
5995 "properties": {
5996 "path": { "type": "string", "description": "PDF path relative to the workspace root." },
5997 "fields": { "type": "object", "description": "Field name to value.", "additionalProperties": { "type": "string" } }
5998 },
5999 "required": ["path", "fields"]
6000 }),
6001 },
6002 ]);
6003 #[cfg(feature = "barcode")]
6004 v.push(ToolSpec {
6005 name: BARCODE_DECODE_TOOL.to_string(),
6006 description: "Decode barcodes and QR codes from a PNG or JPEG in the workspace. Reports plainly when the image contains none.".to_string(),
6007 parameters: json!({
6008 "type": "object",
6009 "properties": { "path": { "type": "string", "description": "Image path relative to the workspace root." } },
6010 "required": ["path"]
6011 }),
6012 });
6013 v
6014}