Skip to main content

agentd/runtime/
waits.rs

1// SPDX-License-Identifier: AGPL-3.0-only
2//! **Orchestration steps** — the integration, intelligence and control kinds:
3//! `wait` (on a resource update, a CEL condition, a signal, a run, a subagent,
4//! a conversation message, or a deadline), `join` (fan-in of runs/subagents),
5//! `workflow` (a child run: `sync | async | detached`, `cascade`),
6//! `workflow.signal` / `workflow.wait` / `workflow.cancel`, `subagent`,
7//! `human` (through the `ask_human` contract), `mcp.resource`
8//! (`read | list | prompt | complete`), `a2a.delegate` (the outbound A2A
9//! client),
10//! the `think` presets (`classify | extract | summarize | judge | route`), and
11//! step `cache` (memoized outputs by input hash). Waits suspend the step
12//! durably (`StepState.wait`) and are resolved by the loop's tick.
13
14use super::events::kinds;
15use super::reactor::{PendingKind, Runtime, Target};
16use super::tools::{ToolCaller, ToolOutcome};
17use crate::context::ROOT;
18use crate::engine::model::Step;
19use crate::engine::run::StepStatus;
20use crate::engine::template::Data;
21use crate::state::{Kind, now_ms};
22use serde_json::{Map, Value, json};
23#[cfg(feature = "a2a")]
24use std::time::Duration;
25
26/// A durable wait record kept in `StepState.wait`.
27/// The message id an `idempotency:` declaration asks for — the step's derived
28/// key, or the declared `value` (an application-level identity, which is
29/// stronger: it also collides two different RUNS attempting the same
30/// real-world operation). `None` when the step declared nothing, keeping
31/// today's unique-per-send minting: a `goto` re-entry is a LOGICALLY new send,
32/// and silently deduping it on a peer would lose notifications — so retry
33///-safety here is opt-in, per node, like everywhere else.
34#[cfg(feature = "a2a")]
35fn idempotency_message_id(
36    spec: &Map<String, Value>,
37    run_id: &str,
38    step_id: &str,
39) -> Option<String> {
40    let idem = spec.get("idempotency")?;
41    if idem.as_bool() == Some(false) {
42        return None;
43    }
44    Some(
45        idem.get("value")
46            .and_then(Value::as_str)
47            .map(str::to_string)
48            .unwrap_or_else(|| crate::engine::run::idempotency_key(run_id, step_id)),
49    )
50}
51
52pub(crate) fn wait_record(kind: &str, extra: Value, timeout_ms: Option<u64>) -> Value {
53    let mut w = json!({"kind": kind, "since_ms": now_ms()});
54    if let Some(t) = timeout_ms {
55        w["deadline_ms"] = json!(now_ms() + t);
56    }
57    if let Value::Object(o) = extra {
58        for (k, v) in o {
59            w[k] = v;
60        }
61    }
62    w
63}
64
65/// How many events one stream's waiters advance through per tick. Bounds the
66/// single-writer loop's time on a busy stream; the anchor persists, so a
67/// backlog is worked off across ticks rather than dropped.
68const EVENT_SCAN: usize = 256;
69
70/// The durable key of one event — mirrors the stream module's layout.
71fn event_key(stream: &str, seq: u64) -> String {
72    format!("{stream}/e{seq:020}")
73}
74
75impl Runtime {
76    /// Execute one of the orchestration kinds (called from `execute_step`).
77    pub(crate) fn execute_orchestration_step(
78        &mut self,
79        run_id: &str,
80        step_id: &str,
81        step: &Step,
82        spec: &Map<String, Value>,
83        data: &Data,
84        caller: &ToolCaller,
85    ) {
86        match step.kind.as_str() {
87            "wait" => self.step_wait(run_id, step_id, step, spec, data),
88            "join" => {
89                let handles: Vec<String> = spec
90                    .get("handles")
91                    .map(|h| match h {
92                        Value::Array(a) => a
93                            .iter()
94                            .filter_map(Value::as_str)
95                            .map(str::to_string)
96                            .collect(),
97                        Value::String(s) => vec![s.clone()],
98                        _ => Vec::new(),
99                    })
100                    .unwrap_or_default();
101                let timeout = spec
102                    .get("timeout")
103                    .and_then(crate::engine::model::duration_ms_opt);
104                let min = spec.get("min").and_then(Value::as_u64);
105                self.suspend_wait(run_id, step_id, wait_record("join", json!({"handles": handles, "min": min, "partials": spec.get("partials").and_then(Value::as_bool).unwrap_or(false)}), timeout));
106            }
107            "workflow" => self.step_child_workflow(run_id, step_id, spec, caller),
108            "message" => self.step_message(run_id, step_id, spec),
109            "workflow.signal" => {
110                let name = spec
111                    .get("name")
112                    .and_then(Value::as_str)
113                    .unwrap_or("")
114                    .to_string();
115                let payload = spec.get("payload").cloned().unwrap_or(Value::Null);
116                let target_run = spec.get("run").and_then(Value::as_str).map(str::to_string);
117                let delivered =
118                    self.deliver_signal(&name, payload, target_run.as_deref(), Some(run_id));
119                self.finish_step_pub(
120                    run_id,
121                    step_id,
122                    StepStatus::Done,
123                    Some(json!({"delivered": delivered})),
124                    None,
125                    0,
126                );
127            }
128            "workflow.wait" => {
129                let target = spec
130                    .get("run")
131                    .and_then(Value::as_str)
132                    .unwrap_or("")
133                    .to_string();
134                let timeout = spec
135                    .get("timeout")
136                    .and_then(crate::engine::model::duration_ms_opt);
137                self.suspend_wait(
138                    run_id,
139                    step_id,
140                    wait_record("run", json!({"run": target}), timeout),
141                );
142            }
143            "workflow.cancel" => {
144                let target = spec
145                    .get("run")
146                    .and_then(Value::as_str)
147                    .unwrap_or("")
148                    .to_string();
149                let reason = spec
150                    .get("reason")
151                    .and_then(Value::as_str)
152                    .unwrap_or("cancelled by workflow.cancel")
153                    .to_string();
154                if self.runs.contains_key(&target) {
155                    self.cancel_run(&target, &reason);
156                    self.finish_step_pub(
157                        run_id,
158                        step_id,
159                        StepStatus::Done,
160                        Some(json!({"ok": true, "run": target})),
161                        None,
162                        0,
163                    );
164                } else {
165                    self.finish_step_pub(
166                        run_id,
167                        step_id,
168                        StepStatus::Failed,
169                        None,
170                        Some(format!("no such run {target:?}")),
171                        0,
172                    );
173                }
174            }
175            "subagent" => {
176                let mut args = json!({});
177                for k in [
178                    "instruction",
179                    "template",
180                    "params",
181                    "mode",
182                    "tools",
183                    "servers",
184                    "limits",
185                    "priority",
186                    "context",
187                    "output_contract",
188                    "output_schema",
189                    "skills",
190                    "durable",
191                ] {
192                    if let Some(v) = spec.get(k) {
193                        args[k] = v.clone();
194                    }
195                }
196                let mode = args["mode"].as_str().unwrap_or("sync").to_string();
197                match self.subagent_tool(caller, "subagent.run", args) {
198                    ToolOutcome::Ready(v, is_error) => {
199                        let err = is_error.then(|| v.to_string());
200                        // async/detached: the step's output is the handle record.
201                        self.finish_step_pub(
202                            run_id,
203                            step_id,
204                            if is_error {
205                                StepStatus::Failed
206                            } else {
207                                StepStatus::Done
208                            },
209                            Some(v),
210                            err,
211                            0,
212                        );
213                    }
214                    ToolOutcome::Deferred(kind) => {
215                        let handle = match &kind {
216                            PendingKind::Subagent { handle } => handle.clone(),
217                            _ => String::new(),
218                        };
219                        self.suspend_wait(
220                            run_id,
221                            step_id,
222                            wait_record(
223                                "subagent",
224                                json!({"handle": handle, "mode": mode}),
225                                step.timeout_ms,
226                            ),
227                        );
228                        self.push_pending(super::reactor::PendingTool {
229                            target: Target::Step(run_id.to_string(), step_id.to_string()),
230                            name: "subagent".into(),
231                            kind,
232                            started_ms: now_ms(),
233                        });
234                    }
235                    ToolOutcome::Executing => {}
236                }
237            }
238            "human" => {
239                let mut args =
240                    json!({"question": spec.get("question").cloned().unwrap_or(Value::Null)});
241                for k in ["schema", "to", "timeout"] {
242                    if let Some(v) = spec.get(k) {
243                        args[k] = v.clone();
244                    }
245                }
246                match self.execute_tool(caller, "ask_human", args) {
247                    ToolOutcome::Ready(v, is_error) => {
248                        let err = is_error.then(|| v.to_string());
249                        self.finish_step_pub(
250                            run_id,
251                            step_id,
252                            if is_error {
253                                StepStatus::Failed
254                            } else {
255                                StepStatus::Done
256                            },
257                            Some(v),
258                            err,
259                            0,
260                        );
261                    }
262                    ToolOutcome::Deferred(kind) => {
263                        // The gate's ENFORCEMENT travels in the durable wait
264                        // record, not just in the in-memory pending ask. A
265                        // restart rebuilds the pending from this record, so
266                        // anything omitted here is silently dropped on
267                        // restart — which for `schema` meant a gate that
268                        // demanded `{decision: "file"|"hold"}` would accept
269                        // "maybe later" after a restart, and for `to` would
270                        // let anyone holding the task answer a gate that
271                        // named a decider.
272                        let mut detail = json!({});
273                        for k in ["schema", "to"] {
274                            if let Some(v) = spec.get(k).filter(|v| !v.is_null()) {
275                                detail[k] = v.clone();
276                            }
277                        }
278                        self.suspend_wait(
279                            run_id,
280                            step_id,
281                            wait_record("human", detail, step.timeout_ms),
282                        );
283                        self.push_pending(super::reactor::PendingTool {
284                            target: Target::Step(run_id.to_string(), step_id.to_string()),
285                            name: "human".into(),
286                            kind,
287                            started_ms: now_ms(),
288                        });
289                    }
290                    ToolOutcome::Executing => {
291                        self.executing
292                            .insert(format!("{run_id}/{step_id}"), std::time::Instant::now());
293                    }
294                }
295            }
296            "mcp.resource" => self.step_mcp_resource(run_id, step_id, spec),
297            #[cfg(feature = "a2a")]
298            "a2a.delegate" => self.step_a2a_delegate(run_id, step_id, spec),
299            #[cfg(not(feature = "a2a"))]
300            "a2a.delegate" => self.finish_step_pub(
301                run_id,
302                step_id,
303                StepStatus::Failed,
304                None,
305                Some("a2a.delegate requires the 'a2a' build feature".into()),
306                0,
307            ),
308            #[cfg(feature = "a2a")]
309            "a2a.send" => self.step_a2a_send(run_id, step_id, spec),
310            #[cfg(not(feature = "a2a"))]
311            "a2a.send" => self.finish_step_pub(
312                run_id,
313                step_id,
314                StepStatus::Failed,
315                None,
316                Some("a2a.send requires the 'a2a' build feature".into()),
317                0,
318            ),
319            // `a2a.wait` suspends on a CONVERSATION, exactly as `wait {on:
320            // message}` does — same durable wait record, so the same arrival
321            // hook resolves both and a restart resumes either. The spelling
322            // exists because a workflow that sent with `a2a.send` reads better
323            // awaiting with `a2a.wait` than with a generic `wait`.
324            "a2a.wait" => {
325                let conv = spec
326                    .get("conversation")
327                    .and_then(Value::as_str)
328                    .unwrap_or("")
329                    .to_string();
330                let timeout = spec
331                    .get("timeout")
332                    .and_then(crate::engine::model::duration_ms_opt);
333                self.suspend_wait(
334                    run_id,
335                    step_id,
336                    wait_record("message", json!({"conversation": conv}), timeout),
337                );
338            }
339            "classify" | "extract" | "summarize" | "judge" | "route" => {
340                self.step_preset(run_id, step_id, step, spec, data)
341            }
342            other => self.finish_step_pub(
343                run_id,
344                step_id,
345                StepStatus::Failed,
346                None,
347                Some(format!(
348                    "step kind {other:?} is not executable in this build"
349                )),
350                0,
351            ),
352        }
353    }
354
355    /// Suspend a step with a durable wait record.
356    pub(crate) fn suspend_wait(&mut self, run_id: &str, step_id: &str, wait: Value) {
357        if let Some(r) = self.runs.get_mut(run_id) {
358            r.suspend_step(step_id, wait);
359        }
360        crate::state::kill_point("wait.armed");
361        self.checkpoint(false);
362    }
363
364    /// `wait {on: resource|condition|signal|run|subagent|message, …, timeout}`.
365    fn step_wait(
366        &mut self,
367        run_id: &str,
368        step_id: &str,
369        step: &Step,
370        spec: &Map<String, Value>,
371        _data: &Data,
372    ) {
373        let on = spec
374            .get("on")
375            .and_then(Value::as_str)
376            .unwrap_or("")
377            .to_string();
378        let timeout = spec
379            .get("timeout")
380            .and_then(crate::engine::model::duration_ms_opt)
381            .or(step.timeout_ms);
382        match on.as_str() {
383            "resource" => {
384                let server = spec.get("server").and_then(Value::as_str).unwrap_or("").to_string();
385                let uri = spec.get("uri").and_then(Value::as_str).unwrap_or("").to_string();
386                // Subscribe (notify-then-read on the loop's notification poll).
387                match self.mcp.get(&server).cloned() {
388                    Some(c) => {
389                        if let Err(e) = c.subscribe(&uri) {
390                            self.finish_step_pub(run_id, step_id, StepStatus::Failed, None, Some(format!("wait resource: subscribe {uri}: {e}")), 0);
391                            return;
392                        }
393                        self.suspend_wait(run_id, step_id, wait_record("resource", json!({"server": server, "uri": uri}), timeout));
394                    }
395                    None => self.finish_step_pub(run_id, step_id, StepStatus::Failed, None, Some(format!("wait resource: server {server:?} is not connected")), 0),
396                }
397            }
398            "condition" => {
399                let cond = step.field_str("condition").unwrap_or("false").to_string();
400                if let Err(e) = crate::cel::compile_check(cond.trim().trim_start_matches("CEL:").trim()) {
401                    self.finish_step_pub(run_id, step_id, StepStatus::Failed, None, Some(format!("wait condition: {e}")), 0);
402                    return;
403                }
404                self.suspend_wait(run_id, step_id, wait_record("condition", json!({"condition": cond}), timeout));
405            }
406            "signal" => {
407                let name = spec.get("signal").and_then(Value::as_str).unwrap_or("").to_string();
408                self.suspend_wait(run_id, step_id, wait_record("signal", json!({"signal": name}), timeout));
409            }
410            "run" => {
411                let target = spec.get("run").and_then(Value::as_str).unwrap_or("").to_string();
412                self.suspend_wait(run_id, step_id, wait_record("run", json!({"run": target}), timeout));
413            }
414            "subagent" => {
415                let handle = spec.get("subagent").and_then(Value::as_str).unwrap_or("").to_string();
416                self.suspend_wait(run_id, step_id, wait_record("subagent", json!({"handle": handle}), timeout));
417            }
418            "message" => {
419                let conv = spec.get("conversation").and_then(Value::as_str).map(str::to_string).or_else(|| self.runs.get(run_id).and_then(|r| r.conversation.clone()));
420                self.suspend_wait(run_id, step_id, wait_record("message", json!({"conversation": conv}), timeout));
421            }
422            // Park on the durable log. This is the one edge in the system that
423            // is ordered, replayable and correlated, and until now it could
424            // only START a run — so every pattern past "one run per event"
425            // (sagas, absence, quorum) had to be two workflows plus hand-rolled
426            // bookkeeping. The reason is narrow and specific: a `stream` start's
427            // filter sees only `{event}`, and there was nowhere to match an
428            // event against THIS run's inputs. `match` closes that.
429            "event" => {
430                let Some(stream) = spec.get("stream").and_then(Value::as_str) else {
431                    self.finish_step_pub(run_id, step_id, StepStatus::Failed, None, Some("wait on: event requires `stream`".into()), 0);
432                    return;
433                };
434                if !self.settings.streams.contains_key(stream) {
435                    self.finish_step_pub(run_id, step_id, StepStatus::Failed, None, Some(format!("stream {stream:?} is not declared (add it under `streams:`)")), 0);
436                    return;
437                }
438                // Anchored at NOW, never earlier. A wait that could resolve on
439                // an event predating the run would break the at-least-once
440                // contract for everything downstream of it: the step would
441                // "succeed" on work that happened before anyone asked, and its
442                // idempotency key would cover a different world. There is
443                // deliberately no `from: earliest` here — that belongs to
444                // consumers, which own a durable offset.
445                let anchor = self.durable.manifest().streams.get(stream).map(|m| m.seq).unwrap_or(0);
446                let rec = json!({
447                    "stream": stream,
448                    "subject": spec.get("subject").and_then(Value::as_str),
449                    "match": spec.get("match").and_then(Value::as_str),
450                    "anchor": anchor,
451                });
452                self.suspend_wait(run_id, step_id, wait_record("event", rec, timeout));
453            }
454            #[cfg(feature = "a2a")]
455            "webhook" => self.webhook_wait(run_id, step_id, spec, timeout),
456            "deadline" | "" if timeout.is_some() => {
457                self.suspend_wait(run_id, step_id, wait_record("deadline", json!({}), timeout));
458            }
459            other => self.finish_step_pub(run_id, step_id, StepStatus::Failed, None, Some(format!("wait: on must be resource|condition|signal|run|subagent|message|event|webhook (got {other:?})")), 0),
460        }
461    }
462
463    /// Every tick: resolve `wait {on: event}` steps against the durable log.
464    ///
465    /// **Inverted on purpose.** The obvious shape — each parked wait scanning
466    /// its own stream every tick — costs (waiters × events) durable reads on
467    /// the single-writer thread, for waits that may sit for days. A hundred
468    /// parked sagas would be a hundred independent walks per tick of a log the
469    /// consumer loop is already walking. So this groups waiters BY STREAM and
470    /// reads each event exactly once per tick no matter how many runs are
471    /// parked on it, the same way `poll_stream_starts` batches.
472    ///
473    /// Each waiter's anchor advances past what it scanned, so a quiet stream
474    /// costs one manifest lookup and a busy one never rescans. The advance is
475    /// an ordinary run mutation, so it rides the tick's checkpoint rather than
476    /// forcing a write of its own.
477    pub(crate) fn poll_event_waits(&mut self) {
478        // One waiter: where it is parked, what it is looking for, and the
479        // run-local data its `match` expression is allowed to see.
480        struct Waiter {
481            run: String,
482            step: String,
483            anchor: u64,
484            subject: Option<String>,
485            expr: Option<String>,
486            inputs: Value,
487            vars: Value,
488        }
489        let mut by_stream: std::collections::BTreeMap<String, Vec<Waiter>> =
490            std::collections::BTreeMap::new();
491        for (rid, run) in &self.runs {
492            if run.status.is_terminal() {
493                continue;
494            }
495            for (sid, st) in &run.steps {
496                if st.status != StepStatus::Suspended {
497                    continue;
498                }
499                let Some(w) = &st.wait else { continue };
500                if w["kind"].as_str() != Some("event") {
501                    continue;
502                }
503                let Some(stream) = w["stream"].as_str() else {
504                    continue;
505                };
506                by_stream
507                    .entry(stream.to_string())
508                    .or_default()
509                    .push(Waiter {
510                        run: rid.clone(),
511                        step: sid.clone(),
512                        anchor: w["anchor"].as_u64().unwrap_or(0),
513                        subject: w["subject"].as_str().map(str::to_string),
514                        expr: w["match"].as_str().map(str::to_string),
515                        inputs: run.inputs.clone(),
516                        vars: Value::Object(run.vars.clone()),
517                    });
518            }
519        }
520        if by_stream.is_empty() {
521            return;
522        }
523        // (run, step, event) for the ones that matched, and (run, step, seq)
524        // for every waiter's new anchor.
525        let mut resolved: Vec<(String, String, Value)> = Vec::new();
526        let mut advanced: Vec<(String, String, u64)> = Vec::new();
527        for (stream, waiters) in &by_stream {
528            let head = self
529                .durable
530                .manifest()
531                .streams
532                .get(stream)
533                .map(|m| m.seq)
534                .unwrap_or(0);
535            let first = self
536                .durable
537                .manifest()
538                .streams
539                .get(stream)
540                .map(|m| m.first)
541                .unwrap_or(1);
542            let Some(low) = waiters.iter().map(|w| w.anchor).min() else {
543                continue;
544            };
545            if low >= head {
546                continue; // nothing new on this stream since the earliest anchor
547            }
548            let mut pending: Vec<&Waiter> = waiters.iter().collect();
549            let mut seq = low.max(first.saturating_sub(1));
550            let mut scanned = 0usize;
551            while seq < head && scanned < EVENT_SCAN && !pending.is_empty() {
552                seq += 1;
553                scanned += 1;
554                // One read, shared by every waiter on this stream.
555                let Some(env) = self
556                    .durable
557                    .get(Kind::Event, &event_key(stream, seq))
558                    .ok()
559                    .flatten()
560                else {
561                    continue; // trimmed underneath us
562                };
563                let event = env.state;
564                let subject = event.get("subject").and_then(Value::as_str).unwrap_or("");
565                pending.retain(|w| {
566                    if w.anchor >= seq {
567                        return true; // this waiter armed after this event
568                    }
569                    if let Some(pat) = &w.subject
570                        && !super::streams::subject_matches(pat, subject)
571                    {
572                        return true;
573                    }
574                    if let Some(expr) = &w.expr {
575                        // The point of the whole node: the expression sees this
576                        // RUN's inputs beside the event, so "the shipment for
577                        // the order this run is about" is expressible.
578                        let vars: Vec<(&str, &Value)> =
579                            vec![("event", &event), ("inputs", &w.inputs), ("vars", &w.vars)];
580                        if crate::cel::eval_bool(
581                            expr.trim().trim_start_matches("CEL:").trim(),
582                            &vars,
583                        ) != Ok(true)
584                        {
585                            return true;
586                        }
587                    }
588                    resolved.push((w.run.clone(), w.step.clone(), event.clone()));
589                    false
590                });
591            }
592            // Everything still parked has now seen the log up to `seq`.
593            for w in pending {
594                if seq > w.anchor {
595                    advanced.push((w.run.clone(), w.step.clone(), seq));
596                }
597            }
598        }
599        for (rid, sid, seq) in advanced {
600            if let Some(w) = self
601                .runs
602                .get_mut(&rid)
603                .and_then(|r| r.steps.get_mut(&sid))
604                .and_then(|s| s.wait.as_mut())
605            {
606                w["anchor"] = json!(seq);
607            }
608        }
609        for (rid, sid, event) in resolved {
610            self.log.info(
611                "wait.resolved",
612                json!({"run": rid, "step": sid, "kind": "event",
613                       "stream": event.get("stream"), "subject": event.get("subject")}),
614            );
615            self.finish_step_pub(&rid, &sid, StepStatus::Done, Some(event), None, 0);
616        }
617    }
618
619    /// Every tick: resolve suspended waits (conditions, run/subagent completion,
620    /// joins, deadlines, resource updates that arrived).
621    pub(crate) fn poll_waits(&mut self) {
622        let now = now_ms();
623        let mut resolve: Vec<(String, String, StepStatus, Value, Option<String>)> = Vec::new();
624        let await_data = self.await_data_view();
625        for (rid, run) in &self.runs {
626            if run.status.is_terminal() {
627                continue;
628            }
629            for (sid, st) in &run.steps {
630                if st.status != StepStatus::Suspended {
631                    continue;
632                }
633                let Some(w) = &st.wait else { continue };
634                let deadline = w["deadline_ms"].as_u64();
635                let timed_out = deadline.is_some_and(|d| now >= d);
636                match w["kind"].as_str() {
637                    Some("condition") => {
638                        let cond = w["condition"].as_str().unwrap_or("false").trim().trim_start_matches("CEL:").trim().to_string();
639                        let vars: Vec<(&str, &Value)> = await_data.iter().map(|(k, v)| (k.as_str(), v)).collect();
640                        match crate::cel::eval_bool(&cond, &vars) {
641                            Ok(true) => resolve.push((rid.clone(), sid.clone(), StepStatus::Done, json!({"satisfied": true}), None)),
642                            Ok(false) if timed_out => resolve.push((rid.clone(), sid.clone(), StepStatus::Timeout, json!({"satisfied": false, "timed_out": true}), Some("wait timed out".into()))),
643                            Ok(false) => {}
644                            Err(e) => resolve.push((rid.clone(), sid.clone(), StepStatus::Failed, Value::Null, Some(format!("wait condition: {e}")))),
645                        }
646                    }
647                    Some("run") => {
648                        let target = w["run"].as_str().unwrap_or("");
649                        match self.runs.get(target) {
650                            Some(t) if t.status.is_terminal() => resolve.push((rid.clone(), sid.clone(), StepStatus::Done, json!({"run": target, "status": t.status, "output": t.output, "error": t.error}), None)),
651                            None => resolve.push((rid.clone(), sid.clone(), StepStatus::Failed, Value::Null, Some(format!("wait run: no such run {target:?}")))),
652                            _ if timed_out => resolve.push((rid.clone(), sid.clone(), StepStatus::Timeout, json!({"run": target, "timed_out": true}), Some("wait timed out".into()))),
653                            _ => {}
654                        }
655                    }
656                    Some("subagent") => {
657                        let handle = w["handle"].as_str().unwrap_or("");
658                        match self.subagents.get(handle) {
659                            Some(s) if super::reactor::is_terminal_status(&s.status) => resolve.push((rid.clone(), sid.clone(), if s.status == "completed" { StepStatus::Done } else { StepStatus::Failed }, json!({"handle": handle, "status": s.status, "result": s.result, "error": s.error}), (s.status != "completed").then(|| s.error.clone().unwrap_or_else(|| format!("subagent {}", s.status))))),
660                            None if !handle.is_empty() => resolve.push((rid.clone(), sid.clone(), StepStatus::Failed, Value::Null, Some(format!("wait subagent: no such subagent {handle:?}")))),
661                            _ if timed_out => resolve.push((rid.clone(), sid.clone(), StepStatus::Timeout, json!({"handle": handle, "timed_out": true}), Some("wait timed out".into()))),
662                            _ => {}
663                        }
664                    }
665                    Some("join") => {
666                        let handles: Vec<String> = w["handles"].as_array().map(|a| a.iter().filter_map(Value::as_str).map(str::to_string).collect()).unwrap_or_default();
667                        let min = w["min"].as_u64().map(|m| m as usize).unwrap_or(handles.len());
668                        let mut results = Map::new();
669                        let mut done = 0usize;
670                        for h in &handles {
671                            if let Some(t) = self.runs.get(h) {
672                                if t.status.is_terminal() {
673                                    done += 1;
674                                    results.insert(h.clone(), json!({"kind": "run", "status": t.status, "output": t.output, "error": t.error}));
675                                }
676                            } else if let Some(s) = self.subagents.get(h) {
677                                if super::reactor::is_terminal_status(&s.status) {
678                                    done += 1;
679                                    results.insert(h.clone(), json!({"kind": "subagent", "status": s.status, "result": s.result, "error": s.error}));
680                                }
681                            } else {
682                                results.insert(h.clone(), json!({"error": "unknown handle"}));
683                                done += 1;
684                            }
685                        }
686                        if done >= handles.len() || done >= min {
687                            resolve.push((rid.clone(), sid.clone(), StepStatus::Done, Value::Object(results), None));
688                        } else if timed_out {
689                            let partials = w["partials"].as_bool().unwrap_or(false);
690                            if partials {
691                                resolve.push((rid.clone(), sid.clone(), StepStatus::Done, json!({"partial": true, "results": results}), None));
692                            } else {
693                                resolve.push((rid.clone(), sid.clone(), StepStatus::Timeout, Value::Object(results), Some("join timed out".into())));
694                            }
695                        }
696                    }
697                    Some("deadline") if timed_out => resolve.push((rid.clone(), sid.clone(), StepStatus::Done, json!({"waited_ms": now.saturating_sub(w["since_ms"].as_u64().unwrap_or(now))}), None)),
698                    // Absence IS the branch: an event that never arrives routes
699                    // through `on_timeout` like any other deadline, which is
700                    // what makes "alert fired, no recovery in ten minutes"
701                    // expressible without a polling loop.
702                    Some("resource") | Some("signal") | Some("message") | Some("human") | Some("child_run") | Some("event") if timed_out => {
703                        resolve.push((rid.clone(), sid.clone(), StepStatus::Timeout, json!({"timed_out": true}), Some("wait timed out".into())));
704                    }
705                    _ => {}
706                }
707            }
708        }
709        for (rid, sid, status, out, err) in resolve {
710            self.log.info(
711                "wait.resolved",
712                json!({"run": rid, "step": sid, "status": status}),
713            );
714            self.finish_step_pub(&rid, &sid, status, Some(out), err, 0);
715        }
716    }
717
718    /// The variables an `await`/`wait condition` sees.
719    pub(crate) fn await_data_view(&self) -> Data {
720        let mut d = Data::new();
721        d.insert(
722            "runs".into(),
723            Value::Object(
724                self.runs
725                    .iter()
726                    .map(|(k, r)| (k.clone(), json!({"status": r.status, "output": r.output})))
727                    .collect(),
728            ),
729        );
730        d.insert(
731            "subagents".into(),
732            Value::Object(
733                self.subagents
734                    .iter()
735                    .map(|(k, s)| (k.clone(), json!({"status": s.status, "result": s.result})))
736                    .collect(),
737            ),
738        );
739        d.insert("now_ms".into(), json!(now_ms()));
740        d.insert(
741            "signals".into(),
742            Value::Object(
743                self.recent_signals
744                    .iter()
745                    .map(|(k, v)| (k.clone(), v.clone()))
746                    .collect(),
747            ),
748        );
749        d
750    }
751
752    /// A resource update arrived: resolve `wait resource` steps on it.
753    ///
754    /// The notify-then-read runs on an executor thread, never here. This is the
755    /// single-writer loop: an MCP `resources/read` is a network round trip
756    /// bounded only by the server's own patience, so reading inline hands a slow
757    /// (or hostile) server the whole daemon for the length of one read — no
758    /// timer fires, no checkpoint lands, the drain does not progress and SIGTERM
759    /// is not observed. Subscriptions are agentd's reactivity story, so this is
760    /// the hot path. The thread reports back through `events_tx` exactly like
761    /// `step_mcp_resource` and every other asynchronous effect, and the loop
762    /// resolves the waits when that event arrives.
763    pub(crate) fn on_resource_updated(&mut self, server: &str, uri: &str) {
764        // Each hit carries its own deadline: the loop keeps running while we
765        // read, so a `wait … timeout` may resolve the step as Timeout first, and
766        // a late read must not resurrect a step the loop has already finished.
767        let mut hits: Vec<(String, String, Option<u64>)> = Vec::new();
768        for (rid, run) in &self.runs {
769            for (sid, st) in &run.steps {
770                if st.status == StepStatus::Suspended
771                    && let Some(w) = &st.wait
772                    && w["kind"] == "resource"
773                    && w["server"] == server
774                    && w["uri"] == uri
775                {
776                    hits.push((rid.clone(), sid.clone(), w["deadline_ms"].as_u64()));
777                }
778            }
779        }
780        if hits.is_empty() {
781            return;
782        }
783        let Some(client) = self.mcp.get(server).cloned() else {
784            return; // the server went away between the notification and here
785        };
786        let tx = self.events_tx.clone();
787        let (srv, u) = (server.to_string(), uri.to_string());
788        std::thread::Builder::new()
789            .name(format!("mcp.updated:{server}"))
790            .spawn(move || {
791                // A failed read still resolves the wait, as the inline read did:
792                // the update itself is the event, and `content: null` says the
793                // follow-up read did not land.
794                let content = client.read_resource(&u).ok().map(|r| {
795                    let t = r.text();
796                    serde_json::from_str::<Value>(&t).unwrap_or(Value::String(t))
797                });
798                for (run, step, deadline_ms) in hits {
799                    if deadline_ms.is_some_and(|d| now_ms() > d) {
800                        continue;
801                    }
802                    let _ = tx.send(super::events::Event::StepDone {
803                        run,
804                        step,
805                        output: json!({"uri": u, "server": srv, "content": content}),
806                        is_error: false,
807                        error: None,
808                        tokens: 0,
809                    });
810                }
811            })
812            .ok();
813    }
814
815    /// An A2A message arrived on a conversation: resolve every step suspended
816    /// waiting for one. Returns how many were woken.
817    ///
818    /// This is what makes the asynchronous half of an A2A conversation
819    /// expressible: `wait {on: message}` and `a2a.wait` both suspend on a
820    /// `{kind: message, conversation}` record, and without a resolver they
821    /// could only ever end by timing out — a workflow could send but never be
822    /// woken by the reply.
823    ///
824    /// A wait with an EMPTY conversation matches any conversation, which is how
825    /// a workflow awaits "the next thing anyone says" without knowing the id in
826    /// advance.
827    pub(crate) fn deliver_a2a_message(
828        &mut self,
829        conversation: &str,
830        message: &Value,
831        principal: Option<&str>,
832    ) -> u64 {
833        let mut hits: Vec<(String, String)> = Vec::new();
834        for (rid, run) in &self.runs {
835            for (sid, st) in &run.steps {
836                if st.status == StepStatus::Suspended
837                    && let Some(w) = &st.wait
838                    && w["kind"] == "message"
839                {
840                    let want = w["conversation"].as_str().unwrap_or("");
841                    if want.is_empty() || want == conversation {
842                        hits.push((rid.clone(), sid.clone()));
843                    }
844                }
845            }
846        }
847        let mut delivered = 0u64;
848        for (rid, sid) in hits {
849            delivered += 1;
850            self.finish_step_pub(
851                &rid,
852                &sid,
853                StepStatus::Done,
854                Some(json!({
855                    "conversation": conversation,
856                    "message": message,
857                    "principal": principal,
858                })),
859                None,
860                0,
861            );
862        }
863        delivered
864    }
865
866    /// Deliver a named signal: to `wait signal` steps (any run, or only the
867    /// `run` named as the target), to `signal` start nodes, and into the
868    /// recent-signals view. Returns how many waiting steps were woken.
869    pub(crate) fn deliver_signal(
870        &mut self,
871        name: &str,
872        payload: Value,
873        target_run: Option<&str>,
874        from_run: Option<&str>,
875    ) -> u64 {
876        let mut delivered = 0u64;
877        let mut hits: Vec<(String, String)> = Vec::new();
878        for (rid, run) in &self.runs {
879            if let Some(t) = target_run
880                && rid != t
881            {
882                continue;
883            }
884            for (sid, st) in &run.steps {
885                if st.status == StepStatus::Suspended
886                    && let Some(w) = &st.wait
887                    && w["kind"] == "signal"
888                    && w["signal"] == name
889                {
890                    hits.push((rid.clone(), sid.clone()));
891                }
892            }
893        }
894        for (rid, sid) in hits {
895            delivered += 1;
896            self.finish_step_pub(
897                &rid,
898                &sid,
899                StepStatus::Done,
900                Some(json!({"signal": name, "payload": payload, "from": from_run})),
901                None,
902                0,
903            );
904        }
905        self.recent_signals.insert(
906            name.to_string(),
907            json!({"payload": payload, "ts": now_ms(), "from": from_run}),
908        );
909        if self.recent_signals.len() > 64 {
910            let first = self.recent_signals.keys().next().cloned();
911            if let Some(k) = first {
912                self.recent_signals.remove(&k);
913            }
914        }
915        // Signal start nodes.
916        delivered += self.fire_signal_starts(name, &payload, target_run.is_none());
917        // When this signal is the configured `lifecycle.until_signal` it is the
918        // retirement trigger: stop admitting, drain live runs, exit cleanly.
919        // Delivery to whatever was parked on the signal happens first (above),
920        // so an all-clear both completes the waiting run and retires the
921        // instance.
922        if self
923            .settings
924            .lifecycle
925            .until_signal
926            .as_deref()
927            .is_some_and(|u| u == name)
928        {
929            self.log
930                .info("lifecycle.until_signal", json!({"signal": name}));
931            self.begin_drain("until_signal");
932        }
933        delivered
934    }
935
936    /// `message` step: deliver into one of this instance's own conversations,
937    /// so a run can hand work to the agent instead of only the reverse.
938    ///
939    /// The delivery is an ordinary inbound message event — the same one the A2A
940    /// listener produces — so it takes the same three readers in the same
941    /// order (a step waiting on the conversation, then a matching `a2a` start,
942    /// then a turn), and inherits write-ahead durability, crash replay, the
943    /// per-context lock and pressure shedding without any of them being taught
944    /// about this node.
945    ///
946    /// `wait: reply` parks on the answer using the `message` wait that already
947    /// exists; without it the step completes as soon as the delivery is durable
948    /// and the turn happens on its own schedule.
949    fn step_message(&mut self, run_id: &str, step_id: &str, spec: &Map<String, Value>) {
950        let to = spec
951            .get("to")
952            .and_then(Value::as_str)
953            .unwrap_or(ROOT)
954            .trim();
955        // `to: new` opens a fresh conversation, which is how a run gets a
956        // clean transcript without borrowing the operator's.
957        let ctx = if to.eq_ignore_ascii_case("new") {
958            format!("run-{}", crate::state::ulid::new())
959        } else if to.is_empty() {
960            ROOT.to_string()
961        } else {
962            to.to_string()
963        };
964        let text = spec
965            .get("text")
966            .and_then(Value::as_str)
967            .unwrap_or("")
968            .to_string();
969        let parts = spec.get("parts").cloned();
970        if text.trim().is_empty() && parts.is_none() {
971            self.finish_step_pub(
972                run_id,
973                step_id,
974                StepStatus::Failed,
975                None,
976                Some("message: one of text or parts is required".into()),
977                0,
978            );
979            return;
980        }
981        // The hop guard. A run inherits the depth of the work that caused it,
982        // so a chain that came back around to its own conversation is refused
983        // HERE — before the delivery is durable — rather than being noticed
984        // once the loop is already running.
985        let depth = self.runs.get(run_id).map(|r| r.msg_depth).unwrap_or(0) + 1;
986        let cap = self.settings.limits.message_depth();
987        if depth > cap {
988            self.log.warn(
989                "message.too_deep",
990                json!({"run": run_id, "step": step_id, "conversation": ctx,
991                       "depth": depth, "max": cap}),
992            );
993            self.finish_step_pub(
994                run_id,
995                step_id,
996                StepStatus::Failed,
997                None,
998                Some(format!(
999                    "message refused: {depth} chained deliveries exceeds limits.max_message_depth ({cap}) — \
1000                     a message that causes a turn that causes this message is a loop, not a conversation"
1001                )),
1002                0,
1003            );
1004            return;
1005        }
1006        let mut payload = json!({"text": text, "context_id": ctx, "msg_depth": depth});
1007        if let Some(p) = parts {
1008            payload["parts"] = p;
1009        }
1010        let principal = self.runs.get(run_id).and_then(|r| r.principal.clone());
1011        if let Err(e) = self.accept_event(
1012            crate::runtime::events::kinds::A2A_MESSAGE,
1013            principal,
1014            payload,
1015        ) {
1016            self.finish_step_pub(
1017                run_id,
1018                step_id,
1019                StepStatus::Failed,
1020                None,
1021                Some(format!("message: {e}")),
1022                0,
1023            );
1024            return;
1025        }
1026        let wants_reply = match spec.get("wait") {
1027            Some(Value::Bool(b)) => *b,
1028            Some(Value::String(s)) => {
1029                let s = s.trim();
1030                !s.is_empty() && !s.eq_ignore_ascii_case("none") && !s.eq_ignore_ascii_case("false")
1031            }
1032            Some(Value::Object(o)) => o
1033                .get("for")
1034                .and_then(Value::as_str)
1035                .is_some_and(|f| f.eq_ignore_ascii_case("reply")),
1036            _ => false,
1037        };
1038        if !wants_reply {
1039            self.finish_step_pub(
1040                run_id,
1041                step_id,
1042                StepStatus::Done,
1043                Some(json!({"delivered": true, "conversation": ctx, "depth": depth})),
1044                None,
1045                0,
1046            );
1047            return;
1048        }
1049        // Park on the answer. `timeout` may sit on the node or inside `wait`,
1050        // and `on_timeout` routes an unanswered message the same way every
1051        // other suspending kind routes one.
1052        let timeout = spec
1053            .get("timeout")
1054            .or_else(|| spec.get("wait").and_then(|w| w.get("timeout")))
1055            .and_then(crate::engine::model::duration_ms_opt);
1056        self.suspend_wait(
1057            run_id,
1058            step_id,
1059            wait_record("message", json!({"conversation": ctx}), timeout),
1060        );
1061    }
1062
1063    /// `workflow` step: a child run (`mode: sync|async|detached`, `cascade`).
1064    fn step_child_workflow(
1065        &mut self,
1066        run_id: &str,
1067        step_id: &str,
1068        spec: &Map<String, Value>,
1069        caller: &ToolCaller,
1070    ) {
1071        let name = spec
1072            .get("name")
1073            .and_then(Value::as_str)
1074            .unwrap_or("")
1075            .to_string();
1076        let mode = spec
1077            .get("mode")
1078            .and_then(Value::as_str)
1079            .unwrap_or("sync")
1080            .to_string();
1081        let inputs = spec.get("inputs").cloned().unwrap_or(json!({}));
1082        let start = spec
1083            .get("start")
1084            .and_then(Value::as_str)
1085            .map(str::to_string);
1086        if let Some(pin) = spec.get("version").and_then(Value::as_str)
1087            && let Some(w) = self.workflows.get(&name)
1088            && !w.hash.starts_with(pin)
1089        {
1090            self.finish_step_pub(
1091                run_id,
1092                step_id,
1093                StepStatus::Failed,
1094                None,
1095                Some(format!(
1096                    "workflow {name:?} hash {} does not match the pinned version {pin}",
1097                    &w.hash[..12]
1098                )),
1099                0,
1100            );
1101            return;
1102        }
1103        let Some(w) = self.workflows.get(&name) else {
1104            self.finish_step_pub(
1105                run_id,
1106                step_id,
1107                StepStatus::Failed,
1108                None,
1109                Some(format!("no such workflow {name:?}")),
1110                0,
1111            );
1112            return;
1113        };
1114        let start_node = match start {
1115            Some(s) => s,
1116            None => {
1117                let starts = w.start_steps();
1118                match starts
1119                    .iter()
1120                    .find(|s| s.kind == "manual")
1121                    .or_else(|| starts.first())
1122                {
1123                    Some(s) => s.id.clone(),
1124                    None => {
1125                        self.finish_step_pub(
1126                            run_id,
1127                            step_id,
1128                            StepStatus::Failed,
1129                            None,
1130                            Some(format!("workflow {name:?} has no start node")),
1131                            0,
1132                        );
1133                        return;
1134                    }
1135                }
1136            }
1137        };
1138        let cascade = spec.get("cascade").and_then(Value::as_bool).unwrap_or(true);
1139        let payload = json!({"workflow": name, "node": start_node, "payload": {"requested_by": caller.label_pub()}, "inputs": inputs, "parent": {"run": run_id, "step": step_id, "cascade": cascade}, "conversation": self.runs.get(run_id).and_then(|r| r.conversation.clone()), "task": self.runs.get(run_id).and_then(|r| r.task.clone()), "msg_depth": self.runs.get(run_id).map(|r| r.msg_depth).unwrap_or(0)});
1140        match self.accept_event(
1141            kinds::WORKFLOW_RUN,
1142            self.runs.get(run_id).and_then(|r| r.principal.clone()),
1143            payload,
1144        ) {
1145            Ok(_) => {}
1146            Err(e) => {
1147                self.finish_step_pub(run_id, step_id, StepStatus::Failed, None, Some(e), 0);
1148                return;
1149            }
1150        }
1151        // Process the event now so the child id is known.
1152        let child_id = if let Some(ev) = self.inbox_queue.pop_back() {
1153            let before: std::collections::BTreeSet<String> = self.runs.keys().cloned().collect();
1154            let done = self.on_start_event(&ev);
1155            if done {
1156                self.inbox_done(&ev.id);
1157            }
1158            self.runs.keys().find(|k| !before.contains(*k)).cloned()
1159        } else {
1160            None
1161        };
1162        let Some(child_id) = child_id else {
1163            // Queued (concurrency) — wait for it to appear: fall back to a run-wait
1164            // on the parent link. Simplest: suspend as a `child_run` wait resolved
1165            // when a run with our parent link finishes.
1166            self.suspend_wait(
1167                run_id,
1168                step_id,
1169                wait_record("child_run", json!({"workflow": name, "mode": mode}), None),
1170            );
1171            return;
1172        };
1173        if let Some(r) = self.runs.get_mut(run_id) {
1174            r.children.push(child_id.clone());
1175            r.touch();
1176        }
1177        match mode.as_str() {
1178            "sync" => self.suspend_wait(
1179                run_id,
1180                step_id,
1181                wait_record(
1182                    "run",
1183                    json!({"run": child_id, "child": true}),
1184                    spec.get("timeout")
1185                        .and_then(crate::engine::model::duration_ms_opt),
1186                ),
1187            ),
1188            _ => self.finish_step_pub(
1189                run_id,
1190                step_id,
1191                StepStatus::Done,
1192                Some(json!({"run": child_id, "workflow": name, "mode": mode})),
1193                None,
1194                0,
1195            ),
1196        }
1197    }
1198
1199    /// `mcp.resource {server, op: read|list|prompt|complete, …}`.
1200    fn step_mcp_resource(&mut self, run_id: &str, step_id: &str, spec: &Map<String, Value>) {
1201        let server = spec
1202            .get("server")
1203            .and_then(Value::as_str)
1204            .unwrap_or("")
1205            .to_string();
1206        let op = spec
1207            .get("op")
1208            .and_then(Value::as_str)
1209            .unwrap_or("read")
1210            .to_string();
1211        let Some(client) = self.mcp.get(&server).cloned() else {
1212            self.finish_step_pub(
1213                run_id,
1214                step_id,
1215                StepStatus::Failed,
1216                None,
1217                Some(format!("mcp.resource: server {server:?} is not connected")),
1218                0,
1219            );
1220            return;
1221        };
1222        let uri = spec
1223            .get("uri")
1224            .and_then(Value::as_str)
1225            .unwrap_or("")
1226            .to_string();
1227        let name = spec
1228            .get("name")
1229            .and_then(Value::as_str)
1230            .unwrap_or("")
1231            .to_string();
1232        let arguments = spec.get("arguments").cloned();
1233        let reference = spec.get("reference").cloned();
1234        let argument = spec.get("argument").cloned();
1235        let tx = self.events_tx.clone();
1236        let (r, s) = (run_id.to_string(), step_id.to_string());
1237        self.executing
1238            .insert(format!("{run_id}/{step_id}"), std::time::Instant::now());
1239        std::thread::Builder::new()
1240            .name(format!("mcp.resource:{server}"))
1241            .spawn(move || {
1242                let res: Result<Value, String> = match op.as_str() {
1243                    "read" => client.read_resource(&uri).map(|r| {
1244                        let t = r.text();
1245                        json!({"uri": uri, "text": t, "json": serde_json::from_str::<Value>(&t).ok(), "contents": r.contents})
1246                    }).map_err(|e| e.to_string()),
1247                    "list" => client.list_resources().map(|l| json!({"resources": l})).map_err(|e| e.to_string()),
1248                    "prompt" => client.get_prompt(&name, arguments).map(|p| json!({"description": p.description, "messages": p.messages, "text": crate::context::skills::prompt_messages_text(&p.messages)})).map_err(|e| e.to_string()),
1249                    "complete" => client.complete(reference.unwrap_or(Value::Null), argument.unwrap_or(Value::Null)).map(|c| serde_json::to_value(c).unwrap_or(Value::Null)).map_err(|e| e.to_string()),
1250                    "templates" => client.list_resource_templates().map(|l| json!({"templates": l})).map_err(|e| e.to_string()),
1251                    other => Err(format!("mcp.resource: op must be read|list|prompt|complete|templates (got {other:?})")),
1252                };
1253                let (output, is_error, error) = match res {
1254                    Ok(v) => (v, false, None),
1255                    Err(e) => (Value::Null, true, Some(e)),
1256                };
1257                let _ = tx.send(super::events::Event::StepDone { run: r, step: s, output, is_error, error, tokens: 0 });
1258            })
1259            .ok();
1260    }
1261
1262    /// Resolve a configured A2A peer into a dialable endpoint plus its client
1263    /// auth (bearer headers, a per-request SigV4 signer, an mTLS identity).
1264    ///
1265    /// Shared by `a2a.delegate` and `a2a.send`, which differ only in what they
1266    /// do once connected — everything up to the socket is identical, and it is
1267    /// ~140 lines of credential plumbing that must not diverge between the two.
1268    #[cfg(feature = "a2a")]
1269    // `timeout` bounds only the interactive credential fetch, which lives behind
1270    // `oauth`; without that feature there is nothing to bound and the parameter
1271    // is genuinely unused. Keeping it in the signature keeps both callers
1272    // identical across feature sets.
1273    #[cfg_attr(not(feature = "oauth"), allow(unused_variables))]
1274    fn a2a_peer_conn(
1275        &self,
1276        peer_name: &str,
1277        timeout: Duration,
1278        what: &str,
1279    ) -> Result<(crate::config::A2aEndpoint, crate::mcp::a2a_client::PeerAuth), String> {
1280        let configured = self
1281            .settings
1282            .a2a
1283            .peers
1284            .iter()
1285            .find(|p| p.name == peer_name)
1286            .cloned();
1287        // Live instance children are dialable as peers too — by handle, or by
1288        // template name when that template has a single instance. A configured
1289        // peer wins the name, so an operator can always override the lookup.
1290        let peer = match configured {
1291            Some(p) => p,
1292            None => match self.instance_peer_endpoint(peer_name) {
1293                Some(endpoint) => crate::config::v2::A2aPeer {
1294                    name: peer_name.to_string(),
1295                    endpoint,
1296                    service: None,
1297                    headers: std::collections::BTreeMap::new(),
1298                    client_cert: None,
1299                    client_key: None,
1300                    auth: None,
1301                },
1302                None => {
1303                    return Err(format!(
1304                        "{what}: no such peer {peer_name:?} (a2a.peers or a live instance child)"
1305                    ));
1306                }
1307            },
1308        };
1309        let spec_v1 = crate::config::A2aPeerSpec {
1310            name: peer.name.clone(),
1311            endpoint: peer.endpoint.clone(),
1312            headers: peer
1313                .headers
1314                .iter()
1315                .map(|(k, v)| (k.clone(), v.clone()))
1316                .collect(),
1317            client_cert: peer.client_cert.clone(),
1318            client_key: peer.client_key.clone(),
1319        };
1320        let endpoint = spec_v1
1321            .endpoint_of()
1322            .map_err(|e| format!("{what}: peer endpoint: {e}"))?;
1323        #[allow(unused_mut)]
1324        let mut headers = crate::mcp::auth::resolve_headers(&spec_v1.headers)
1325            .map_err(|e| format!("{what}: peer headers: {e}"))?;
1326        // A peer `auth:` block resolves at dial time. A body-INDEPENDENT
1327        // bearer (static / oauth2 device-login / spiffe jwt) is baked into the
1328        // static headers; SigV4 (`kind: aws`) covers the exact body, so it rides
1329        // as a PER-REQUEST signer on `PeerAuth` (re-run on every POST).
1330        #[cfg(feature = "oauth")]
1331        let mut peer_signer: Option<std::sync::Arc<dyn ::mcp::http::RequestSigner>> = None;
1332        #[cfg(feature = "oauth")]
1333        if let Some(a) = &peer.auth {
1334            let aspec = a.to_spec();
1335            // A peer that names a catalog `service:` caches its credential
1336            // under `service:<entry>`, shared with every other consumer of that
1337            // entry, so one `agentd login service:<entry>` serves them all.
1338            // Without one the cache is private to this peer's name.
1339            let target = match &peer.service {
1340                Some(svc) => format!("service:{svc}"),
1341                None => format!("a2a:{}", peer.name),
1342            };
1343            if aspec.kind == "aws" {
1344                let s = crate::auth::aws::SigV4Signer::from_spec(&aspec, &target)
1345                    .map_err(|e| format!("{what}: peer aws auth: {e}"))?;
1346                peer_signer = Some(s as std::sync::Arc<dyn ::mcp::http::RequestSigner>);
1347            } else if let Some(signer) = crate::auth::device::signer_for(&aspec, &target, timeout)
1348                .map_err(|e| format!("{what}: peer auth: {e}"))?
1349            {
1350                for (k, v) in signer.sign("POST", &peer.endpoint, "/", &[]) {
1351                    headers.push((k, v));
1352                }
1353            }
1354        }
1355        #[allow(unused_mut)]
1356        let mut auth = crate::mcp::a2a_client::PeerAuth {
1357            headers,
1358            ..Default::default()
1359        };
1360        #[cfg(feature = "oauth")]
1361        {
1362            auth.signer = peer_signer;
1363        }
1364        #[cfg(feature = "tls")]
1365        if let (Some(cert), Some(key)) = (&spec_v1.client_cert, &spec_v1.client_key) {
1366            let id = std::fs::read(cert)
1367                .and_then(|c| std::fs::read(key).map(|k| (c, k)))
1368                .map_err(|e| e.to_string())
1369                .and_then(|(c, k)| {
1370                    crate::net::tls::ClientIdentity::from_pem(&c, &k).map_err(|e| e.to_string())
1371                })
1372                .map_err(|e| format!("{what}: peer mtls: {e}"))?;
1373            auth.identity = Some(id);
1374        }
1375        Ok((endpoint, auth))
1376    }
1377
1378    /// `a2a.send {to, parts, context?, timeout?}` — notify a peer, do not wait.
1379    ///
1380    /// The step completes when the peer ACCEPTS the message, not when it has
1381    /// done anything about it. That is the difference from `a2a.delegate`, and
1382    /// it is what makes the asynchronous shape expressible: send, keep working,
1383    /// and pick the reply up later with `a2a.wait` on the same conversation.
1384    #[cfg(feature = "a2a")]
1385    fn step_a2a_send(&mut self, run_id: &str, step_id: &str, spec: &Map<String, Value>) {
1386        let peer_name = spec
1387            .get("to")
1388            .and_then(Value::as_str)
1389            .unwrap_or("")
1390            .to_string();
1391        // `command` + `args` build the typed DataPart the peer's `a2a` start
1392        // nodes match on; `parts` text (if any) rides along for the humans
1393        // reading the transcript. Without `command`, `parts` goes as before.
1394        let parts = match spec.get("command").and_then(Value::as_str) {
1395            Some(cmd) => {
1396                let mut env = serde_json::Map::new();
1397                env.insert("op".into(), json!(cmd));
1398                if let Some(args) = spec.get("args") {
1399                    match args.as_object() {
1400                        Some(o) => {
1401                            for (k, v) in o {
1402                                env.insert(k.clone(), v.clone());
1403                            }
1404                        }
1405                        None => {
1406                            self.finish_step_pub(
1407                                run_id,
1408                                step_id,
1409                                StepStatus::Failed,
1410                                None,
1411                                Some("a2a.send: args must be a mapping".into()),
1412                                0,
1413                            );
1414                            return;
1415                        }
1416                    }
1417                }
1418                let mut arr = vec![json!({"data": {"agentd": Value::Object(env)}})];
1419                if let Some(t) = spec.get("parts").and_then(Value::as_str) {
1420                    arr.push(json!({"text": t}));
1421                }
1422                Value::Array(arr)
1423            }
1424            None => spec.get("parts").cloned().unwrap_or(Value::Null),
1425        };
1426        let context = spec
1427            .get("context")
1428            .and_then(Value::as_str)
1429            .map(str::to_string);
1430        let timeout = spec
1431            .get("timeout")
1432            .and_then(crate::engine::model::duration_ms_opt)
1433            .map(Duration::from_millis)
1434            .unwrap_or(Duration::from_secs(30));
1435        let (endpoint, auth) = match self.a2a_peer_conn(&peer_name, timeout, "a2a.send") {
1436            Ok(v) => v,
1437            Err(e) => {
1438                self.finish_step_pub(run_id, step_id, StepStatus::Failed, None, Some(e), 0);
1439                return;
1440            }
1441        };
1442        let message_id = idempotency_message_id(spec, run_id, step_id);
1443        let tx = self.events_tx.clone();
1444        let (r, st) = (run_id.to_string(), step_id.to_string());
1445        self.executing
1446            .insert(format!("{run_id}/{step_id}"), std::time::Instant::now());
1447        self.log.info(
1448            "a2a.send",
1449            json!({"run": run_id, "step": step_id, "to": peer_name}),
1450        );
1451        std::thread::Builder::new()
1452            .name(format!("a2a.send:{peer_name}"))
1453            .spawn(move || {
1454                let deadline = std::time::Instant::now() + timeout;
1455                let (output, is_error, error) = match crate::mcp::a2a_client::send(
1456                    &endpoint,
1457                    auth,
1458                    &parts,
1459                    context.as_deref(),
1460                    message_id.as_deref(),
1461                    deadline,
1462                ) {
1463                    Ok(v) => (v, false, None),
1464                    Err(e) => (Value::Null, true, Some(e)),
1465                };
1466                let _ = tx.send(super::events::Event::StepDone {
1467                    run: r,
1468                    step: st,
1469                    output,
1470                    is_error,
1471                    error,
1472                    tokens: 0,
1473                });
1474            })
1475            .ok();
1476    }
1477
1478    /// `a2a.delegate {peer, objective, output_contract?, timeout?}`: hand a unit
1479    /// of work to a peer agent over A2A and suspend until it answers.
1480    #[cfg(feature = "a2a")]
1481    fn step_a2a_delegate(&mut self, run_id: &str, step_id: &str, spec: &Map<String, Value>) {
1482        let peer_name = spec
1483            .get("peer")
1484            .and_then(Value::as_str)
1485            .unwrap_or("")
1486            .to_string();
1487        let objective = spec
1488            .get("objective")
1489            .and_then(Value::as_str)
1490            .unwrap_or("")
1491            .to_string();
1492        // The typed form: `command` + `args` become the DataPart the peer's
1493        // `a2a` start matches; `objective` text (optional here) rides along.
1494        let command_env = match spec.get("command").and_then(Value::as_str) {
1495            Some(cmd) => {
1496                let mut env = serde_json::Map::new();
1497                env.insert("op".into(), json!(cmd));
1498                if let Some(args) = spec.get("args") {
1499                    match args.as_object() {
1500                        Some(o) => {
1501                            for (k, v) in o {
1502                                env.insert(k.clone(), v.clone());
1503                            }
1504                        }
1505                        None => {
1506                            self.finish_step_pub(
1507                                run_id,
1508                                step_id,
1509                                StepStatus::Failed,
1510                                None,
1511                                Some("a2a.delegate: args must be a mapping".into()),
1512                                0,
1513                            );
1514                            return;
1515                        }
1516                    }
1517                }
1518                Some(Value::Object(env))
1519            }
1520            None => None,
1521        };
1522        let contract = spec
1523            .get("output_contract")
1524            .and_then(Value::as_str)
1525            .map(str::to_string);
1526        let timeout = spec
1527            .get("timeout")
1528            .and_then(crate::engine::model::duration_ms_opt)
1529            .map(Duration::from_millis)
1530            .unwrap_or(Duration::from_secs(120));
1531        let (endpoint, auth) = match self.a2a_peer_conn(&peer_name, timeout, "a2a.delegate") {
1532            Ok(v) => v,
1533            Err(e) => {
1534                self.finish_step_pub(run_id, step_id, StepStatus::Failed, None, Some(e), 0);
1535                return;
1536            }
1537        };
1538        let message_id = idempotency_message_id(spec, run_id, step_id);
1539        let tx = self.events_tx.clone();
1540        let (r, s) = (run_id.to_string(), step_id.to_string());
1541        self.executing
1542            .insert(format!("{run_id}/{step_id}"), std::time::Instant::now());
1543        self.log.info(
1544            "a2a.delegate",
1545            json!({"run": run_id, "step": step_id, "peer": peer_name}),
1546        );
1547        std::thread::Builder::new()
1548            .name(format!("a2a.delegate:{peer_name}"))
1549            .spawn(move || {
1550                let deadline = std::time::Instant::now() + timeout;
1551                let (output, is_error, error) = match crate::mcp::a2a_client::delegate(
1552                    &endpoint,
1553                    auth,
1554                    &objective,
1555                    command_env.as_ref(),
1556                    contract.as_deref(),
1557                    message_id.as_deref(),
1558                    deadline,
1559                ) {
1560                    crate::mcp::a2a_client::DelegateOutcome::Distillate(text) => (
1561                        serde_json::from_str::<Value>(&text).unwrap_or(Value::String(text)),
1562                        false,
1563                        None,
1564                    ),
1565                    crate::mcp::a2a_client::DelegateOutcome::Error(e) => {
1566                        (Value::Null, true, Some(e))
1567                    }
1568                };
1569                let _ = tx.send(super::events::Event::StepDone {
1570                    run: r,
1571                    step: s,
1572                    output,
1573                    is_error,
1574                    error,
1575                    tokens: 0,
1576                });
1577            })
1578            .ok();
1579    }
1580
1581    /// The `think` presets: sugar over `think` with a fixed prompt frame + schema.
1582    fn step_preset(
1583        &mut self,
1584        run_id: &str,
1585        step_id: &str,
1586        step: &Step,
1587        spec: &Map<String, Value>,
1588        data: &Data,
1589    ) {
1590        let input = spec.get("input").cloned().unwrap_or(Value::Null);
1591        let input_text = match &input {
1592            Value::String(s) => s.clone(),
1593            other => other.to_string(),
1594        };
1595        let extra = spec
1596            .get("prompt")
1597            .and_then(Value::as_str)
1598            .map(|p| format!("\n\nAdditional guidance: {p}"))
1599            .unwrap_or_default();
1600        let (prompt, schema) = match step.kind.as_str() {
1601            "classify" => {
1602                let classes: Vec<String> = spec
1603                    .get("classes")
1604                    .and_then(Value::as_array)
1605                    .map(|a| {
1606                        a.iter()
1607                            .filter_map(Value::as_str)
1608                            .map(str::to_string)
1609                            .collect()
1610                    })
1611                    .unwrap_or_default();
1612                (
1613                    format!(
1614                        "Classify the input into exactly one of these classes: {}.{extra}\n\nInput:\n{input_text}\n\nReply with ONLY a JSON object {{\"class\": <one of the classes>, \"confidence\": <0..1>, \"reason\": <short>}}.",
1615                        classes.join(", ")
1616                    ),
1617                    json!({"type": "object", "properties": {"class": {"enum": classes}, "confidence": {"type": "number"}, "reason": {"type": "string"}}, "required": ["class"]}),
1618                )
1619            }
1620            "extract" => {
1621                let schema = spec
1622                    .get("output_schema")
1623                    .cloned()
1624                    .unwrap_or(json!({"type": "object"}));
1625                (
1626                    format!(
1627                        "Extract the structured data described by this JSON Schema from the input.{extra}\n\nSchema:\n{schema}\n\nInput:\n{input_text}\n\nReply with ONLY one JSON object matching the schema."
1628                    ),
1629                    schema,
1630                )
1631            }
1632            "summarize" => {
1633                let length = spec
1634                    .get("length")
1635                    .and_then(Value::as_str)
1636                    .unwrap_or("a short paragraph");
1637                (
1638                    format!(
1639                        "Summarize the input in {length}. Keep facts, names, numbers and identifiers verbatim.{extra}\n\nInput:\n{input_text}\n\nReply with ONLY a JSON object {{\"summary\": <text>}}."
1640                    ),
1641                    json!({"type": "object", "properties": {"summary": {"type": "string"}}, "required": ["summary"]}),
1642                )
1643            }
1644            "judge" => {
1645                let rubric = spec.get("rubric").cloned().unwrap_or(Value::Null);
1646                (
1647                    format!(
1648                        "Judge the input against the rubric.{extra}\n\nRubric:\n{rubric}\n\nInput:\n{input_text}\n\nReply with ONLY a JSON object {{\"verdict\": \"pass\"|\"fail\", \"score\": <0..10>, \"reasons\": [<short strings>]}}."
1649                    ),
1650                    json!({"type": "object", "properties": {"verdict": {"enum": ["pass", "fail"]}, "score": {"type": "number"}, "reasons": {"type": "array", "items": {"type": "string"}}}, "required": ["verdict"]}),
1651                )
1652            }
1653            _ => {
1654                let choices: Vec<String> = match spec.get("choices") {
1655                    Some(Value::Array(a)) => a
1656                        .iter()
1657                        .filter_map(Value::as_str)
1658                        .map(str::to_string)
1659                        .collect(),
1660                    Some(Value::Object(o)) => o.keys().cloned().collect(),
1661                    _ => Vec::new(),
1662                };
1663                (
1664                    format!(
1665                        "Route the input to exactly one of these choices: {}.{extra}\n\nInput:\n{input_text}\n\nReply with ONLY a JSON object {{\"choice\": <one of the choices>, \"reason\": <short>}}.",
1666                        choices.join(", ")
1667                    ),
1668                    json!({"type": "object", "properties": {"choice": {"enum": choices}, "reason": {"type": "string"}}, "required": ["choice"]}),
1669                )
1670            }
1671        };
1672        // Delegate to the think machinery with a synthesized spec. Only the
1673        // fields that survive the rewrite are carried: the preset OWNS the
1674        // prompt and the schema (that is what makes it a preset), while
1675        // `skills` and `model` are the author's, so they travel. Forgetting
1676        // `model` here is what made the tier catalogue useless on exactly the
1677        // cheap shaping kinds it was introduced for — the field was refused by
1678        // the parser, so the omission read as a deliberate limit rather than a
1679        // dropped line.
1680        let mut think = step.clone();
1681        think.kind = "think".into();
1682        let mut think_spec = Map::new();
1683        think_spec.insert("prompt".into(), Value::String(prompt));
1684        think_spec.insert("output_schema".into(), schema);
1685        for carried in ["skills", "model"] {
1686            if let Some(v) = spec.get(carried) {
1687                think_spec.insert(carried.into(), v.clone());
1688            }
1689        }
1690        self.step_turn_pub(run_id, step_id, &think, &think_spec, data);
1691    }
1692
1693    /// Step `cache {key, ttl}`: a memoized output by key (memory `_cache/<hash>`).
1694    pub(crate) fn cache_lookup(
1695        &mut self,
1696        step: &Step,
1697        spec: &Map<String, Value>,
1698        data: &Data,
1699    ) -> Option<(String, Option<Value>)> {
1700        let cache = step.cache.as_ref()?;
1701        let key_expr = cache.get("key").and_then(Value::as_str).unwrap_or("");
1702        let key_material = if key_expr.is_empty() {
1703            Value::Object(spec.clone()).to_string()
1704        } else {
1705            match crate::engine::template::render_str(key_expr, data) {
1706                Ok(v) => v.to_string(),
1707                Err(_) => return None,
1708            }
1709        };
1710        let hash = crate::sha::sha256_hex(
1711            format!("{}:{}:{}", step.kind, step.id, key_material).as_bytes(),
1712        );
1713        let ttl_ms = cache
1714            .get("ttl")
1715            .and_then(crate::engine::model::duration_ms_opt);
1716        let cache_key = format!("_cache/{hash}");
1717        let hit = self
1718            .durable
1719            .get(Kind::Memory, &cache_key)
1720            .ok()
1721            .flatten()
1722            .and_then(|env| {
1723                let ts = env.state.get("ts").and_then(Value::as_u64).unwrap_or(0);
1724                let fresh = ttl_ms.is_none_or(|t| now_ms() < ts + t);
1725                fresh.then(|| env.state.get("value").cloned()).flatten()
1726            });
1727        Some((cache_key, hit))
1728    }
1729
1730    pub(crate) fn cache_store(&mut self, cache_key: &str, output: &Value) {
1731        let _ = self.durable.put(
1732            Kind::Memory,
1733            cache_key,
1734            json!({"value": output, "ts": now_ms()}),
1735            None,
1736        );
1737    }
1738}