Skip to main content

agentd/runtime/
reactor.rs

1// SPDX-License-Identifier: AGPL-3.0-only
2//! The **runtime state + event loop**: one single-threaded reactor over child
3//! frames, reaped children, executor results, timers, the durable inbox and
4//! signals.
5//!
6//! State mutation happens only here. Being the single writer is what makes the
7//! rest of the runtime reasonable about: no lock ordering, no torn reads, and
8//! an answer computed for one caller is computed against one consistent view.
9//! Every mutation is followed by a checkpoint decision, so durable state never
10//! trails the in-memory state by more than one loop turn.
11//!
12//! The other `runtime::*` modules add `impl Runtime` blocks for turns, tools,
13//! steps and subagents; this file owns construction, the loop, lifecycle and
14//! the status view.
15
16use super::artifacts::Artifacts;
17use super::children::{ChildKind, Children};
18use super::events::{Event, kinds};
19use super::timers::Timers;
20use crate::config::v2::{RunUntil, Settings};
21use crate::context::memory::Memory;
22use crate::context::{Contexts, skills, tokens};
23use crate::engine::{RunState, RunStatus, Workflow};
24use crate::governor::Governor;
25use crate::mcp::client::McpClient;
26use crate::obs::log::Logger;
27use crate::registry::Registry;
28use crate::state::{Durable, InboxEvent, Kind, now_ms};
29use crate::subagent::protocol::AgentMsg;
30use crate::supervisor::reap::Reaped;
31use crate::supervisor::tree::NodeId;
32use serde_json::{Value, json};
33use std::collections::{BTreeMap, VecDeque};
34use std::sync::Arc;
35use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender};
36use std::time::{Duration, Instant};
37
38/// The reactor tick.
39pub const TICK: Duration = Duration::from_millis(200);
40/// Extra grace after the drain deadline before children are abandoned.
41pub const ABANDON_GRACE: Duration = Duration::from_secs(3);
42
43/// Who receives a deferred tool's answer.
44#[derive(Debug, Clone, PartialEq, Eq)]
45pub enum Target {
46    /// A child's `ToolRequest` (answered with `ToolResult`).
47    Child(NodeId, u64),
48    /// A workflow step (answered as the step's outcome).
49    Step(String, String),
50}
51
52/// A deferred internal-tool request (answered when its wait resolves).
53#[derive(Debug, Clone)]
54pub struct PendingTool {
55    pub target: Target,
56    pub name: String,
57    pub kind: PendingKind,
58    pub started_ms: u64,
59}
60
61#[derive(Debug, Clone)]
62pub enum PendingKind {
63    /// A durable timer (`sleep`).
64    Timer { id: String },
65    /// A subagent result (`subagent.run` sync / `subagent.await`).
66    Subagent { handle: String },
67    /// A think child (`think` tool / `context.compact`).
68    Think { child: NodeId },
69    /// A run's terminal state (`workflow.run wait` / `workflow.wait`).
70    Run { run: String, deadline_ms: u64 },
71    /// A CEL condition polled each tick (`await`).
72    Await { condition: String, deadline_ms: u64 },
73    /// A human's answer (`ask_human` / the `human` node): the
74    /// A2A task `task` sits in `input-required`; a `SendMessage` carrying its
75    /// `taskId` resolves this with the reply text. With no interface to answer
76    /// on, `task` is a synthetic ask id (no A2A task exists).
77    Human {
78        task: String,
79        question: String,
80        deadline_ms: u64,
81        /// The task exists ONLY for this ask (no A2A caller/run owns it) —
82        /// complete it when the answer lands.
83        standalone: bool,
84        /// The `auto` fallback judge is running (or already ran) for this ask.
85        auto_fired: bool,
86        /// The answer's declared shape (`human.schema` / `ask_human.schema`).
87        ///
88        /// Carried on the pending ask so the reply can be validated against it
89        /// when it lands. Forwarding the schema to clients only makes them
90        /// render the right form; a gate that declares it wants
91        /// `{decision: "file"|"hold"}` must also refuse "maybe later", or the
92        /// run proceeds on an answer it never asked for.
93        schema: Option<Value>,
94        /// Who must answer (`to:`). `None` ⇒ whoever holds the task, which is
95        /// the ordinary case. Enforced when the answer lands, for the same
96        /// reason the schema is: a gate that names a decider and then accepts
97        /// anyone records something that did not happen.
98        addressee: Option<crate::a2a::principals::Addressee>,
99    },
100}
101
102/// A queued root/conversation turn, waiting for a worker slot and for its
103/// context to be free. One context runs at most one turn at a time, so turns
104/// for the same conversation queue behind each other rather than interleaving
105/// into the same history.
106#[derive(Debug, Clone)]
107pub struct TurnJob {
108    pub ctx: String,
109    /// The triggering inbox event (marked done when the turn completes).
110    pub event: Option<String>,
111    pub principal: Option<String>,
112    /// The message appended to the context before the turn (already appended
113    /// when `None`).
114    pub message: Option<crate::context::Msg>,
115    /// Skill references to preload.
116    pub skills: Vec<String>,
117    /// The user text (for preflight / knowledge retrieval).
118    pub text: String,
119    /// Preflight ran (or was not needed).
120    pub preflight_done: bool,
121    /// Knowledge auto-context ran (or was not needed).
122    pub knowledge_done: bool,
123    /// The retrieved knowledge block (system message) for this turn.
124    pub knowledge: Option<String>,
125    /// The message-hop depth this turn inherits (see `RunState::msg_depth`).
126    /// A message from a person is depth 0; one a `message` step delivered
127    /// carries that step's depth, and anything this turn starts inherits it.
128    pub msg_depth: u32,
129}
130
131impl TurnJob {
132    pub fn new(
133        ctx: String,
134        event: Option<String>,
135        principal: Option<String>,
136        message: Option<crate::context::Msg>,
137        skills: Vec<String>,
138        text: String,
139    ) -> TurnJob {
140        TurnJob {
141            ctx,
142            event,
143            principal,
144            message,
145            skills,
146            text,
147            preflight_done: false,
148            knowledge_done: false,
149            knowledge: None,
150            msg_depth: 0,
151        }
152    }
153    /// The same job, carrying a delivered message's hop depth.
154    pub fn at_depth(mut self, depth: u32) -> TurnJob {
155        self.msg_depth = depth;
156        self
157    }
158}
159
160/// A subagent registry record, persisted as `subagent/<handle>` so a child's
161/// identity and result outlive both the child and this process.
162#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
163pub struct SubagentRecord {
164    pub handle: String,
165    pub instruction: String,
166    pub mode: String,
167    pub status: String,
168    #[serde(default)]
169    pub attempt: u32,
170    #[serde(default, skip_serializing_if = "Option::is_none")]
171    pub result: Option<Value>,
172    #[serde(default, skip_serializing_if = "Option::is_none")]
173    pub error: Option<String>,
174    #[serde(default, skip_serializing_if = "Option::is_none")]
175    pub requested_by: Option<Value>,
176    #[serde(default)]
177    pub tokens: u64,
178    #[serde(default)]
179    pub created: u64,
180    #[serde(default)]
181    pub updated: u64,
182    /// The payload (secret-free) for restore re-spawn.
183    #[serde(default, skip_serializing_if = "Option::is_none")]
184    pub payload: Option<Value>,
185    /// The template this child was instantiated from, and its tier
186    /// (`flat` | `instance`). A freeform spawn carries neither.
187    #[serde(default, skip_serializing_if = "Option::is_none")]
188    pub template: Option<String>,
189    #[serde(default, skip_serializing_if = "Option::is_none")]
190    pub tier: Option<String>,
191    /// Instance tier: the child daemon's pid, config path, A2A socket and
192    /// (epoch-ms) retire-at deadline.
193    #[serde(default, skip_serializing_if = "Option::is_none")]
194    pub pid: Option<i32>,
195    #[serde(default, skip_serializing_if = "Option::is_none")]
196    pub config_path: Option<String>,
197    #[serde(default, skip_serializing_if = "Option::is_none")]
198    pub socket: Option<String>,
199    #[serde(default, skip_serializing_if = "Option::is_none")]
200    pub retire_at: Option<u64>,
201    /// Instance tier: set when retirement began (SIGTERM sent); the tick
202    /// escalates to SIGKILL after the drain window.
203    #[serde(default, skip_serializing_if = "Option::is_none")]
204    pub retiring_since: Option<u64>,
205    /// Durability class (default true). `false` ⇒ the record is memory-only:
206    /// never persisted, never restore-respawned — the fast path for throwaway
207    /// workers. Restored records (all persisted by construction) default true.
208    #[serde(default = "record_durable_default")]
209    pub durable: bool,
210    #[serde(skip)]
211    pub node: Option<NodeId>,
212    #[serde(skip)]
213    pub dirty: bool,
214}
215
216fn record_durable_default() -> bool {
217    true
218}
219
220/// The instruction in force. `version` increments on every change, so a
221/// consumer can tell a re-read from a genuinely new instruction.
222#[derive(Debug, Clone)]
223pub struct Instruction {
224    pub text: String,
225    pub source: &'static str,
226    pub uri: Option<String>,
227    pub server: Option<String>,
228    pub version: u64,
229}
230
231/// Counters for status/reports.
232#[derive(Debug, Default, Clone)]
233pub struct Counters {
234    pub turns: u64,
235    pub tool_calls: u64,
236    pub runs_started: u64,
237    pub runs_finished: u64,
238    pub inbox_processed: u64,
239    pub tokens_in: u64,
240    pub tokens_out: u64,
241}
242
243pub struct Runtime {
244    /// Resource pressure (disk headroom, cgroup memory): consulted at every
245    /// ADMISSION gate — start-node firing, webhook accept, `workflow.run`,
246    /// turn dispatch, subagent spawn — never on work already in flight.
247    pub(crate) pressure: std::sync::Arc<super::pressure::Pressure>,
248    /// The last level the tick reported, so transitions log exactly once.
249    pub(crate) pressure_seen: super::pressure::Level,
250    /// A step reached a terminal state since the last scheduling pass — its
251    /// dependents may be ready NOW (the same-iteration re-schedule fixpoint).
252    pub(crate) resched: bool,
253    /// Reaps already deferred once for frame ordering (by pid) — see
254    /// [`Runtime::on_reaped`].
255    pub(crate) reap_deferred: std::collections::HashSet<i32>,
256    /// Outbound token buckets for steps that declare `rate:`, keyed like the
257    /// breaker (`workflow/unscoped-step`). In-memory on purpose: a rate is a
258    /// statement about LIVE traffic, and a restart briefly refilling the burst
259    /// is harmless where a durable bucket would be bookkeeping for its own
260    /// sake. The paired f64 is the window seconds, for computing the wait.
261    pub(crate) step_rates:
262        std::collections::HashMap<String, (crate::supervisor::tree::TokenBucket, f64, u32)>,
263    pub(crate) settings: Settings,
264    /// The merged document the settings came from (restart-only diff base).
265    pub(crate) settings_doc: Value,
266    /// The invocation (for reload).
267    pub(crate) args: Vec<String>,
268    pub(crate) env: Vec<(String, String)>,
269    /// Workflow definitions pinned by live runs after a reload (hash → definition).
270    pub(crate) pinned: BTreeMap<String, std::sync::Arc<Workflow>>,
271    /// Retired definitions still owning live runs (`runtime::retire`), by hash.
272    pub(crate) retiring: BTreeMap<String, super::retire::Retiring>,
273    /// Definition hashes whose durable pin was written this life (one write
274    /// per version; see `retire::ensure_pin`).
275    pub(crate) pin_written: std::collections::HashSet<String>,
276    /// The last payload per signal name (for `await`/`wait condition` views).
277    pub(crate) recent_signals: BTreeMap<String, Value>,
278    /// Memoized `memory.<key>` references per definition content hash: the
279    /// scan walks the whole definition and `run_data` runs per step.
280    pub(crate) memory_keys: std::collections::HashMap<String, Vec<String>>,
281    /// An `emit` appended since the last stream poll (same-iteration wake).
282    pub(crate) stream_dirty: bool,
283    pub(crate) log: Logger,
284    pub(crate) instance: String,
285    pub(crate) run_id: String,
286    pub(crate) durable: Durable,
287    pub(crate) mcp: BTreeMap<String, Arc<McpClient>>,
288    pub(crate) mcp_specs: BTreeMap<String, crate::config::McpServerSpec>,
289    pub(crate) registry: Registry,
290    pub(crate) contexts: Contexts,
291    pub(crate) memory: Memory,
292    pub(crate) artifacts: Artifacts,
293    pub(crate) skills: skills::Catalogue,
294    pub(crate) governor: Governor,
295    /// Per-principal budgets and rate quotas, indexed by principal id when one
296    /// is first seen. `a2a.principals[].quotas` parsed and validated for a
297    /// long time without anything reading it; these are its readers.
298    pub(crate) principal_budgets: BTreeMap<String, crate::config::v2::Budget>,
299    /// Only the A2A listener admits callers, so a build without it has
300    /// nowhere to spend an arrival quota.
301    #[cfg_attr(not(feature = "a2a"), allow(dead_code))]
302    pub(crate) principal_rates: BTreeMap<String, crate::supervisor::tree::TokenBucket>,
303    /// Labels an id acts under, for `_meta` and audit.
304    pub(crate) principal_labels: BTreeMap<String, BTreeMap<String, String>>,
305    pub(crate) workflows: BTreeMap<String, std::sync::Arc<Workflow>>,
306    pub(crate) runs: BTreeMap<String, RunState>,
307    pub(crate) children: Children,
308    pub(crate) timers: Timers,
309    pub(crate) events_rx: Receiver<Event>,
310    pub(crate) events_tx: Sender<Event>,
311    pub(crate) reap_rx: Receiver<Reaped>,
312    pub(crate) pending: Vec<PendingTool>,
313    pub(crate) turn_queue: VecDeque<TurnJob>,
314    /// Turn jobs parked while their preflight think / knowledge retrieval runs.
315    pub(crate) staged_turns: BTreeMap<u64, TurnJob>,
316    pub(crate) inbox_queue: VecDeque<InboxEvent>,
317    pub(crate) subagents: BTreeMap<String, SubagentRecord>,
318    pub(crate) instruction: Instruction,
319    pub(crate) job_shape: bool,
320    pub(crate) exit: Option<i32>,
321    pub(crate) draining: bool,
322    /// Operator-held (a2a.pause): intake continues; no new turns dispatch and
323    /// no steps schedule until a2a.resume. Reversible, unlike drain.
324    pub(crate) paused: bool,
325    pub(crate) drain_started: Option<Instant>,
326    pub(crate) drain_reason: String,
327    pub(crate) idle_since: Option<Instant>,
328    pub(crate) intel_uri: String,
329    pub(crate) intel_token: Option<String>,
330    /// Resolved `intelligence.headers`, pushed on every LLM dial and threaded
331    /// to subagents via the spawn payload so a child dials identically.
332    pub(crate) intel_headers: Vec<(String, String)>,
333    /// An optional intelligence credential provider: a closure returning the
334    /// current bearer, refreshed from the device-login cache. Its resolved
335    /// bearer overrides `intel_token`, and is threaded to subagents fresh at
336    /// each spawn so no child carries a stale one. `None` when no
337    /// `intelligence.auth` oauth2 block is configured.
338    pub(crate) intel_bearer: Option<std::sync::Arc<dyn Fn() -> Option<String> + Send + Sync>>,
339    pub(crate) model: String,
340    pub(crate) trace_id: Option<String>,
341    pub(crate) started: Instant,
342    pub(crate) seq: u64,
343    pub(crate) counters: Counters,
344    /// The `once`-started run(s) whose finish decides a job's exit code.
345    pub(crate) job_runs: Vec<String>,
346    /// Steps executing on executor threads (`run/step` → started).
347    pub(crate) executing: BTreeMap<String, Instant>,
348    pub(crate) last_manifest_flush: Instant,
349    /// Unix-ms a goal LLM judge was dispatched (so overlapping checks don't spawn
350    /// duplicate judges); `None` = none in flight.
351    pub(crate) goal_judge_at: Option<u64>,
352    /// Durable A2A tasks, keyed by task id.
353    #[cfg(feature = "a2a")]
354    pub(crate) tasks: BTreeMap<String, crate::a2a::Task>,
355    /// Inbox-event id → the A2A task it answers (a conversation turn).
356    #[cfg(feature = "a2a")]
357    pub(crate) event_to_task: BTreeMap<String, String>,
358    /// The task snapshot the A2A listener threads read (None ⇒ not serving).
359    #[cfg(feature = "a2a")]
360    /// The interface event feed. `None` means the interface is disabled.
361    #[cfg(feature = "a2a")]
362    pub(crate) a2a_feed: Option<std::sync::Arc<super::a2a_server::SharedFeed>>,
363    /// Pairing-code login state. `None` means pairing is disabled.
364    #[cfg(feature = "a2a")]
365    pub(crate) a2a_pairing: Option<std::sync::Arc<super::a2a_server::PairingState>>,
366    /// The id the listener reserved for the task the request being served will
367    /// create. Taken by the first `task_create` of that request, and cleared
368    /// after it — an id belongs to one request only.
369    #[cfg(feature = "a2a")]
370    pub(crate) reserved_task_id: Option<String>,
371    /// Where a task transition is published so A2A subscribers see it.
372    #[cfg(feature = "a2a")]
373    pub(crate) a2a_sink: Option<std::sync::Arc<crate::a2a::ports::StreamSink>>,
374    /// The live listener. Held, not used: dropping it stops serving.
375    #[cfg(feature = "a2a")]
376    pub(crate) a2a_listener: Option<crate::a2a::serve::Listener>,
377    /// The listener's bridge, so a reload can swap rebuilt principal rules in.
378    #[cfg(feature = "a2a")]
379    pub(crate) a2a_bridge: Option<std::sync::Arc<super::a2a_server::A2aBridge>>,
380    /// The webhook listener's handler, so a reload can swap rebuilt routes in.
381    #[cfg(feature = "a2a")]
382    pub(crate) webhook_handler: Option<std::sync::Arc<super::webhooks::WebhookHandler>>,
383    /// The listener's live CORS allowlist, so a reload can revise it.
384    #[cfg(feature = "a2a")]
385    pub(crate) a2a_origins: Option<crate::a2a::serve::OriginList>,
386    /// Live per-unit activity, keyed by child node id.
387    pub(crate) activity: BTreeMap<u64, super::activity::Activity>,
388    /// The newest root-context reply, so a `--prompt` job can print its answer
389    /// (a prompt runs as a turn, not as a `once` run with an output).
390    pub(crate) last_root_reply: Option<String>,
391    /// Per-item fingerprints behind the feed's section diffing (`feed_tick`).
392    #[cfg(feature = "a2a")]
393    pub(crate) feed_marks: BTreeMap<String, u64>,
394    /// The last section-diff pass (rate-limits `feed_tick`).
395    #[cfg(feature = "a2a")]
396    pub(crate) feed_last: Instant,
397    /// The `wait: {on: webhook}` await-callback registry, shared with the webhook
398    /// listener threads.
399    #[cfg(feature = "a2a")]
400    pub(crate) webhook_callbacks: super::webhooks::SharedCallbacks,
401    /// Pending `respond: sync` webhook replies, keyed by the run id they await.
402    #[cfg(feature = "a2a")]
403    pub(crate) webhook_sync: std::collections::HashMap<
404        String,
405        std::sync::mpsc::SyncSender<super::webhooks::WebhookReply>,
406    >,
407}
408
409impl Runtime {
410    /// A fresh id (turn ids, handles).
411    /// The deployment's default durability class for work (runs + subagent
412    /// records): `store.durability.work: ephemeral` ⇒ false.
413    pub(crate) fn work_durable_default(&self) -> bool {
414        !matches!(
415            self.settings.store.durability.work,
416            Some(crate::config::v2::WorkDurability::Ephemeral)
417        )
418    }
419
420    pub(crate) fn next_id(&mut self, prefix: &str) -> String {
421        self.seq += 1;
422        format!("{prefix}-{}", self.seq)
423    }
424
425    // ---- the loop ----------------------------------------------------------
426
427    /// Run until exit. Returns the process exit code.
428    pub fn run_loop(&mut self) -> i32 {
429        self.log.info("proc.ready", json!({"instance": self.instance, "job_shape": self.job_shape, "workflows": self.workflows.len(), "runs": self.runs.len(), "inbox_pending": self.inbox_queue.len()}));
430        loop {
431            crate::obs::health::tick();
432            // Pressure transitions are logged HERE, once per change, so the
433            // per-request gates can refuse silently instead of each writing its
434            // own line per refusal — under real pressure that would be a log
435            // flood on top of a disk that is already full.
436            {
437                let level = self.pressure.level();
438                if level != self.pressure_seen {
439                    let free = self
440                        .pressure
441                        .disk_free
442                        .load(std::sync::atomic::Ordering::Relaxed);
443                    let detail = json!({
444                        "level": level.as_str(),
445                        "cause": self.pressure.cause(),
446                        "disk_free_bytes": if free == u64::MAX { Value::Null } else { json!(free) },
447                    });
448                    match level {
449                        super::pressure::Level::Ok => self.log.info("pressure.cleared", detail),
450                        super::pressure::Level::Warn => self.log.warn("pressure.warn", detail),
451                        super::pressure::Level::Shed => self.log.warn("pressure.shed", detail),
452                    }
453                    self.pressure_seen = level;
454                }
455            }
456            // 1. Child frames.
457            // (child frames arrive as Event::Child on the main channel — they
458            // wake the parked loop instead of waiting for the tick)
459            // 2. Reaped children.
460            let _ = crate::signals::take_child_exit();
461            crate::supervisor::reaper::reap_and_dispatch();
462            while let Ok(r) = self.reap_rx.try_recv() {
463                self.on_reaped(r);
464            }
465            // 3. Executor / internal events.
466            while let Ok(ev) = self.events_rx.try_recv() {
467                self.on_event(ev);
468            }
469            // 3.5. Retiring workflows whose drain deadline passed.
470            self.retire_tick();
471            // 4. Timers.
472            let now = now_ms();
473            for t in self.timers.fire(&self.durable, now) {
474                self.on_timer(t);
475            }
476            // 4.9. The daemon's own events, queued by the tap since the last
477            // tick, become appends — so a tripped breaker or a shed admission
478            // can start a run. Done BEFORE the inbox and the start poll so
479            // this tick's consumers see this tick's telemetry.
480            self.drain_runtime_events();
481            // 5. The inbox.
482            self.process_inbox();
483            // 6. Start nodes + runs (+ suspended waits).
484            self.poll_starts();
485            self.poll_stream_starts();
486            // Runs parked on the log resolve in the same pass that advances
487            // consumers, so a produce→wait hop costs a tick, not a timeout.
488            self.poll_event_waits();
489            self.poll_waits();
490            self.schedule_runs();
491            // Inline steps (assign/map/template/switch…) complete synchronously
492            // inside that pass, which makes their dependents ready NOW — without
493            // this fixpoint a pure data pipeline advanced ONE step per 200 ms
494            // tick (measured: 200 chained assigns = 42 s; with it, milliseconds).
495            // Bounded for the loop's honesty: effectful steps complete via
496            // events, so only inline chains re-enter here, and `limits.run.steps`
497            // already caps how long one can be.
498            let mut passes = 0;
499            while std::mem::take(&mut self.resched) && passes < 1024 {
500                self.schedule_runs();
501                passes += 1;
502            }
503            // 6.6. Streams appended in this iteration fire their consumers
504            // NOW: a same-process produce->consume pipeline advances at
505            // engine speed instead of paying the tick park per hop. Bounded
506            // like the fixpoint; an emit inside a fired consumer re-enters
507            // here, and `limits.run.steps` caps how deep that can go.
508            let mut stream_rounds = 0;
509            while std::mem::take(&mut self.stream_dirty) && stream_rounds < 64 {
510                self.poll_stream_starts();
511                // A run parked on the log is a consumer too: without this, a
512                // saga whose awaited event was emitted by a step in this very
513                // iteration would park until the next tick.
514                self.poll_event_waits();
515                self.schedule_runs();
516                let mut passes = 0;
517                while std::mem::take(&mut self.resched) && passes < 1024 {
518                    self.schedule_runs();
519                    passes += 1;
520                }
521                stream_rounds += 1;
522            }
523            // 7. Turns.
524            self.dispatch_turns();
525            // 8. Pending waits + MCP notifications.
526            self.poll_pending();
527            self.poll_mcp_notifications();
528            // 9. Children maintenance.
529            for (node, health) in self.children.tick() {
530                self.on_unhealthy_child(node, health);
531            }
532            // 9b. Instance-tier children: ttl retirement, plus the
533            // SIGTERM→SIGKILL escalation for children that ignored the drain.
534            self.instances_tick();
535            // 10. Checkpoints + the point-in-time observability gauges.
536            self.checkpoint(false);
537            crate::obs::metrics::set_inbox_pending(self.inbox_queue.len() as u64);
538            crate::obs::metrics::set_context_tokens(self.contexts.max_est_tokens());
539            {
540                let free = self
541                    .pressure
542                    .disk_free
543                    .load(std::sync::atomic::Ordering::Relaxed);
544                crate::obs::metrics::set_pressure(
545                    self.pressure_seen as u64,
546                    (free != u64::MAX).then_some(free),
547                );
548                crate::obs::metrics::set_work_backlog(
549                    self.runs
550                        .values()
551                        .filter(|r| !r.status.is_terminal())
552                        .count() as u64,
553                    self.turn_queue.len() as u64,
554                );
555            }
556            // 10.5. The interface feed's section diff: publish
557            // run/conversation/subagent/child/status deltas to attached display
558            // clients. A no-op unless `interface.enabled`; rate-limited inside.
559            #[cfg(feature = "a2a")]
560            self.feed_tick();
561            // 11. Signals + lifecycle.
562            self.check_signals();
563            if let Some(code) = self.lifecycle_step() {
564                self.shutdown(code);
565                return code;
566            }
567            // 12. Wait for the next event, bounded by the tick or the nearest
568            // imminent deadline (a timer, a schedule/loop start, a pending wait)
569            // so time-based work fires promptly rather than at tick granularity.
570            crate::signals::drain_wakeup();
571            let wait = self.next_wake().min(TICK);
572            match self.events_rx.recv_timeout(wait) {
573                Ok(ev) => self.on_event(ev),
574                Err(RecvTimeoutError::Timeout) | Err(RecvTimeoutError::Disconnected) => {}
575            }
576        }
577    }
578
579    fn on_event(&mut self, ev: Event) {
580        match ev {
581            Event::Child(node, msg) => self.on_child_frame(node, msg),
582            Event::Reaped(r) => self.on_reaped(r),
583            Event::StepDone {
584                run,
585                step,
586                output,
587                is_error,
588                error,
589                tokens,
590            } => self.on_step_done(&run, &step, output, is_error, error, tokens),
591            Event::ToolDone {
592                node,
593                req,
594                result,
595                is_error,
596            } => self.on_tool_done(node, req, result, is_error),
597            Event::KnowledgeDone { job, block } => self.on_knowledge_done(job, block),
598            Event::TimerFired { id, owner, payload } => self.on_timer(crate::state::TimerRecord {
599                id,
600                deadline_ms: now_ms(),
601                owner,
602                payload,
603            }),
604            Event::Inbox(ev) => self.inbox_queue.push_back(ev),
605            #[cfg(feature = "a2a")]
606            Event::A2a(req) => self.on_a2a_request(*req),
607            #[cfg(feature = "a2a")]
608            Event::Webhook(req) => self.on_webhook_request(*req),
609            Event::Background { id, result } if id == "goal.judge" => self.on_goal_judge(&result),
610            Event::Background { id, result } if id.starts_with("human.judge:") => {
611                let ask = id.trim_start_matches("human.judge:").to_string();
612                self.on_human_judge(&ask, &result);
613            }
614            Event::SubscribeRead {
615                server,
616                uri,
617                content,
618            } => self.on_subscribe_read(&server, &uri, content),
619            Event::Background { .. } | Event::Tick => {}
620        }
621    }
622
623    // ---- inbox -------------------------------------------------------------
624
625    /// Accept a durable event: write it to the store first, then queue it for
626    /// the loop. Write-ahead is the whole point — once acceptance is
627    /// acknowledged to the outside world, a crash before the event is acted on
628    /// must replay it rather than drop it.
629    pub(crate) fn accept_event(
630        &mut self,
631        kind: &str,
632        principal: Option<String>,
633        payload: Value,
634    ) -> Result<String, String> {
635        let ev = InboxEvent::new(kind, principal, payload);
636        self.durable
637            .inbox_put(&ev)
638            .map_err(|e| format!("inbox: {e}"))?;
639        let id = ev.id.clone();
640        self.log
641            .info("inbox.accepted", json!({"inbox_event": id, "kind": kind}));
642        self.inbox_queue.push_back(ev);
643        Ok(id)
644    }
645
646    fn process_inbox(&mut self) {
647        // Drain a SNAPSHOT, never the live deque: a start event that overflows
648        // its workflow's concurrency cap re-queues itself (`on_overflow: queue`,
649        // the default), and the cap can only be relieved by `schedule_runs` — a
650        // LATER step of this tick. Popping from the same deque the requeue
651        // pushes onto re-offers the event immediately and the single-writer
652        // reactor spins at 100% CPU forever: no timers, no checkpoint, no
653        // SIGTERM. Requeued (and newly accepted) events land in the fresh
654        // `self.inbox_queue` and are retried on the next tick instead.
655        let mut batch = std::mem::take(&mut self.inbox_queue);
656        while let Some(ev) = batch.pop_front() {
657            if self.draining {
658                // Keep it durable for the next life; stop intake — with one
659                // exception: the start event of a `lifecycle.shutdown` deinit
660                // workflow exists to run DURING the drain, and the drain gate
661                // is waiting for it. Everything else waits for the next life.
662                let deinit = ev.kind == kinds::START_FIRED
663                    && ev.payload["workflow"]
664                        .as_str()
665                        .and_then(|n| self.workflows.get(n))
666                        .is_some_and(|w| {
667                            w.start_steps().iter().any(|s| {
668                                s.kind == "event" && s.field_str("on") == Some("lifecycle.shutdown")
669                            })
670                        });
671                if !deinit {
672                    self.inbox_queue.push_back(ev);
673                    continue;
674                }
675            }
676            self.counters.inbox_processed += 1;
677            match ev.kind.as_str() {
678                kinds::START_FIRED | kinds::WORKFLOW_RUN => {
679                    let done = self.on_start_event(&ev);
680                    if done {
681                        self.inbox_done(&ev.id);
682                    }
683                }
684                kinds::A2A_MESSAGE => {
685                    // Handled the same whether it arrived live or was replayed
686                    // from the inbox after a restart.
687                    self.on_a2a_message_event(&ev);
688                }
689                kinds::SIGNAL => {
690                    let name = ev.payload["name"].as_str().unwrap_or("").to_string();
691                    let payload = ev.payload.get("payload").cloned().unwrap_or(Value::Null);
692                    let target = ev
693                        .payload
694                        .get("run")
695                        .and_then(Value::as_str)
696                        .map(str::to_string);
697                    let from = ev
698                        .payload
699                        .get("from")
700                        .and_then(Value::as_str)
701                        .map(str::to_string);
702                    let delivered =
703                        self.deliver_signal(&name, payload, target.as_deref(), from.as_deref());
704                    self.log.info(
705                        "signal.received",
706                        json!({"inbox_event": ev.id, "name": name, "delivered": delivered}),
707                    );
708                    self.inbox_done(&ev.id);
709                }
710                other => {
711                    self.log.warn(
712                        "inbox.unknown_kind",
713                        json!({"inbox_event": ev.id, "kind": other}),
714                    );
715                    self.inbox_done(&ev.id);
716                }
717            }
718        }
719        // Whatever the drain did not consume keeps its place ahead of the
720        // events requeued (or accepted) while the batch was processing.
721        batch.append(&mut self.inbox_queue);
722        self.inbox_queue = batch;
723    }
724
725    pub(crate) fn inbox_done(&mut self, id: &str) {
726        if let Err(e) = self.durable.inbox_done(id) {
727            self.log.warn(
728                "inbox.done.fail",
729                json!({"inbox_event": id, "err": e.to_string()}),
730            );
731        }
732    }
733
734    /// An A2A message event, routed to whichever reader owns it. Control-plane
735    /// ops are consumed first, then a waiting step, then a start node, and only
736    /// what is left becomes a conversation turn.
737    fn on_a2a_message_event(&mut self, ev: &InboxEvent) {
738        let ctx = ev.payload["context_id"]
739            .as_str()
740            .unwrap_or("default")
741            .to_string();
742        let text = ev.payload["text"]
743            .as_str()
744            .map(str::to_string)
745            .unwrap_or_else(|| ev.payload["parts"].to_string());
746        let principal = ev.principal.clone();
747        // Re-link a replayed message to its durable task (crash recovery).
748        #[cfg(feature = "a2a")]
749        if let Some(task_id) = ev.payload["task"].as_str() {
750            self.event_to_task
751                .insert(ev.id.clone(), task_id.to_string());
752        }
753        // `_instance.*` ops are a child reporting home. The runtime consumes
754        // them BEFORE any reader, so they can never be mistaken for a wait's
755        // answer, a start's request, or a conversational turn — control-plane
756        // traffic must not reach a model.
757        #[cfg(feature = "a2a")]
758        if self.handle_instance_op(ev) {
759            return;
760        }
761        // An inbound message has three possible readers, in this order. Only one
762        // takes it: a message that woke a waiting step is an ANSWER, and a
763        // message that fired a workflow is a REQUEST — neither should also
764        // become a conversational turn, or the agent replies to itself.
765        //
766        // 1. A step suspended on this conversation (`a2a.wait` / `wait {on:
767        //    message}`) — the reply half of an asynchronous exchange.
768        let msg = json!({"parts": ev.payload.get("parts").cloned().unwrap_or(Value::Null),
769                         "text": text, "message_id": ev.payload.get("message_id").cloned()});
770        if self.deliver_a2a_message(&ctx, &msg, principal.as_deref()) > 0 {
771            self.log.info(
772                "a2a.message.delivered",
773                json!({"inbox_event": ev.id, "conversation": ctx}),
774            );
775            return;
776        }
777        // 2. An `a2a` START node whose command and roles match — a peer or an
778        //    operator asking for a workflow rather than a conversation.
779        if self.fire_a2a_start(ev, &ctx) {
780            return;
781        }
782        // 3. Otherwise it is what it looks like: something to answer.
783        #[allow(unused)]
784        let skills = self.skills.references(&text);
785        let depth = ev.payload["msg_depth"].as_u64().unwrap_or(0) as u32;
786        self.turn_queue.push_back(
787            TurnJob::new(
788                ctx,
789                Some(ev.id.clone()),
790                principal.clone(),
791                Some(crate::context::Msg::user(text.clone(), principal)),
792                skills,
793                text,
794            )
795            .at_depth(depth),
796        );
797    }
798
799    /// Without the `a2a` feature there is no listener to deliver a message, so a
800    /// replayed event simply degrades to a turn.
801    #[cfg(not(feature = "a2a"))]
802    fn fire_a2a_start(&mut self, _ev: &InboxEvent, _ctx: &str) -> bool {
803        false
804    }
805
806    /// Match an inbound A2A message against every `a2a` start node and fire the
807    /// first that accepts it. Returns whether a run was started.
808    ///
809    /// `command` selects on the command DataPart's `op` — absent means "any
810    /// message", which is how a workflow takes plain conversation as its
811    /// trigger. `roles` restricts which principals may fire it, and defaults to
812    /// no restriction beyond the authorization the listener already applied:
813    /// the start node narrows, it never widens.
814    #[cfg(feature = "a2a")]
815    fn fire_a2a_start(&mut self, ev: &InboxEvent, ctx: &str) -> bool {
816        let op = ev.payload.get("parts").and_then(|parts| {
817            crate::runtime::a2a_server::command_op(&json!({"parts": parts.clone()}))
818        });
819        // The typed command payload, `op` removed: a workflow reads
820        // `{{ steps.cmd.output.args.<field> }}` instead of parsing parts.
821        let args = ev.payload.get("parts").and_then(|parts| {
822            crate::runtime::a2a_server::command_data(&json!({"parts": parts.clone()})).map(
823                |mut d| {
824                    if let Some(o) = d.as_object_mut() {
825                        o.remove("op");
826                    }
827                    d
828                },
829            )
830        });
831        let role = ev.payload["role"].as_str().unwrap_or("");
832        let specs: Vec<(String, String, serde_json::Map<String, Value>)> = self
833            .workflows
834            .values()
835            .flat_map(|w| {
836                w.start_steps()
837                    .into_iter()
838                    .filter(|s| s.kind == "a2a")
839                    .map(|s| (w.name.clone(), s.id.clone(), s.spec.clone()))
840                    .collect::<Vec<_>>()
841            })
842            .collect();
843        for (workflow, node, spec) in specs {
844            if let Some(want) = spec.get("command").and_then(Value::as_str)
845                && Some(want) != op.as_deref()
846            {
847                continue;
848            }
849            if let Some(roles) = spec.get("roles").and_then(Value::as_array)
850                && !roles.is_empty()
851                && !roles.iter().any(|r| r.as_str() == Some(role))
852            {
853                continue;
854            }
855            let payload = json!({
856                "conversation": ctx,
857                "principal": ev.principal,
858                "role": role,
859                "command": op,
860                "args": args.clone().unwrap_or(Value::Null),
861                // The A2A task tracking this message: carried onto the run so
862                // its terminal status completes the task — which is what lets
863                // a peer's `a2a.delegate {command}` BLOCK on the answer.
864                "task": ev.payload.get("task").cloned().unwrap_or(Value::Null),
865                "parts": ev.payload.get("parts").cloned().unwrap_or(Value::Null),
866                "text": ev.payload.get("text").cloned().unwrap_or(Value::Null),
867                "message_id": ev.payload.get("message_id").cloned().unwrap_or(Value::Null),
868                // The message-hop depth rides through this reader too. Without
869                // it a chain routed through an `a2a` start would reset to zero
870                // on every hop, and the cap would never bite — the run this
871                // fires can `message` again, and that is the same loop.
872                "msg_depth": ev.payload.get("msg_depth").cloned().unwrap_or(json!(0)),
873            });
874            self.log.info(
875                "start.a2a.fired",
876                json!({"workflow": workflow, "node": node, "conversation": ctx,
877                       "command": op, "role": role}),
878            );
879            self.fire_start(&workflow, &node, &spec, payload, "a2a");
880            return true;
881        }
882        false
883    }
884
885    // ---- children ----------------------------------------------------------
886
887    fn on_child_frame(&mut self, node: NodeId, msg: AgentMsg) {
888        if !self.children.on_frame(node, &msg) {
889            return; // a late frame from a reaped child
890        }
891        match msg {
892            AgentMsg::Ready
893            | AgentMsg::Pong { .. }
894            | AgentMsg::Gate { .. }
895            | AgentMsg::GateClosed { .. } => {}
896            // Coarse progress from the child: what this unit is doing right
897            // now, for the display clients' working row.
898            AgentMsg::Event { event, fields } => self.on_child_progress(node, &event, &fields),
899            AgentMsg::Usage(u) => {
900                self.counters.tokens_in += u.input_tokens;
901                self.counters.tokens_out += u.output_tokens;
902                crate::obs::metrics::record_tokens(u.input_tokens, u.output_tokens);
903                // A subagent's usage is charged as it reports; turn usage is
904                // settled on TurnDone against its reservation.
905                if let Some(ChildKind::Subagent { .. }) = self.children.get(node).map(|c| &c.kind) {
906                    self.governor.charge(u, &[]);
907                }
908            }
909            AgentMsg::IntelHealth { all_down, .. } => {
910                if crate::signals::set_intel_all_down(all_down) {
911                    self.log.warn("intel.health", json!({"all_down": all_down}));
912                }
913            }
914            AgentMsg::ToolRequest { id, name, args } => self.on_tool_request(node, id, &name, args),
915            AgentMsg::BudgetRequest { id, estimate } => self.on_budget_request(node, id, estimate),
916            AgentMsg::TurnDone { turn } => self.on_turn_done(node, *turn),
917            AgentMsg::Turn { outcome } => self.on_subagent_turn(node, outcome),
918            AgentMsg::Result { outcome } => self.on_subagent_result(node, Ok(outcome)),
919            AgentMsg::Failed { error } => {
920                let kind = self.children.get(node).map(|c| c.kind.clone());
921                match kind {
922                    Some(ChildKind::Subagent { .. }) => self.on_subagent_result(node, Err(error)),
923                    Some(_) => self.on_turn_failed(node, error),
924                    None => {}
925                }
926            }
927        }
928    }
929
930    fn on_reaped(&mut self, r: Reaped) {
931        // Frames-before-reap. A child's terminal frame rides the same event
932        // queue as everything else (that is what makes its arrival WAKE the
933        // loop), so a reap racing ahead of it would read as "worker exited
934        // without a result". Restore the invariant by construction: join the
935        // child's reader thread — bounded, its pipe has already EOF'd — so
936        // every frame it ever wrote is IN the queue, then requeue the reap
937        // BEHIND them. FIFO does the rest; one deferral suffices.
938        if !self.reap_deferred.remove(&r.pid) && self.children.has_pid(r.pid) {
939            self.children.join_reader_of(r.pid);
940            self.reap_deferred.insert(r.pid);
941            let _ = self.events_tx.send(Event::Reaped(r));
942            return;
943        }
944        // An instance-tier daemon child has no control channel and no node in
945        // the child table, so its exit closes the subagent record directly.
946        if !self.children.has_pid(r.pid) && self.on_instance_reaped(&r) {
947            return;
948        }
949        let Some((node, child)) = self.children.on_reaped(&r) else {
950            return;
951        };
952        self.activity_end(node);
953        self.log.info("child.exit", json!({"node": node.0, "pid": r.pid, "kind": super::children::kind_label(&child.kind), "outcome": format!("{:?}", r.outcome)}));
954        // A child that died without its terminal frame: fail its unit.
955        match child.kind {
956            // Ask the STEP, not the child table, whether this worker died
957            // owing a result. The child table cannot answer it here: a
958            // `TurnDone` settles the step but leaves the child in the table
959            // until it is reaped, and `Children::on_reaped` above has already
960            // removed the entry — so "is the child in the table?" reads the
961            // same for a settled worker and an orphaned one. The step is
962            // unambiguous: it is Running and still owned by THIS worker only
963            // when no terminal frame ever landed.
964            ChildKind::StepTurn {
965                ref run,
966                ref step,
967                reservation,
968            } => {
969                let node_owned = node.0.to_string();
970                let orphaned = self
971                    .runs
972                    .get(run)
973                    .and_then(|st| st.step(step))
974                    .is_some_and(|s| {
975                        s.status == crate::engine::StepStatus::Running
976                            && s.worker.as_deref() == Some(node_owned.as_str())
977                    });
978                if orphaned {
979                    // `on_turn_failed` would route this, but it re-reads the
980                    // child table too and returns early on the reaped node; the
981                    // reservation it would have released is released here.
982                    if let Some(res) = reservation {
983                        self.governor.release(res);
984                    }
985                    self.log.warn(
986                        "turn.failed",
987                        json!({"node": node.0, "kind": super::children::kind_label(&child.kind), "err": "worker exited without a result"}),
988                    );
989                    self.on_step_turn_done(
990                        run,
991                        step,
992                        crate::subagent::protocol::TurnResult {
993                            status: "failed".into(),
994                            error: Some(format!(
995                                "worker exited without a result ({:?})",
996                                r.outcome
997                            )),
998                            ..Default::default()
999                        },
1000                    );
1001                }
1002            }
1003            // A root turn and a think expose no equivalent state to test
1004            // here, so they ask `pending_turn_exists`, which answers from the
1005            // settled marker `on_turn_done` / `on_turn_failed` leave on the
1006            // child record rather than from the child's presence in the table.
1007            // Presence cannot answer it: `on_reaped` has already removed the
1008            // child by the time this runs, and a normally-settled worker also
1009            // stays in the table until it is reaped, so presence reads the same
1010            // for settled and orphaned workers alike.
1011            ChildKind::RootTurn { .. } | ChildKind::Think { .. } => {
1012                if self.pending_turn_exists(node) {
1013                    self.on_turn_failed(
1014                        node,
1015                        format!("worker exited without a result ({:?})", r.outcome),
1016                    );
1017                }
1018            }
1019            ChildKind::Subagent { ref handle } => {
1020                if self
1021                    .subagents
1022                    .get(handle)
1023                    .is_some_and(|s| !is_terminal_status(&s.status))
1024                {
1025                    self.on_subagent_result(
1026                        node,
1027                        Err(format!(
1028                            "subagent exited without a result ({:?})",
1029                            r.outcome
1030                        )),
1031                    );
1032                }
1033            }
1034        }
1035        // Answer any tool request that was waiting on this child (a think).
1036        let waiting: Vec<PendingTool> = self
1037            .pending
1038            .iter()
1039            .filter(|p| matches!(&p.kind, PendingKind::Think { child } if *child == node))
1040            .cloned()
1041            .collect();
1042        for p in waiting {
1043            self.pending.retain(|q| q.target != p.target);
1044            self.reply(
1045                &p.target,
1046                Value::String("think worker exited without a result".into()),
1047                true,
1048            );
1049        }
1050    }
1051
1052    fn on_unhealthy_child(&mut self, node: NodeId, health: crate::supervisor::liveness::Health) {
1053        self.log.warn(
1054            "child.unhealthy",
1055            json!({"node": node.0, "health": format!("{health:?}")}),
1056        );
1057        self.children.cancel(node, &format!("{health:?}"));
1058        // Escalate: give it a moment, then kill.
1059        let started = self
1060            .children
1061            .get(node)
1062            .map(|c| c.started)
1063            .unwrap_or_else(Instant::now);
1064        if started.elapsed() > Duration::from_secs(1) {
1065            self.children.kill(node);
1066        }
1067    }
1068
1069    // ---- lifecycle ---------------------------------------------------------
1070
1071    fn check_signals(&mut self) {
1072        if crate::signals::draining() && !self.draining {
1073            self.begin_drain("signal");
1074        }
1075        if crate::signals::reload_requested() {
1076            crate::signals::clear_reload();
1077            self.on_reload_requested();
1078        }
1079    }
1080
1081    pub(crate) fn begin_drain(&mut self, reason: &str) {
1082        if self.draining {
1083            return;
1084        }
1085        self.draining = true;
1086        self.drain_started = Some(Instant::now());
1087        self.drain_reason = reason.to_string();
1088        crate::signals::set_lame_duck(true);
1089        self.log.info("drain.start", json!({"reason": reason, "children": self.children.len(), "runs": self.runs.values().filter(|r| !r.status.is_terminal()).count()}));
1090        crate::obs::metrics::record_drain("started");
1091        // Tell every attached display client, so a client can stop offering
1092        // actions the daemon will now refuse.
1093        #[cfg(feature = "a2a")]
1094        self.feed_push(
1095            "lifecycle",
1096            super::a2a_server::FeedVis::All,
1097            json!({"draining": true, "reason": reason}),
1098        );
1099        self.children.begin_drain(reason);
1100        // Deinitialization workflows: `event {on: lifecycle.shutdown}` starts
1101        // fire NOW — releasing a claimed webhook route, deregistering from a
1102        // service, flushing a summary — and the drain below WAITS for exactly
1103        // those runs (bounded by drain_timeout like everything else). The
1104        // mirror of `once {policy: always}`, which is the init workflow.
1105        self.fire_event_starts("lifecycle.shutdown", &json!({"reason": reason}));
1106    }
1107
1108    /// Non-terminal runs of workflows that declare a `lifecycle.shutdown`
1109    /// start — the runs drain must wait for. (Any of the workflow's runs
1110    /// counts: an in-flight ordinary run of a deinit-capable workflow is not
1111    /// distinguishable from the deinit run by the time both must finish.)
1112    fn shutdown_runs_live(&self) -> usize {
1113        let capable = |name: &str, hash: &str| {
1114            self.definition_for_run_ref(name, hash).is_some_and(|w| {
1115                w.start_steps()
1116                    .iter()
1117                    .any(|s| s.kind == "event" && s.field_str("on") == Some("lifecycle.shutdown"))
1118            })
1119        };
1120        let live = self
1121            .runs
1122            .values()
1123            .filter(|r| !r.status.is_terminal())
1124            .filter(|r| capable(&r.workflow, &r.workflow_hash))
1125            .count();
1126        // A fired-but-not-yet-created run is still in the inbox for a tick —
1127        // the gate must not slip through that window.
1128        let queued = self
1129            .inbox_queue
1130            .iter()
1131            .filter(|e| e.kind == super::events::kinds::START_FIRED)
1132            .filter(|e| {
1133                e.payload["workflow"]
1134                    .as_str()
1135                    .and_then(|n| self.workflows.get(n))
1136                    .is_some_and(|w| {
1137                        w.start_steps().iter().any(|s| {
1138                            s.kind == "event" && s.field_str("on") == Some("lifecycle.shutdown")
1139                        })
1140                    })
1141            })
1142            .count();
1143        live + queued
1144    }
1145
1146    /// Decide whether to exit now. Returns the exit code when done.
1147    fn lifecycle_step(&mut self) -> Option<i32> {
1148        if let Some(code) = self.exit {
1149            // A `finish {exit: true}` or a fatal store failure asked to exit:
1150            // drain first.
1151            if !self.draining {
1152                self.begin_drain("exit");
1153            }
1154            if self.children.is_empty() {
1155                return Some(code);
1156            }
1157        }
1158        if self.draining {
1159            let timeout = self.settings.lifecycle.drain_timeout();
1160            let started = self.drain_started.unwrap_or_else(Instant::now);
1161            let force = crate::signals::force() || started.elapsed() >= timeout;
1162            let done =
1163                self.children.drive_drain(force) && (force || self.shutdown_runs_live() == 0);
1164            if done || started.elapsed() >= timeout + ABANDON_GRACE {
1165                if !done {
1166                    self.log
1167                        .warn("drain.abandon", json!({"children": self.children.len()}));
1168                    self.children.abandon();
1169                }
1170                crate::obs::metrics::record_drain("completed");
1171                self.checkpoint(true);
1172                self.log
1173                    .info("drain.done", json!({"reason": self.drain_reason}));
1174                return Some(self.exit.unwrap_or(crate::exit::SUCCESS));
1175            }
1176            return None;
1177        }
1178        // Job shape / idle policy.
1179        let run_until = self.settings.lifecycle.run_until;
1180        // `auto` re-reads the LIVE workflow set, not just the configured one:
1181        // a long-lived workflow the agent defined at runtime (`workflow.create`
1182        // — the self-setup shape, where a `--prompt` tells it to build its own
1183        // loop/schedule/subscribe) turns the one-shot job into a daemon exactly
1184        // as a configured one would have. Without this the instance idle-exits
1185        // out from under the thing it was just asked to set up.
1186        let job_now = self.job_shape && !self.workflows.values().any(|w| w.is_long_lived());
1187        let idle_policy = match run_until {
1188            RunUntil::Idle => true,
1189            RunUntil::Drained => false,
1190            RunUntil::Auto => job_now,
1191        };
1192        if !idle_policy {
1193            return None;
1194        }
1195        let busy = self.paused // a paused instance never idle-exits underneath the operator
1196            || !self.children.is_empty()
1197            || !self.turn_queue.is_empty()
1198            || !self.staged_turns.is_empty()
1199            || !self.inbox_queue.is_empty()
1200            || !self.pending.is_empty()
1201            || !self.executing.is_empty()
1202            || self.runs.values().any(|r| !r.status.is_terminal())
1203            || !self.timers.is_empty();
1204        if busy {
1205            self.idle_since = None;
1206            return None;
1207        }
1208        let since = *self.idle_since.get_or_insert_with(Instant::now);
1209        if since.elapsed() >= self.settings.lifecycle.idle_grace() || job_now {
1210            let code = self.job_exit_code();
1211            self.log.info(
1212                "lifecycle.idle_exit",
1213                json!({"code": code, "job_shape": self.job_shape}),
1214            );
1215            self.checkpoint(true);
1216            return Some(code);
1217        }
1218        None
1219    }
1220
1221    /// The exit code of a job-shaped instance, mapped from the `once`-started
1222    /// workflow's finish status. With several such runs the worst outcome
1223    /// wins, so a partial success is never reported as a clean exit. A daemon
1224    /// is not job-shaped and drains to 0.
1225    fn job_exit_code(&self) -> i32 {
1226        let mut code = crate::exit::SUCCESS;
1227        for id in &self.job_runs {
1228            if let Some(r) = self.runs.get(id) {
1229                let c = run_exit_code(r);
1230                if c != crate::exit::SUCCESS {
1231                    code = c;
1232                }
1233            }
1234        }
1235        if self.job_runs.is_empty() && self.job_shape {
1236            // Nothing ever ran (no workflow fired) — a configuration edge; report success.
1237            return crate::exit::SUCCESS;
1238        }
1239        crate::exit::apply_budget_remap(
1240            code,
1241            self.settings
1242                .lifecycle
1243                .exit_code_map
1244                .get(&code.to_string())
1245                .copied(),
1246        )
1247    }
1248
1249    fn shutdown(&mut self, code: i32) {
1250        self.children.abandon();
1251        let _ = self.durable.flush(true);
1252        self.log.info("proc.exit", json!({"code": code, "uptime_ms": self.started.elapsed().as_millis() as u64, "turns": self.counters.turns, "tool_calls": self.counters.tool_calls, "runs": self.counters.runs_finished, "tokens_in": self.counters.tokens_in, "tokens_out": self.counters.tokens_out}));
1253    }
1254
1255    /// The job's result (the once-started run's output), for stdout.
1256    pub fn job_output(&self) -> Option<Value> {
1257        self.job_runs
1258            .iter()
1259            .rev()
1260            .filter_map(|id| self.runs.get(id))
1261            .find_map(|r| r.output.clone())
1262            // A `--prompt` job has no `once` run to carry an output: its answer
1263            // is the root turn's reply.
1264            .or_else(|| self.last_root_reply.clone().map(Value::String))
1265    }
1266
1267    // ---- checkpoints ---------------------------------------------------------
1268
1269    /// Persist dirty runs/contexts/subagents; flush the manifest (debounced,
1270    /// forced at drain). A halting store error triggers an exit.
1271    pub(crate) fn checkpoint(&mut self, force: bool) {
1272        let mut failed: Option<String> = None;
1273        for run in self.runs.values_mut() {
1274            if run.dirty {
1275                // A non-durable run (workflow `durable: false`, or the
1276                // `store.durability.work: ephemeral` default) is memory-only:
1277                // no serialization, no write, gone after a restart.
1278                if !run.durable {
1279                    run.dirty = false;
1280                    continue;
1281                }
1282                crate::state::kill_point("step.before_done");
1283                match self.durable.put(
1284                    Kind::Run,
1285                    &run.id,
1286                    serde_json::to_value(&*run).unwrap_or(Value::Null),
1287                    Some(run.workflow_hash.clone()),
1288                ) {
1289                    Ok(_) => run.dirty = false,
1290                    Err(e) => failed = Some(format!("run {}: {e}", run.id)),
1291                }
1292            }
1293        }
1294        if let Err(e) = self.contexts.checkpoint(&self.durable) {
1295            failed = Some(format!("context: {e}"));
1296        }
1297        for s in self.subagents.values_mut() {
1298            if s.dirty {
1299                if !s.durable {
1300                    s.dirty = false;
1301                    continue;
1302                }
1303                match self.durable.put(
1304                    Kind::Subagent,
1305                    &s.handle,
1306                    serde_json::to_value(&*s).unwrap_or(Value::Null),
1307                    None,
1308                ) {
1309                    Ok(_) => s.dirty = false,
1310                    Err(e) => failed = Some(format!("subagent {}: {e}", s.handle)),
1311                }
1312            }
1313        }
1314        // Manifest: budget counters + lifecycle, debounced.
1315        let budget = self.governor.to_value();
1316        self.durable.manifest_update(|m| {
1317            m.budget = budget;
1318        });
1319        match self.durable.flush(force) {
1320            Ok(_) => {}
1321            Err(e) => failed = Some(format!("manifest: {e}")),
1322        }
1323        if let Some(e) = failed {
1324            self.log.error("store.checkpoint.fail", json!({"err": e}));
1325            if !self.durable.is_degraded() {
1326                // Halt policy: refuse new intake, drain.
1327                self.exit = Some(crate::exit::GENERIC);
1328            }
1329        }
1330    }
1331
1332    // ---- status ------------------------------------------------------------
1333
1334    /// `status` tool / `agent://status`.
1335    pub(crate) fn status_value(&self) -> Value {
1336        json!({
1337            "instance": self.instance,
1338            "run_id": self.run_id,
1339            "uptime_ms": self.started.elapsed().as_millis() as u64,
1340            "job_shape": self.job_shape,
1341            "draining": self.draining,
1342            "paused": self.paused,
1343            "store": {"kind": self.durable.store_kind(), "degraded": self.durable.is_degraded(), "generation": self.durable.manifest().generation},
1344            "workflows": self.workflows.values().map(|w| json!({"name": w.name, "hash": w.hash, "armed": w.armed, "starts": w.start_steps().iter().map(|s| s.kind.clone()).collect::<Vec<_>>()})).collect::<Vec<_>>(),
1345            "runs": self.runs.values().map(RunState::summary).collect::<Vec<_>>(),
1346            "conversations": self.contexts.status(),
1347            "subagents": self.subagents.values().map(|s| json!({"handle": s.handle, "mode": s.mode, "status": s.status, "tokens": s.tokens, "template": s.template, "tier": s.tier, "pid": s.pid, "retire_at": s.retire_at})).collect::<Vec<_>>(),
1348            "children": self.children.status(),
1349            "timers": self.timers.status(),
1350            "inbox_pending": self.inbox_queue.len(),
1351            "budget": self.governor.status(now_ms()),
1352            "tools": self.registry.len(),
1353            "skills": self.skills.names(),
1354            "counters": {"turns": self.counters.turns, "tool_calls": self.counters.tool_calls, "runs_started": self.counters.runs_started, "runs_finished": self.counters.runs_finished, "tokens_in": self.counters.tokens_in, "tokens_out": self.counters.tokens_out},
1355            "instruction": {"source": self.instruction.source, "uri": self.instruction.uri, "version": self.instruction.version, "bytes": self.instruction.text.len()},
1356            "model": self.model,
1357            "activity": self.activity_value(),
1358        })
1359    }
1360
1361    /// The shortest time until the next time-based wake (a timer, an armed
1362    /// schedule/loop start, a suspended wait deadline, a budget wait). Bounded
1363    /// below at 5 ms so a due deadline is serviced on the next pass without a
1364    /// busy spin.
1365    fn next_wake(&self) -> Duration {
1366        let now = now_ms();
1367        let mut soonest = now + 200;
1368        if let Some(t) = self.timers.next_deadline() {
1369            soonest = soonest.min(t);
1370        }
1371        for st in self.durable.manifest().starts.values() {
1372            for k in ["next_ms", "debounce_until"] {
1373                if let Some(n) = st[k].as_u64() {
1374                    soonest = soonest.min(n);
1375                }
1376            }
1377        }
1378        for run in self.runs.values() {
1379            if run.status.is_terminal() {
1380                continue;
1381            }
1382            for step in run.steps.values() {
1383                if let Some(w) = &step.wait
1384                    && let Some(d) = w["deadline_ms"].as_u64()
1385                {
1386                    soonest = soonest.min(d);
1387                }
1388            }
1389        }
1390        if !self.pending.is_empty() || !self.turn_queue.is_empty() {
1391            soonest = soonest.min(now + 50);
1392        }
1393        Duration::from_millis(soonest.saturating_sub(now).max(5))
1394    }
1395
1396    /// The model window (compaction threshold base): `context.model_window`
1397    /// when set, else inferred from the model name.
1398    /// The model window (compaction threshold base).
1399    ///
1400    /// `context.model_window` wins, then the active tier's declared `window`,
1401    /// and only then the guess from the model NAME — a substring match that is
1402    /// simply wrong for any provider whose naming does not happen to match.
1403    /// A tier that declares its window replaces the guess with a fact.
1404    pub(crate) fn model_window(&self) -> u64 {
1405        if let Some(w) = self.settings.context.model_window {
1406            return w;
1407        }
1408        if let Some(w) = self
1409            .settings
1410            .intelligence
1411            .default_reference()
1412            .and_then(|r| self.settings.intelligence.tier(&r).and_then(|t| t.window))
1413        {
1414            return w;
1415        }
1416        tokens::window_for_model(&self.model)
1417    }
1418}
1419
1420pub(crate) fn is_terminal_status(s: &str) -> bool {
1421    matches!(
1422        s,
1423        "completed" | "failed" | "cancelled" | "refused" | "killed" | "crashed" | "retired"
1424    )
1425}
1426
1427/// Map a finished run's status onto a process exit code, so a caller can tell
1428/// *how* a job ended without parsing its output: refusal, budget exhaustion,
1429/// a missed deadline and an unreachable model each get their own code, and
1430/// anything still unfinished reports as partial.
1431pub fn run_exit_code(r: &RunState) -> i32 {
1432    match r.status {
1433        RunStatus::Completed => crate::exit::SUCCESS,
1434        RunStatus::Refused => crate::exit::REFUSED,
1435        RunStatus::Stalled => crate::exit::PARTIAL,
1436        RunStatus::Failed => {
1437            let e = r.error.as_deref().unwrap_or("");
1438            if e.contains("exhausted") || e.contains("budget") {
1439                crate::exit::BUDGET
1440            } else if e.contains("deadline") {
1441                crate::exit::DEADLINE
1442            } else if e.contains("intel") {
1443                crate::exit::INTEL_UNAVAILABLE
1444            } else {
1445                crate::exit::GENERIC
1446            }
1447        }
1448        RunStatus::Cancelled => crate::exit::GENERIC,
1449        _ => crate::exit::PARTIAL,
1450    }
1451}