agentd/runtime/reactor.rs
1// SPDX-License-Identifier: AGPL-3.0-only
2//! The **runtime state + event loop**: one single-threaded reactor over child
3//! frames, reaped children, executor results, timers, the durable inbox and
4//! signals.
5//!
6//! State mutation happens only here. Being the single writer is what makes the
7//! rest of the runtime reasonable about: no lock ordering, no torn reads, and
8//! an answer computed for one caller is computed against one consistent view.
9//! Every mutation is followed by a checkpoint decision, so durable state never
10//! trails the in-memory state by more than one loop turn.
11//!
12//! The other `runtime::*` modules add `impl Runtime` blocks for turns, tools,
13//! steps and subagents; this file owns construction, the loop, lifecycle and
14//! the status view.
15
16use super::artifacts::Artifacts;
17use super::children::{ChildKind, Children};
18use super::events::{Event, kinds};
19use super::timers::Timers;
20use crate::config::v2::{RunUntil, Settings};
21use crate::context::memory::Memory;
22use crate::context::{Contexts, skills, tokens};
23use crate::engine::{RunState, RunStatus, Workflow};
24use crate::governor::Governor;
25use crate::mcp::client::McpClient;
26use crate::obs::log::Logger;
27use crate::registry::Registry;
28use crate::state::{Durable, InboxEvent, Kind, now_ms};
29use crate::subagent::protocol::AgentMsg;
30use crate::supervisor::reap::Reaped;
31use crate::supervisor::tree::NodeId;
32use serde_json::{Value, json};
33use std::collections::{BTreeMap, VecDeque};
34use std::sync::Arc;
35use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender};
36use std::time::{Duration, Instant};
37
38/// The reactor tick.
39pub const TICK: Duration = Duration::from_millis(200);
40/// Extra grace after the drain deadline before children are abandoned.
41pub const ABANDON_GRACE: Duration = Duration::from_secs(3);
42
43/// Who receives a deferred tool's answer.
44#[derive(Debug, Clone, PartialEq, Eq)]
45pub enum Target {
46 /// A child's `ToolRequest` (answered with `ToolResult`).
47 Child(NodeId, u64),
48 /// A workflow step (answered as the step's outcome).
49 Step(String, String),
50}
51
52/// A deferred internal-tool request (answered when its wait resolves).
53#[derive(Debug, Clone)]
54pub struct PendingTool {
55 pub target: Target,
56 pub name: String,
57 pub kind: PendingKind,
58 pub started_ms: u64,
59}
60
61#[derive(Debug, Clone)]
62pub enum PendingKind {
63 /// A durable timer (`sleep`).
64 Timer { id: String },
65 /// A subagent result (`subagent.run` sync / `subagent.await`).
66 Subagent { handle: String },
67 /// A think child (`think` tool / `context.compact`).
68 Think { child: NodeId },
69 /// A run's terminal state (`workflow.run wait` / `workflow.wait`).
70 Run { run: String, deadline_ms: u64 },
71 /// A CEL condition polled each tick (`await`).
72 Await { condition: String, deadline_ms: u64 },
73 /// A human's answer (`ask_human` / the `human` node): the
74 /// A2A task `task` sits in `input-required`; a `SendMessage` carrying its
75 /// `taskId` resolves this with the reply text. With no interface to answer
76 /// on, `task` is a synthetic ask id (no A2A task exists).
77 Human {
78 task: String,
79 question: String,
80 deadline_ms: u64,
81 /// The task exists ONLY for this ask (no A2A caller/run owns it) —
82 /// complete it when the answer lands.
83 standalone: bool,
84 /// The `auto` fallback judge is running (or already ran) for this ask.
85 auto_fired: bool,
86 /// The answer's declared shape (`human.schema` / `ask_human.schema`).
87 ///
88 /// Carried on the pending ask so the reply can be validated against it
89 /// when it lands. Forwarding the schema to clients only makes them
90 /// render the right form; a gate that declares it wants
91 /// `{decision: "file"|"hold"}` must also refuse "maybe later", or the
92 /// run proceeds on an answer it never asked for.
93 schema: Option<Value>,
94 /// Who must answer (`to:`). `None` ⇒ whoever holds the task, which is
95 /// the ordinary case. Enforced when the answer lands, for the same
96 /// reason the schema is: a gate that names a decider and then accepts
97 /// anyone records something that did not happen.
98 addressee: Option<crate::a2a::principals::Addressee>,
99 },
100}
101
102/// A queued root/conversation turn, waiting for a worker slot and for its
103/// context to be free. One context runs at most one turn at a time, so turns
104/// for the same conversation queue behind each other rather than interleaving
105/// into the same history.
106#[derive(Debug, Clone)]
107pub struct TurnJob {
108 pub ctx: String,
109 /// The triggering inbox event (marked done when the turn completes).
110 pub event: Option<String>,
111 pub principal: Option<String>,
112 /// The message appended to the context before the turn (already appended
113 /// when `None`).
114 pub message: Option<crate::context::Msg>,
115 /// Skill references to preload.
116 pub skills: Vec<String>,
117 /// The user text (for preflight / knowledge retrieval).
118 pub text: String,
119 /// Preflight ran (or was not needed).
120 pub preflight_done: bool,
121 /// Knowledge auto-context ran (or was not needed).
122 pub knowledge_done: bool,
123 /// The retrieved knowledge block (system message) for this turn.
124 pub knowledge: Option<String>,
125 /// The message-hop depth this turn inherits (see `RunState::msg_depth`).
126 /// A message from a person is depth 0; one a `message` step delivered
127 /// carries that step's depth, and anything this turn starts inherits it.
128 pub msg_depth: u32,
129}
130
131impl TurnJob {
132 pub fn new(
133 ctx: String,
134 event: Option<String>,
135 principal: Option<String>,
136 message: Option<crate::context::Msg>,
137 skills: Vec<String>,
138 text: String,
139 ) -> TurnJob {
140 TurnJob {
141 ctx,
142 event,
143 principal,
144 message,
145 skills,
146 text,
147 preflight_done: false,
148 knowledge_done: false,
149 knowledge: None,
150 msg_depth: 0,
151 }
152 }
153 /// The same job, carrying a delivered message's hop depth.
154 pub fn at_depth(mut self, depth: u32) -> TurnJob {
155 self.msg_depth = depth;
156 self
157 }
158}
159
160/// A subagent registry record, persisted as `subagent/<handle>` so a child's
161/// identity and result outlive both the child and this process.
162#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
163pub struct SubagentRecord {
164 pub handle: String,
165 pub instruction: String,
166 pub mode: String,
167 pub status: String,
168 #[serde(default)]
169 pub attempt: u32,
170 #[serde(default, skip_serializing_if = "Option::is_none")]
171 pub result: Option<Value>,
172 #[serde(default, skip_serializing_if = "Option::is_none")]
173 pub error: Option<String>,
174 #[serde(default, skip_serializing_if = "Option::is_none")]
175 pub requested_by: Option<Value>,
176 #[serde(default)]
177 pub tokens: u64,
178 #[serde(default)]
179 pub created: u64,
180 #[serde(default)]
181 pub updated: u64,
182 /// The payload (secret-free) for restore re-spawn.
183 #[serde(default, skip_serializing_if = "Option::is_none")]
184 pub payload: Option<Value>,
185 /// The template this child was instantiated from, and its tier
186 /// (`flat` | `instance`). A freeform spawn carries neither.
187 #[serde(default, skip_serializing_if = "Option::is_none")]
188 pub template: Option<String>,
189 #[serde(default, skip_serializing_if = "Option::is_none")]
190 pub tier: Option<String>,
191 /// Instance tier: the child daemon's pid, config path, A2A socket and
192 /// (epoch-ms) retire-at deadline.
193 #[serde(default, skip_serializing_if = "Option::is_none")]
194 pub pid: Option<i32>,
195 #[serde(default, skip_serializing_if = "Option::is_none")]
196 pub config_path: Option<String>,
197 #[serde(default, skip_serializing_if = "Option::is_none")]
198 pub socket: Option<String>,
199 #[serde(default, skip_serializing_if = "Option::is_none")]
200 pub retire_at: Option<u64>,
201 /// Instance tier: set when retirement began (SIGTERM sent); the tick
202 /// escalates to SIGKILL after the drain window.
203 #[serde(default, skip_serializing_if = "Option::is_none")]
204 pub retiring_since: Option<u64>,
205 /// Durability class (default true). `false` ⇒ the record is memory-only:
206 /// never persisted, never restore-respawned — the fast path for throwaway
207 /// workers. Restored records (all persisted by construction) default true.
208 #[serde(default = "record_durable_default")]
209 pub durable: bool,
210 #[serde(skip)]
211 pub node: Option<NodeId>,
212 #[serde(skip)]
213 pub dirty: bool,
214}
215
216fn record_durable_default() -> bool {
217 true
218}
219
220/// The instruction in force. `version` increments on every change, so a
221/// consumer can tell a re-read from a genuinely new instruction.
222#[derive(Debug, Clone)]
223pub struct Instruction {
224 pub text: String,
225 pub source: &'static str,
226 pub uri: Option<String>,
227 pub server: Option<String>,
228 pub version: u64,
229}
230
231/// Counters for status/reports.
232#[derive(Debug, Default, Clone)]
233pub struct Counters {
234 pub turns: u64,
235 pub tool_calls: u64,
236 pub runs_started: u64,
237 pub runs_finished: u64,
238 pub inbox_processed: u64,
239 pub tokens_in: u64,
240 pub tokens_out: u64,
241}
242
243pub struct Runtime {
244 /// Resource pressure (disk headroom, cgroup memory): consulted at every
245 /// ADMISSION gate — start-node firing, webhook accept, `workflow.run`,
246 /// turn dispatch, subagent spawn — never on work already in flight.
247 pub(crate) pressure: std::sync::Arc<super::pressure::Pressure>,
248 /// The last level the tick reported, so transitions log exactly once.
249 pub(crate) pressure_seen: super::pressure::Level,
250 /// A step reached a terminal state since the last scheduling pass — its
251 /// dependents may be ready NOW (the same-iteration re-schedule fixpoint).
252 pub(crate) resched: bool,
253 /// Reaps already deferred once for frame ordering (by pid) — see
254 /// [`Runtime::on_reaped`].
255 pub(crate) reap_deferred: std::collections::HashSet<i32>,
256 /// Outbound token buckets for steps that declare `rate:`, keyed like the
257 /// breaker (`workflow/unscoped-step`). In-memory on purpose: a rate is a
258 /// statement about LIVE traffic, and a restart briefly refilling the burst
259 /// is harmless where a durable bucket would be bookkeeping for its own
260 /// sake. The paired f64 is the window seconds, for computing the wait.
261 pub(crate) step_rates:
262 std::collections::HashMap<String, (crate::supervisor::tree::TokenBucket, f64, u32)>,
263 pub(crate) settings: Settings,
264 /// The merged document the settings came from (restart-only diff base).
265 pub(crate) settings_doc: Value,
266 /// The invocation (for reload).
267 pub(crate) args: Vec<String>,
268 pub(crate) env: Vec<(String, String)>,
269 /// Workflow definitions pinned by live runs after a reload (hash → definition).
270 pub(crate) pinned: BTreeMap<String, std::sync::Arc<Workflow>>,
271 /// Retired definitions still owning live runs (`runtime::retire`), by hash.
272 pub(crate) retiring: BTreeMap<String, super::retire::Retiring>,
273 /// Definition hashes whose durable pin was written this life (one write
274 /// per version; see `retire::ensure_pin`).
275 pub(crate) pin_written: std::collections::HashSet<String>,
276 /// The last payload per signal name (for `await`/`wait condition` views).
277 pub(crate) recent_signals: BTreeMap<String, Value>,
278 /// Memoized `memory.<key>` references per definition content hash: the
279 /// scan walks the whole definition and `run_data` runs per step.
280 pub(crate) memory_keys: std::collections::HashMap<String, Vec<String>>,
281 /// An `emit` appended since the last stream poll (same-iteration wake).
282 pub(crate) stream_dirty: bool,
283 pub(crate) log: Logger,
284 pub(crate) instance: String,
285 pub(crate) run_id: String,
286 pub(crate) durable: Durable,
287 pub(crate) mcp: BTreeMap<String, Arc<McpClient>>,
288 pub(crate) mcp_specs: BTreeMap<String, crate::config::McpServerSpec>,
289 pub(crate) registry: Registry,
290 pub(crate) contexts: Contexts,
291 pub(crate) memory: Memory,
292 pub(crate) artifacts: Artifacts,
293 pub(crate) skills: skills::Catalogue,
294 pub(crate) governor: Governor,
295 /// Per-principal budgets and rate quotas, indexed by principal id when one
296 /// is first seen. `a2a.principals[].quotas` parsed and validated for a
297 /// long time without anything reading it; these are its readers.
298 pub(crate) principal_budgets: BTreeMap<String, crate::config::v2::Budget>,
299 /// Only the A2A listener admits callers, so a build without it has
300 /// nowhere to spend an arrival quota.
301 #[cfg_attr(not(feature = "a2a"), allow(dead_code))]
302 pub(crate) principal_rates: BTreeMap<String, crate::supervisor::tree::TokenBucket>,
303 /// Labels an id acts under, for `_meta` and audit.
304 pub(crate) principal_labels: BTreeMap<String, BTreeMap<String, String>>,
305 pub(crate) workflows: BTreeMap<String, std::sync::Arc<Workflow>>,
306 pub(crate) runs: BTreeMap<String, RunState>,
307 pub(crate) children: Children,
308 pub(crate) timers: Timers,
309 pub(crate) events_rx: Receiver<Event>,
310 pub(crate) events_tx: Sender<Event>,
311 pub(crate) reap_rx: Receiver<Reaped>,
312 pub(crate) pending: Vec<PendingTool>,
313 pub(crate) turn_queue: VecDeque<TurnJob>,
314 /// Turn jobs parked while their preflight think / knowledge retrieval runs.
315 pub(crate) staged_turns: BTreeMap<u64, TurnJob>,
316 pub(crate) inbox_queue: VecDeque<InboxEvent>,
317 pub(crate) subagents: BTreeMap<String, SubagentRecord>,
318 pub(crate) instruction: Instruction,
319 pub(crate) job_shape: bool,
320 pub(crate) exit: Option<i32>,
321 pub(crate) draining: bool,
322 /// Operator-held (a2a.pause): intake continues; no new turns dispatch and
323 /// no steps schedule until a2a.resume. Reversible, unlike drain.
324 pub(crate) paused: bool,
325 pub(crate) drain_started: Option<Instant>,
326 pub(crate) drain_reason: String,
327 pub(crate) idle_since: Option<Instant>,
328 pub(crate) intel_uri: String,
329 pub(crate) intel_token: Option<String>,
330 /// Resolved `intelligence.headers`, pushed on every LLM dial and threaded
331 /// to subagents via the spawn payload so a child dials identically.
332 pub(crate) intel_headers: Vec<(String, String)>,
333 /// An optional intelligence credential provider: a closure returning the
334 /// current bearer, refreshed from the device-login cache. Its resolved
335 /// bearer overrides `intel_token`, and is threaded to subagents fresh at
336 /// each spawn so no child carries a stale one. `None` when no
337 /// `intelligence.auth` oauth2 block is configured.
338 pub(crate) intel_bearer: Option<std::sync::Arc<dyn Fn() -> Option<String> + Send + Sync>>,
339 pub(crate) model: String,
340 pub(crate) trace_id: Option<String>,
341 pub(crate) started: Instant,
342 pub(crate) seq: u64,
343 pub(crate) counters: Counters,
344 /// The `once`-started run(s) whose finish decides a job's exit code.
345 pub(crate) job_runs: Vec<String>,
346 /// Steps executing on executor threads (`run/step` → started).
347 pub(crate) executing: BTreeMap<String, Instant>,
348 pub(crate) last_manifest_flush: Instant,
349 /// Unix-ms a goal LLM judge was dispatched (so overlapping checks don't spawn
350 /// duplicate judges); `None` = none in flight.
351 pub(crate) goal_judge_at: Option<u64>,
352 /// Durable A2A tasks, keyed by task id.
353 #[cfg(feature = "a2a")]
354 pub(crate) tasks: BTreeMap<String, crate::a2a::Task>,
355 /// Inbox-event id → the A2A task it answers (a conversation turn).
356 #[cfg(feature = "a2a")]
357 pub(crate) event_to_task: BTreeMap<String, String>,
358 /// The task snapshot the A2A listener threads read (None ⇒ not serving).
359 #[cfg(feature = "a2a")]
360 /// The interface event feed. `None` means the interface is disabled.
361 #[cfg(feature = "a2a")]
362 pub(crate) a2a_feed: Option<std::sync::Arc<super::a2a_server::SharedFeed>>,
363 /// Pairing-code login state. `None` means pairing is disabled.
364 #[cfg(feature = "a2a")]
365 pub(crate) a2a_pairing: Option<std::sync::Arc<super::a2a_server::PairingState>>,
366 /// The id the listener reserved for the task the request being served will
367 /// create. Taken by the first `task_create` of that request, and cleared
368 /// after it — an id belongs to one request only.
369 #[cfg(feature = "a2a")]
370 pub(crate) reserved_task_id: Option<String>,
371 /// Where a task transition is published so A2A subscribers see it.
372 #[cfg(feature = "a2a")]
373 pub(crate) a2a_sink: Option<std::sync::Arc<crate::a2a::ports::StreamSink>>,
374 /// The live listener. Held, not used: dropping it stops serving.
375 #[cfg(feature = "a2a")]
376 pub(crate) a2a_listener: Option<crate::a2a::serve::Listener>,
377 /// The listener's bridge, so a reload can swap rebuilt principal rules in.
378 #[cfg(feature = "a2a")]
379 pub(crate) a2a_bridge: Option<std::sync::Arc<super::a2a_server::A2aBridge>>,
380 /// The webhook listener's handler, so a reload can swap rebuilt routes in.
381 #[cfg(feature = "a2a")]
382 pub(crate) webhook_handler: Option<std::sync::Arc<super::webhooks::WebhookHandler>>,
383 /// The listener's live CORS allowlist, so a reload can revise it.
384 #[cfg(feature = "a2a")]
385 pub(crate) a2a_origins: Option<crate::a2a::serve::OriginList>,
386 /// Live per-unit activity, keyed by child node id.
387 pub(crate) activity: BTreeMap<u64, super::activity::Activity>,
388 /// The newest root-context reply, so a `--prompt` job can print its answer
389 /// (a prompt runs as a turn, not as a `once` run with an output).
390 pub(crate) last_root_reply: Option<String>,
391 /// Per-item fingerprints behind the feed's section diffing (`feed_tick`).
392 #[cfg(feature = "a2a")]
393 pub(crate) feed_marks: BTreeMap<String, u64>,
394 /// The last section-diff pass (rate-limits `feed_tick`).
395 #[cfg(feature = "a2a")]
396 pub(crate) feed_last: Instant,
397 /// The `wait: {on: webhook}` await-callback registry, shared with the webhook
398 /// listener threads.
399 #[cfg(feature = "a2a")]
400 pub(crate) webhook_callbacks: super::webhooks::SharedCallbacks,
401 /// Pending `respond: sync` webhook replies, keyed by the run id they await.
402 #[cfg(feature = "a2a")]
403 pub(crate) webhook_sync: std::collections::HashMap<
404 String,
405 std::sync::mpsc::SyncSender<super::webhooks::WebhookReply>,
406 >,
407}
408
409impl Runtime {
410 /// A fresh id (turn ids, handles).
411 /// The deployment's default durability class for work (runs + subagent
412 /// records): `store.durability.work: ephemeral` ⇒ false.
413 pub(crate) fn work_durable_default(&self) -> bool {
414 !matches!(
415 self.settings.store.durability.work,
416 Some(crate::config::v2::WorkDurability::Ephemeral)
417 )
418 }
419
420 pub(crate) fn next_id(&mut self, prefix: &str) -> String {
421 self.seq += 1;
422 format!("{prefix}-{}", self.seq)
423 }
424
425 // ---- the loop ----------------------------------------------------------
426
427 /// Run until exit. Returns the process exit code.
428 pub fn run_loop(&mut self) -> i32 {
429 self.log.info("proc.ready", json!({"instance": self.instance, "job_shape": self.job_shape, "workflows": self.workflows.len(), "runs": self.runs.len(), "inbox_pending": self.inbox_queue.len()}));
430 loop {
431 crate::obs::health::tick();
432 // Pressure transitions are logged HERE, once per change, so the
433 // per-request gates can refuse silently instead of each writing its
434 // own line per refusal — under real pressure that would be a log
435 // flood on top of a disk that is already full.
436 {
437 let level = self.pressure.level();
438 if level != self.pressure_seen {
439 let free = self
440 .pressure
441 .disk_free
442 .load(std::sync::atomic::Ordering::Relaxed);
443 let detail = json!({
444 "level": level.as_str(),
445 "cause": self.pressure.cause(),
446 "disk_free_bytes": if free == u64::MAX { Value::Null } else { json!(free) },
447 });
448 match level {
449 super::pressure::Level::Ok => self.log.info("pressure.cleared", detail),
450 super::pressure::Level::Warn => self.log.warn("pressure.warn", detail),
451 super::pressure::Level::Shed => self.log.warn("pressure.shed", detail),
452 }
453 self.pressure_seen = level;
454 }
455 }
456 // 1. Child frames.
457 // (child frames arrive as Event::Child on the main channel — they
458 // wake the parked loop instead of waiting for the tick)
459 // 2. Reaped children.
460 let _ = crate::signals::take_child_exit();
461 crate::supervisor::reaper::reap_and_dispatch();
462 while let Ok(r) = self.reap_rx.try_recv() {
463 self.on_reaped(r);
464 }
465 // 3. Executor / internal events.
466 while let Ok(ev) = self.events_rx.try_recv() {
467 self.on_event(ev);
468 }
469 // 3.5. Retiring workflows whose drain deadline passed.
470 self.retire_tick();
471 // 4. Timers.
472 let now = now_ms();
473 for t in self.timers.fire(&self.durable, now) {
474 self.on_timer(t);
475 }
476 // 4.9. The daemon's own events, queued by the tap since the last
477 // tick, become appends — so a tripped breaker or a shed admission
478 // can start a run. Done BEFORE the inbox and the start poll so
479 // this tick's consumers see this tick's telemetry.
480 self.drain_runtime_events();
481 // 5. The inbox.
482 self.process_inbox();
483 // 6. Start nodes + runs (+ suspended waits).
484 self.poll_starts();
485 self.poll_stream_starts();
486 // Joins consume the same stream, so they advance in the same pass —
487 // and their window sweep runs every tick, which is what makes an
488 // `on_timeout: fire_partial` escalation fire on time rather than on
489 // the next event to arrive.
490 self.poll_correlate_starts();
491 // Runs parked on the log resolve in the same pass that advances
492 // consumers, so a produce→wait hop costs a tick, not a timeout.
493 self.poll_event_waits();
494 self.poll_waits();
495 self.schedule_runs();
496 // Inline steps (assign/map/template/switch…) complete synchronously
497 // inside that pass, which makes their dependents ready NOW — without
498 // this fixpoint a pure data pipeline advanced ONE step per 200 ms
499 // tick (measured: 200 chained assigns = 42 s; with it, milliseconds).
500 // Bounded for the loop's honesty: effectful steps complete via
501 // events, so only inline chains re-enter here, and `limits.run.steps`
502 // already caps how long one can be.
503 let mut passes = 0;
504 while std::mem::take(&mut self.resched) && passes < 1024 {
505 self.schedule_runs();
506 passes += 1;
507 }
508 // 6.6. Streams appended in this iteration fire their consumers
509 // NOW: a same-process produce->consume pipeline advances at
510 // engine speed instead of paying the tick park per hop. Bounded
511 // like the fixpoint; an emit inside a fired consumer re-enters
512 // here, and `limits.run.steps` caps how deep that can go.
513 let mut stream_rounds = 0;
514 while std::mem::take(&mut self.stream_dirty) && stream_rounds < 64 {
515 self.poll_stream_starts();
516 self.poll_correlate_starts();
517 // A run parked on the log is a consumer too: without this, a
518 // saga whose awaited event was emitted by a step in this very
519 // iteration would park until the next tick.
520 self.poll_event_waits();
521 self.schedule_runs();
522 let mut passes = 0;
523 while std::mem::take(&mut self.resched) && passes < 1024 {
524 self.schedule_runs();
525 passes += 1;
526 }
527 stream_rounds += 1;
528 }
529 // 7. Turns.
530 self.dispatch_turns();
531 // 8. Pending waits + MCP notifications.
532 self.poll_pending();
533 self.poll_mcp_notifications();
534 // 9. Children maintenance.
535 for (node, health) in self.children.tick() {
536 self.on_unhealthy_child(node, health);
537 }
538 // 9b. Instance-tier children: ttl retirement, plus the
539 // SIGTERM→SIGKILL escalation for children that ignored the drain.
540 self.instances_tick();
541 // 10. Checkpoints + the point-in-time observability gauges.
542 self.checkpoint(false);
543 crate::obs::metrics::set_inbox_pending(self.inbox_queue.len() as u64);
544 crate::obs::metrics::set_context_tokens(self.contexts.max_est_tokens());
545 {
546 let free = self
547 .pressure
548 .disk_free
549 .load(std::sync::atomic::Ordering::Relaxed);
550 crate::obs::metrics::set_pressure(
551 self.pressure_seen as u64,
552 (free != u64::MAX).then_some(free),
553 );
554 crate::obs::metrics::set_work_backlog(
555 self.runs
556 .values()
557 .filter(|r| !r.status.is_terminal())
558 .count() as u64,
559 self.turn_queue.len() as u64,
560 );
561 }
562 // 10.5. The interface feed's section diff: publish
563 // run/conversation/subagent/child/status deltas to attached display
564 // clients. A no-op unless `interface.enabled`; rate-limited inside.
565 #[cfg(feature = "a2a")]
566 self.feed_tick();
567 // 11. Signals + lifecycle.
568 self.check_signals();
569 if let Some(code) = self.lifecycle_step() {
570 self.shutdown(code);
571 return code;
572 }
573 // 12. Wait for the next event, bounded by the tick or the nearest
574 // imminent deadline (a timer, a schedule/loop start, a pending wait)
575 // so time-based work fires promptly rather than at tick granularity.
576 crate::signals::drain_wakeup();
577 let wait = self.next_wake().min(TICK);
578 match self.events_rx.recv_timeout(wait) {
579 Ok(ev) => self.on_event(ev),
580 Err(RecvTimeoutError::Timeout) | Err(RecvTimeoutError::Disconnected) => {}
581 }
582 }
583 }
584
585 fn on_event(&mut self, ev: Event) {
586 match ev {
587 Event::Child(node, msg) => self.on_child_frame(node, msg),
588 Event::Reaped(r) => self.on_reaped(r),
589 Event::StepDone {
590 run,
591 step,
592 output,
593 is_error,
594 error,
595 tokens,
596 } => self.on_step_done(&run, &step, output, is_error, error, tokens),
597 Event::ToolDone {
598 node,
599 req,
600 result,
601 is_error,
602 } => self.on_tool_done(node, req, result, is_error),
603 Event::KnowledgeDone { job, block } => self.on_knowledge_done(job, block),
604 Event::TimerFired { id, owner, payload } => self.on_timer(crate::state::TimerRecord {
605 id,
606 deadline_ms: now_ms(),
607 owner,
608 payload,
609 }),
610 Event::Inbox(ev) => self.inbox_queue.push_back(ev),
611 #[cfg(feature = "a2a")]
612 Event::A2a(req) => self.on_a2a_request(*req),
613 #[cfg(feature = "a2a")]
614 Event::Webhook(req) => self.on_webhook_request(*req),
615 Event::Background { id, result } if id == "goal.judge" => self.on_goal_judge(&result),
616 Event::Background { id, result } if id.starts_with("human.judge:") => {
617 let ask = id.trim_start_matches("human.judge:").to_string();
618 self.on_human_judge(&ask, &result);
619 }
620 Event::SubscribeRead {
621 server,
622 uri,
623 content,
624 } => self.on_subscribe_read(&server, &uri, content),
625 Event::Background { .. } | Event::Tick => {}
626 }
627 }
628
629 // ---- inbox -------------------------------------------------------------
630
631 /// Accept a durable event: write it to the store first, then queue it for
632 /// the loop. Write-ahead is the whole point — once acceptance is
633 /// acknowledged to the outside world, a crash before the event is acted on
634 /// must replay it rather than drop it.
635 pub(crate) fn accept_event(
636 &mut self,
637 kind: &str,
638 principal: Option<String>,
639 payload: Value,
640 ) -> Result<String, String> {
641 let ev = InboxEvent::new(kind, principal, payload);
642 self.durable
643 .inbox_put(&ev)
644 .map_err(|e| format!("inbox: {e}"))?;
645 let id = ev.id.clone();
646 self.log
647 .info("inbox.accepted", json!({"inbox_event": id, "kind": kind}));
648 self.inbox_queue.push_back(ev);
649 Ok(id)
650 }
651
652 fn process_inbox(&mut self) {
653 // Drain a SNAPSHOT, never the live deque: a start event that overflows
654 // its workflow's concurrency cap re-queues itself (`on_overflow: queue`,
655 // the default), and the cap can only be relieved by `schedule_runs` — a
656 // LATER step of this tick. Popping from the same deque the requeue
657 // pushes onto re-offers the event immediately and the single-writer
658 // reactor spins at 100% CPU forever: no timers, no checkpoint, no
659 // SIGTERM. Requeued (and newly accepted) events land in the fresh
660 // `self.inbox_queue` and are retried on the next tick instead.
661 let mut batch = std::mem::take(&mut self.inbox_queue);
662 while let Some(ev) = batch.pop_front() {
663 if self.draining {
664 // Keep it durable for the next life; stop intake — with one
665 // exception: the start event of a `lifecycle.shutdown` deinit
666 // workflow exists to run DURING the drain, and the drain gate
667 // is waiting for it. Everything else waits for the next life.
668 let deinit = ev.kind == kinds::START_FIRED
669 && ev.payload["workflow"]
670 .as_str()
671 .and_then(|n| self.workflows.get(n))
672 .is_some_and(|w| {
673 w.start_steps().iter().any(|s| {
674 s.kind == "event" && s.field_str("on") == Some("lifecycle.shutdown")
675 })
676 });
677 if !deinit {
678 self.inbox_queue.push_back(ev);
679 continue;
680 }
681 }
682 self.counters.inbox_processed += 1;
683 match ev.kind.as_str() {
684 kinds::START_FIRED | kinds::WORKFLOW_RUN => {
685 let done = self.on_start_event(&ev);
686 if done {
687 self.inbox_done(&ev.id);
688 }
689 }
690 kinds::A2A_MESSAGE => {
691 // Handled the same whether it arrived live or was replayed
692 // from the inbox after a restart.
693 self.on_a2a_message_event(&ev);
694 }
695 kinds::SIGNAL => {
696 let name = ev.payload["name"].as_str().unwrap_or("").to_string();
697 let payload = ev.payload.get("payload").cloned().unwrap_or(Value::Null);
698 let target = ev
699 .payload
700 .get("run")
701 .and_then(Value::as_str)
702 .map(str::to_string);
703 let from = ev
704 .payload
705 .get("from")
706 .and_then(Value::as_str)
707 .map(str::to_string);
708 let delivered =
709 self.deliver_signal(&name, payload, target.as_deref(), from.as_deref());
710 self.log.info(
711 "signal.received",
712 json!({"inbox_event": ev.id, "name": name, "delivered": delivered}),
713 );
714 self.inbox_done(&ev.id);
715 }
716 other => {
717 self.log.warn(
718 "inbox.unknown_kind",
719 json!({"inbox_event": ev.id, "kind": other}),
720 );
721 self.inbox_done(&ev.id);
722 }
723 }
724 }
725 // Whatever the drain did not consume keeps its place ahead of the
726 // events requeued (or accepted) while the batch was processing.
727 batch.append(&mut self.inbox_queue);
728 self.inbox_queue = batch;
729 }
730
731 pub(crate) fn inbox_done(&mut self, id: &str) {
732 if let Err(e) = self.durable.inbox_done(id) {
733 self.log.warn(
734 "inbox.done.fail",
735 json!({"inbox_event": id, "err": e.to_string()}),
736 );
737 }
738 }
739
740 /// An A2A message event, routed to whichever reader owns it. Control-plane
741 /// ops are consumed first, then a waiting step, then a start node, and only
742 /// what is left becomes a conversation turn.
743 fn on_a2a_message_event(&mut self, ev: &InboxEvent) {
744 let ctx = ev.payload["context_id"]
745 .as_str()
746 .unwrap_or("default")
747 .to_string();
748 let text = ev.payload["text"]
749 .as_str()
750 .map(str::to_string)
751 .unwrap_or_else(|| ev.payload["parts"].to_string());
752 let principal = ev.principal.clone();
753 // Re-link a replayed message to its durable task (crash recovery).
754 #[cfg(feature = "a2a")]
755 if let Some(task_id) = ev.payload["task"].as_str() {
756 self.event_to_task
757 .insert(ev.id.clone(), task_id.to_string());
758 }
759 // `_instance.*` ops are a child reporting home. The runtime consumes
760 // them BEFORE any reader, so they can never be mistaken for a wait's
761 // answer, a start's request, or a conversational turn — control-plane
762 // traffic must not reach a model.
763 #[cfg(feature = "a2a")]
764 if self.handle_instance_op(ev) {
765 return;
766 }
767 // An inbound message has three possible readers, in this order. Only one
768 // takes it: a message that woke a waiting step is an ANSWER, and a
769 // message that fired a workflow is a REQUEST — neither should also
770 // become a conversational turn, or the agent replies to itself.
771 //
772 // 1. A step suspended on this conversation (`a2a.wait` / `wait {on:
773 // message}`) — the reply half of an asynchronous exchange.
774 let msg = json!({"parts": ev.payload.get("parts").cloned().unwrap_or(Value::Null),
775 "text": text, "message_id": ev.payload.get("message_id").cloned()});
776 if self.deliver_a2a_message(&ctx, &msg, principal.as_deref()) > 0 {
777 self.log.info(
778 "a2a.message.delivered",
779 json!({"inbox_event": ev.id, "conversation": ctx}),
780 );
781 return;
782 }
783 // 2. An `a2a` START node whose command and roles match — a peer or an
784 // operator asking for a workflow rather than a conversation.
785 if self.fire_a2a_start(ev, &ctx) {
786 return;
787 }
788 // 3. Otherwise it is what it looks like: something to answer.
789 #[allow(unused)]
790 let skills = self.skills.references(&text);
791 let depth = ev.payload["msg_depth"].as_u64().unwrap_or(0) as u32;
792 self.turn_queue.push_back(
793 TurnJob::new(
794 ctx,
795 Some(ev.id.clone()),
796 principal.clone(),
797 Some(crate::context::Msg::user(text.clone(), principal)),
798 skills,
799 text,
800 )
801 .at_depth(depth),
802 );
803 }
804
805 /// Without the `a2a` feature there is no listener to deliver a message, so a
806 /// replayed event simply degrades to a turn.
807 #[cfg(not(feature = "a2a"))]
808 fn fire_a2a_start(&mut self, _ev: &InboxEvent, _ctx: &str) -> bool {
809 false
810 }
811
812 /// Match an inbound A2A message against every `a2a` start node and fire the
813 /// first that accepts it. Returns whether a run was started.
814 ///
815 /// `command` selects on the command DataPart's `op` — absent means "any
816 /// message", which is how a workflow takes plain conversation as its
817 /// trigger. `roles` restricts which principals may fire it, and defaults to
818 /// no restriction beyond the authorization the listener already applied:
819 /// the start node narrows, it never widens.
820 #[cfg(feature = "a2a")]
821 fn fire_a2a_start(&mut self, ev: &InboxEvent, ctx: &str) -> bool {
822 let op = ev.payload.get("parts").and_then(|parts| {
823 crate::runtime::a2a_server::command_op(&json!({"parts": parts.clone()}))
824 });
825 // The typed command payload, `op` removed: a workflow reads
826 // `{{ steps.cmd.output.args.<field> }}` instead of parsing parts.
827 let args = ev.payload.get("parts").and_then(|parts| {
828 crate::runtime::a2a_server::command_data(&json!({"parts": parts.clone()})).map(
829 |mut d| {
830 if let Some(o) = d.as_object_mut() {
831 o.remove("op");
832 }
833 d
834 },
835 )
836 });
837 let role = ev.payload["role"].as_str().unwrap_or("");
838 let specs: Vec<(String, String, serde_json::Map<String, Value>)> = self
839 .workflows
840 .values()
841 .flat_map(|w| {
842 w.start_steps()
843 .into_iter()
844 .filter(|s| s.kind == "a2a")
845 .map(|s| (w.name.clone(), s.id.clone(), s.spec.clone()))
846 .collect::<Vec<_>>()
847 })
848 .collect();
849 for (workflow, node, spec) in specs {
850 if let Some(want) = spec.get("command").and_then(Value::as_str)
851 && Some(want) != op.as_deref()
852 {
853 continue;
854 }
855 if let Some(roles) = spec.get("roles").and_then(Value::as_array)
856 && !roles.is_empty()
857 && !roles.iter().any(|r| r.as_str() == Some(role))
858 {
859 continue;
860 }
861 let payload = json!({
862 "conversation": ctx,
863 "principal": ev.principal,
864 "role": role,
865 "command": op,
866 "args": args.clone().unwrap_or(Value::Null),
867 // The A2A task tracking this message: carried onto the run so
868 // its terminal status completes the task — which is what lets
869 // a peer's `a2a.delegate {command}` BLOCK on the answer.
870 "task": ev.payload.get("task").cloned().unwrap_or(Value::Null),
871 "parts": ev.payload.get("parts").cloned().unwrap_or(Value::Null),
872 "text": ev.payload.get("text").cloned().unwrap_or(Value::Null),
873 "message_id": ev.payload.get("message_id").cloned().unwrap_or(Value::Null),
874 // The message-hop depth rides through this reader too. Without
875 // it a chain routed through an `a2a` start would reset to zero
876 // on every hop, and the cap would never bite — the run this
877 // fires can `message` again, and that is the same loop.
878 "msg_depth": ev.payload.get("msg_depth").cloned().unwrap_or(json!(0)),
879 });
880 // `into: {stream, subject}` — APPEND the message instead of
881 // firing a run (RFC 0035 §5), so a fleet peer can feed a stream
882 // over mTLS (or the co-located unix-socket lane) and get the same
883 // replay-after-downtime a webhook `into` gives. Authorization has
884 // already happened: the principal was resolved and its `roles`
885 // filter applied above, so this is the last step, not a bypass.
886 if let Some(into) = spec.get("into") {
887 let stream = into.get("stream").and_then(Value::as_str).unwrap_or("");
888 let subject = into.get("subject").and_then(Value::as_str).unwrap_or("");
889 let id = ev
890 .payload
891 .get("message_id")
892 .and_then(Value::as_str)
893 .map(str::to_string)
894 .unwrap_or_else(|| crate::state::ulid::new().to_string());
895 match self.append_event(stream, subject, Some(ctx), payload, &id, &workflow) {
896 Ok(seq) => self.log.info(
897 "start.a2a.into",
898 json!({"workflow": workflow, "node": node, "conversation": ctx,
899 "stream": stream, "subject": subject, "seq": seq}),
900 ),
901 Err(e) => self.log.warn(
902 "start.a2a.into.refused",
903 json!({"workflow": workflow, "node": node, "stream": stream,
904 "err": e}),
905 ),
906 }
907 return true;
908 }
909 self.log.info(
910 "start.a2a.fired",
911 json!({"workflow": workflow, "node": node, "conversation": ctx,
912 "command": op, "role": role}),
913 );
914 self.fire_start(&workflow, &node, &spec, payload, "a2a");
915 return true;
916 }
917 false
918 }
919
920 // ---- children ----------------------------------------------------------
921
922 fn on_child_frame(&mut self, node: NodeId, msg: AgentMsg) {
923 if !self.children.on_frame(node, &msg) {
924 return; // a late frame from a reaped child
925 }
926 match msg {
927 AgentMsg::Ready
928 | AgentMsg::Pong { .. }
929 | AgentMsg::Gate { .. }
930 | AgentMsg::GateClosed { .. } => {}
931 // Coarse progress from the child: what this unit is doing right
932 // now, for the display clients' working row.
933 AgentMsg::Event { event, fields } => self.on_child_progress(node, &event, &fields),
934 AgentMsg::Usage(u) => {
935 self.counters.tokens_in += u.input_tokens;
936 self.counters.tokens_out += u.output_tokens;
937 crate::obs::metrics::record_tokens(u.input_tokens, u.output_tokens);
938 // A subagent's usage is charged as it reports; turn usage is
939 // settled on TurnDone against its reservation.
940 if let Some(ChildKind::Subagent { .. }) = self.children.get(node).map(|c| &c.kind) {
941 self.governor.charge(u, &[]);
942 }
943 }
944 AgentMsg::IntelHealth { all_down, .. } => {
945 if crate::signals::set_intel_all_down(all_down) {
946 self.log.warn("intel.health", json!({"all_down": all_down}));
947 }
948 }
949 AgentMsg::ToolRequest { id, name, args } => self.on_tool_request(node, id, &name, args),
950 AgentMsg::BudgetRequest { id, estimate } => self.on_budget_request(node, id, estimate),
951 AgentMsg::TurnDone { turn } => self.on_turn_done(node, *turn),
952 AgentMsg::Turn { outcome } => self.on_subagent_turn(node, outcome),
953 AgentMsg::Result { outcome } => self.on_subagent_result(node, Ok(outcome)),
954 AgentMsg::Failed { error } => {
955 let kind = self.children.get(node).map(|c| c.kind.clone());
956 match kind {
957 Some(ChildKind::Subagent { .. }) => self.on_subagent_result(node, Err(error)),
958 Some(_) => self.on_turn_failed(node, error),
959 None => {}
960 }
961 }
962 }
963 }
964
965 fn on_reaped(&mut self, r: Reaped) {
966 // Frames-before-reap. A child's terminal frame rides the same event
967 // queue as everything else (that is what makes its arrival WAKE the
968 // loop), so a reap racing ahead of it would read as "worker exited
969 // without a result". Restore the invariant by construction: join the
970 // child's reader thread — bounded, its pipe has already EOF'd — so
971 // every frame it ever wrote is IN the queue, then requeue the reap
972 // BEHIND them. FIFO does the rest; one deferral suffices.
973 if !self.reap_deferred.remove(&r.pid) && self.children.has_pid(r.pid) {
974 self.children.join_reader_of(r.pid);
975 self.reap_deferred.insert(r.pid);
976 let _ = self.events_tx.send(Event::Reaped(r));
977 return;
978 }
979 // An instance-tier daemon child has no control channel and no node in
980 // the child table, so its exit closes the subagent record directly.
981 if !self.children.has_pid(r.pid) && self.on_instance_reaped(&r) {
982 return;
983 }
984 let Some((node, child)) = self.children.on_reaped(&r) else {
985 return;
986 };
987 self.activity_end(node);
988 self.log.info("child.exit", json!({"node": node.0, "pid": r.pid, "kind": super::children::kind_label(&child.kind), "outcome": format!("{:?}", r.outcome)}));
989 // A child that died without its terminal frame: fail its unit.
990 match child.kind {
991 // Ask the STEP, not the child table, whether this worker died
992 // owing a result. The child table cannot answer it here: a
993 // `TurnDone` settles the step but leaves the child in the table
994 // until it is reaped, and `Children::on_reaped` above has already
995 // removed the entry — so "is the child in the table?" reads the
996 // same for a settled worker and an orphaned one. The step is
997 // unambiguous: it is Running and still owned by THIS worker only
998 // when no terminal frame ever landed.
999 ChildKind::StepTurn {
1000 ref run,
1001 ref step,
1002 reservation,
1003 } => {
1004 let node_owned = node.0.to_string();
1005 let orphaned = self
1006 .runs
1007 .get(run)
1008 .and_then(|st| st.step(step))
1009 .is_some_and(|s| {
1010 s.status == crate::engine::StepStatus::Running
1011 && s.worker.as_deref() == Some(node_owned.as_str())
1012 });
1013 if orphaned {
1014 // `on_turn_failed` would route this, but it re-reads the
1015 // child table too and returns early on the reaped node; the
1016 // reservation it would have released is released here.
1017 if let Some(res) = reservation {
1018 self.governor.release(res);
1019 }
1020 self.log.warn(
1021 "turn.failed",
1022 json!({"node": node.0, "kind": super::children::kind_label(&child.kind), "err": "worker exited without a result"}),
1023 );
1024 self.on_step_turn_done(
1025 run,
1026 step,
1027 crate::subagent::protocol::TurnResult {
1028 status: "failed".into(),
1029 error: Some(format!(
1030 "worker exited without a result ({:?})",
1031 r.outcome
1032 )),
1033 ..Default::default()
1034 },
1035 );
1036 }
1037 }
1038 // A root turn and a think expose no equivalent state to test
1039 // here, so they ask `pending_turn_exists`, which answers from the
1040 // settled marker `on_turn_done` / `on_turn_failed` leave on the
1041 // child record rather than from the child's presence in the table.
1042 // Presence cannot answer it: `on_reaped` has already removed the
1043 // child by the time this runs, and a normally-settled worker also
1044 // stays in the table until it is reaped, so presence reads the same
1045 // for settled and orphaned workers alike.
1046 ChildKind::RootTurn { .. } | ChildKind::Think { .. } => {
1047 if self.pending_turn_exists(node) {
1048 self.on_turn_failed(
1049 node,
1050 format!("worker exited without a result ({:?})", r.outcome),
1051 );
1052 }
1053 }
1054 ChildKind::Subagent { ref handle } => {
1055 if self
1056 .subagents
1057 .get(handle)
1058 .is_some_and(|s| !is_terminal_status(&s.status))
1059 {
1060 self.on_subagent_result(
1061 node,
1062 Err(format!(
1063 "subagent exited without a result ({:?})",
1064 r.outcome
1065 )),
1066 );
1067 }
1068 }
1069 }
1070 // Answer any tool request that was waiting on this child (a think).
1071 let waiting: Vec<PendingTool> = self
1072 .pending
1073 .iter()
1074 .filter(|p| matches!(&p.kind, PendingKind::Think { child } if *child == node))
1075 .cloned()
1076 .collect();
1077 for p in waiting {
1078 self.pending.retain(|q| q.target != p.target);
1079 self.reply(
1080 &p.target,
1081 Value::String("think worker exited without a result".into()),
1082 true,
1083 );
1084 }
1085 }
1086
1087 fn on_unhealthy_child(&mut self, node: NodeId, health: crate::supervisor::liveness::Health) {
1088 self.log.warn(
1089 "child.unhealthy",
1090 json!({"node": node.0, "health": format!("{health:?}")}),
1091 );
1092 self.children.cancel(node, &format!("{health:?}"));
1093 // Escalate: give it a moment, then kill.
1094 let started = self
1095 .children
1096 .get(node)
1097 .map(|c| c.started)
1098 .unwrap_or_else(Instant::now);
1099 if started.elapsed() > Duration::from_secs(1) {
1100 self.children.kill(node);
1101 }
1102 }
1103
1104 // ---- lifecycle ---------------------------------------------------------
1105
1106 fn check_signals(&mut self) {
1107 if crate::signals::draining() && !self.draining {
1108 self.begin_drain("signal");
1109 }
1110 if crate::signals::reload_requested() {
1111 crate::signals::clear_reload();
1112 self.on_reload_requested();
1113 }
1114 }
1115
1116 pub(crate) fn begin_drain(&mut self, reason: &str) {
1117 if self.draining {
1118 return;
1119 }
1120 self.draining = true;
1121 self.drain_started = Some(Instant::now());
1122 self.drain_reason = reason.to_string();
1123 crate::signals::set_lame_duck(true);
1124 self.log.info("drain.start", json!({"reason": reason, "children": self.children.len(), "runs": self.runs.values().filter(|r| !r.status.is_terminal()).count()}));
1125 crate::obs::metrics::record_drain("started");
1126 // Tell every attached display client, so a client can stop offering
1127 // actions the daemon will now refuse.
1128 #[cfg(feature = "a2a")]
1129 self.feed_push(
1130 "lifecycle",
1131 super::a2a_server::FeedVis::All,
1132 json!({"draining": true, "reason": reason}),
1133 );
1134 self.children.begin_drain(reason);
1135 // Deinitialization workflows: `event {on: lifecycle.shutdown}` starts
1136 // fire NOW — releasing a claimed webhook route, deregistering from a
1137 // service, flushing a summary — and the drain below WAITS for exactly
1138 // those runs (bounded by drain_timeout like everything else). The
1139 // mirror of `once {policy: always}`, which is the init workflow.
1140 self.fire_event_starts("lifecycle.shutdown", &json!({"reason": reason}));
1141 }
1142
1143 /// Non-terminal runs of workflows that declare a `lifecycle.shutdown`
1144 /// start — the runs drain must wait for. (Any of the workflow's runs
1145 /// counts: an in-flight ordinary run of a deinit-capable workflow is not
1146 /// distinguishable from the deinit run by the time both must finish.)
1147 fn shutdown_runs_live(&self) -> usize {
1148 let capable = |name: &str, hash: &str| {
1149 self.definition_for_run_ref(name, hash).is_some_and(|w| {
1150 w.start_steps()
1151 .iter()
1152 .any(|s| s.kind == "event" && s.field_str("on") == Some("lifecycle.shutdown"))
1153 })
1154 };
1155 let live = self
1156 .runs
1157 .values()
1158 .filter(|r| !r.status.is_terminal())
1159 .filter(|r| capable(&r.workflow, &r.workflow_hash))
1160 .count();
1161 // A fired-but-not-yet-created run is still in the inbox for a tick —
1162 // the gate must not slip through that window.
1163 let queued = self
1164 .inbox_queue
1165 .iter()
1166 .filter(|e| e.kind == super::events::kinds::START_FIRED)
1167 .filter(|e| {
1168 e.payload["workflow"]
1169 .as_str()
1170 .and_then(|n| self.workflows.get(n))
1171 .is_some_and(|w| {
1172 w.start_steps().iter().any(|s| {
1173 s.kind == "event" && s.field_str("on") == Some("lifecycle.shutdown")
1174 })
1175 })
1176 })
1177 .count();
1178 live + queued
1179 }
1180
1181 /// Decide whether to exit now. Returns the exit code when done.
1182 fn lifecycle_step(&mut self) -> Option<i32> {
1183 if let Some(code) = self.exit {
1184 // A `finish {exit: true}` or a fatal store failure asked to exit:
1185 // drain first.
1186 if !self.draining {
1187 self.begin_drain("exit");
1188 }
1189 if self.children.is_empty() {
1190 return Some(code);
1191 }
1192 }
1193 if self.draining {
1194 let timeout = self.settings.lifecycle.drain_timeout();
1195 let started = self.drain_started.unwrap_or_else(Instant::now);
1196 let force = crate::signals::force() || started.elapsed() >= timeout;
1197 let done =
1198 self.children.drive_drain(force) && (force || self.shutdown_runs_live() == 0);
1199 if done || started.elapsed() >= timeout + ABANDON_GRACE {
1200 if !done {
1201 self.log
1202 .warn("drain.abandon", json!({"children": self.children.len()}));
1203 self.children.abandon();
1204 }
1205 crate::obs::metrics::record_drain("completed");
1206 self.checkpoint(true);
1207 self.log
1208 .info("drain.done", json!({"reason": self.drain_reason}));
1209 return Some(self.exit.unwrap_or(crate::exit::SUCCESS));
1210 }
1211 return None;
1212 }
1213 // Job shape / idle policy.
1214 let run_until = self.settings.lifecycle.run_until;
1215 // `auto` re-reads the LIVE workflow set, not just the configured one:
1216 // a long-lived workflow the agent defined at runtime (`workflow.create`
1217 // — the self-setup shape, where a `--prompt` tells it to build its own
1218 // loop/schedule/subscribe) turns the one-shot job into a daemon exactly
1219 // as a configured one would have. Without this the instance idle-exits
1220 // out from under the thing it was just asked to set up.
1221 let job_now = self.job_shape && !self.workflows.values().any(|w| w.is_long_lived());
1222 let idle_policy = match run_until {
1223 RunUntil::Idle => true,
1224 RunUntil::Drained => false,
1225 RunUntil::Auto => job_now,
1226 };
1227 if !idle_policy {
1228 return None;
1229 }
1230 let busy = self.paused // a paused instance never idle-exits underneath the operator
1231 || !self.children.is_empty()
1232 || !self.turn_queue.is_empty()
1233 || !self.staged_turns.is_empty()
1234 || !self.inbox_queue.is_empty()
1235 || !self.pending.is_empty()
1236 || !self.executing.is_empty()
1237 || self.runs.values().any(|r| !r.status.is_terminal())
1238 || !self.timers.is_empty();
1239 if busy {
1240 self.idle_since = None;
1241 return None;
1242 }
1243 let since = *self.idle_since.get_or_insert_with(Instant::now);
1244 if since.elapsed() >= self.settings.lifecycle.idle_grace() || job_now {
1245 let code = self.job_exit_code();
1246 self.log.info(
1247 "lifecycle.idle_exit",
1248 json!({"code": code, "job_shape": self.job_shape}),
1249 );
1250 self.checkpoint(true);
1251 return Some(code);
1252 }
1253 None
1254 }
1255
1256 /// The exit code of a job-shaped instance, mapped from the `once`-started
1257 /// workflow's finish status. With several such runs the worst outcome
1258 /// wins, so a partial success is never reported as a clean exit. A daemon
1259 /// is not job-shaped and drains to 0.
1260 fn job_exit_code(&self) -> i32 {
1261 let mut code = crate::exit::SUCCESS;
1262 for id in &self.job_runs {
1263 if let Some(r) = self.runs.get(id) {
1264 let c = run_exit_code(r);
1265 if c != crate::exit::SUCCESS {
1266 code = c;
1267 }
1268 }
1269 }
1270 if self.job_runs.is_empty() && self.job_shape {
1271 // Nothing ever ran (no workflow fired) — a configuration edge; report success.
1272 return crate::exit::SUCCESS;
1273 }
1274 crate::exit::apply_budget_remap(
1275 code,
1276 self.settings
1277 .lifecycle
1278 .exit_code_map
1279 .get(&code.to_string())
1280 .copied(),
1281 )
1282 }
1283
1284 fn shutdown(&mut self, code: i32) {
1285 self.children.abandon();
1286 let _ = self.durable.flush(true);
1287 self.log.info("proc.exit", json!({"code": code, "uptime_ms": self.started.elapsed().as_millis() as u64, "turns": self.counters.turns, "tool_calls": self.counters.tool_calls, "runs": self.counters.runs_finished, "tokens_in": self.counters.tokens_in, "tokens_out": self.counters.tokens_out}));
1288 }
1289
1290 /// The job's result (the once-started run's output), for stdout.
1291 pub fn job_output(&self) -> Option<Value> {
1292 self.job_runs
1293 .iter()
1294 .rev()
1295 .filter_map(|id| self.runs.get(id))
1296 .find_map(|r| r.output.clone())
1297 // A `--prompt` job has no `once` run to carry an output: its answer
1298 // is the root turn's reply.
1299 .or_else(|| self.last_root_reply.clone().map(Value::String))
1300 }
1301
1302 // ---- checkpoints ---------------------------------------------------------
1303
1304 /// Persist dirty runs/contexts/subagents; flush the manifest (debounced,
1305 /// forced at drain). A halting store error triggers an exit.
1306 pub(crate) fn checkpoint(&mut self, force: bool) {
1307 let mut failed: Option<String> = None;
1308 for run in self.runs.values_mut() {
1309 if run.dirty {
1310 // A non-durable run (workflow `durable: false`, or the
1311 // `store.durability.work: ephemeral` default) is memory-only:
1312 // no serialization, no write, gone after a restart.
1313 if !run.durable {
1314 run.dirty = false;
1315 continue;
1316 }
1317 crate::state::kill_point("step.before_done");
1318 match self.durable.put(
1319 Kind::Run,
1320 &run.id,
1321 serde_json::to_value(&*run).unwrap_or(Value::Null),
1322 Some(run.workflow_hash.clone()),
1323 ) {
1324 Ok(_) => run.dirty = false,
1325 Err(e) => failed = Some(format!("run {}: {e}", run.id)),
1326 }
1327 }
1328 }
1329 if let Err(e) = self.contexts.checkpoint(&self.durable) {
1330 failed = Some(format!("context: {e}"));
1331 }
1332 for s in self.subagents.values_mut() {
1333 if s.dirty {
1334 if !s.durable {
1335 s.dirty = false;
1336 continue;
1337 }
1338 match self.durable.put(
1339 Kind::Subagent,
1340 &s.handle,
1341 serde_json::to_value(&*s).unwrap_or(Value::Null),
1342 None,
1343 ) {
1344 Ok(_) => s.dirty = false,
1345 Err(e) => failed = Some(format!("subagent {}: {e}", s.handle)),
1346 }
1347 }
1348 }
1349 // Manifest: budget counters + lifecycle, debounced.
1350 let budget = self.governor.to_value();
1351 self.durable.manifest_update(|m| {
1352 m.budget = budget;
1353 });
1354 match self.durable.flush(force) {
1355 Ok(_) => {}
1356 Err(e) => failed = Some(format!("manifest: {e}")),
1357 }
1358 if let Some(e) = failed {
1359 self.log.error("store.checkpoint.fail", json!({"err": e}));
1360 if !self.durable.is_degraded() {
1361 // Halt policy: refuse new intake, drain.
1362 self.exit = Some(crate::exit::GENERIC);
1363 }
1364 }
1365 }
1366
1367 // ---- status ------------------------------------------------------------
1368
1369 /// `status` tool / `agent://status`.
1370 pub(crate) fn status_value(&self) -> Value {
1371 json!({
1372 "instance": self.instance,
1373 "run_id": self.run_id,
1374 "uptime_ms": self.started.elapsed().as_millis() as u64,
1375 "job_shape": self.job_shape,
1376 "draining": self.draining,
1377 "paused": self.paused,
1378 "store": {"kind": self.durable.store_kind(), "degraded": self.durable.is_degraded(), "generation": self.durable.manifest().generation},
1379 "workflows": self.workflows.values().map(|w| json!({"name": w.name, "hash": w.hash, "armed": w.armed, "starts": w.start_steps().iter().map(|s| s.kind.clone()).collect::<Vec<_>>()})).collect::<Vec<_>>(),
1380 "runs": self.runs.values().map(RunState::summary).collect::<Vec<_>>(),
1381 "conversations": self.contexts.status(),
1382 "subagents": self.subagents.values().map(|s| json!({"handle": s.handle, "mode": s.mode, "status": s.status, "tokens": s.tokens, "template": s.template, "tier": s.tier, "pid": s.pid, "retire_at": s.retire_at})).collect::<Vec<_>>(),
1383 "children": self.children.status(),
1384 "timers": self.timers.status(),
1385 "inbox_pending": self.inbox_queue.len(),
1386 "budget": self.governor.status(now_ms()),
1387 "tools": self.registry.len(),
1388 "skills": self.skills.names(),
1389 "counters": {"turns": self.counters.turns, "tool_calls": self.counters.tool_calls, "runs_started": self.counters.runs_started, "runs_finished": self.counters.runs_finished, "tokens_in": self.counters.tokens_in, "tokens_out": self.counters.tokens_out},
1390 "instruction": {"source": self.instruction.source, "uri": self.instruction.uri, "version": self.instruction.version, "bytes": self.instruction.text.len()},
1391 "model": self.model,
1392 "activity": self.activity_value(),
1393 })
1394 }
1395
1396 /// The shortest time until the next time-based wake (a timer, an armed
1397 /// schedule/loop start, a suspended wait deadline, a budget wait). Bounded
1398 /// below at 5 ms so a due deadline is serviced on the next pass without a
1399 /// busy spin.
1400 fn next_wake(&self) -> Duration {
1401 let now = now_ms();
1402 let mut soonest = now + 200;
1403 if let Some(t) = self.timers.next_deadline() {
1404 soonest = soonest.min(t);
1405 }
1406 for st in self.durable.manifest().starts.values() {
1407 for k in ["next_ms", "debounce_until"] {
1408 if let Some(n) = st[k].as_u64() {
1409 soonest = soonest.min(n);
1410 }
1411 }
1412 }
1413 for run in self.runs.values() {
1414 if run.status.is_terminal() {
1415 continue;
1416 }
1417 for step in run.steps.values() {
1418 if let Some(w) = &step.wait
1419 && let Some(d) = w["deadline_ms"].as_u64()
1420 {
1421 soonest = soonest.min(d);
1422 }
1423 }
1424 }
1425 if !self.pending.is_empty() || !self.turn_queue.is_empty() {
1426 soonest = soonest.min(now + 50);
1427 }
1428 Duration::from_millis(soonest.saturating_sub(now).max(5))
1429 }
1430
1431 /// The model window (compaction threshold base): `context.model_window`
1432 /// when set, else inferred from the model name.
1433 /// The model window (compaction threshold base).
1434 ///
1435 /// `context.model_window` wins, then the active tier's declared `window`,
1436 /// and only then the guess from the model NAME — a substring match that is
1437 /// simply wrong for any provider whose naming does not happen to match.
1438 /// A tier that declares its window replaces the guess with a fact.
1439 pub(crate) fn model_window(&self) -> u64 {
1440 if let Some(w) = self.settings.context.model_window {
1441 return w;
1442 }
1443 if let Some(w) = self
1444 .settings
1445 .intelligence
1446 .default_reference()
1447 .and_then(|r| self.settings.intelligence.tier(&r).and_then(|t| t.window))
1448 {
1449 return w;
1450 }
1451 tokens::window_for_model(&self.model)
1452 }
1453}
1454
1455pub(crate) fn is_terminal_status(s: &str) -> bool {
1456 matches!(
1457 s,
1458 "completed" | "failed" | "cancelled" | "refused" | "killed" | "crashed" | "retired"
1459 )
1460}
1461
1462/// Map a finished run's status onto a process exit code, so a caller can tell
1463/// *how* a job ended without parsing its output: refusal, budget exhaustion,
1464/// a missed deadline and an unreachable model each get their own code, and
1465/// anything still unfinished reports as partial.
1466pub fn run_exit_code(r: &RunState) -> i32 {
1467 match r.status {
1468 RunStatus::Completed => crate::exit::SUCCESS,
1469 RunStatus::Refused => crate::exit::REFUSED,
1470 RunStatus::Stalled => crate::exit::PARTIAL,
1471 RunStatus::Failed => {
1472 let e = r.error.as_deref().unwrap_or("");
1473 if e.contains("exhausted") || e.contains("budget") {
1474 crate::exit::BUDGET
1475 } else if e.contains("deadline") {
1476 crate::exit::DEADLINE
1477 } else if e.contains("intel") {
1478 crate::exit::INTEL_UNAVAILABLE
1479 } else {
1480 crate::exit::GENERIC
1481 }
1482 }
1483 RunStatus::Cancelled => crate::exit::GENERIC,
1484 _ => crate::exit::PARTIAL,
1485 }
1486}