1#[cfg(feature = "a2a")]
15pub mod a2a_server;
16pub mod activity;
17pub mod artifacts;
18pub mod audit;
19pub mod breaker;
20pub mod children;
21pub mod env; pub mod events;
23#[cfg(feature = "exec")]
24pub mod exec; pub mod goal;
26pub mod http_node;
27pub mod human; pub(crate) mod instances; pub mod nested;
30pub mod pressure; pub mod reactor;
32pub mod reload;
33pub(crate) mod retire;
34pub mod starts;
35pub mod steps;
36pub(crate) mod streams;
37pub mod subagents;
38pub mod timers;
39pub mod tools;
40pub mod turns;
41pub mod waits;
42#[cfg(feature = "a2a")]
43pub mod webhooks;
44pub mod worker;
45
46pub use reactor::Runtime;
47
48use crate::config::v2::{Loaded, StoreKind};
49use crate::context::memory::Memory;
50use crate::context::{Contexts, skills, tokens};
51use crate::engine::run::StepStatus;
52use crate::governor::Governor;
53use crate::mcp::client::McpClient;
54use crate::obs::log::{Comp, Level, LogCtx, Logger};
55use crate::registry::{Registry, ServerTools};
56use crate::state::{Durable, Kind, Policy, now_ms};
57use serde_json::{Value, json};
58use std::collections::BTreeMap;
59use std::sync::Arc;
60use std::time::{Duration, Instant};
61
62fn reference_preflight(
72 doc: &Value,
73 settings: &crate::config::v2::Settings,
74 at: &str,
75 log: &Logger,
76) -> Option<i32> {
77 let mut missing = crate::config::v2::missing_references(doc, at, &settings.vars);
78 if missing.is_empty() {
79 return None;
80 }
81 if crate::config::prompt::prompt_missing_requested() {
82 let mut found = Vec::new();
83 crate::config::v2::scan_references(doc, at, &mut found);
84 let mut names: Vec<String> = found
85 .into_iter()
86 .filter(|r| r.kind == "secret" && !crate::sec::secret::secret_available(&r.name))
87 .map(|r| r.name)
88 .collect();
89 names.sort();
90 names.dedup();
91 for name in names {
92 match crate::config::prompt::read_secret_from_tty(&format!("{name} (secret)")) {
93 Ok(v) => crate::sec::secret::set_prompted(&name, v),
94 Err(e) => {
95 log.error("prompt.failed", json!({"secret": name, "err": e}));
96 break;
97 }
98 }
99 }
100 missing = crate::config::v2::missing_references(doc, at, &settings.vars);
101 if missing.is_empty() {
102 return None;
103 }
104 }
105 for m in &missing {
106 log.error("config.invalid", json!({"error": m}));
107 }
108 log.error(
109 "proc.exit",
110 json!({"code": crate::exit::USAGE, "err": format!("{} unresolved reference(s)", missing.len())}),
111 );
112 Some(crate::exit::USAGE)
113}
114
115pub fn run(loaded: &Loaded, args: &[String], env: &[(String, String)]) -> i32 {
120 let settings = loaded.settings.clone();
121 let instance = settings.instance_name();
122 let run_id = match settings.lifecycle.run_id.clone() {
123 Some(r) => r,
124 None => crate::state::ulid::new(),
125 };
126 let trace = crate::obs::trace::resolve(&run_id, settings.observability.traceparent.as_deref());
127 let level = settings
128 .observability
129 .log_level
130 .as_deref()
131 .and_then(Level::parse)
132 .unwrap_or(Level::Info);
133 let log = Logger::new(
134 LogCtx {
135 run_id: run_id.clone(),
136 agent_id: "sup".into(),
137 agent_path: "0".into(),
138 comp: Comp::Supervisor,
139 pid: std::process::id(),
140 trace_id: Some(trace.trace_id.clone()),
141 },
142 level,
143 )
144 .with_content(settings.observability.log_content);
145 log.info("proc.start", json!({"version": crate::VERSION, "runtime": "1", "instance": instance, "config_files": loaded.files.iter().map(|(p, _)| p.clone()).collect::<Vec<_>>()}));
146 for w in &loaded.warnings {
147 log.warn("config.warning", json!({"warning": w}));
148 }
149 crate::signals::install();
150 crate::supervisor::reap::set_child_subreaper();
151
152 if let Some(code) = reference_preflight(&loaded.doc, &settings, "config", &log) {
157 return code;
158 }
159 let envmap = |k: &str| env.iter().find(|(n, _)| n == k).map(|(_, v)| v.clone());
160
161 #[cfg(feature = "tls")]
163 if let Some(path) = settings.security.tls_ca.as_deref()
164 && let Err(e) = std::fs::read(path).and_then(|pem| crate::net::tls::install_extra_ca(&pem))
165 {
166 log.error(
167 "proc.exit",
168 json!({"code": crate::exit::USAGE, "err": format!("security.tls_ca {path}: {e}")}),
169 );
170 return crate::exit::USAGE;
171 }
172 #[cfg(feature = "aauth")]
175 if let Some(a) = &settings.security.aauth {
176 let v1 = crate::config::AAuthSettings {
177 provider: a.provider.clone(),
178 key_file: a
179 .key_file
180 .clone()
181 .unwrap_or_else(|| "/var/lib/agentd/aauth-key".into()),
182 enrollment_token: a.enroll_token.as_ref().map(|s| s.0.clone()),
183 enroll_assertion_file: a.enroll_assertion_file.clone(),
184 person_server: a.person_server.clone(),
185 };
186 if let Err(e) = crate::aauth::setup(&v1, Duration::from_secs(30)) {
187 log.error(
188 "proc.exit",
189 json!({"code": crate::exit::USAGE, "err": format!("aauth: {e}")}),
190 );
191 return crate::exit::USAGE;
192 }
193 }
194
195 #[cfg(unix)]
200 if settings.security.cgroup.spec.is_some() {
201 let c = &settings.security.cgroup;
202 if let Some(configured) = crate::supervisor::cgroup::configure(
203 c.spec.as_deref(),
204 c.memory_max.as_deref(),
205 c.pids_max.as_deref(),
206 ) {
207 log.info("cgroup.armed", json!({"parent": configured.parent.display().to_string(), "limits_unavailable": configured.limits_unavailable}));
208 }
209 }
210
211 let intel_uri = settings.intelligence.endpoint_list().unwrap_or_default();
213 let intel_token = match resolve_intel_token(&settings, &envmap) {
214 Ok(t) => t,
215 Err(e) => {
216 log.error("proc.exit", json!({"code": crate::exit::USAGE, "err": e}));
217 return crate::exit::USAGE;
218 }
219 };
220 let model = settings
225 .intelligence
226 .default_reference()
227 .map(|r| settings.intelligence.wire_model(&r))
228 .unwrap_or_default();
229 let intel_headers: Vec<(String, String)> = settings
234 .intelligence
235 .headers
236 .iter()
237 .filter_map(|(k, v)| {
238 crate::sec::secret::resolve(v, &envmap)
239 .ok()
240 .map(|r| (k.clone(), r))
241 })
242 .collect();
243 let intel_bearer = intel_bearer_provider(&settings);
244
245 let mut mcp: BTreeMap<String, Arc<McpClient>> = BTreeMap::new();
248 let mut mcp_specs = BTreeMap::new();
249 let mut server_tools: Vec<ServerTools> = Vec::new();
250 let mcp_timeout = settings
251 .mcp
252 .default_timeout
253 .map(|d| d.0)
254 .unwrap_or(Duration::from_secs(60));
255 for s in &settings.mcp.servers {
256 if let Err(e) = crate::config::v2::egress_allows(
260 &settings.services,
261 settings.security.egress,
262 crate::config::v2::ServiceKind::Mcp,
263 &s.endpoint,
264 ) {
265 log.error("proc.exit", json!({"code": crate::exit::USAGE, "err": e}));
266 return crate::exit::USAGE;
267 }
268 let spec = match s.to_spec() {
269 Ok(sp) => sp,
270 Err(e) => {
271 log.error("proc.exit", json!({"code": crate::exit::USAGE, "err": e}));
272 return crate::exit::USAGE;
273 }
274 };
275 let per_timeout = s.timeout.map(|d| d.0).unwrap_or(mcp_timeout);
276 match crate::mcp::from_spec(&spec, per_timeout).and_then(|mut c| c.initialize().map(|()| c))
277 {
278 Ok(mut c) => {
279 let mut meta = json!({"agent/run_id": run_id, "agent/instance": instance});
280 meta["traceparent"] =
281 crate::obs::trace::outbound_traceparent(&trace.trace_id).into();
282 c.set_tool_meta(meta);
283 let tools = c.list_tools().unwrap_or_default();
284 log.info(
285 "mcp.connect",
286 json!({"server": s.name, "tools": tools.len()}),
287 );
288 server_tools.push(ServerTools {
289 name: s.name.clone(),
290 ns: s.ns.clone(),
291 tags: spec.tags.clone(),
292 tools,
293 });
294 mcp.insert(s.name.clone(), Arc::new(c));
295 }
296 Err(e) => {
297 log.warn(
298 "mcp.connect.fail",
299 json!({"server": s.name, "err": e.to_string()}),
300 );
301 crate::obs::metrics::record_mcp_connect_failure(&s.name);
302 }
303 }
304 mcp_specs.insert(s.name.clone(), spec);
305 }
306
307 let store = match settings.store.kind {
312 StoreKind::None => {
313 log.warn(
314 "store.none",
315 json!({"note": "no durable store: state lives in this process only (job shape)"}),
316 );
317 Arc::new(crate::store::memory::MemoryStore::new()) as crate::store::SharedStore
318 }
319 _ => {
320 let mcp_ref = mcp.clone();
321 match crate::store::open(&settings.store, &|name: &str| {
322 mcp_ref
323 .get(name)
324 .map(|c| c.clone() as Arc<dyn crate::store::mcp::McpCall>)
325 }) {
326 Ok(Some(s)) => s,
327 Ok(None) => Arc::new(crate::store::memory::MemoryStore::new()),
328 Err(e) => {
329 log.error("proc.exit", json!({"code": crate::exit::MCP_REQUIRED_DOWN, "err": format!("store: {e}")}));
330 return crate::exit::MCP_REQUIRED_DOWN;
331 }
332 }
333 }
334 };
335 let durable = Durable::new(
336 store,
337 settings.store.prefix(),
338 &instance,
339 Policy::from_settings(&settings.store),
340 Some(log.clone()),
341 );
342
343 let restored = match durable.restore() {
347 Ok(r) => r,
348 Err(e) => {
349 log.error(
350 "proc.exit",
351 json!({"code": crate::exit::MCP_REQUIRED_DOWN, "err": format!("restore: {e}")}),
352 );
353 return crate::exit::MCP_REQUIRED_DOWN;
354 }
355 };
356 if settings.store.kind == StoreKind::File {
365 let root = crate::config::v2::file_store_root(&settings.store);
366 log.info(
367 "store.file",
368 json!({
369 "path": root.display().to_string(),
370 "generation": restored.manifest.as_ref().map(|m| m.generation).unwrap_or(1),
371 "defaulted": loaded.doc.pointer("/store/kind").is_none(),
374 "msg": "durable state is on the local filesystem; it survives a restart of this process but not a move to another host — use store.kind mcp|http for a fleet",
375 }),
376 );
377 }
378
379 let registry = match Registry::build(&settings, &server_tools) {
383 Ok(r) => r,
384 Err(errs) => {
385 for e in &errs {
386 log.error("config.invalid", json!({"error": e}));
387 }
388 log.error(
389 "proc.exit",
390 json!({"code": crate::exit::USAGE, "err": "tool registry"}),
391 );
392 return crate::exit::USAGE;
393 }
394 };
395 for w in ®istry.warnings {
396 log.warn("registry.warning", json!({"warning": w}));
397 }
398
399 let mut catalogue = skills::Catalogue::new(
401 settings
402 .skills
403 .reference_prefix
404 .as_deref()
405 .unwrap_or(skills::DEFAULT_PREFIX),
406 settings.skills.max_bytes.unwrap_or(32_768) as usize,
407 );
408 for src in &settings.skills.sources {
409 match mcp.get(&src.server) {
410 Some(c) => {
411 let mode = match src.discover {
412 crate::config::v2::Discover::Prompts => skills::Discover::Prompts,
413 crate::config::v2::Discover::Resources => skills::Discover::Resources,
414 crate::config::v2::Discover::Auto => skills::Discover::Auto,
415 };
416 let found = catalogue.discover(&**c, mode, src.filter.as_deref());
417 log.info(
418 "skills.discovered",
419 json!({"server": src.server, "count": found.len(), "skills": found}),
420 );
421 }
422 None => log.warn("skills.source.unavailable", json!({"server": src.server})),
423 }
424 }
425 if let Some(dir) = &settings.skills.dir {
426 let (names, errs) = catalogue.add_dir(std::path::Path::new(dir));
427 for e in errs {
428 log.warn("skills.file.unreadable", json!({"err": e}));
429 }
430 if !names.is_empty() {
431 log.info(
432 "skills.discovered",
433 json!({"server": "file", "dir": dir, "count": names.len(), "skills": names}),
434 );
435 }
436 }
437 if !settings.agent.inline_skills.is_empty() {
438 let names = catalogue.add_inline(&settings.agent.inline_skills);
439 log.info(
440 "skills.discovered",
441 json!({"server": "instruction", "count": names.len(), "skills": names}),
442 );
443 }
444
445 let (events_tx, events_rx) = std::sync::mpsc::channel();
447 let child_tx: crate::supervisor::spawn::FrameSink = {
459 let events_tx = events_tx.clone();
460 std::sync::Arc::new(move |node, msg| {
461 events_tx.send(events::Event::Child(node, msg)).is_ok()
462 })
463 };
464 let (reap_tx, reap_rx) = std::sync::mpsc::channel();
465 let exe = std::env::current_exe().unwrap_or_else(|_| std::path::PathBuf::from("agentd"));
466
467 let model_window = settings
468 .context
469 .model_window
470 .or_else(|| {
474 settings
475 .intelligence
476 .default_reference()
477 .and_then(|r| settings.intelligence.tier(&r).and_then(|t| t.window))
478 })
479 .unwrap_or_else(|| {
480 if model.is_empty() {
481 tokens::DEFAULT_MODEL_WINDOW
482 } else {
483 tokens::window_for_model(&model)
484 }
485 });
486 let pressure = {
491 use crate::config::v2::StoreKind;
492 let (path, shed) = if settings.store.kind == StoreKind::File {
493 let root = crate::config::v2::file_store_root(&settings.store);
494 let min = settings
495 .store
496 .file
497 .as_ref()
498 .and_then(|f| f.min_free.as_deref())
499 .map(super::runtime::pressure::parse_bytes)
500 .transpose()
501 .unwrap_or_else(|e| {
502 log.warn("config.warning", json!({"warning": format!("store.file.min_free: {e}; using the 256MB default")}));
503 None
504 })
505 .unwrap_or(256 << 20);
506 (Some(root), min)
507 } else {
508 (None, 0)
509 };
510 std::sync::Arc::new(pressure::Pressure::new(path, shed))
511 };
512
513 let mut rt = Runtime {
514 instance: instance.clone(),
515 run_id: run_id.clone(),
516 durable,
517 mcp,
518 mcp_specs,
519 registry,
520 contexts: Contexts::new(model_window),
521 memory: Memory::new(
522 settings.memory.max_value_bytes.unwrap_or(65_536) as usize,
523 settings.memory.list_default_limit.unwrap_or(100) as usize,
524 ),
525 artifacts: artifacts::Artifacts::new(),
526 skills: catalogue,
527 governor: Governor::new(&settings.intelligence.budget),
528 workflows: BTreeMap::new(),
529 runs: BTreeMap::new(),
530 children: children::Children::new(exe, child_tx, reap_tx),
531 timers: timers::Timers::new(),
532 events_rx,
533 events_tx,
534 reap_rx,
535 pending: Vec::new(),
536 turn_queue: Default::default(),
537 staged_turns: BTreeMap::new(),
538 inbox_queue: Default::default(),
539 subagents: BTreeMap::new(),
540 instruction: reactor::Instruction {
541 text: String::new(),
542 source: "static",
543 uri: None,
544 server: None,
545 version: 1,
546 },
547 job_shape: false,
548 principal_budgets: BTreeMap::new(),
552 principal_rates: BTreeMap::new(),
553 principal_labels: BTreeMap::new(),
554 exit: None,
555 draining: false,
556 paused: false,
557 drain_started: None,
558 drain_reason: String::new(),
559 idle_since: None,
560 intel_uri,
561 intel_token,
562 intel_headers,
563 intel_bearer,
564 model,
565 trace_id: Some(trace.trace_id.clone()),
566 started: Instant::now(),
567 seq: 0,
568 counters: Default::default(),
569 job_runs: Vec::new(),
570 executing: BTreeMap::new(),
571 last_manifest_flush: Instant::now(),
572 goal_judge_at: None,
573 #[cfg(feature = "a2a")]
574 tasks: BTreeMap::new(),
575 #[cfg(feature = "a2a")]
576 event_to_task: BTreeMap::new(),
577 #[cfg(feature = "a2a")]
578 #[cfg(feature = "a2a")]
579 a2a_feed: None,
580 #[cfg(feature = "a2a")]
581 a2a_pairing: None,
582 #[cfg(feature = "a2a")]
583 reserved_task_id: None,
584 #[cfg(feature = "a2a")]
585 a2a_sink: None,
586 #[cfg(feature = "a2a")]
587 a2a_listener: None,
588 #[cfg(feature = "a2a")]
589 a2a_bridge: None,
590 #[cfg(feature = "a2a")]
591 webhook_handler: None,
592 #[cfg(feature = "a2a")]
593 a2a_origins: None,
594 activity: BTreeMap::new(),
595 last_root_reply: None,
596 #[cfg(feature = "a2a")]
597 feed_marks: BTreeMap::new(),
598 #[cfg(feature = "a2a")]
599 feed_last: Instant::now(),
600 #[cfg(feature = "a2a")]
601 webhook_callbacks: std::sync::Arc::new(std::sync::Mutex::new(
602 std::collections::HashMap::new(),
603 )),
604 #[cfg(feature = "a2a")]
605 webhook_sync: std::collections::HashMap::new(),
606 pressure: pressure.clone(),
607 pressure_seen: pressure::Level::Ok,
608 resched: false,
609 reap_deferred: Default::default(),
610 step_rates: Default::default(),
611 settings_doc: loaded.doc.clone(),
612 args: args.to_vec(),
613 env: env.to_vec(),
614 pinned: BTreeMap::new(),
615 retiring: BTreeMap::new(),
616 pin_written: Default::default(),
617 recent_signals: BTreeMap::new(),
618 memory_keys: std::collections::HashMap::new(),
619 stream_dirty: false,
620 settings,
621 log: log.clone(),
622 };
623
624 let lost_ctx = rt.contexts.restore(restored.of(Kind::Context));
626 if !lost_ctx.is_empty() {
627 log.warn("restore.context.lost", json!({"ids": lost_ctx}));
628 }
629 rt.timers.restore(restored.timers());
630 rt.artifacts.restore(restored.of(Kind::Artifact));
631 let mut replayed: Vec<(String, String)> = Vec::new();
632 for env in restored.of(Kind::Run) {
633 match serde_json::from_value::<crate::engine::RunState>(env.state.clone()) {
634 Ok(mut r) => {
635 r.dirty = false;
636 if !r.status.is_terminal() {
637 for (id, st) in r.steps.iter_mut() {
642 if st.status == StepStatus::Running {
643 log.info(
644 "restore.step.replay",
645 json!({"run": r.id, "step": id, "attempt": st.attempt}),
646 );
647 st.status = StepStatus::Pending;
648 st.worker = None;
649 replayed.push((r.id.clone(), id.clone()));
653 }
654 }
655 r.status = crate::engine::RunStatus::Running;
656 r.dirty = true;
657 }
658 rt.runs.insert(r.id.clone(), r);
659 }
660 Err(e) => log.warn(
661 "restore.run.corrupt",
662 json!({"id": env.id, "err": e.to_string()}),
663 ),
664 }
665 }
666 for env in restored.of(Kind::Subagent) {
667 match serde_json::from_value::<reactor::SubagentRecord>(env.state.clone()) {
668 Ok(s) => {
669 rt.subagents.insert(s.handle.clone(), s);
670 }
671 Err(e) => log.warn(
672 "restore.subagent.corrupt",
673 json!({"id": env.id, "err": e.to_string()}),
674 ),
675 }
676 }
677 #[cfg(feature = "a2a")]
678 rt.restore_a2a_tasks(restored.of(Kind::Task));
679 if let Some(m) = &restored.manifest {
680 rt.governor.restore(&m.budget, now_ms());
681 }
682 for ev in restored.inbox_pending() {
683 rt.inbox_queue.push_back(ev);
684 }
685 if restored.manifest.is_some() {
686 log.info("restore.adopted", json!({"runs": rt.runs.len(), "contexts": rt.contexts.len(), "subagents": rt.subagents.len(), "timers": rt.timers.len(), "artifacts": rt.artifacts.len(), "inbox_pending": rt.inbox_queue.len(), "lost": restored.lost.len()}));
687 rt.restore_pins();
688 rt.audit(audit::AuditEvent {
692 action: "restore",
693 target: json!({"runs": rt.runs.len(), "subagents": rt.subagents.len(), "inbox_pending": rt.inbox_queue.len(), "lost": restored.lost.len()}),
694 outcome: if restored.lost.is_empty() { "restored" } else { "restored_with_loss" },
695 principal: Some("system"),
696 role: Some("system"),
697 request_id: None,
698 });
699 }
700
701 if let Some(text) = rt.settings.agent.instruction.clone() {
704 if crate::config::v2::looks_like_resource_uri(&text) {
705 match rt.subscribe_instruction(&text) {
706 Ok(()) => {}
707 Err(e) => {
708 log.error("proc.exit", json!({"code": crate::exit::MCP_REQUIRED_DOWN, "err": format!("agent.instruction {text}: {e}")}));
709 return crate::exit::MCP_REQUIRED_DOWN;
710 }
711 }
712 } else {
713 rt.instruction.text = text;
714 }
715 }
716
717 if let Err(errs) = rt.load_workflows() {
718 for e in &errs {
719 log.error("config.invalid", json!({"error": e}));
720 }
721 log.error(
722 "proc.exit",
723 json!({"code": crate::exit::USAGE, "err": "workflow definitions"}),
724 );
725 return crate::exit::USAGE;
726 }
727 {
733 let defs: Vec<&crate::engine::Workflow> =
734 rt.workflows.values().map(|w| w.as_ref()).collect();
735 let errs = rt.registry.register_workflow_tools(&defs);
736 if !errs.is_empty() {
737 for e in &errs {
738 log.error("config.invalid", json!({"error": e}));
739 }
740 log.error(
741 "proc.exit",
742 json!({"code": crate::exit::USAGE, "err": "workflow tool registration"}),
743 );
744 return crate::exit::USAGE;
745 }
746 for w in &defs {
751 let Some(t) = &w.tool else { continue };
752 log.info(
753 "registry.workflow_tools",
754 json!({"tool": t.name, "workflow": w.name,
755 "mode": if t.mode == crate::engine::model::WorkflowToolMode::Sync { "sync" } else { "async" },
756 "tags": rt.registry.get(&t.name).map(|s| s.tags.clone()).unwrap_or_default(),
757 "arguments": rt.registry.get(&t.name).map(|s| s.input_schema.clone())}),
758 );
759 }
760 }
761 {
769 let mut all = serde_json::Map::new();
770 for (name, wf) in &rt.workflows {
771 let mut steps = serde_json::Map::new();
772 for (sid, step) in &wf.steps {
773 steps.insert(
774 sid.clone(),
775 Value::Object(step.spec.clone().into_iter().collect()),
776 );
777 }
778 all.insert(name.clone(), Value::Object(steps));
779 }
780 if let Some(code) =
781 reference_preflight(&Value::Object(all), &rt.settings, "workflows", &rt.log)
782 {
783 return code;
784 }
785 }
786
787 if !replayed.is_empty() {
792 let policies: Vec<(String, String, crate::engine::model::OnReplay)> = replayed
793 .iter()
794 .filter_map(|(rid, sid)| {
795 let wf_name = rt.runs.get(rid)?.workflow.clone();
796 let step = rt.workflows.get(&wf_name)?.steps.get(sid)?;
797 Some((rid.clone(), sid.clone(), step.on_replay))
798 })
799 .collect();
800 for (rid, sid, policy) in policies {
801 match policy {
802 crate::engine::model::OnReplay::Retry => {}
803 crate::engine::model::OnReplay::Skip => {
804 if let Some(r) = rt.runs.get_mut(&rid) {
805 r.end_step(&sid, StepStatus::Skipped, None, None);
806 }
807 rt.log.info(
808 "restore.step.skipped",
809 json!({"run": rid, "step": sid, "on_replay": "skip"}),
810 );
811 }
812 crate::engine::model::OnReplay::Fail => {
813 if let Some(r) = rt.runs.get_mut(&rid) {
814 r.end_step(
815 &sid,
816 StepStatus::Failed,
817 None,
818 Some(
819 "step was in flight when the process died and its \
820 on_replay policy is `fail`"
821 .into(),
822 ),
823 );
824 }
825 rt.log.warn(
826 "restore.step.failed",
827 json!({"run": rid, "step": sid, "on_replay": "fail"}),
828 );
829 }
830 }
831 }
832 }
833 {
838 let re = rt.settings.observability.runtime_events.clone();
839 let audit_stream = rt.settings.observability.audit.stream.clone();
840 let audit_wants_stream = rt
841 .settings
842 .observability
843 .audit
844 .sink
845 .as_ref()
846 .is_some_and(|s| {
847 s.iter()
848 .any(|x| matches!(x, crate::config::v2::AuditSink::Stream))
849 });
850 if re.is_some() || (audit_wants_stream && audit_stream.is_some()) {
851 let (stream, include, sampled, cap) = match &re {
852 Some(r) => (
853 r.stream.clone().unwrap_or_default(),
854 r.include.clone(),
855 r.sampled.clone(),
856 r.queue_cap(),
857 ),
858 None => (
859 String::new(),
860 Vec::new(),
861 Vec::new(),
862 crate::config::v2::DEFAULT_TAP_QUEUE as usize,
863 ),
864 };
865 crate::obs::log::install_runtime_tap(&stream, include.clone(), sampled.clone(), cap);
866 rt.log.info(
867 "stream.tap",
868 json!({"stream": stream, "include": include, "sampled": sampled,
869 "queue": cap, "audit_stream": audit_stream}),
870 );
871 }
872 }
873 rt.job_shape = match rt.settings.lifecycle.run_until {
878 crate::config::v2::RunUntil::Drained => false,
879 crate::config::v2::RunUntil::Idle => true,
880 crate::config::v2::RunUntil::Auto => {
881 rt.settings.a2a.listen.is_none() && !rt.workflows.values().any(|w| w.is_long_lived())
882 }
883 };
884 for r in rt.runs.values() {
886 if rt.job_shape
887 && rt
888 .workflows
889 .get(&r.workflow)
890 .and_then(|w| w.step(&r.start.node))
891 .is_some_and(|s| s.kind == "once")
892 {
893 rt.job_runs.push(r.id.clone());
894 }
895 }
896 let refs = rt.skills.references(&rt.instruction.text.clone());
898 if !refs.is_empty() {
899 let unknown = rt.preload_skills(crate::context::ROOT, &refs, None);
900 for u in unknown {
901 rt.note_root(format!(
902 "skill.unknown: {u:?} referenced by the instruction is not in the catalogue"
903 ));
904 }
905 }
906 #[cfg(all(unix, feature = "config-watch"))]
909 if rt.settings.lifecycle.watch_config {
910 for (path, _) in &loaded.files {
911 crate::config::watch::spawn_config_watcher(std::path::Path::new(path), &log);
912 }
913 }
914 rt.arm_workflows();
915 rt.arm_long_lived_starts();
916 rt.arm_goal();
917 rt.respawn_restored_subagents();
918 rt.respawn_restored_instances();
919 #[cfg(feature = "a2a")]
923 if rt.settings.a2a.listen.is_some() {
924 let resolver = match crate::a2a::Resolver::build(&rt.settings.a2a, &envmap) {
925 Ok(r) => r,
926 Err(e) => {
927 log.error(
928 "proc.exit",
929 json!({"code": crate::exit::USAGE, "err": format!("a2a principals: {e}")}),
930 );
931 return crate::exit::USAGE;
932 }
933 };
934 let write_timeout = rt.settings.lifecycle.drain_timeout();
935 match a2a_server::spawn_a2a_listener(
936 &rt.settings.a2a,
937 &rt.settings.interface,
938 rt.events_tx.clone(),
939 resolver,
940 &envmap,
941 write_timeout,
942 log.clone(),
943 ) {
944 Ok(serving) => {
945 rt.a2a_feed = serving.feed;
946 rt.a2a_pairing = serving.pairing;
947 rt.a2a_sink = Some(std::sync::Arc::clone(&serving.listener.sink));
948 rt.a2a_listener = Some(serving.listener);
951 rt.a2a_bridge = Some(serving.bridge);
952 rt.a2a_origins = Some(serving.origins);
953 if rt.settings.interface.enabled && rt.settings.interface.debug {
957 let cap = rt
958 .settings
959 .observability
960 .events_ring
961 .map(|n| n as usize)
962 .unwrap_or(crate::obs::log::EVENTS_RING_DEFAULT);
963 crate::obs::log::install_event_ring(cap);
964 log.info("interface.debug", json!({"events_ring": cap}));
965 }
966 for id in rt.tasks.keys().cloned().collect::<Vec<_>>() {
968 rt.task_sync(&id);
969 }
970 }
971 Err(e) => {
972 log.error(
973 "proc.exit",
974 json!({"code": crate::exit::USAGE, "err": format!("a2a listen: {e}")}),
975 );
976 return crate::exit::USAGE;
977 }
978 }
979 }
980 #[cfg(feature = "a2a")]
984 if rt.settings.webhooks.listen.is_some() {
985 let nodes = rt.webhook_nodes();
986 let write_timeout = rt.settings.lifecycle.drain_timeout();
987 match webhooks::spawn_webhook_listener(
988 &rt.settings.webhooks,
989 nodes,
990 rt.webhook_callbacks.clone(),
991 rt.events_tx.clone(),
992 &envmap,
993 write_timeout,
994 rt.pressure.clone(),
995 log.clone(),
996 ) {
997 Ok(h) => rt.webhook_handler = Some(h),
999 Err(e) => {
1000 log.error(
1001 "proc.exit",
1002 json!({"code": crate::exit::USAGE, "err": format!("webhooks listen: {e}")}),
1003 );
1004 return crate::exit::USAGE;
1005 }
1006 }
1007 }
1008 #[cfg(feature = "metrics")]
1011 if let Some(addr) = rt.settings.observability.metrics_addr.clone()
1012 && let Err(e) = crate::obs::serve::spawn(&addr, log.clone())
1013 {
1014 log.warn(
1015 "metrics.serve.fail",
1016 json!({"addr": addr, "err": e.to_string()}),
1017 );
1018 }
1019 if let Some(path) = rt.settings.observability.health_file.clone() {
1020 crate::obs::health::spawn_writer(
1021 std::path::PathBuf::from(path),
1022 run_id.clone(),
1023 "1".into(),
1024 std::time::Duration::from_secs(10),
1025 );
1026 }
1027 #[cfg(feature = "otel")]
1030 if rt.settings.observability.otel.logs == Some(true)
1031 && let Some(ep) = rt
1032 .settings
1033 .observability
1034 .otel
1035 .endpoint
1036 .clone()
1037 .or_else(|| std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok())
1038 {
1039 crate::obs::otel::arm_logs(&ep, "agentd", crate::VERSION);
1040 log.info("otel.logs.armed", json!({"endpoint": ep}));
1041 }
1042 if let Some(prompt) = rt.settings.agent.prompt.clone()
1049 && !prompt.trim().is_empty()
1050 && let Err(err) = rt.accept_event(
1051 events::kinds::A2A_MESSAGE,
1052 Some("operator".into()),
1053 json!({"text": prompt, "context_id": crate::context::ROOT}),
1054 )
1055 {
1056 log.warn("prompt.reject", json!({"err": err}));
1057 }
1058 #[cfg(any(feature = "internal-mocks", debug_assertions))]
1062 if let Ok(path) = std::env::var("AGENTD_TEST_INBOX_FILE") {
1063 match std::fs::read_to_string(&path).map_err(|e| e.to_string()).and_then(|t| serde_json::from_str::<Value>(&t).map_err(|e| e.to_string())) {
1064 Ok(Value::Array(events)) => {
1065 for e in events {
1066 let kind = e["kind"].as_str().unwrap_or(events::kinds::A2A_MESSAGE).to_string();
1067 let principal = e["principal"].as_str().map(str::to_string);
1068 let payload = e.get("payload").cloned().unwrap_or(Value::Null);
1069 if let Err(err) = rt.accept_event(&kind, principal, payload) {
1070 log.warn("test.inbox.reject", json!({"err": err}));
1071 }
1072 }
1073 let _ = std::fs::remove_file(&path);
1074 }
1075 other => log.warn("test.inbox.bad_file", json!({"path": path, "err": format!("{other:?}").chars().take(200).collect::<String>()})),
1076 }
1077 }
1078 rt.checkpoint(true);
1079 let code = rt.run_loop();
1080 let _ = &rt.last_manifest_flush;
1081 if rt.job_shape
1084 && let Some(out) = rt.job_output()
1085 {
1086 match out {
1087 Value::String(s) => println!("{s}"),
1088 Value::Null => {}
1089 other => println!(
1090 "{}",
1091 serde_json::to_string_pretty(&other).unwrap_or_default()
1092 ),
1093 }
1094 }
1095 code
1096}
1097
1098pub fn capabilities(loaded: &Loaded) -> Value {
1104 let start_kinds = crate::engine::model::start_kinds();
1109 let s = &loaded.settings;
1110 let workflows: Vec<Value> = s
1111 .workflows
1112 .iter()
1113 .map(|w| {
1114 let starts: Vec<String> = w["steps"]
1115 .as_object()
1116 .map(|steps| steps.values().filter_map(|st| st["kind"].as_str()).filter(|k| start_kinds.contains(k)).map(str::to_string).collect())
1117 .unwrap_or_default();
1118 json!({"name": w["name"].as_str().unwrap_or(""), "description": w.get("description").and_then(Value::as_str), "start_kinds": starts, "inputs_schema": w.get("inputs").is_some()})
1119 })
1120 .collect();
1121 let a2a = s.a2a.listen.as_ref().map(|listen| {
1122 let principals: Vec<Value> = s
1123 .a2a
1124 .principals
1125 .iter()
1126 .map(|p| json!({"role": format!("{:?}", p.role).to_lowercase(), "match": principal_match_desc(&p.matcher), "grants": p.grants}))
1127 .collect();
1128 let mut methods = vec![
1129 "SendMessage",
1130 "SendStreamingMessage",
1131 "GetTask",
1132 "CancelTask",
1133 "ListTasks",
1134 "SubscribeToTask",
1135 "GetAgentCard",
1136 ];
1137 let mut command_ops = vec![
1138 "status",
1139 "config",
1140 "workflow.run",
1141 "workflow.status",
1142 "workflow.cancel",
1143 "workflow.signal",
1144 "subagent.send",
1145 "subagent.kill",
1146 "subagent.status",
1147 "plan.get",
1148 ];
1149 if s.interface.enabled {
1150 methods.push("SubscribeToEvents");
1151 command_ops.push("interface.info");
1152 if s.interface.debug {
1153 command_ops.extend(["conversation.get", "run.get", "debug.events"]);
1154 }
1155 }
1156 json!({
1157 "listen": listen,
1158 "tls": s.a2a.tls.cert.is_some(),
1159 "mtls": s.a2a.tls.client_ca.is_some(),
1160 "bearer": s.a2a.bearer.is_some(),
1161 "methods": methods,
1162 "admin": ["a2a.drain", "a2a.lameduck", "a2a.cancel", "a2a.pause", "a2a.resume"],
1163 "command_ops": command_ops,
1164 "principals": principals,
1165 "loopback_operator": s.a2a.principals.is_empty(),
1166 })
1167 });
1168 json!({
1169 "runtime": "1",
1170 "version": crate::VERSION,
1171 "agent": {"name": s.instance_name(), "instruction": s.agent.instruction.is_some(), "preflight": format!("{:?}", s.agent.preflight).to_lowercase()},
1172 "intelligence": {"model": s.intelligence.model, "endpoints": s.intelligence.endpoints.len()},
1173 "mcp_servers": s.mcp.servers.iter().map(|m| m.name.clone()).collect::<Vec<_>>(),
1174 "internal_tools": crate::registry::internal::names(),
1175 "tools": {"overrides": s.tools.overrides.keys().cloned().collect::<Vec<_>>(), "disabled": s.tools.disabled},
1176 "workflows": workflows,
1177 "knowledge": {"server": s.knowledge.server},
1178 "search": {"server": s.search.server},
1179 "skills": {"sources": s.skills.sources.len()},
1180 "a2a": a2a,
1181 "interface": {"enabled": s.interface.enabled, "debug": s.interface.debug, "origins": s.interface.origins.len(), "pairing": s.interface.pairing.enabled, "display": {"top": s.interface.display.top, "bottom": s.interface.display.bottom}},
1182 "store": format!("{:?}", s.store.kind).to_lowercase(),
1183 "store_file": (s.store.kind == StoreKind::File).then(|| json!({
1189 "path": crate::config::v2::file_store_root(&s.store).display().to_string(),
1190 "defaulted": loaded.doc.pointer("/store/kind").is_none(),
1191 })),
1192 "lifecycle": {
1197 "run_until": format!("{:?}", s.lifecycle.run_until).to_lowercase(),
1198 "daemon": s.a2a.listen.is_some() || s.workflows.iter().any(|w| {
1199 w["steps"].as_object().is_some_and(|st| {
1200 st.values().any(|n| {
1201 n["kind"].as_str().is_some_and(crate::engine::model::is_long_lived_start)
1202 })
1203 })
1204 }),
1205 },
1206 "webhooks": s.webhooks.listen.as_ref().map(|l| json!({
1210 "listen": l,
1211 "default_auth": s.webhooks.default_auth.is_some(),
1212 "routes": s.workflows.iter().flat_map(|w| {
1213 let wf = w["name"].as_str().unwrap_or("").to_string();
1214 w["steps"].as_object().into_iter().flatten()
1215 .filter(|(_, n)| n["kind"] == "webhook")
1216 .map(move |(id, n)| json!({
1217 "workflow": wf, "node": id,
1218 "path": n["path"], "methods": n["methods"],
1219 "auth": n.get("auth").is_some(),
1222 })).collect::<Vec<_>>()
1223 }).collect::<Vec<_>>(),
1224 })),
1225 "surfaces": {
1230 "exit_codes": crate::exit::EXIT_CODES,
1231 "config_schema": crate::config::v2::schema::CONFIG_VERSION,
1232 },
1233 })
1234}
1235
1236fn principal_match_desc(m: &crate::config::v2::PrincipalMatch) -> Value {
1238 if m.any {
1239 json!({"any": true})
1240 } else if let Some(s) = &m.san {
1241 json!({"san": s})
1242 } else if let Some(s) = &m.sub {
1243 json!({"sub": s})
1244 } else if m.bearer_ref.is_some() {
1245 json!({"bearer_ref": "***"})
1246 } else if let Some(a) = &m.aauth_agent {
1247 json!({"aauth_agent": a})
1248 } else {
1249 json!({})
1250 }
1251}
1252
1253fn intel_bearer_provider(
1260 settings: &crate::config::v2::Settings,
1261) -> Option<std::sync::Arc<dyn Fn() -> Option<String> + Send + Sync>> {
1262 #[cfg(feature = "oauth")]
1263 {
1264 let auth = settings.intelligence.auth.as_ref()?;
1265 let spec = auth.to_spec();
1266 if spec.kind == "aws" {
1271 return None;
1272 }
1273 let signer = crate::auth::device::signer_for(
1277 &spec,
1278 "intelligence",
1279 std::time::Duration::from_secs(30),
1280 )
1281 .ok()??;
1282 Some(std::sync::Arc::new(move || {
1283 signer
1284 .sign("POST", "", "", &[])
1285 .into_iter()
1286 .find(|(k, _)| k.eq_ignore_ascii_case("authorization"))
1287 .map(|(_, v)| v.strip_prefix("Bearer ").unwrap_or(&v).to_string())
1288 }))
1289 }
1290 #[cfg(not(feature = "oauth"))]
1291 {
1292 let _ = settings;
1293 None
1294 }
1295}
1296
1297impl Runtime {
1298 pub(crate) fn current_intel_bearer(&self) -> Option<String> {
1305 self.intel_bearer
1306 .as_ref()
1307 .and_then(|f| f())
1308 .or_else(|| self.intel_token.clone())
1309 }
1310
1311 pub(crate) fn intel_aws_auth(&self) -> Option<crate::config::AuthSpec> {
1316 let a = self.settings.intelligence.auth.as_ref()?;
1317 (a.kind == crate::config::v2::AuthKind::Aws).then(|| a.to_spec())
1318 }
1319
1320 pub(crate) fn intel_dialect(&self) -> Option<String> {
1324 self.settings.intelligence.dialect.clone()
1325 }
1326}
1327
1328fn resolve_intel_token(
1330 settings: &crate::config::v2::Settings,
1331 env: &dyn Fn(&str) -> Option<String>,
1332) -> Result<Option<String>, String> {
1333 if let Some(t) = &settings.intelligence.token {
1334 let resolved = crate::sec::secret::resolve(&t.0, env)
1335 .map_err(|e| format!("intelligence.token: {e}"))?;
1336 return Ok(Some(resolved));
1337 }
1338 if let Some(p) = &settings.intelligence.token_file {
1339 return crate::sec::secret::read_token_file(p)
1340 .map(Some)
1341 .map_err(|e| format!("intelligence.token_file: {e}"));
1342 }
1343 Ok(None)
1346}
1347
1348impl Runtime {
1349 pub(crate) fn subscribe_instruction(&mut self, uri: &str) -> Result<(), String> {
1352 let (server, res) = match uri.strip_prefix("mcp://").and_then(|r| r.split_once('/')) {
1353 Some((s, r)) => (Some(s.to_string()), r.to_string()),
1354 None => (None, uri.to_string()),
1355 };
1356 let candidates: Vec<(String, Arc<McpClient>)> = match &server {
1358 Some(s) => self
1359 .mcp
1360 .get(s)
1361 .map(|c| vec![(s.clone(), c.clone())])
1362 .unwrap_or_default(),
1363 None => self
1364 .mcp
1365 .iter()
1366 .map(|(n, c)| (n.clone(), c.clone()))
1367 .collect(),
1368 };
1369 let mut last_err = String::from("no connected MCP server serves it");
1370 for (name, c) in candidates {
1371 match c.read_resource(&res) {
1372 Ok(r) => {
1373 let text = r.text();
1374 if c.capabilities().supports_resources()
1375 && let Err(e) = c.subscribe(&res)
1376 {
1377 self.log.warn(
1378 "instruction.subscribe.fail",
1379 json!({"server": name, "uri": res, "err": e.to_string()}),
1380 );
1381 }
1382 let changed = self.instruction.text != text;
1383 self.instruction = reactor::Instruction {
1384 text,
1385 source: "resource",
1386 uri: Some(res.clone()),
1387 server: Some(name.clone()),
1388 version: self.instruction.version + u64::from(changed),
1389 };
1390 self.log.info("instruction.loaded", json!({"server": name, "uri": res, "bytes": self.instruction.text.len(), "version": self.instruction.version}));
1391 return Ok(());
1392 }
1393 Err(e) => last_err = e.to_string(),
1394 }
1395 }
1396 Err(last_err)
1397 }
1398
1399 pub(crate) fn poll_mcp_notifications(&mut self) {
1405 let mut updated_instruction = false;
1406 let mut tools_changed = Vec::new();
1407 let mut resource_updates: Vec<(String, String)> = Vec::new();
1408 for (name, c) in &self.mcp {
1409 for n in c.drain_notifications() {
1410 match n.method.as_str() {
1411 ::mcp::wire::method::NOTIFY_RESOURCES_UPDATED => {
1412 let uri = n
1413 .params
1414 .as_ref()
1415 .and_then(|p| p.get("uri"))
1416 .and_then(Value::as_str)
1417 .unwrap_or("");
1418 if self.instruction.uri.as_deref() == Some(uri)
1419 && self.instruction.server.as_deref() == Some(name.as_str())
1420 {
1421 updated_instruction = true;
1422 }
1423 resource_updates.push((name.clone(), uri.to_string()));
1424 }
1425 ::mcp::wire::method::NOTIFY_TOOLS_LIST_CHANGED => {
1426 tools_changed.push(name.clone())
1427 }
1428 _ => {}
1429 }
1430 }
1431 }
1432 if updated_instruction && let Some(uri) = self.instruction.uri.clone() {
1433 let full = match &self.instruction.server {
1434 Some(s) => format!("mcp://{s}/{uri}"),
1435 None => uri,
1436 };
1437 let before = self.instruction.version;
1438 if self.subscribe_instruction(&full).is_ok() && self.instruction.version != before {
1439 self.log.info(
1440 "instruction.updated",
1441 json!({"version": self.instruction.version}),
1442 );
1443 if self
1444 .settings
1445 .agent
1446 .wake_on()
1447 .contains(&crate::config::v2::WakeEvent::InstructionUpdated)
1448 {
1449 self.note_root("instruction.updated: the instruction resource changed; re-read it with instruction.read".into());
1450 }
1451 }
1452 }
1453 for (server, uri) in resource_updates {
1454 self.on_resource_updated(&server, &uri); self.on_subscribe_resource(&server, &uri); }
1457 for s in tools_changed {
1458 self.log.info("mcp.tools_changed", json!({"server": s, "note": "recorded only; the tool catalogue is rebuilt at the next config reload"}));
1459 }
1460 }
1461}