1#[cfg(feature = "a2a")]
15pub mod a2a_server;
16pub mod activity;
17pub mod artifacts;
18pub mod audit;
19pub mod breaker;
20pub mod children;
21pub mod env; pub mod events;
23#[cfg(feature = "exec")]
24pub mod exec; pub mod goal;
26pub mod http_node;
27pub mod human; pub(crate) mod instances; pub mod nested;
30pub mod pressure; pub mod reactor;
32pub mod reload;
33pub(crate) mod retire;
34pub mod starts;
35pub mod steps;
36pub(crate) mod streams;
37pub mod subagents;
38pub mod timers;
39pub mod tools;
40pub mod turns;
41pub mod waits;
42#[cfg(feature = "a2a")]
43pub mod webhooks;
44pub mod worker;
45
46pub use reactor::Runtime;
47
48use crate::config::v2::{Loaded, StoreKind};
49use crate::context::memory::Memory;
50use crate::context::{Contexts, skills, tokens};
51use crate::engine::run::StepStatus;
52use crate::governor::Governor;
53use crate::mcp::client::McpClient;
54use crate::obs::log::{Comp, Level, LogCtx, Logger};
55use crate::registry::{Registry, ServerTools};
56use crate::state::{Durable, Kind, Policy, now_ms};
57use serde_json::{Value, json};
58use std::collections::BTreeMap;
59use std::sync::Arc;
60use std::time::{Duration, Instant};
61
62fn reference_preflight(
72 doc: &Value,
73 settings: &crate::config::v2::Settings,
74 at: &str,
75 log: &Logger,
76) -> Option<i32> {
77 let mut missing = crate::config::v2::missing_references(doc, at, &settings.vars);
78 if missing.is_empty() {
79 return None;
80 }
81 if crate::config::prompt::prompt_missing_requested() {
82 let mut found = Vec::new();
83 crate::config::v2::scan_references(doc, at, &mut found);
84 let mut names: Vec<String> = found
85 .into_iter()
86 .filter(|r| r.kind == "secret" && !crate::sec::secret::secret_available(&r.name))
87 .map(|r| r.name)
88 .collect();
89 names.sort();
90 names.dedup();
91 for name in names {
92 match crate::config::prompt::read_secret_from_tty(&format!("{name} (secret)")) {
93 Ok(v) => crate::sec::secret::set_prompted(&name, v),
94 Err(e) => {
95 log.error("prompt.failed", json!({"secret": name, "err": e}));
96 break;
97 }
98 }
99 }
100 missing = crate::config::v2::missing_references(doc, at, &settings.vars);
101 if missing.is_empty() {
102 return None;
103 }
104 }
105 for m in &missing {
106 log.error("config.invalid", json!({"error": m}));
107 }
108 log.error(
109 "proc.exit",
110 json!({"code": crate::exit::USAGE, "err": format!("{} unresolved reference(s)", missing.len())}),
111 );
112 Some(crate::exit::USAGE)
113}
114
115pub fn run(loaded: &Loaded, args: &[String], env: &[(String, String)]) -> i32 {
120 let settings = loaded.settings.clone();
121 let instance = settings.instance_name();
122 let run_id = match settings.lifecycle.run_id.clone() {
123 Some(r) => r,
124 None => crate::state::ulid::new(),
125 };
126 let trace = crate::obs::trace::resolve(&run_id, settings.observability.traceparent.as_deref());
127 let level = settings
128 .observability
129 .log_level
130 .as_deref()
131 .and_then(Level::parse)
132 .unwrap_or(Level::Info);
133 let log = Logger::new(
134 LogCtx {
135 run_id: run_id.clone(),
136 agent_id: "sup".into(),
137 agent_path: "0".into(),
138 comp: Comp::Supervisor,
139 pid: std::process::id(),
140 trace_id: Some(trace.trace_id.clone()),
141 },
142 level,
143 )
144 .with_content(settings.observability.log_content);
145 log.info("proc.start", json!({"version": crate::VERSION, "runtime": "1", "instance": instance, "config_files": loaded.files.iter().map(|(p, _)| p.clone()).collect::<Vec<_>>()}));
146 for w in &loaded.warnings {
147 log.warn("config.warning", json!({"warning": w}));
148 }
149 crate::signals::install();
150 crate::supervisor::reap::set_child_subreaper();
151
152 if let Some(code) = reference_preflight(&loaded.doc, &settings, "config", &log) {
157 return code;
158 }
159 let envmap = |k: &str| env.iter().find(|(n, _)| n == k).map(|(_, v)| v.clone());
160
161 #[cfg(feature = "tls")]
163 if let Some(path) = settings.security.tls_ca.as_deref()
164 && let Err(e) = std::fs::read(path).and_then(|pem| crate::net::tls::install_extra_ca(&pem))
165 {
166 log.error(
167 "proc.exit",
168 json!({"code": crate::exit::USAGE, "err": format!("security.tls_ca {path}: {e}")}),
169 );
170 return crate::exit::USAGE;
171 }
172 #[cfg(feature = "aauth")]
175 if let Some(a) = &settings.security.aauth {
176 let v1 = crate::config::AAuthSettings {
177 provider: a.provider.clone(),
178 key_file: a
179 .key_file
180 .clone()
181 .unwrap_or_else(|| "/var/lib/agentd/aauth-key".into()),
182 enrollment_token: a.enroll_token.as_ref().map(|s| s.0.clone()),
183 enroll_assertion_file: a.enroll_assertion_file.clone(),
184 person_server: a.person_server.clone(),
185 };
186 if let Err(e) = crate::aauth::setup(&v1, Duration::from_secs(30)) {
187 log.error(
188 "proc.exit",
189 json!({"code": crate::exit::USAGE, "err": format!("aauth: {e}")}),
190 );
191 return crate::exit::USAGE;
192 }
193 }
194
195 #[cfg(unix)]
200 if settings.security.cgroup.spec.is_some() {
201 let c = &settings.security.cgroup;
202 if let Some(configured) = crate::supervisor::cgroup::configure(
203 c.spec.as_deref(),
204 c.memory_max.as_deref(),
205 c.pids_max.as_deref(),
206 ) {
207 log.info("cgroup.armed", json!({"parent": configured.parent.display().to_string(), "limits_unavailable": configured.limits_unavailable}));
208 }
209 }
210
211 let intel_uri = settings.intelligence.endpoint_list().unwrap_or_default();
213 let intel_token = match resolve_intel_token(&settings, &envmap) {
214 Ok(t) => t,
215 Err(e) => {
216 log.error("proc.exit", json!({"code": crate::exit::USAGE, "err": e}));
217 return crate::exit::USAGE;
218 }
219 };
220 let model = settings
225 .intelligence
226 .default_reference()
227 .map(|r| settings.intelligence.wire_model(&r))
228 .unwrap_or_default();
229 let intel_headers: Vec<(String, String)> = settings
234 .intelligence
235 .headers
236 .iter()
237 .filter_map(|(k, v)| {
238 crate::sec::secret::resolve(v, &envmap)
239 .ok()
240 .map(|r| (k.clone(), r))
241 })
242 .collect();
243 let intel_bearer = intel_bearer_provider(&settings);
244
245 let mut mcp: BTreeMap<String, Arc<McpClient>> = BTreeMap::new();
248 let mut mcp_specs = BTreeMap::new();
249 let mut server_tools: Vec<ServerTools> = Vec::new();
250 let mcp_timeout = settings
251 .mcp
252 .default_timeout
253 .map(|d| d.0)
254 .unwrap_or(Duration::from_secs(60));
255 for s in &settings.mcp.servers {
256 if let Err(e) = crate::config::v2::egress_allows(
260 &settings.services,
261 settings.security.egress,
262 crate::config::v2::ServiceKind::Mcp,
263 &s.endpoint,
264 ) {
265 log.error("proc.exit", json!({"code": crate::exit::USAGE, "err": e}));
266 return crate::exit::USAGE;
267 }
268 let spec = match s.to_spec() {
269 Ok(sp) => sp,
270 Err(e) => {
271 log.error("proc.exit", json!({"code": crate::exit::USAGE, "err": e}));
272 return crate::exit::USAGE;
273 }
274 };
275 let per_timeout = s.timeout.map(|d| d.0).unwrap_or(mcp_timeout);
276 match crate::mcp::from_spec(&spec, per_timeout).and_then(|mut c| c.initialize().map(|()| c))
277 {
278 Ok(mut c) => {
279 let mut meta = json!({"agent/run_id": run_id, "agent/instance": instance});
280 meta["traceparent"] =
281 crate::obs::trace::outbound_traceparent(&trace.trace_id).into();
282 c.set_tool_meta(meta);
283 let tools = c.list_tools().unwrap_or_default();
284 log.info(
285 "mcp.connect",
286 json!({"server": s.name, "tools": tools.len()}),
287 );
288 server_tools.push(ServerTools {
289 name: s.name.clone(),
290 ns: s.ns.clone(),
291 tags: spec.tags.clone(),
292 tools,
293 });
294 mcp.insert(s.name.clone(), Arc::new(c));
295 }
296 Err(e) => {
297 log.warn(
298 "mcp.connect.fail",
299 json!({"server": s.name, "err": e.to_string()}),
300 );
301 crate::obs::metrics::record_mcp_connect_failure(&s.name);
302 }
303 }
304 mcp_specs.insert(s.name.clone(), spec);
305 }
306
307 let store = match settings.store.kind {
312 StoreKind::None => {
313 log.warn(
314 "store.none",
315 json!({"note": "no durable store: state lives in this process only (job shape)"}),
316 );
317 Arc::new(crate::store::memory::MemoryStore::new()) as crate::store::SharedStore
318 }
319 _ => {
320 let mcp_ref = mcp.clone();
321 match crate::store::open(&settings.store, &|name: &str| {
322 mcp_ref
323 .get(name)
324 .map(|c| c.clone() as Arc<dyn crate::store::mcp::McpCall>)
325 }) {
326 Ok(Some(s)) => s,
327 Ok(None) => Arc::new(crate::store::memory::MemoryStore::new()),
328 Err(e) => {
329 log.error("proc.exit", json!({"code": crate::exit::MCP_REQUIRED_DOWN, "err": format!("store: {e}")}));
330 return crate::exit::MCP_REQUIRED_DOWN;
331 }
332 }
333 }
334 };
335 let durable = Durable::new(
336 store,
337 settings.store.prefix(),
338 &instance,
339 Policy::from_settings(&settings.store),
340 Some(log.clone()),
341 );
342
343 let restored = match durable.restore() {
347 Ok(r) => r,
348 Err(e) => {
349 log.error(
350 "proc.exit",
351 json!({"code": crate::exit::MCP_REQUIRED_DOWN, "err": format!("restore: {e}")}),
352 );
353 return crate::exit::MCP_REQUIRED_DOWN;
354 }
355 };
356 if settings.store.kind == StoreKind::File {
365 let root = crate::config::v2::file_store_root(&settings.store);
366 log.info(
367 "store.file",
368 json!({
369 "path": root.display().to_string(),
370 "generation": restored.manifest.as_ref().map(|m| m.generation).unwrap_or(1),
371 "defaulted": loaded.doc.pointer("/store/kind").is_none(),
374 "msg": "durable state is on the local filesystem; it survives a restart of this process but not a move to another host — use store.kind mcp|http for a fleet",
375 }),
376 );
377 }
378
379 let registry = match Registry::build(&settings, &server_tools) {
383 Ok(r) => r,
384 Err(errs) => {
385 for e in &errs {
386 log.error("config.invalid", json!({"error": e}));
387 }
388 log.error(
389 "proc.exit",
390 json!({"code": crate::exit::USAGE, "err": "tool registry"}),
391 );
392 return crate::exit::USAGE;
393 }
394 };
395 for w in ®istry.warnings {
396 log.warn("registry.warning", json!({"warning": w}));
397 }
398
399 let mut catalogue = skills::Catalogue::new(
401 settings
402 .skills
403 .reference_prefix
404 .as_deref()
405 .unwrap_or(skills::DEFAULT_PREFIX),
406 settings.skills.max_bytes.unwrap_or(32_768) as usize,
407 );
408 for src in &settings.skills.sources {
409 match mcp.get(&src.server) {
410 Some(c) => {
411 let mode = match src.discover {
412 crate::config::v2::Discover::Prompts => skills::Discover::Prompts,
413 crate::config::v2::Discover::Resources => skills::Discover::Resources,
414 crate::config::v2::Discover::Auto => skills::Discover::Auto,
415 };
416 let found = catalogue.discover(&**c, mode, src.filter.as_deref());
417 log.info(
418 "skills.discovered",
419 json!({"server": src.server, "count": found.len(), "skills": found}),
420 );
421 }
422 None => log.warn("skills.source.unavailable", json!({"server": src.server})),
423 }
424 }
425 if let Some(dir) = &settings.skills.dir {
426 let (names, errs) = catalogue.add_dir(std::path::Path::new(dir));
427 for e in errs {
428 log.warn("skills.file.unreadable", json!({"err": e}));
429 }
430 if !names.is_empty() {
431 log.info(
432 "skills.discovered",
433 json!({"server": "file", "dir": dir, "count": names.len(), "skills": names}),
434 );
435 }
436 }
437 if !settings.agent.inline_skills.is_empty() {
438 let names = catalogue.add_inline(&settings.agent.inline_skills);
439 log.info(
440 "skills.discovered",
441 json!({"server": "instruction", "count": names.len(), "skills": names}),
442 );
443 }
444
445 let (events_tx, events_rx) = std::sync::mpsc::channel();
447 let child_tx: crate::supervisor::spawn::FrameSink = {
459 let events_tx = events_tx.clone();
460 std::sync::Arc::new(move |node, msg| {
461 events_tx.send(events::Event::Child(node, msg)).is_ok()
462 })
463 };
464 let (reap_tx, reap_rx) = std::sync::mpsc::channel();
465 let exe = std::env::current_exe().unwrap_or_else(|_| std::path::PathBuf::from("agentd"));
466
467 let model_window = settings
468 .context
469 .model_window
470 .or_else(|| {
474 settings
475 .intelligence
476 .default_reference()
477 .and_then(|r| settings.intelligence.tier(&r).and_then(|t| t.window))
478 })
479 .unwrap_or_else(|| {
480 if model.is_empty() {
481 tokens::DEFAULT_MODEL_WINDOW
482 } else {
483 tokens::window_for_model(&model)
484 }
485 });
486 let pressure = {
491 use crate::config::v2::StoreKind;
492 let (path, shed) = if settings.store.kind == StoreKind::File {
493 let root = crate::config::v2::file_store_root(&settings.store);
494 let min = settings
495 .store
496 .file
497 .as_ref()
498 .and_then(|f| f.min_free.as_deref())
499 .map(super::runtime::pressure::parse_bytes)
500 .transpose()
501 .unwrap_or_else(|e| {
502 log.warn("config.warning", json!({"warning": format!("store.file.min_free: {e}; using the 256MB default")}));
503 None
504 })
505 .unwrap_or(256 << 20);
506 (Some(root), min)
507 } else {
508 (None, 0)
509 };
510 std::sync::Arc::new(pressure::Pressure::new(path, shed))
511 };
512
513 let mut rt = Runtime {
514 instance: instance.clone(),
515 run_id: run_id.clone(),
516 durable,
517 mcp,
518 mcp_specs,
519 registry,
520 contexts: Contexts::new(model_window),
521 memory: Memory::new(
522 settings.memory.max_value_bytes.unwrap_or(65_536) as usize,
523 settings.memory.list_default_limit.unwrap_or(100) as usize,
524 ),
525 artifacts: artifacts::Artifacts::new(),
526 skills: catalogue,
527 governor: Governor::new(&settings.intelligence.budget),
528 workflows: BTreeMap::new(),
529 runs: BTreeMap::new(),
530 children: children::Children::new(exe, child_tx, reap_tx),
531 timers: timers::Timers::new(),
532 events_rx,
533 events_tx,
534 reap_rx,
535 pending: Vec::new(),
536 turn_queue: Default::default(),
537 staged_turns: BTreeMap::new(),
538 inbox_queue: Default::default(),
539 subagents: BTreeMap::new(),
540 instruction: reactor::Instruction {
541 text: String::new(),
542 source: "static",
543 uri: None,
544 server: None,
545 version: 1,
546 },
547 job_shape: false,
548 principal_budgets: BTreeMap::new(),
552 principal_rates: BTreeMap::new(),
553 principal_labels: BTreeMap::new(),
554 exit: None,
555 draining: false,
556 paused: false,
557 drain_started: None,
558 drain_reason: String::new(),
559 idle_since: None,
560 intel_uri,
561 intel_token,
562 intel_headers,
563 intel_bearer,
564 model,
565 trace_id: Some(trace.trace_id.clone()),
566 started: Instant::now(),
567 seq: 0,
568 counters: Default::default(),
569 job_runs: Vec::new(),
570 executing: BTreeMap::new(),
571 last_manifest_flush: Instant::now(),
572 goal_judge_at: None,
573 #[cfg(feature = "a2a")]
574 tasks: BTreeMap::new(),
575 #[cfg(feature = "a2a")]
576 event_to_task: BTreeMap::new(),
577 #[cfg(feature = "a2a")]
578 #[cfg(feature = "a2a")]
579 a2a_feed: None,
580 #[cfg(feature = "a2a")]
581 a2a_pairing: None,
582 #[cfg(feature = "a2a")]
583 reserved_task_id: None,
584 #[cfg(feature = "a2a")]
585 a2a_sink: None,
586 #[cfg(feature = "a2a")]
587 a2a_listener: None,
588 activity: BTreeMap::new(),
589 last_root_reply: None,
590 #[cfg(feature = "a2a")]
591 feed_marks: BTreeMap::new(),
592 #[cfg(feature = "a2a")]
593 feed_last: Instant::now(),
594 #[cfg(feature = "a2a")]
595 webhook_callbacks: std::sync::Arc::new(std::sync::Mutex::new(
596 std::collections::HashMap::new(),
597 )),
598 #[cfg(feature = "a2a")]
599 webhook_sync: std::collections::HashMap::new(),
600 pressure: pressure.clone(),
601 pressure_seen: pressure::Level::Ok,
602 resched: false,
603 reap_deferred: Default::default(),
604 step_rates: Default::default(),
605 settings_doc: loaded.doc.clone(),
606 args: args.to_vec(),
607 env: env.to_vec(),
608 pinned: BTreeMap::new(),
609 retiring: BTreeMap::new(),
610 pin_written: Default::default(),
611 recent_signals: BTreeMap::new(),
612 memory_keys: std::collections::HashMap::new(),
613 stream_dirty: false,
614 settings,
615 log: log.clone(),
616 };
617
618 let lost_ctx = rt.contexts.restore(restored.of(Kind::Context));
620 if !lost_ctx.is_empty() {
621 log.warn("restore.context.lost", json!({"ids": lost_ctx}));
622 }
623 rt.timers.restore(restored.timers());
624 rt.artifacts.restore(restored.of(Kind::Artifact));
625 let mut replayed: Vec<(String, String)> = Vec::new();
626 for env in restored.of(Kind::Run) {
627 match serde_json::from_value::<crate::engine::RunState>(env.state.clone()) {
628 Ok(mut r) => {
629 r.dirty = false;
630 if !r.status.is_terminal() {
631 for (id, st) in r.steps.iter_mut() {
636 if st.status == StepStatus::Running {
637 log.info(
638 "restore.step.replay",
639 json!({"run": r.id, "step": id, "attempt": st.attempt}),
640 );
641 st.status = StepStatus::Pending;
642 st.worker = None;
643 replayed.push((r.id.clone(), id.clone()));
647 }
648 }
649 r.status = crate::engine::RunStatus::Running;
650 r.dirty = true;
651 }
652 rt.runs.insert(r.id.clone(), r);
653 }
654 Err(e) => log.warn(
655 "restore.run.corrupt",
656 json!({"id": env.id, "err": e.to_string()}),
657 ),
658 }
659 }
660 for env in restored.of(Kind::Subagent) {
661 match serde_json::from_value::<reactor::SubagentRecord>(env.state.clone()) {
662 Ok(s) => {
663 rt.subagents.insert(s.handle.clone(), s);
664 }
665 Err(e) => log.warn(
666 "restore.subagent.corrupt",
667 json!({"id": env.id, "err": e.to_string()}),
668 ),
669 }
670 }
671 #[cfg(feature = "a2a")]
672 rt.restore_a2a_tasks(restored.of(Kind::Task));
673 if let Some(m) = &restored.manifest {
674 rt.governor.restore(&m.budget, now_ms());
675 }
676 for ev in restored.inbox_pending() {
677 rt.inbox_queue.push_back(ev);
678 }
679 if restored.manifest.is_some() {
680 log.info("restore.adopted", json!({"runs": rt.runs.len(), "contexts": rt.contexts.len(), "subagents": rt.subagents.len(), "timers": rt.timers.len(), "artifacts": rt.artifacts.len(), "inbox_pending": rt.inbox_queue.len(), "lost": restored.lost.len()}));
681 rt.restore_pins();
682 rt.audit(audit::AuditEvent {
686 action: "restore",
687 target: json!({"runs": rt.runs.len(), "subagents": rt.subagents.len(), "inbox_pending": rt.inbox_queue.len(), "lost": restored.lost.len()}),
688 outcome: if restored.lost.is_empty() { "restored" } else { "restored_with_loss" },
689 principal: Some("system"),
690 role: Some("system"),
691 request_id: None,
692 });
693 }
694
695 if let Some(text) = rt.settings.agent.instruction.clone() {
698 if crate::config::v2::looks_like_resource_uri(&text) {
699 match rt.subscribe_instruction(&text) {
700 Ok(()) => {}
701 Err(e) => {
702 log.error("proc.exit", json!({"code": crate::exit::MCP_REQUIRED_DOWN, "err": format!("agent.instruction {text}: {e}")}));
703 return crate::exit::MCP_REQUIRED_DOWN;
704 }
705 }
706 } else {
707 rt.instruction.text = text;
708 }
709 }
710
711 if let Err(errs) = rt.load_workflows() {
712 for e in &errs {
713 log.error("config.invalid", json!({"error": e}));
714 }
715 log.error(
716 "proc.exit",
717 json!({"code": crate::exit::USAGE, "err": "workflow definitions"}),
718 );
719 return crate::exit::USAGE;
720 }
721 {
727 let defs: Vec<&crate::engine::Workflow> =
728 rt.workflows.values().map(|w| w.as_ref()).collect();
729 let errs = rt.registry.register_workflow_tools(&defs);
730 if !errs.is_empty() {
731 for e in &errs {
732 log.error("config.invalid", json!({"error": e}));
733 }
734 log.error(
735 "proc.exit",
736 json!({"code": crate::exit::USAGE, "err": "workflow tool registration"}),
737 );
738 return crate::exit::USAGE;
739 }
740 for w in &defs {
745 let Some(t) = &w.tool else { continue };
746 log.info(
747 "registry.workflow_tools",
748 json!({"tool": t.name, "workflow": w.name,
749 "mode": if t.mode == crate::engine::model::WorkflowToolMode::Sync { "sync" } else { "async" },
750 "tags": rt.registry.get(&t.name).map(|s| s.tags.clone()).unwrap_or_default(),
751 "arguments": rt.registry.get(&t.name).map(|s| s.input_schema.clone())}),
752 );
753 }
754 }
755 {
763 let mut all = serde_json::Map::new();
764 for (name, wf) in &rt.workflows {
765 let mut steps = serde_json::Map::new();
766 for (sid, step) in &wf.steps {
767 steps.insert(
768 sid.clone(),
769 Value::Object(step.spec.clone().into_iter().collect()),
770 );
771 }
772 all.insert(name.clone(), Value::Object(steps));
773 }
774 if let Some(code) =
775 reference_preflight(&Value::Object(all), &rt.settings, "workflows", &rt.log)
776 {
777 return code;
778 }
779 }
780
781 if !replayed.is_empty() {
786 let policies: Vec<(String, String, crate::engine::model::OnReplay)> = replayed
787 .iter()
788 .filter_map(|(rid, sid)| {
789 let wf_name = rt.runs.get(rid)?.workflow.clone();
790 let step = rt.workflows.get(&wf_name)?.steps.get(sid)?;
791 Some((rid.clone(), sid.clone(), step.on_replay))
792 })
793 .collect();
794 for (rid, sid, policy) in policies {
795 match policy {
796 crate::engine::model::OnReplay::Retry => {}
797 crate::engine::model::OnReplay::Skip => {
798 if let Some(r) = rt.runs.get_mut(&rid) {
799 r.end_step(&sid, StepStatus::Skipped, None, None);
800 }
801 rt.log.info(
802 "restore.step.skipped",
803 json!({"run": rid, "step": sid, "on_replay": "skip"}),
804 );
805 }
806 crate::engine::model::OnReplay::Fail => {
807 if let Some(r) = rt.runs.get_mut(&rid) {
808 r.end_step(
809 &sid,
810 StepStatus::Failed,
811 None,
812 Some(
813 "step was in flight when the process died and its \
814 on_replay policy is `fail`"
815 .into(),
816 ),
817 );
818 }
819 rt.log.warn(
820 "restore.step.failed",
821 json!({"run": rid, "step": sid, "on_replay": "fail"}),
822 );
823 }
824 }
825 }
826 }
827 {
832 let re = rt.settings.observability.runtime_events.clone();
833 let audit_stream = rt.settings.observability.audit.stream.clone();
834 let audit_wants_stream = rt
835 .settings
836 .observability
837 .audit
838 .sink
839 .as_ref()
840 .is_some_and(|s| {
841 s.iter()
842 .any(|x| matches!(x, crate::config::v2::AuditSink::Stream))
843 });
844 if re.is_some() || (audit_wants_stream && audit_stream.is_some()) {
845 let (stream, include, sampled, cap) = match &re {
846 Some(r) => (
847 r.stream.clone().unwrap_or_default(),
848 r.include.clone(),
849 r.sampled.clone(),
850 r.queue_cap(),
851 ),
852 None => (
853 String::new(),
854 Vec::new(),
855 Vec::new(),
856 crate::config::v2::DEFAULT_TAP_QUEUE as usize,
857 ),
858 };
859 crate::obs::log::install_runtime_tap(&stream, include.clone(), sampled.clone(), cap);
860 rt.log.info(
861 "stream.tap",
862 json!({"stream": stream, "include": include, "sampled": sampled,
863 "queue": cap, "audit_stream": audit_stream}),
864 );
865 }
866 }
867 rt.job_shape = match rt.settings.lifecycle.run_until {
872 crate::config::v2::RunUntil::Drained => false,
873 crate::config::v2::RunUntil::Idle => true,
874 crate::config::v2::RunUntil::Auto => {
875 rt.settings.a2a.listen.is_none() && !rt.workflows.values().any(|w| w.is_long_lived())
876 }
877 };
878 for r in rt.runs.values() {
880 if rt.job_shape
881 && rt
882 .workflows
883 .get(&r.workflow)
884 .and_then(|w| w.step(&r.start.node))
885 .is_some_and(|s| s.kind == "once")
886 {
887 rt.job_runs.push(r.id.clone());
888 }
889 }
890 let refs = rt.skills.references(&rt.instruction.text.clone());
892 if !refs.is_empty() {
893 let unknown = rt.preload_skills(crate::context::ROOT, &refs, None);
894 for u in unknown {
895 rt.note_root(format!(
896 "skill.unknown: {u:?} referenced by the instruction is not in the catalogue"
897 ));
898 }
899 }
900 #[cfg(all(unix, feature = "config-watch"))]
903 if rt.settings.lifecycle.watch_config {
904 for (path, _) in &loaded.files {
905 crate::config::watch::spawn_config_watcher(std::path::Path::new(path), &log);
906 }
907 }
908 rt.arm_workflows();
909 rt.arm_long_lived_starts();
910 rt.arm_goal();
911 rt.respawn_restored_subagents();
912 rt.respawn_restored_instances();
913 #[cfg(feature = "a2a")]
917 if rt.settings.a2a.listen.is_some() {
918 let resolver = match crate::a2a::Resolver::build(&rt.settings.a2a, &envmap) {
919 Ok(r) => r,
920 Err(e) => {
921 log.error(
922 "proc.exit",
923 json!({"code": crate::exit::USAGE, "err": format!("a2a principals: {e}")}),
924 );
925 return crate::exit::USAGE;
926 }
927 };
928 let write_timeout = rt.settings.lifecycle.drain_timeout();
929 match a2a_server::spawn_a2a_listener(
930 &rt.settings.a2a,
931 &rt.settings.interface,
932 rt.events_tx.clone(),
933 resolver,
934 &envmap,
935 write_timeout,
936 log.clone(),
937 ) {
938 Ok(serving) => {
939 rt.a2a_feed = serving.feed;
940 rt.a2a_pairing = serving.pairing;
941 rt.a2a_sink = Some(std::sync::Arc::clone(&serving.listener.sink));
942 rt.a2a_listener = Some(serving.listener);
945 if rt.settings.interface.enabled && rt.settings.interface.debug {
949 let cap = rt
950 .settings
951 .observability
952 .events_ring
953 .map(|n| n as usize)
954 .unwrap_or(crate::obs::log::EVENTS_RING_DEFAULT);
955 crate::obs::log::install_event_ring(cap);
956 log.info("interface.debug", json!({"events_ring": cap}));
957 }
958 for id in rt.tasks.keys().cloned().collect::<Vec<_>>() {
960 rt.task_sync(&id);
961 }
962 }
963 Err(e) => {
964 log.error(
965 "proc.exit",
966 json!({"code": crate::exit::USAGE, "err": format!("a2a listen: {e}")}),
967 );
968 return crate::exit::USAGE;
969 }
970 }
971 }
972 #[cfg(feature = "a2a")]
976 if rt.settings.webhooks.listen.is_some() {
977 let nodes: Vec<(
978 String,
979 String,
980 serde_json::Map<String, serde_json::Value>,
981 bool,
982 )> = rt
983 .workflows
984 .values()
985 .flat_map(|wf| {
986 let low = wf.priority == crate::engine::model::Priority::Low;
987 wf.steps
988 .values()
989 .filter(|s| s.kind == "webhook")
990 .map(|s| (wf.name.clone(), s.id.clone(), s.spec.clone(), low))
991 .collect::<Vec<_>>()
992 })
993 .collect();
994 let write_timeout = rt.settings.lifecycle.drain_timeout();
995 if let Err(e) = webhooks::spawn_webhook_listener(
996 &rt.settings.webhooks,
997 nodes,
998 rt.webhook_callbacks.clone(),
999 rt.events_tx.clone(),
1000 &envmap,
1001 write_timeout,
1002 rt.pressure.clone(),
1003 log.clone(),
1004 ) {
1005 log.error(
1006 "proc.exit",
1007 json!({"code": crate::exit::USAGE, "err": format!("webhooks listen: {e}")}),
1008 );
1009 return crate::exit::USAGE;
1010 }
1011 }
1012 #[cfg(feature = "metrics")]
1015 if let Some(addr) = rt.settings.observability.metrics_addr.clone()
1016 && let Err(e) = crate::obs::serve::spawn(&addr, log.clone())
1017 {
1018 log.warn(
1019 "metrics.serve.fail",
1020 json!({"addr": addr, "err": e.to_string()}),
1021 );
1022 }
1023 if let Some(path) = rt.settings.observability.health_file.clone() {
1024 crate::obs::health::spawn_writer(
1025 std::path::PathBuf::from(path),
1026 run_id.clone(),
1027 "1".into(),
1028 std::time::Duration::from_secs(10),
1029 );
1030 }
1031 #[cfg(feature = "otel")]
1034 if rt.settings.observability.otel.logs == Some(true)
1035 && let Some(ep) = rt
1036 .settings
1037 .observability
1038 .otel
1039 .endpoint
1040 .clone()
1041 .or_else(|| std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok())
1042 {
1043 crate::obs::otel::arm_logs(&ep, "agentd", crate::VERSION);
1044 log.info("otel.logs.armed", json!({"endpoint": ep}));
1045 }
1046 if let Some(prompt) = rt.settings.agent.prompt.clone()
1053 && !prompt.trim().is_empty()
1054 && let Err(err) = rt.accept_event(
1055 events::kinds::A2A_MESSAGE,
1056 Some("operator".into()),
1057 json!({"text": prompt, "context_id": crate::context::ROOT}),
1058 )
1059 {
1060 log.warn("prompt.reject", json!({"err": err}));
1061 }
1062 #[cfg(any(feature = "internal-mocks", debug_assertions))]
1066 if let Ok(path) = std::env::var("AGENTD_TEST_INBOX_FILE") {
1067 match std::fs::read_to_string(&path).map_err(|e| e.to_string()).and_then(|t| serde_json::from_str::<Value>(&t).map_err(|e| e.to_string())) {
1068 Ok(Value::Array(events)) => {
1069 for e in events {
1070 let kind = e["kind"].as_str().unwrap_or(events::kinds::A2A_MESSAGE).to_string();
1071 let principal = e["principal"].as_str().map(str::to_string);
1072 let payload = e.get("payload").cloned().unwrap_or(Value::Null);
1073 if let Err(err) = rt.accept_event(&kind, principal, payload) {
1074 log.warn("test.inbox.reject", json!({"err": err}));
1075 }
1076 }
1077 let _ = std::fs::remove_file(&path);
1078 }
1079 other => log.warn("test.inbox.bad_file", json!({"path": path, "err": format!("{other:?}").chars().take(200).collect::<String>()})),
1080 }
1081 }
1082 rt.checkpoint(true);
1083 let code = rt.run_loop();
1084 let _ = &rt.last_manifest_flush;
1085 if rt.job_shape
1088 && let Some(out) = rt.job_output()
1089 {
1090 match out {
1091 Value::String(s) => println!("{s}"),
1092 Value::Null => {}
1093 other => println!(
1094 "{}",
1095 serde_json::to_string_pretty(&other).unwrap_or_default()
1096 ),
1097 }
1098 }
1099 code
1100}
1101
1102pub fn capabilities(loaded: &Loaded) -> Value {
1108 let start_kinds = crate::engine::model::start_kinds();
1113 let s = &loaded.settings;
1114 let workflows: Vec<Value> = s
1115 .workflows
1116 .iter()
1117 .map(|w| {
1118 let starts: Vec<String> = w["steps"]
1119 .as_object()
1120 .map(|steps| steps.values().filter_map(|st| st["kind"].as_str()).filter(|k| start_kinds.contains(k)).map(str::to_string).collect())
1121 .unwrap_or_default();
1122 json!({"name": w["name"].as_str().unwrap_or(""), "description": w.get("description").and_then(Value::as_str), "start_kinds": starts, "inputs_schema": w.get("inputs").is_some()})
1123 })
1124 .collect();
1125 let a2a = s.a2a.listen.as_ref().map(|listen| {
1126 let principals: Vec<Value> = s
1127 .a2a
1128 .principals
1129 .iter()
1130 .map(|p| json!({"role": format!("{:?}", p.role).to_lowercase(), "match": principal_match_desc(&p.matcher), "grants": p.grants}))
1131 .collect();
1132 let mut methods = vec![
1133 "SendMessage",
1134 "SendStreamingMessage",
1135 "GetTask",
1136 "CancelTask",
1137 "ListTasks",
1138 "SubscribeToTask",
1139 "GetAgentCard",
1140 ];
1141 let mut command_ops = vec![
1142 "status",
1143 "config",
1144 "workflow.run",
1145 "workflow.status",
1146 "workflow.cancel",
1147 "workflow.signal",
1148 "subagent.send",
1149 "subagent.kill",
1150 "subagent.status",
1151 "plan.get",
1152 ];
1153 if s.interface.enabled {
1154 methods.push("SubscribeToEvents");
1155 command_ops.push("interface.info");
1156 if s.interface.debug {
1157 command_ops.extend(["conversation.get", "run.get", "debug.events"]);
1158 }
1159 }
1160 json!({
1161 "listen": listen,
1162 "tls": s.a2a.tls.cert.is_some(),
1163 "mtls": s.a2a.tls.client_ca.is_some(),
1164 "bearer": s.a2a.bearer.is_some(),
1165 "methods": methods,
1166 "admin": ["a2a.drain", "a2a.lameduck", "a2a.cancel", "a2a.pause", "a2a.resume"],
1167 "command_ops": command_ops,
1168 "principals": principals,
1169 "loopback_operator": s.a2a.principals.is_empty(),
1170 })
1171 });
1172 json!({
1173 "runtime": "1",
1174 "version": crate::VERSION,
1175 "agent": {"name": s.instance_name(), "instruction": s.agent.instruction.is_some(), "preflight": format!("{:?}", s.agent.preflight).to_lowercase()},
1176 "intelligence": {"model": s.intelligence.model, "endpoints": s.intelligence.endpoints.len()},
1177 "mcp_servers": s.mcp.servers.iter().map(|m| m.name.clone()).collect::<Vec<_>>(),
1178 "internal_tools": crate::registry::internal::names(),
1179 "tools": {"overrides": s.tools.overrides.keys().cloned().collect::<Vec<_>>(), "disabled": s.tools.disabled},
1180 "workflows": workflows,
1181 "knowledge": {"server": s.knowledge.server},
1182 "search": {"server": s.search.server},
1183 "skills": {"sources": s.skills.sources.len()},
1184 "a2a": a2a,
1185 "interface": {"enabled": s.interface.enabled, "debug": s.interface.debug, "origins": s.interface.origins.len(), "pairing": s.interface.pairing.enabled, "display": {"top": s.interface.display.top, "bottom": s.interface.display.bottom}},
1186 "store": format!("{:?}", s.store.kind).to_lowercase(),
1187 "store_file": (s.store.kind == StoreKind::File).then(|| json!({
1193 "path": crate::config::v2::file_store_root(&s.store).display().to_string(),
1194 "defaulted": loaded.doc.pointer("/store/kind").is_none(),
1195 })),
1196 "lifecycle": {"run_until": format!("{:?}", s.lifecycle.run_until).to_lowercase(), "daemon": s.a2a.listen.is_some() || s.workflows.iter().any(|w| w["steps"].as_object().is_some_and(|st| st.values().any(|n| n["kind"].as_str().is_some_and(|k| matches!(k, "loop" | "schedule" | "subscribe" | "signal" | "event")))))},
1197 })
1198}
1199
1200fn principal_match_desc(m: &crate::config::v2::PrincipalMatch) -> Value {
1202 if m.any {
1203 json!({"any": true})
1204 } else if let Some(s) = &m.san {
1205 json!({"san": s})
1206 } else if let Some(s) = &m.sub {
1207 json!({"sub": s})
1208 } else if m.bearer_ref.is_some() {
1209 json!({"bearer_ref": "***"})
1210 } else if let Some(a) = &m.aauth_agent {
1211 json!({"aauth_agent": a})
1212 } else {
1213 json!({})
1214 }
1215}
1216
1217fn intel_bearer_provider(
1224 settings: &crate::config::v2::Settings,
1225) -> Option<std::sync::Arc<dyn Fn() -> Option<String> + Send + Sync>> {
1226 #[cfg(feature = "oauth")]
1227 {
1228 let auth = settings.intelligence.auth.as_ref()?;
1229 let spec = auth.to_spec();
1230 if spec.kind == "aws" {
1235 return None;
1236 }
1237 let signer = crate::auth::device::signer_for(
1241 &spec,
1242 "intelligence",
1243 std::time::Duration::from_secs(30),
1244 )
1245 .ok()??;
1246 Some(std::sync::Arc::new(move || {
1247 signer
1248 .sign("POST", "", "", &[])
1249 .into_iter()
1250 .find(|(k, _)| k.eq_ignore_ascii_case("authorization"))
1251 .map(|(_, v)| v.strip_prefix("Bearer ").unwrap_or(&v).to_string())
1252 }))
1253 }
1254 #[cfg(not(feature = "oauth"))]
1255 {
1256 let _ = settings;
1257 None
1258 }
1259}
1260
1261impl Runtime {
1262 pub(crate) fn current_intel_bearer(&self) -> Option<String> {
1269 self.intel_bearer
1270 .as_ref()
1271 .and_then(|f| f())
1272 .or_else(|| self.intel_token.clone())
1273 }
1274
1275 pub(crate) fn intel_aws_auth(&self) -> Option<crate::config::AuthSpec> {
1280 let a = self.settings.intelligence.auth.as_ref()?;
1281 (a.kind == crate::config::v2::AuthKind::Aws).then(|| a.to_spec())
1282 }
1283
1284 pub(crate) fn intel_dialect(&self) -> Option<String> {
1288 self.settings.intelligence.dialect.clone()
1289 }
1290}
1291
1292fn resolve_intel_token(
1294 settings: &crate::config::v2::Settings,
1295 env: &dyn Fn(&str) -> Option<String>,
1296) -> Result<Option<String>, String> {
1297 if let Some(t) = &settings.intelligence.token {
1298 let resolved = crate::sec::secret::resolve(&t.0, env)
1299 .map_err(|e| format!("intelligence.token: {e}"))?;
1300 return Ok(Some(resolved));
1301 }
1302 if let Some(p) = &settings.intelligence.token_file {
1303 return crate::sec::secret::read_token_file(p)
1304 .map(Some)
1305 .map_err(|e| format!("intelligence.token_file: {e}"));
1306 }
1307 Ok(None)
1310}
1311
1312impl Runtime {
1313 pub(crate) fn subscribe_instruction(&mut self, uri: &str) -> Result<(), String> {
1316 let (server, res) = match uri.strip_prefix("mcp://").and_then(|r| r.split_once('/')) {
1317 Some((s, r)) => (Some(s.to_string()), r.to_string()),
1318 None => (None, uri.to_string()),
1319 };
1320 let candidates: Vec<(String, Arc<McpClient>)> = match &server {
1322 Some(s) => self
1323 .mcp
1324 .get(s)
1325 .map(|c| vec![(s.clone(), c.clone())])
1326 .unwrap_or_default(),
1327 None => self
1328 .mcp
1329 .iter()
1330 .map(|(n, c)| (n.clone(), c.clone()))
1331 .collect(),
1332 };
1333 let mut last_err = String::from("no connected MCP server serves it");
1334 for (name, c) in candidates {
1335 match c.read_resource(&res) {
1336 Ok(r) => {
1337 let text = r.text();
1338 if c.capabilities().supports_resources()
1339 && let Err(e) = c.subscribe(&res)
1340 {
1341 self.log.warn(
1342 "instruction.subscribe.fail",
1343 json!({"server": name, "uri": res, "err": e.to_string()}),
1344 );
1345 }
1346 let changed = self.instruction.text != text;
1347 self.instruction = reactor::Instruction {
1348 text,
1349 source: "resource",
1350 uri: Some(res.clone()),
1351 server: Some(name.clone()),
1352 version: self.instruction.version + u64::from(changed),
1353 };
1354 self.log.info("instruction.loaded", json!({"server": name, "uri": res, "bytes": self.instruction.text.len(), "version": self.instruction.version}));
1355 return Ok(());
1356 }
1357 Err(e) => last_err = e.to_string(),
1358 }
1359 }
1360 Err(last_err)
1361 }
1362
1363 pub(crate) fn poll_mcp_notifications(&mut self) {
1369 let mut updated_instruction = false;
1370 let mut tools_changed = Vec::new();
1371 let mut resource_updates: Vec<(String, String)> = Vec::new();
1372 for (name, c) in &self.mcp {
1373 for n in c.drain_notifications() {
1374 match n.method.as_str() {
1375 ::mcp::wire::method::NOTIFY_RESOURCES_UPDATED => {
1376 let uri = n
1377 .params
1378 .as_ref()
1379 .and_then(|p| p.get("uri"))
1380 .and_then(Value::as_str)
1381 .unwrap_or("");
1382 if self.instruction.uri.as_deref() == Some(uri)
1383 && self.instruction.server.as_deref() == Some(name.as_str())
1384 {
1385 updated_instruction = true;
1386 }
1387 resource_updates.push((name.clone(), uri.to_string()));
1388 }
1389 ::mcp::wire::method::NOTIFY_TOOLS_LIST_CHANGED => {
1390 tools_changed.push(name.clone())
1391 }
1392 _ => {}
1393 }
1394 }
1395 }
1396 if updated_instruction && let Some(uri) = self.instruction.uri.clone() {
1397 let full = match &self.instruction.server {
1398 Some(s) => format!("mcp://{s}/{uri}"),
1399 None => uri,
1400 };
1401 let before = self.instruction.version;
1402 if self.subscribe_instruction(&full).is_ok() && self.instruction.version != before {
1403 self.log.info(
1404 "instruction.updated",
1405 json!({"version": self.instruction.version}),
1406 );
1407 if self
1408 .settings
1409 .agent
1410 .wake_on()
1411 .contains(&crate::config::v2::WakeEvent::InstructionUpdated)
1412 {
1413 self.note_root("instruction.updated: the instruction resource changed; re-read it with instruction.read".into());
1414 }
1415 }
1416 }
1417 for (server, uri) in resource_updates {
1418 self.on_resource_updated(&server, &uri); self.on_subscribe_resource(&server, &uri); }
1421 for s in tools_changed {
1422 self.log.info("mcp.tools_changed", json!({"server": s, "note": "recorded only; the tool catalogue is rebuilt at the next config reload"}));
1423 }
1424 }
1425}