1use jsrt::sync;
27use serde_json::{Value, json};
28use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
29use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender, channel};
30use std::sync::{Arc, Mutex};
31use std::time::{Duration, Instant};
32
33pub const ASYNC_EVENTS_ENV: &str = "ULTRON_ASYNC_EVENTS";
34
35pub const RUNTIME_EVENT_MESSAGE_TYPE: &str = "ultron-runtime-event";
36pub const DEFAULT_COALESCE: Duration = Duration::from_millis(500);
37pub const DEFAULT_MAX_EVENT_RUNS: usize = 16;
38pub const RETRY_AFTER: Duration = Duration::from_millis(1000);
41
42#[derive(Clone, Debug, PartialEq)]
44pub struct RuntimeEvent {
45 pub kind: String,
47 pub id: String,
48 pub status: String,
49 pub summary: String,
50 pub fetch: String,
52 pub lane: String,
54 pub root_id: Option<String>,
56}
57
58pub fn async_events_enabled(value: Option<&str>) -> bool {
60 match value.map(str::trim) {
61 None | Some("") => true,
62 Some(value) => !["off", "0", "false", "no"].contains(&value.to_lowercase().as_str()),
63 }
64}
65
66pub fn max_event_runs_from_env(value: Option<&str>) -> usize {
68 value
69 .map(str::trim)
70 .filter(|v| !v.is_empty())
71 .and_then(|v| v.parse::<usize>().ok())
72 .unwrap_or(DEFAULT_MAX_EVENT_RUNS)
73}
74
75pub type MaskText = Arc<dyn Fn(&str) -> String + Send + Sync>;
78
79pub fn default_mask() -> MaskText {
81 Arc::new(|text: &str| host::secrets::OutputMasker::default().mask_counted(text, None).0)
82}
83
84fn attribute(value: &str) -> String {
86 value
87 .split_whitespace()
88 .collect::<Vec<_>>()
89 .join(" ")
90 .replace('&', "&")
91 .replace('"', """)
92 .replace('<', "<")
93 .replace('>', ">")
94}
95
96fn bounded(text: &str) -> String {
98 let line = jsrt::string::flatten(text);
99 if line.chars().count() > 240 {
100 format!("{}…", line.chars().take(239).collect::<String>())
101 } else {
102 line
103 }
104}
105
106pub fn runtime_event_line(kind: &str, id: &str, status: &str, summary: &str, fetch: &str, mask: &MaskText) -> String {
109 format!(
110 "<runtime_event kind=\"{}\" id=\"{}\" status=\"{}\" summary=\"{}\" fetch=\"{}\" />",
111 attribute(kind),
112 attribute(id),
113 attribute(status),
114 attribute(&bounded(&mask(summary))),
115 attribute(fetch)
116 )
117}
118
119pub fn runtime_event_text(events: &[RuntimeEvent], mask: &MaskText) -> String {
122 events
123 .iter()
124 .map(|e| {
125 let summary = jsrt::utf16::head(&e.summary, 4 * 240).to_string();
126 runtime_event_line(&e.kind, &e.id, &e.status, &summary, &e.fetch, mask)
127 })
128 .collect::<Vec<_>>()
129 .join("\n")
130}
131
132pub fn runtime_event_message_with(events: &[RuntimeEvent], now: u64, mask: &MaskText) -> Value {
134 json!({
135 "role": "custom",
136 "customType": RUNTIME_EVENT_MESSAGE_TYPE,
137 "content": runtime_event_text(events, mask),
138 "display": true,
139 "details": {"events": events.iter().map(|e| json!({"kind": e.kind, "id": e.id, "status": e.status})).collect::<Vec<_>>()},
140 "timestamp": now,
141 })
142}
143
144#[derive(Clone, Copy, Debug, PartialEq, Eq)]
146pub enum Execution {
147 Idle,
148 Run,
149 Other,
151}
152
153#[derive(Clone, Debug, PartialEq, Eq)]
155pub enum Accepted {
156 Run(String),
158 Busy,
160 Refused(String),
161}
162
163#[derive(Clone, Copy, Debug, PartialEq, Eq)]
165pub enum Cancelled {
166 Cancelled,
167 AlreadyConsumed,
168 NotFound,
169}
170
171pub trait EventLanes: Send + Sync {
173 fn execution(&self, lane: &str) -> Execution;
174 fn accept(&self, lane: &str, message: &Value, root: &str) -> Accepted;
176 fn steer(&self, lane: &str, message: &Value) -> Result<String, String>;
178 fn next_run(&self, lane: &str, message: &Value) -> Result<String, String>;
180 fn cancel_queued(&self, lane: &str, entry: &str) -> Result<Cancelled, String>;
181}
182
183#[derive(Clone, Debug, PartialEq)]
185pub struct Delivery {
186 pub lane: String,
187 pub mode: &'static str,
189 pub events: Vec<RuntimeEvent>,
190 pub reason: Option<String>,
191}
192
193pub type RootRefusal = Arc<dyn Fn(&str) -> Option<String> + Send + Sync>;
194pub type RootCallback = Arc<dyn Fn(&str) + Send + Sync>;
195pub type DeliveryCallback = Arc<dyn Fn(&Delivery) + Send + Sync>;
196pub type RootOfRun = Arc<dyn Fn(&str) -> String + Send + Sync>;
197pub type EventNote = Arc<dyn Fn(&str, &str) -> Option<String> + Send + Sync>;
199
200#[derive(Clone, Debug, Default, PartialEq)]
205pub struct Chains {
206 pub aborted: BTreeSet<String>,
207 pub runs: BTreeMap<String, usize>,
208}
209
210pub const CHAINS_MODULE: &str = "async-events";
212
213impl Chains {
214 pub fn from_value(value: Option<&Value>) -> Chains {
216 let Some(value) = value else {
217 return Chains::default();
218 };
219 Chains {
220 aborted: value["abortedRoots"]
221 .as_array()
222 .into_iter()
223 .flatten()
224 .filter_map(|root| root.as_str().map(str::to_string))
225 .collect(),
226 runs: value["chainRuns"]
227 .as_object()
228 .into_iter()
229 .flatten()
230 .filter_map(|(root, runs)| Some((root.clone(), runs.as_u64()? as usize)))
231 .collect(),
232 }
233 }
234
235 pub fn to_value(&self) -> Value {
236 json!({"version": 1, "abortedRoots": self.aborted, "chainRuns": self.runs})
237 }
238}
239
240pub type SaveChains = Arc<dyn Fn(&Chains) + Send + Sync>;
242
243#[derive(Clone)]
245pub struct DispatcherOptions {
246 pub enabled: bool,
247 pub coalesce: Duration,
248 pub max_runs: usize,
250 pub root_of_run: RootOfRun,
252 pub refuse: Option<RootRefusal>,
254 pub on_root_aborted: Option<RootCallback>,
256 pub on_delivery: Option<DeliveryCallback>,
257 pub retry_after: Duration,
258 pub note: Option<EventNote>,
260 pub mask: Option<MaskText>,
262 pub chains: Chains,
264 pub save_chains: Option<SaveChains>,
265}
266
267impl DispatcherOptions {
268 pub fn new(root_of_run: RootOfRun) -> DispatcherOptions {
269 DispatcherOptions {
270 enabled: true,
271 coalesce: DEFAULT_COALESCE,
272 max_runs: DEFAULT_MAX_EVENT_RUNS,
273 root_of_run,
274 refuse: None,
275 on_root_aborted: None,
276 on_delivery: None,
277 retry_after: RETRY_AFTER,
278 note: None,
279 mask: None,
280 chains: Chains::default(),
281 save_chains: None,
282 }
283 }
284}
285
286enum Command {
287 Publish(RuntimeEvent),
288 RunEnded {
289 lane: String,
290 run_id: String,
291 aborted: bool,
292 },
293 Drain(Sender<()>),
294 Close,
295}
296
297#[derive(Default)]
298struct Counts {
299 pending: HashMap<String, usize>,
300 steered: HashMap<String, usize>,
301 in_flight: usize,
302 aborted: HashSet<String>,
303}
304
305pub struct Dispatcher {
307 tx: Mutex<Option<Sender<Command>>>,
308 counts: Arc<Mutex<Counts>>,
309 enabled: bool,
310 thread: Mutex<Option<std::thread::JoinHandle<()>>>,
311}
312
313impl Dispatcher {
314 pub fn new(options: DispatcherOptions, lanes: Arc<dyn EventLanes>) -> Arc<Dispatcher> {
315 let (tx, rx) = channel();
316 let counts = Arc::new(Mutex::new(Counts {
317 aborted: options.chains.aborted.iter().cloned().collect(),
318 ..Counts::default()
319 }));
320 let enabled = options.enabled;
321 let chain_runs = options.chains.runs.clone().into_iter().collect();
322 let mut worker = Worker {
323 options,
324 lanes,
325 counts: counts.clone(),
326 pending: HashMap::new(),
327 retries: Vec::new(),
328 steered: Vec::new(),
329 chain_runs,
330 notes: HashMap::new(),
331 now: || jsrt::date::now_ms() as u64,
332 };
333 let thread = std::thread::spawn(move || worker.run(&rx));
334 Arc::new(Dispatcher {
335 tx: Mutex::new(Some(tx)),
336 counts,
337 enabled,
338 thread: Mutex::new(Some(thread)),
339 })
340 }
341
342 pub fn enabled(&self) -> bool {
343 self.enabled
344 }
345
346 fn send(&self, command: Command) -> bool {
347 sync::lock(&self.tx).as_ref().is_some_and(|tx| tx.send(command).is_ok())
348 }
349
350 pub fn publish(&self, event: RuntimeEvent) {
352 if !self.enabled {
353 return;
354 }
355 *sync::lock(&self.counts).pending.entry(event.lane.clone()).or_default() += 1;
356 if !self.send(Command::Publish(event.clone())) {
357 let mut counts = sync::lock(&self.counts);
358 if let Some(n) = counts.pending.get_mut(&event.lane) {
359 *n = n.saturating_sub(1);
360 }
361 }
362 }
363
364 pub fn run_ended(&self, lane: &str, run_id: &str, aborted: bool) {
367 self.send(Command::RunEnded {
368 lane: lane.into(),
369 run_id: run_id.into(),
370 aborted,
371 });
372 }
373
374 pub fn drain(&self) {
376 let (ack, done) = channel();
377 if self.send(Command::Drain(ack)) {
378 let _ = done.recv();
379 }
380 }
381
382 pub fn pending_for(&self, lane: &str) -> usize {
384 let counts = sync::lock(&self.counts);
385 counts.pending.get(lane).copied().unwrap_or(0)
386 + counts.steered.get(lane).copied().unwrap_or(0)
387 + counts.in_flight
388 }
389
390 pub fn root_aborted(&self, root: Option<&str>) -> bool {
392 root.is_some_and(|root| sync::lock(&self.counts).aborted.contains(root))
393 }
394
395 pub fn close(&self) {
398 if let Some(tx) = sync::lock(&self.tx).take() {
399 let _ = tx.send(Command::Close);
400 }
401 sync::lock(&self.thread).take();
402 }
403}
404
405impl Drop for Dispatcher {
406 fn drop(&mut self) {
407 self.close();
408 }
409}
410
411struct Steered {
412 lane: String,
413 entry: String,
414 events: Vec<RuntimeEvent>,
415}
416
417struct Worker {
418 options: DispatcherOptions,
419 lanes: Arc<dyn EventLanes>,
420 counts: Arc<Mutex<Counts>>,
421 pending: HashMap<String, (Vec<RuntimeEvent>, Instant)>,
423 retries: Vec<(Instant, Vec<RuntimeEvent>)>,
424 steered: Vec<Steered>,
425 chain_runs: HashMap<String, usize>,
426 notes: HashMap<String, Option<String>>,
428 now: fn() -> u64,
429}
430
431impl Worker {
432 fn run(&mut self, rx: &Receiver<Command>) {
433 loop {
434 let next = self
435 .pending
436 .values()
437 .map(|(_, at)| *at)
438 .chain(self.retries.iter().map(|(at, _)| *at))
439 .min();
440 let received = match next {
441 Some(at) => rx.recv_timeout(at.saturating_duration_since(Instant::now())),
442 None => rx.recv().map_err(|_| RecvTimeoutError::Disconnected),
443 };
444 match received {
445 Ok(Command::Publish(event)) => {
446 let coalesce = self.options.coalesce;
447 self.pending
448 .entry(event.lane.clone())
449 .or_insert_with(|| (Vec::new(), Instant::now() + coalesce))
450 .0
451 .push(event);
452 }
453 Ok(Command::RunEnded { lane, run_id, aborted }) => {
454 if lane == session_journal::run::MAIN_LANE && aborted {
455 let root = (self.options.root_of_run)(&run_id);
456 if sync::lock(&self.counts).aborted.insert(root.clone()) {
457 self.save_chains();
458 }
459 if let Some(callback) = &self.options.on_root_aborted {
460 callback(&root);
461 }
462 }
463 if self.steered.iter().any(|s| s.lane == lane) {
464 self.reclaim(&lane, aborted);
465 }
466 }
467 Ok(Command::Drain(ack)) => {
468 let lanes: Vec<String> = self.pending.keys().cloned().collect();
469 for lane in lanes {
470 self.flush(&lane);
471 }
472 let _ = ack.send(());
473 }
474 Ok(Command::Close) | Err(RecvTimeoutError::Disconnected) => return,
475 Err(RecvTimeoutError::Timeout) => {}
476 }
477 let now = Instant::now();
478 let due: Vec<String> = self
479 .pending
480 .iter()
481 .filter(|(_, (_, at))| *at <= now)
482 .map(|(lane, _)| lane.clone())
483 .collect();
484 for lane in due {
485 self.flush(&lane);
486 }
487 let (retry, waiting): (Vec<_>, Vec<_>) = std::mem::take(&mut self.retries)
488 .into_iter()
489 .partition(|(at, _)| *at <= now);
490 self.retries = waiting;
491 for (_, events) in retry {
492 for event in events {
494 let coalesce = self.options.coalesce;
495 self.pending
496 .entry(event.lane.clone())
497 .or_insert_with(|| (Vec::new(), Instant::now() + coalesce))
498 .0
499 .push(event);
500 }
501 }
502 }
503 }
504
505 fn flush(&mut self, lane: &str) {
506 let Some((events, _)) = self.pending.remove(lane) else {
507 return;
508 };
509 {
510 let mut counts = sync::lock(&self.counts);
511 if let Some(n) = counts.pending.get_mut(lane) {
512 *n = n.saturating_sub(events.len());
513 }
514 counts.in_flight += 1;
515 }
516 self.deliver(lane, events);
517 sync::lock(&self.counts).in_flight -= 1;
518 }
519
520 fn save_chains(&self) {
521 if let Some(save) = &self.options.save_chains {
522 let chains = Chains {
523 aborted: sync::lock(&self.counts).aborted.iter().cloned().collect(),
524 runs: self
525 .chain_runs
526 .iter()
527 .map(|(root, runs)| (root.clone(), *runs))
528 .collect(),
529 };
530 save(&chains);
531 }
532 }
533
534 fn message(&mut self, lane: &str, events: &[RuntimeEvent]) -> Value {
535 let mask = self.options.mask.clone().unwrap_or_else(default_mask);
536 let mut message = runtime_event_message_with(events, (self.now)(), &mask);
537 let cells: Vec<&str> = events
538 .iter()
539 .filter(|e| e.kind == "cell_done")
540 .map(|e| e.id.as_str())
541 .collect();
542 if let (Some(note), false) = (&self.options.note, cells.is_empty()) {
543 let text = message["content"].as_str().unwrap_or_default().to_string();
544 let line = self
545 .notes
546 .entry(cells.join("\n"))
547 .or_insert_with(|| note(lane, &text))
548 .clone();
549 if let Some(line) = line {
550 message["content"] = Value::String(format!("{text}\n{line}"));
551 }
552 }
553 message
554 }
555
556 fn delivered(&self, lane: &str, mode: &'static str, events: &[RuntimeEvent], reason: Option<String>) {
557 if let Some(callback) = &self.options.on_delivery {
558 callback(&Delivery {
559 lane: lane.into(),
560 mode,
561 events: events.to_vec(),
562 reason,
563 });
564 }
565 }
566
567 fn deliver(&mut self, lane: &str, events: Vec<RuntimeEvent>) {
568 if events.is_empty() {
569 return;
570 }
571 match self.lanes.execution(lane) {
572 Execution::Other => return self.retry_later(events),
573 Execution::Run => return self.steer(lane, events),
574 Execution::Idle => {}
575 }
576 let message = self.message(lane, &events);
577 let reason = self.refusal(lane, &events);
578 if reason.is_none() {
579 let root = chain_root(&events).unwrap_or_default();
580 match self.lanes.accept(lane, &message, &root) {
581 Accepted::Run(_) => {
582 *self.chain_runs.entry(root).or_default() += 1;
583 self.save_chains();
584 self.delivered(lane, "run", &events, None);
585 return;
586 }
587 Accepted::Busy => return self.steer(lane, events),
588 Accepted::Refused(_) => {}
589 }
590 }
591 if self.lanes.next_run(lane, &message).is_ok() {
592 self.delivered(lane, "nextRun", &events, reason);
593 }
594 }
595
596 fn refusal(&self, lane: &str, events: &[RuntimeEvent]) -> Option<String> {
598 if lane != session_journal::run::MAIN_LANE {
599 return Some("child lanes are never re-invoked".into());
600 }
601 let Some(root) = chain_root(events) else {
602 return Some("no root turn owns this work".into());
603 };
604 if sync::lock(&self.counts).aborted.contains(&root) {
605 return Some("the root turn was aborted".into());
606 }
607 let runs = self.chain_runs.get(&root).copied().unwrap_or(0);
608 let max = self.options.max_runs;
609 if runs >= max {
610 return Some(format!("{runs} of {max} automatic runs used for {root}"));
611 }
612 self.options.refuse.as_ref().and_then(|refuse| refuse(&root))
613 }
614
615 fn steer(&mut self, lane: &str, events: Vec<RuntimeEvent>) {
616 let message = self.message(lane, &events);
617 let entry = match self.lanes.steer(lane, &message) {
618 Ok(entry) => entry,
619 Err(_) => return self.retry_later(events),
620 };
621 self.delivered(lane, "steer", &events, None);
622 *sync::lock(&self.counts).steered.entry(lane.to_string()).or_default() += 1;
623 self.steered.push(Steered {
624 lane: lane.into(),
625 entry,
626 events,
627 });
628 if self.lanes.execution(lane) == Execution::Idle {
630 self.reclaim(lane, false);
631 }
632 }
633
634 fn retry_later(&mut self, events: Vec<RuntimeEvent>) {
637 {
638 let mut counts = sync::lock(&self.counts);
639 for event in &events {
640 *counts.pending.entry(event.lane.clone()).or_default() += 1;
641 }
642 }
643 self.retries.push((Instant::now() + self.options.retry_after, events));
644 }
645
646 fn forget_steer(&mut self, entry: &str) {
647 if let Some(index) = self.steered.iter().position(|s| s.entry == entry) {
648 let steered = self.steered.remove(index);
649 if let Some(n) = sync::lock(&self.counts).steered.get_mut(&steered.lane) {
650 *n = n.saturating_sub(1);
651 }
652 }
653 }
654
655 fn reclaim(&mut self, lane: &str, aborted: bool) {
659 let entries: Vec<(String, Vec<RuntimeEvent>)> = self
660 .steered
661 .iter()
662 .filter(|s| s.lane == lane)
663 .map(|s| (s.entry.clone(), s.events.clone()))
664 .collect();
665 for (entry, events) in entries {
666 match self.lanes.cancel_queued(lane, &entry) {
667 Err(_) | Ok(Cancelled::AlreadyConsumed) => self.forget_steer(&entry),
668 Ok(outcome) if outcome == Cancelled::Cancelled || aborted => {
669 self.forget_steer(&entry);
670 let kept: Vec<RuntimeEvent> = events
671 .into_iter()
672 .filter(|e| !((e.kind == "job_done" || e.kind == "tool_done") && e.status == "cancelled"))
673 .collect();
674 if !kept.is_empty() {
675 self.deliver(lane, kept);
676 }
677 }
678 Ok(_) => {}
679 }
680 }
681 }
682}
683
684fn chain_root(events: &[RuntimeEvent]) -> Option<String> {
686 events.iter().rev().find_map(|e| e.root_id.clone())
687}