use jsrt::sync;
use serde_json::{Value, json};
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender, channel};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
pub const ASYNC_EVENTS_ENV: &str = "ULTRON_ASYNC_EVENTS";
pub const RUNTIME_EVENT_MESSAGE_TYPE: &str = "ultron-runtime-event";
pub const DEFAULT_COALESCE: Duration = Duration::from_millis(500);
pub const DEFAULT_MAX_EVENT_RUNS: usize = 16;
pub const RETRY_AFTER: Duration = Duration::from_millis(1000);
#[derive(Clone, Debug, PartialEq)]
pub struct RuntimeEvent {
pub kind: String,
pub id: String,
pub status: String,
pub summary: String,
pub fetch: String,
pub lane: String,
pub root_id: Option<String>,
}
pub fn async_events_enabled(value: Option<&str>) -> bool {
match value.map(str::trim) {
None | Some("") => true,
Some(value) => !["off", "0", "false", "no"].contains(&value.to_lowercase().as_str()),
}
}
pub fn max_event_runs_from_env(value: Option<&str>) -> usize {
value
.map(str::trim)
.filter(|v| !v.is_empty())
.and_then(|v| v.parse::<usize>().ok())
.unwrap_or(DEFAULT_MAX_EVENT_RUNS)
}
pub type MaskText = Arc<dyn Fn(&str) -> String + Send + Sync>;
pub fn default_mask() -> MaskText {
Arc::new(|text: &str| host::secrets::OutputMasker::default().mask_counted(text, None).0)
}
fn attribute(value: &str) -> String {
value
.split_whitespace()
.collect::<Vec<_>>()
.join(" ")
.replace('&', "&")
.replace('"', """)
.replace('<', "<")
.replace('>', ">")
}
fn bounded(text: &str) -> String {
let line = jsrt::string::flatten(text);
if line.chars().count() > 240 {
format!("{}…", line.chars().take(239).collect::<String>())
} else {
line
}
}
pub fn runtime_event_line(kind: &str, id: &str, status: &str, summary: &str, fetch: &str, mask: &MaskText) -> String {
format!(
"<runtime_event kind=\"{}\" id=\"{}\" status=\"{}\" summary=\"{}\" fetch=\"{}\" />",
attribute(kind),
attribute(id),
attribute(status),
attribute(&bounded(&mask(summary))),
attribute(fetch)
)
}
pub fn runtime_event_text(events: &[RuntimeEvent], mask: &MaskText) -> String {
events
.iter()
.map(|e| {
let summary = jsrt::utf16::head(&e.summary, 4 * 240).to_string();
runtime_event_line(&e.kind, &e.id, &e.status, &summary, &e.fetch, mask)
})
.collect::<Vec<_>>()
.join("\n")
}
pub fn runtime_event_message_with(events: &[RuntimeEvent], now: u64, mask: &MaskText) -> Value {
json!({
"role": "custom",
"customType": RUNTIME_EVENT_MESSAGE_TYPE,
"content": runtime_event_text(events, mask),
"display": true,
"details": {"events": events.iter().map(|e| json!({"kind": e.kind, "id": e.id, "status": e.status})).collect::<Vec<_>>()},
"timestamp": now,
})
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Execution {
Idle,
Run,
Other,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Accepted {
Run(String),
Busy,
Refused(String),
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Cancelled {
Cancelled,
AlreadyConsumed,
NotFound,
}
pub trait EventLanes: Send + Sync {
fn execution(&self, lane: &str) -> Execution;
fn accept(&self, lane: &str, message: &Value, root: &str) -> Accepted;
fn steer(&self, lane: &str, message: &Value) -> Result<String, String>;
fn next_run(&self, lane: &str, message: &Value) -> Result<String, String>;
fn cancel_queued(&self, lane: &str, entry: &str) -> Result<Cancelled, String>;
}
#[derive(Clone, Debug, PartialEq)]
pub struct Delivery {
pub lane: String,
pub mode: &'static str,
pub events: Vec<RuntimeEvent>,
pub reason: Option<String>,
}
pub type RootRefusal = Arc<dyn Fn(&str) -> Option<String> + Send + Sync>;
pub type RootCallback = Arc<dyn Fn(&str) + Send + Sync>;
pub type DeliveryCallback = Arc<dyn Fn(&Delivery) + Send + Sync>;
pub type RootOfRun = Arc<dyn Fn(&str) -> String + Send + Sync>;
pub type EventNote = Arc<dyn Fn(&str, &str) -> Option<String> + Send + Sync>;
#[derive(Clone, Debug, Default, PartialEq)]
pub struct Chains {
pub aborted: BTreeSet<String>,
pub runs: BTreeMap<String, usize>,
}
pub const CHAINS_MODULE: &str = "async-events";
impl Chains {
pub fn from_value(value: Option<&Value>) -> Chains {
let Some(value) = value else {
return Chains::default();
};
Chains {
aborted: value["abortedRoots"]
.as_array()
.into_iter()
.flatten()
.filter_map(|root| root.as_str().map(str::to_string))
.collect(),
runs: value["chainRuns"]
.as_object()
.into_iter()
.flatten()
.filter_map(|(root, runs)| Some((root.clone(), runs.as_u64()? as usize)))
.collect(),
}
}
pub fn to_value(&self) -> Value {
json!({"version": 1, "abortedRoots": self.aborted, "chainRuns": self.runs})
}
}
pub type SaveChains = Arc<dyn Fn(&Chains) + Send + Sync>;
#[derive(Clone)]
pub struct DispatcherOptions {
pub enabled: bool,
pub coalesce: Duration,
pub max_runs: usize,
pub root_of_run: RootOfRun,
pub refuse: Option<RootRefusal>,
pub on_root_aborted: Option<RootCallback>,
pub on_delivery: Option<DeliveryCallback>,
pub retry_after: Duration,
pub note: Option<EventNote>,
pub mask: Option<MaskText>,
pub chains: Chains,
pub save_chains: Option<SaveChains>,
}
impl DispatcherOptions {
pub fn new(root_of_run: RootOfRun) -> DispatcherOptions {
DispatcherOptions {
enabled: true,
coalesce: DEFAULT_COALESCE,
max_runs: DEFAULT_MAX_EVENT_RUNS,
root_of_run,
refuse: None,
on_root_aborted: None,
on_delivery: None,
retry_after: RETRY_AFTER,
note: None,
mask: None,
chains: Chains::default(),
save_chains: None,
}
}
}
enum Command {
Publish(RuntimeEvent),
RunEnded {
lane: String,
run_id: String,
aborted: bool,
},
Drain(Sender<()>),
Close,
}
#[derive(Default)]
struct Counts {
pending: HashMap<String, usize>,
steered: HashMap<String, usize>,
in_flight: usize,
aborted: HashSet<String>,
}
pub struct Dispatcher {
tx: Mutex<Option<Sender<Command>>>,
counts: Arc<Mutex<Counts>>,
enabled: bool,
thread: Mutex<Option<std::thread::JoinHandle<()>>>,
}
impl Dispatcher {
pub fn new(options: DispatcherOptions, lanes: Arc<dyn EventLanes>) -> Arc<Dispatcher> {
let (tx, rx) = channel();
let counts = Arc::new(Mutex::new(Counts {
aborted: options.chains.aborted.iter().cloned().collect(),
..Counts::default()
}));
let enabled = options.enabled;
let chain_runs = options.chains.runs.clone().into_iter().collect();
let mut worker = Worker {
options,
lanes,
counts: counts.clone(),
pending: HashMap::new(),
retries: Vec::new(),
steered: Vec::new(),
chain_runs,
notes: HashMap::new(),
now: || jsrt::date::now_ms() as u64,
};
let thread = std::thread::spawn(move || worker.run(&rx));
Arc::new(Dispatcher {
tx: Mutex::new(Some(tx)),
counts,
enabled,
thread: Mutex::new(Some(thread)),
})
}
pub fn enabled(&self) -> bool {
self.enabled
}
fn send(&self, command: Command) -> bool {
sync::lock(&self.tx).as_ref().is_some_and(|tx| tx.send(command).is_ok())
}
pub fn publish(&self, event: RuntimeEvent) {
if !self.enabled {
return;
}
*sync::lock(&self.counts).pending.entry(event.lane.clone()).or_default() += 1;
if !self.send(Command::Publish(event.clone())) {
let mut counts = sync::lock(&self.counts);
if let Some(n) = counts.pending.get_mut(&event.lane) {
*n = n.saturating_sub(1);
}
}
}
pub fn run_ended(&self, lane: &str, run_id: &str, aborted: bool) {
self.send(Command::RunEnded {
lane: lane.into(),
run_id: run_id.into(),
aborted,
});
}
pub fn drain(&self) {
let (ack, done) = channel();
if self.send(Command::Drain(ack)) {
let _ = done.recv();
}
}
pub fn pending_for(&self, lane: &str) -> usize {
let counts = sync::lock(&self.counts);
counts.pending.get(lane).copied().unwrap_or(0)
+ counts.steered.get(lane).copied().unwrap_or(0)
+ counts.in_flight
}
pub fn root_aborted(&self, root: Option<&str>) -> bool {
root.is_some_and(|root| sync::lock(&self.counts).aborted.contains(root))
}
pub fn close(&self) {
if let Some(tx) = sync::lock(&self.tx).take() {
let _ = tx.send(Command::Close);
}
sync::lock(&self.thread).take();
}
}
impl Drop for Dispatcher {
fn drop(&mut self) {
self.close();
}
}
struct Steered {
lane: String,
entry: String,
events: Vec<RuntimeEvent>,
}
struct Worker {
options: DispatcherOptions,
lanes: Arc<dyn EventLanes>,
counts: Arc<Mutex<Counts>>,
pending: HashMap<String, (Vec<RuntimeEvent>, Instant)>,
retries: Vec<(Instant, Vec<RuntimeEvent>)>,
steered: Vec<Steered>,
chain_runs: HashMap<String, usize>,
notes: HashMap<String, Option<String>>,
now: fn() -> u64,
}
impl Worker {
fn run(&mut self, rx: &Receiver<Command>) {
loop {
let next = self
.pending
.values()
.map(|(_, at)| *at)
.chain(self.retries.iter().map(|(at, _)| *at))
.min();
let received = match next {
Some(at) => rx.recv_timeout(at.saturating_duration_since(Instant::now())),
None => rx.recv().map_err(|_| RecvTimeoutError::Disconnected),
};
match received {
Ok(Command::Publish(event)) => {
let coalesce = self.options.coalesce;
self.pending
.entry(event.lane.clone())
.or_insert_with(|| (Vec::new(), Instant::now() + coalesce))
.0
.push(event);
}
Ok(Command::RunEnded { lane, run_id, aborted }) => {
if lane == session_journal::run::MAIN_LANE && aborted {
let root = (self.options.root_of_run)(&run_id);
if sync::lock(&self.counts).aborted.insert(root.clone()) {
self.save_chains();
}
if let Some(callback) = &self.options.on_root_aborted {
callback(&root);
}
}
if self.steered.iter().any(|s| s.lane == lane) {
self.reclaim(&lane, aborted);
}
}
Ok(Command::Drain(ack)) => {
let lanes: Vec<String> = self.pending.keys().cloned().collect();
for lane in lanes {
self.flush(&lane);
}
let _ = ack.send(());
}
Ok(Command::Close) | Err(RecvTimeoutError::Disconnected) => return,
Err(RecvTimeoutError::Timeout) => {}
}
let now = Instant::now();
let due: Vec<String> = self
.pending
.iter()
.filter(|(_, (_, at))| *at <= now)
.map(|(lane, _)| lane.clone())
.collect();
for lane in due {
self.flush(&lane);
}
let (retry, waiting): (Vec<_>, Vec<_>) = std::mem::take(&mut self.retries)
.into_iter()
.partition(|(at, _)| *at <= now);
self.retries = waiting;
for (_, events) in retry {
for event in events {
let coalesce = self.options.coalesce;
self.pending
.entry(event.lane.clone())
.or_insert_with(|| (Vec::new(), Instant::now() + coalesce))
.0
.push(event);
}
}
}
}
fn flush(&mut self, lane: &str) {
let Some((events, _)) = self.pending.remove(lane) else {
return;
};
{
let mut counts = sync::lock(&self.counts);
if let Some(n) = counts.pending.get_mut(lane) {
*n = n.saturating_sub(events.len());
}
counts.in_flight += 1;
}
self.deliver(lane, events);
sync::lock(&self.counts).in_flight -= 1;
}
fn save_chains(&self) {
if let Some(save) = &self.options.save_chains {
let chains = Chains {
aborted: sync::lock(&self.counts).aborted.iter().cloned().collect(),
runs: self
.chain_runs
.iter()
.map(|(root, runs)| (root.clone(), *runs))
.collect(),
};
save(&chains);
}
}
fn message(&mut self, lane: &str, events: &[RuntimeEvent]) -> Value {
let mask = self.options.mask.clone().unwrap_or_else(default_mask);
let mut message = runtime_event_message_with(events, (self.now)(), &mask);
let cells: Vec<&str> = events
.iter()
.filter(|e| e.kind == "cell_done")
.map(|e| e.id.as_str())
.collect();
if let (Some(note), false) = (&self.options.note, cells.is_empty()) {
let text = message["content"].as_str().unwrap_or_default().to_string();
let line = self
.notes
.entry(cells.join("\n"))
.or_insert_with(|| note(lane, &text))
.clone();
if let Some(line) = line {
message["content"] = Value::String(format!("{text}\n{line}"));
}
}
message
}
fn delivered(&self, lane: &str, mode: &'static str, events: &[RuntimeEvent], reason: Option<String>) {
if let Some(callback) = &self.options.on_delivery {
callback(&Delivery {
lane: lane.into(),
mode,
events: events.to_vec(),
reason,
});
}
}
fn deliver(&mut self, lane: &str, events: Vec<RuntimeEvent>) {
if events.is_empty() {
return;
}
match self.lanes.execution(lane) {
Execution::Other => return self.retry_later(events),
Execution::Run => return self.steer(lane, events),
Execution::Idle => {}
}
let message = self.message(lane, &events);
let reason = self.refusal(lane, &events);
if reason.is_none() {
let root = chain_root(&events).unwrap_or_default();
match self.lanes.accept(lane, &message, &root) {
Accepted::Run(_) => {
*self.chain_runs.entry(root).or_default() += 1;
self.save_chains();
self.delivered(lane, "run", &events, None);
return;
}
Accepted::Busy => return self.steer(lane, events),
Accepted::Refused(_) => {}
}
}
if self.lanes.next_run(lane, &message).is_ok() {
self.delivered(lane, "nextRun", &events, reason);
}
}
fn refusal(&self, lane: &str, events: &[RuntimeEvent]) -> Option<String> {
if lane != session_journal::run::MAIN_LANE {
return Some("child lanes are never re-invoked".into());
}
let Some(root) = chain_root(events) else {
return Some("no root turn owns this work".into());
};
if sync::lock(&self.counts).aborted.contains(&root) {
return Some("the root turn was aborted".into());
}
let runs = self.chain_runs.get(&root).copied().unwrap_or(0);
let max = self.options.max_runs;
if runs >= max {
return Some(format!("{runs} of {max} automatic runs used for {root}"));
}
self.options.refuse.as_ref().and_then(|refuse| refuse(&root))
}
fn steer(&mut self, lane: &str, events: Vec<RuntimeEvent>) {
let message = self.message(lane, &events);
let entry = match self.lanes.steer(lane, &message) {
Ok(entry) => entry,
Err(_) => return self.retry_later(events),
};
self.delivered(lane, "steer", &events, None);
*sync::lock(&self.counts).steered.entry(lane.to_string()).or_default() += 1;
self.steered.push(Steered {
lane: lane.into(),
entry,
events,
});
if self.lanes.execution(lane) == Execution::Idle {
self.reclaim(lane, false);
}
}
fn retry_later(&mut self, events: Vec<RuntimeEvent>) {
{
let mut counts = sync::lock(&self.counts);
for event in &events {
*counts.pending.entry(event.lane.clone()).or_default() += 1;
}
}
self.retries.push((Instant::now() + self.options.retry_after, events));
}
fn forget_steer(&mut self, entry: &str) {
if let Some(index) = self.steered.iter().position(|s| s.entry == entry) {
let steered = self.steered.remove(index);
if let Some(n) = sync::lock(&self.counts).steered.get_mut(&steered.lane) {
*n = n.saturating_sub(1);
}
}
}
fn reclaim(&mut self, lane: &str, aborted: bool) {
let entries: Vec<(String, Vec<RuntimeEvent>)> = self
.steered
.iter()
.filter(|s| s.lane == lane)
.map(|s| (s.entry.clone(), s.events.clone()))
.collect();
for (entry, events) in entries {
match self.lanes.cancel_queued(lane, &entry) {
Err(_) | Ok(Cancelled::AlreadyConsumed) => self.forget_steer(&entry),
Ok(outcome) if outcome == Cancelled::Cancelled || aborted => {
self.forget_steer(&entry);
let kept: Vec<RuntimeEvent> = events
.into_iter()
.filter(|e| !((e.kind == "job_done" || e.kind == "tool_done") && e.status == "cancelled"))
.collect();
if !kept.is_empty() {
self.deliver(lane, kept);
}
}
Ok(_) => {}
}
}
}
}
fn chain_root(events: &[RuntimeEvent]) -> Option<String> {
events.iter().rev().find_map(|e| e.root_id.clone())
}