Skip to main content

session_core/
async_events.rs

1//! Completion events instead of polling: a port of `packages/coding-agent/src/ultron/async-events.ts`.
2//!
3//! When detached work ends (a shell job started with `yield_after`, an `rlm.spawn` child, an `agents.*` task, a
4//! detached cell), the host delivers one short `ultron-runtime-event` custom message to the lane that owns the work:
5//!
6//! ```text
7//! <runtime_event kind="job_done" id="job-1a2b" status="completed" summary="exit 0; 12 passed" fetch="await rlm.job('job-1a2b')" />
8//! ```
9//!
10//! Delivery ([`Dispatcher`]):
11//! - completions within the coalescing window (500 ms) of the first one become one message;
12//! - a lane in the middle of a run gets the message as a steer, placed at the next turn boundary; a steer that a run
13//!   did not consume (it ended first) is taken back and delivered again as below;
14//! - an idle root lane is re-invoked: the message starts a run charged to the root that started the work, so the
15//!   per-root turn, token and cost limits keep counting across the chain;
16//! - otherwise (a child lane, or a root that must not be re-invoked) the message waits as next-run input. Events are
17//!   only ever appended, which keeps the provider's prompt-cache prefix valid.
18//!
19//! Loop guards: only root-owned work re-invokes the root; a root whose turn the user aborted never restarts work (its
20//! jobs are cancelled and their ends are not announced); a refused root (a spent budget) or more than `max_runs`
21//! automatic runs in one chain turn re-invocation into quiet delivery. `ULTRON_ASYNC_EVENTS=off` disables events.
22//!
23//! The dispatcher is one thread: publications, run ends and drains are commands it handles in order, and each lane's
24//! batch is delivered after its coalescing deadline, so delivery decisions never race each other.
25
26use jsrt::sync;
27use serde_json::{Value, json};
28use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
29use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender, channel};
30use std::sync::{Arc, Mutex};
31use std::time::{Duration, Instant};
32
33pub const ASYNC_EVENTS_ENV: &str = "ULTRON_ASYNC_EVENTS";
34
35pub const RUNTIME_EVENT_MESSAGE_TYPE: &str = "ultron-runtime-event";
36pub const DEFAULT_COALESCE: Duration = Duration::from_millis(500);
37pub const DEFAULT_MAX_EVENT_RUNS: usize = 16;
38/// How long a batch that could not be delivered (the lane was compacting, a steer failed) waits before it is tried
39/// again.
40pub const RETRY_AFTER: Duration = Duration::from_millis(1000);
41
42/// One completion.
43#[derive(Clone, Debug, PartialEq)]
44pub struct RuntimeEvent {
45    /// `job_done`, `tool_done`, `child_done`, `task_done` (and the Rust `cell_done`).
46    pub kind: String,
47    pub id: String,
48    pub status: String,
49    pub summary: String,
50    /// The kernel call that returns the full result.
51    pub fetch: String,
52    /// The lane that owns the work and receives the event.
53    pub lane: String,
54    /// The usage root the work was started under.
55    pub root_id: Option<String>,
56}
57
58/// `asyncEventsEnabled`: on unless `off`, `0`, `false` or `no`.
59pub fn async_events_enabled(value: Option<&str>) -> bool {
60    match value.map(str::trim) {
61        None | Some("") => true,
62        Some(value) => !["off", "0", "false", "no"].contains(&value.to_lowercase().as_str()),
63    }
64}
65
66/// `maxEventRunsFromEnv`: a non-negative integer, else [`DEFAULT_MAX_EVENT_RUNS`].
67pub fn max_event_runs_from_env(value: Option<&str>) -> usize {
68    value
69        .map(str::trim)
70        .filter(|v| !v.is_empty())
71        .and_then(|v| v.parse::<usize>().ok())
72        .unwrap_or(DEFAULT_MAX_EVENT_RUNS)
73}
74
75/// Masks secrets in text that reaches the model (`maskCellOutput`: the session's known secrets and the shipped rules,
76/// unless `ULTRON_MASK_SECRETS` turns masking off).
77pub type MaskText = Arc<dyn Fn(&str) -> String + Send + Sync>;
78
79/// The shipped rules only, masking on (no session: nothing registered).
80pub fn default_mask() -> MaskText {
81    Arc::new(|text: &str| host::secrets::OutputMasker::default().mask_counted(text, None).0)
82}
83
84/// `<runtime_event>` attribute text (`attribute` of async-events.ts).
85fn attribute(value: &str) -> String {
86    value
87        .split_whitespace()
88        .collect::<Vec<_>>()
89        .join(" ")
90        .replace('&', "&amp;")
91        .replace('"', "&quot;")
92        .replace('<', "&lt;")
93        .replace('>', "&gt;")
94}
95
96/// `boundedSummary`: one line of at most 240 characters.
97fn bounded(text: &str) -> String {
98    let line = jsrt::string::flatten(text);
99    if line.chars().count() > 240 {
100        format!("{}…", line.chars().take(239).collect::<String>())
101    } else {
102        line
103    }
104}
105
106/// One `<runtime_event>` line (`runtimeEventText` of async-events.ts) with the session's masking
107/// (`maskCellOutput`).
108pub fn runtime_event_line(kind: &str, id: &str, status: &str, summary: &str, fetch: &str, mask: &MaskText) -> String {
109    format!(
110        "<runtime_event kind=\"{}\" id=\"{}\" status=\"{}\" summary=\"{}\" fetch=\"{}\" />",
111        attribute(kind),
112        attribute(id),
113        attribute(status),
114        attribute(&bounded(&mask(summary))),
115        attribute(fetch)
116    )
117}
118
119/// `runtimeEventText` with the session's masking: each summary is cut to 4 × 240 UTF-16 units, masked, then
120/// bounded.
121pub fn runtime_event_text(events: &[RuntimeEvent], mask: &MaskText) -> String {
122    events
123        .iter()
124        .map(|e| {
125            let summary = jsrt::utf16::head(&e.summary, 4 * 240).to_string();
126            runtime_event_line(&e.kind, &e.id, &e.status, &summary, &e.fetch, mask)
127        })
128        .collect::<Vec<_>>()
129        .join("\n")
130}
131
132/// `runtimeEventMessage`: the events as one Pi custom message, with the session's masking.
133pub fn runtime_event_message_with(events: &[RuntimeEvent], now: u64, mask: &MaskText) -> Value {
134    json!({
135        "role": "custom",
136        "customType": RUNTIME_EVENT_MESSAGE_TYPE,
137        "content": runtime_event_text(events, mask),
138        "display": true,
139        "details": {"events": events.iter().map(|e| json!({"kind": e.kind, "id": e.id, "status": e.status})).collect::<Vec<_>>()},
140        "timestamp": now,
141    })
142}
143
144/// What a lane is doing (`inspectExecution`).
145#[derive(Clone, Copy, Debug, PartialEq, Eq)]
146pub enum Execution {
147    Idle,
148    Run,
149    /// Compaction or navigation: delivery waits for it to settle.
150    Other,
151}
152
153/// The outcome of starting a run with an event message (`accept`).
154#[derive(Clone, Debug, PartialEq, Eq)]
155pub enum Accepted {
156    /// The run started (its operation id), charged to the given root.
157    Run(String),
158    /// The lane is running after all (`LaneBusy`).
159    Busy,
160    Refused(String),
161}
162
163/// What taking back a steered message found (`cancelQueued`).
164#[derive(Clone, Copy, Debug, PartialEq, Eq)]
165pub enum Cancelled {
166    Cancelled,
167    AlreadyConsumed,
168    NotFound,
169}
170
171/// The lanes events are delivered to, as the dispatcher uses the harness's lanes.
172pub trait EventLanes: Send + Sync {
173    fn execution(&self, lane: &str) -> Execution;
174    /// Start a run of `lane` with `message`, charged to `root` (`continueRootTurn`).
175    fn accept(&self, lane: &str, message: &Value, root: &str) -> Accepted;
176    /// Queue `message` for the lane's running run; its queued entry id.
177    fn steer(&self, lane: &str, message: &Value) -> Result<String, String>;
178    /// Keep `message` for the lane's next run.
179    fn next_run(&self, lane: &str, message: &Value) -> Result<String, String>;
180    fn cancel_queued(&self, lane: &str, entry: &str) -> Result<Cancelled, String>;
181}
182
183/// A delivery decision, for tests and diagnostics (`onDelivery`).
184#[derive(Clone, Debug, PartialEq)]
185pub struct Delivery {
186    pub lane: String,
187    /// `run`, `steer` or `nextRun`.
188    pub mode: &'static str,
189    pub events: Vec<RuntimeEvent>,
190    pub reason: Option<String>,
191}
192
193pub type RootRefusal = Arc<dyn Fn(&str) -> Option<String> + Send + Sync>;
194pub type RootCallback = Arc<dyn Fn(&str) + Send + Sync>;
195pub type DeliveryCallback = Arc<dyn Fn(&Delivery) + Send + Sync>;
196pub type RootOfRun = Arc<dyn Fn(&str) -> String + Send + Sync>;
197/// A line that ends a lane's event message (`lane`, the message text): the context reminder (crate::context_reminder).
198pub type EventNote = Arc<dyn Fn(&str, &str) -> Option<String> + Send + Sync>;
199
200/// What delivery remembers across workers (a hand-over, a restart): the roots whose turn the user aborted (their
201/// work never re-invokes the model) and how many automatic runs each chain started (its `max_runs` budget). Kept in
202/// the session (`ultron.module/async-events`); in memory only, a successor re-invoked an aborted root for a late
203/// completion and gave every chain a fresh budget.
204#[derive(Clone, Debug, Default, PartialEq)]
205pub struct Chains {
206    pub aborted: BTreeSet<String>,
207    pub runs: BTreeMap<String, usize>,
208}
209
210/// The `ultron.module` key [`Chains`] are stored under.
211pub const CHAINS_MODULE: &str = "async-events";
212
213impl Chains {
214    /// The stored value (absent or unreadable: nothing remembered).
215    pub fn from_value(value: Option<&Value>) -> Chains {
216        let Some(value) = value else {
217            return Chains::default();
218        };
219        Chains {
220            aborted: value["abortedRoots"]
221                .as_array()
222                .into_iter()
223                .flatten()
224                .filter_map(|root| root.as_str().map(str::to_string))
225                .collect(),
226            runs: value["chainRuns"]
227                .as_object()
228                .into_iter()
229                .flatten()
230                .filter_map(|(root, runs)| Some((root.clone(), runs.as_u64()? as usize)))
231                .collect(),
232        }
233    }
234
235    pub fn to_value(&self) -> Value {
236        json!({"version": 1, "abortedRoots": self.aborted, "chainRuns": self.runs})
237    }
238}
239
240/// Keeps [`Chains`] after each change.
241pub type SaveChains = Arc<dyn Fn(&Chains) + Send + Sync>;
242
243/// `AsyncEventOptions`.
244#[derive(Clone)]
245pub struct DispatcherOptions {
246    pub enabled: bool,
247    pub coalesce: Duration,
248    /// Automatic runs one chain (a root and the runs its events started) may start.
249    pub max_runs: usize,
250    /// The usage root of a root-lane run (its own, or the root it continues).
251    pub root_of_run: RootOfRun,
252    /// Why the root may not be re-invoked (a spent budget), or `None` when it may.
253    pub refuse: Option<RootRefusal>,
254    /// A root turn was aborted (Esc): stop its jobs.
255    pub on_root_aborted: Option<RootCallback>,
256    pub on_delivery: Option<DeliveryCallback>,
257    pub retry_after: Duration,
258    /// Ends the message of a batch with a backgrounded cell's end (`cell_done`), as the cell's result would end.
259    pub note: Option<EventNote>,
260    /// Masks the summaries (none: the shipped rules only).
261    pub mask: Option<MaskText>,
262    /// What an earlier worker of the session remembered, and where changes go (none: kept in memory only).
263    pub chains: Chains,
264    pub save_chains: Option<SaveChains>,
265}
266
267impl DispatcherOptions {
268    pub fn new(root_of_run: RootOfRun) -> DispatcherOptions {
269        DispatcherOptions {
270            enabled: true,
271            coalesce: DEFAULT_COALESCE,
272            max_runs: DEFAULT_MAX_EVENT_RUNS,
273            root_of_run,
274            refuse: None,
275            on_root_aborted: None,
276            on_delivery: None,
277            retry_after: RETRY_AFTER,
278            note: None,
279            mask: None,
280            chains: Chains::default(),
281            save_chains: None,
282        }
283    }
284}
285
286enum Command {
287    Publish(RuntimeEvent),
288    RunEnded {
289        lane: String,
290        run_id: String,
291        aborted: bool,
292    },
293    Drain(Sender<()>),
294    Close,
295}
296
297#[derive(Default)]
298struct Counts {
299    pending: HashMap<String, usize>,
300    steered: HashMap<String, usize>,
301    in_flight: usize,
302    aborted: HashSet<String>,
303}
304
305/// `AsyncEventDispatcher`.
306pub struct Dispatcher {
307    tx: Mutex<Option<Sender<Command>>>,
308    counts: Arc<Mutex<Counts>>,
309    enabled: bool,
310    thread: Mutex<Option<std::thread::JoinHandle<()>>>,
311}
312
313impl Dispatcher {
314    pub fn new(options: DispatcherOptions, lanes: Arc<dyn EventLanes>) -> Arc<Dispatcher> {
315        let (tx, rx) = channel();
316        let counts = Arc::new(Mutex::new(Counts {
317            aborted: options.chains.aborted.iter().cloned().collect(),
318            ..Counts::default()
319        }));
320        let enabled = options.enabled;
321        let chain_runs = options.chains.runs.clone().into_iter().collect();
322        let mut worker = Worker {
323            options,
324            lanes,
325            counts: counts.clone(),
326            pending: HashMap::new(),
327            retries: Vec::new(),
328            steered: Vec::new(),
329            chain_runs,
330            notes: HashMap::new(),
331            now: || jsrt::date::now_ms() as u64,
332        };
333        let thread = std::thread::spawn(move || worker.run(&rx));
334        Arc::new(Dispatcher {
335            tx: Mutex::new(Some(tx)),
336            counts,
337            enabled,
338            thread: Mutex::new(Some(thread)),
339        })
340    }
341
342    pub fn enabled(&self) -> bool {
343        self.enabled
344    }
345
346    fn send(&self, command: Command) -> bool {
347        sync::lock(&self.tx).as_ref().is_some_and(|tx| tx.send(command).is_ok())
348    }
349
350    /// A completion: delivered after the coalescing window with the others of its lane.
351    pub fn publish(&self, event: RuntimeEvent) {
352        if !self.enabled {
353            return;
354        }
355        *sync::lock(&self.counts).pending.entry(event.lane.clone()).or_default() += 1;
356        if !self.send(Command::Publish(event.clone())) {
357            let mut counts = sync::lock(&self.counts);
358            if let Some(n) = counts.pending.get_mut(&event.lane) {
359                *n = n.saturating_sub(1);
360            }
361        }
362    }
363
364    /// The harness's `run_end` of `lane`: an aborted root turn's root never restarts work, and steers the run did
365    /// not consume are taken back.
366    pub fn run_ended(&self, lane: &str, run_id: &str, aborted: bool) {
367        self.send(Command::RunEnded {
368            lane: lane.into(),
369            run_id: run_id.into(),
370            aborted,
371        });
372    }
373
374    /// Deliver now, bypassing the coalescing window (tests, shutdown); returns once delivered.
375    pub fn drain(&self) {
376        let (ack, done) = channel();
377        if self.send(Command::Drain(ack)) {
378            let _ = done.recv();
379        }
380    }
381
382    /// Events for `lane` waiting to be delivered, or steered into a run that has not consumed them yet.
383    pub fn pending_for(&self, lane: &str) -> usize {
384        let counts = sync::lock(&self.counts);
385        counts.pending.get(lane).copied().unwrap_or(0)
386            + counts.steered.get(lane).copied().unwrap_or(0)
387            + counts.in_flight
388    }
389
390    /// Whether a root turn was aborted (its work never restarts the model).
391    pub fn root_aborted(&self, root: Option<&str>) -> bool {
392        root.is_some_and(|root| sync::lock(&self.counts).aborted.contains(root))
393    }
394
395    /// Stop delivering (pending events are dropped). The dispatcher thread ends by itself once its current
396    /// delivery returns; this never waits for it, as a delivery may be waiting for the caller (the session actor).
397    pub fn close(&self) {
398        if let Some(tx) = sync::lock(&self.tx).take() {
399            let _ = tx.send(Command::Close);
400        }
401        sync::lock(&self.thread).take();
402    }
403}
404
405impl Drop for Dispatcher {
406    fn drop(&mut self) {
407        self.close();
408    }
409}
410
411struct Steered {
412    lane: String,
413    entry: String,
414    events: Vec<RuntimeEvent>,
415}
416
417struct Worker {
418    options: DispatcherOptions,
419    lanes: Arc<dyn EventLanes>,
420    counts: Arc<Mutex<Counts>>,
421    /// Per lane: its batch and when it is delivered.
422    pending: HashMap<String, (Vec<RuntimeEvent>, Instant)>,
423    retries: Vec<(Instant, Vec<RuntimeEvent>)>,
424    steered: Vec<Steered>,
425    chain_runs: HashMap<String, usize>,
426    /// The note of each batch with cell ends, by its cells, decided once: a batch delivered again keeps it.
427    notes: HashMap<String, Option<String>>,
428    now: fn() -> u64,
429}
430
431impl Worker {
432    fn run(&mut self, rx: &Receiver<Command>) {
433        loop {
434            let next = self
435                .pending
436                .values()
437                .map(|(_, at)| *at)
438                .chain(self.retries.iter().map(|(at, _)| *at))
439                .min();
440            let received = match next {
441                Some(at) => rx.recv_timeout(at.saturating_duration_since(Instant::now())),
442                None => rx.recv().map_err(|_| RecvTimeoutError::Disconnected),
443            };
444            match received {
445                Ok(Command::Publish(event)) => {
446                    let coalesce = self.options.coalesce;
447                    self.pending
448                        .entry(event.lane.clone())
449                        .or_insert_with(|| (Vec::new(), Instant::now() + coalesce))
450                        .0
451                        .push(event);
452                }
453                Ok(Command::RunEnded { lane, run_id, aborted }) => {
454                    if lane == session_journal::run::MAIN_LANE && aborted {
455                        let root = (self.options.root_of_run)(&run_id);
456                        if sync::lock(&self.counts).aborted.insert(root.clone()) {
457                            self.save_chains();
458                        }
459                        if let Some(callback) = &self.options.on_root_aborted {
460                            callback(&root);
461                        }
462                    }
463                    if self.steered.iter().any(|s| s.lane == lane) {
464                        self.reclaim(&lane, aborted);
465                    }
466                }
467                Ok(Command::Drain(ack)) => {
468                    let lanes: Vec<String> = self.pending.keys().cloned().collect();
469                    for lane in lanes {
470                        self.flush(&lane);
471                    }
472                    let _ = ack.send(());
473                }
474                Ok(Command::Close) | Err(RecvTimeoutError::Disconnected) => return,
475                Err(RecvTimeoutError::Timeout) => {}
476            }
477            let now = Instant::now();
478            let due: Vec<String> = self
479                .pending
480                .iter()
481                .filter(|(_, (_, at))| *at <= now)
482                .map(|(lane, _)| lane.clone())
483                .collect();
484            for lane in due {
485                self.flush(&lane);
486            }
487            let (retry, waiting): (Vec<_>, Vec<_>) = std::mem::take(&mut self.retries)
488                .into_iter()
489                .partition(|(at, _)| *at <= now);
490            self.retries = waiting;
491            for (_, events) in retry {
492                // Counted as pending since `retry_later`.
493                for event in events {
494                    let coalesce = self.options.coalesce;
495                    self.pending
496                        .entry(event.lane.clone())
497                        .or_insert_with(|| (Vec::new(), Instant::now() + coalesce))
498                        .0
499                        .push(event);
500                }
501            }
502        }
503    }
504
505    fn flush(&mut self, lane: &str) {
506        let Some((events, _)) = self.pending.remove(lane) else {
507            return;
508        };
509        {
510            let mut counts = sync::lock(&self.counts);
511            if let Some(n) = counts.pending.get_mut(lane) {
512                *n = n.saturating_sub(events.len());
513            }
514            counts.in_flight += 1;
515        }
516        self.deliver(lane, events);
517        sync::lock(&self.counts).in_flight -= 1;
518    }
519
520    fn save_chains(&self) {
521        if let Some(save) = &self.options.save_chains {
522            let chains = Chains {
523                aborted: sync::lock(&self.counts).aborted.iter().cloned().collect(),
524                runs: self
525                    .chain_runs
526                    .iter()
527                    .map(|(root, runs)| (root.clone(), *runs))
528                    .collect(),
529            };
530            save(&chains);
531        }
532    }
533
534    fn message(&mut self, lane: &str, events: &[RuntimeEvent]) -> Value {
535        let mask = self.options.mask.clone().unwrap_or_else(default_mask);
536        let mut message = runtime_event_message_with(events, (self.now)(), &mask);
537        let cells: Vec<&str> = events
538            .iter()
539            .filter(|e| e.kind == "cell_done")
540            .map(|e| e.id.as_str())
541            .collect();
542        if let (Some(note), false) = (&self.options.note, cells.is_empty()) {
543            let text = message["content"].as_str().unwrap_or_default().to_string();
544            let line = self
545                .notes
546                .entry(cells.join("\n"))
547                .or_insert_with(|| note(lane, &text))
548                .clone();
549            if let Some(line) = line {
550                message["content"] = Value::String(format!("{text}\n{line}"));
551            }
552        }
553        message
554    }
555
556    fn delivered(&self, lane: &str, mode: &'static str, events: &[RuntimeEvent], reason: Option<String>) {
557        if let Some(callback) = &self.options.on_delivery {
558            callback(&Delivery {
559                lane: lane.into(),
560                mode,
561                events: events.to_vec(),
562                reason,
563            });
564        }
565    }
566
567    fn deliver(&mut self, lane: &str, events: Vec<RuntimeEvent>) {
568        if events.is_empty() {
569            return;
570        }
571        match self.lanes.execution(lane) {
572            Execution::Other => return self.retry_later(events),
573            Execution::Run => return self.steer(lane, events),
574            Execution::Idle => {}
575        }
576        let message = self.message(lane, &events);
577        let reason = self.refusal(lane, &events);
578        if reason.is_none() {
579            let root = chain_root(&events).unwrap_or_default();
580            match self.lanes.accept(lane, &message, &root) {
581                Accepted::Run(_) => {
582                    *self.chain_runs.entry(root).or_default() += 1;
583                    self.save_chains();
584                    self.delivered(lane, "run", &events, None);
585                    return;
586                }
587                Accepted::Busy => return self.steer(lane, events),
588                Accepted::Refused(_) => {}
589            }
590        }
591        if self.lanes.next_run(lane, &message).is_ok() {
592            self.delivered(lane, "nextRun", &events, reason);
593        }
594    }
595
596    /// Why an idle lane's events must not start a run; `None` when they may.
597    fn refusal(&self, lane: &str, events: &[RuntimeEvent]) -> Option<String> {
598        if lane != session_journal::run::MAIN_LANE {
599            return Some("child lanes are never re-invoked".into());
600        }
601        let Some(root) = chain_root(events) else {
602            return Some("no root turn owns this work".into());
603        };
604        if sync::lock(&self.counts).aborted.contains(&root) {
605            return Some("the root turn was aborted".into());
606        }
607        let runs = self.chain_runs.get(&root).copied().unwrap_or(0);
608        let max = self.options.max_runs;
609        if runs >= max {
610            return Some(format!("{runs} of {max} automatic runs used for {root}"));
611        }
612        self.options.refuse.as_ref().and_then(|refuse| refuse(&root))
613    }
614
615    fn steer(&mut self, lane: &str, events: Vec<RuntimeEvent>) {
616        let message = self.message(lane, &events);
617        let entry = match self.lanes.steer(lane, &message) {
618            Ok(entry) => entry,
619            Err(_) => return self.retry_later(events),
620        };
621        self.delivered(lane, "steer", &events, None);
622        *sync::lock(&self.counts).steered.entry(lane.to_string()).or_default() += 1;
623        self.steered.push(Steered {
624            lane: lane.into(),
625            entry,
626            events,
627        });
628        // The run may have ended between the check and the steer: then nothing will consume it.
629        if self.lanes.execution(lane) == Execution::Idle {
630            self.reclaim(lane, false);
631        }
632    }
633
634    /// Deliver `events` again after `retry_after`. They count as pending meanwhile: uncounted, the session saw no
635    /// work waiting and an unattached worker could retire with them undelivered.
636    fn retry_later(&mut self, events: Vec<RuntimeEvent>) {
637        {
638            let mut counts = sync::lock(&self.counts);
639            for event in &events {
640                *counts.pending.entry(event.lane.clone()).or_default() += 1;
641            }
642        }
643        self.retries.push((Instant::now() + self.options.retry_after, events));
644    }
645
646    fn forget_steer(&mut self, entry: &str) {
647        if let Some(index) = self.steered.iter().position(|s| s.entry == entry) {
648            let steered = self.steered.remove(index);
649            if let Some(n) = sync::lock(&self.counts).steered.get_mut(&steered.lane) {
650                *n = n.saturating_sub(1);
651            }
652        }
653    }
654
655    /// Steered events a finished run did not consume are taken back and delivered again. After an abort, steers are
656    /// dropped with the queue; completions of tasks (not of the jobs the abort cancelled) are kept for the lane's
657    /// next run.
658    fn reclaim(&mut self, lane: &str, aborted: bool) {
659        let entries: Vec<(String, Vec<RuntimeEvent>)> = self
660            .steered
661            .iter()
662            .filter(|s| s.lane == lane)
663            .map(|s| (s.entry.clone(), s.events.clone()))
664            .collect();
665        for (entry, events) in entries {
666            match self.lanes.cancel_queued(lane, &entry) {
667                Err(_) | Ok(Cancelled::AlreadyConsumed) => self.forget_steer(&entry),
668                Ok(outcome) if outcome == Cancelled::Cancelled || aborted => {
669                    self.forget_steer(&entry);
670                    let kept: Vec<RuntimeEvent> = events
671                        .into_iter()
672                        .filter(|e| !((e.kind == "job_done" || e.kind == "tool_done") && e.status == "cancelled"))
673                        .collect();
674                    if !kept.is_empty() {
675                        self.deliver(lane, kept);
676                    }
677                }
678                Ok(_) => {}
679            }
680        }
681    }
682}
683
684/// The root a batch belongs to: the newest event's that names one.
685fn chain_root(events: &[RuntimeEvent]) -> Option<String> {
686    events.iter().rev().find_map(|e| e.root_id.clone())
687}