Skip to main content

infinity_agent_core/system/local/
router.rs

1//! The router: dispatches messages to per-thread drivers, spawning them on
2//! demand. This plus the [drivers](super::driver) is the "actor system" of a
3//! local agent system. The router owns the driver futures directly (one
4//! `FuturesUnordered` pool rather than one spawned task per thread), so a
5//! driver's memory is fully released the moment it goes idle.
6
7use rap_protocol::ThreadId;
8use std::collections::HashMap;
9use std::rc::Rc;
10
11use futures_util::StreamExt;
12use futures_util::stream::FuturesUnordered;
13use tokio::sync::{mpsc, oneshot};
14use tokio_util::sync::CancellationToken;
15
16use crate::message::InputMessage;
17use crate::traits::{ConversationStore, InputSender, StateStore};
18use rap_client::http::HttpClient;
19
20use super::driver::{ActiveThreads, ThreadLifecycleEvent, drive_thread};
21use super::sender::ChannelSender;
22use crate::system::builder::{LocalAgentSystem, SystemInner};
23use crate::system::observer::ThreadObserver;
24use crate::system::thread::is_user_text_input;
25
26/// A subscribe request routed through the local system: the target thread,
27/// the observer-specific request, and an ack fired once the subscriber has
28/// been installed (its replay sent and its registration completed).
29pub(crate) type SubscribeMessage<Sub> = (ThreadId, Sub, oneshot::Sender<()>);
30
31/// A clonable handle for attaching subscribers to a running system's threads.
32pub struct SubscribeHandle<Sub: Send + 'static> {
33    tx: mpsc::UnboundedSender<SubscribeMessage<Sub>>,
34}
35
36impl<Sub: Send + 'static> Clone for SubscribeHandle<Sub> {
37    fn clone(&self) -> Self {
38        Self {
39            tx: self.tx.clone(),
40        }
41    }
42}
43
44impl<Sub: Send + 'static> SubscribeHandle<Sub> {
45    /// Attach a subscriber to a thread. The request is handed to that
46    /// thread's [`ThreadObserver::on_subscribe`] together with a live replay
47    /// snapshot (spawning a driver for the thread if none is running).
48    ///
49    /// Resolves once the subscriber is **installed** — its replay has been
50    /// sent and it is registered for live events — so a caller that
51    /// subscribes and then sends a message is guaranteed to observe that
52    /// message's events. A driver that exits while requests race in hands
53    /// them back to the router, so installation is reliable; `false` is
54    /// returned only if the whole system was shut down.
55    pub async fn subscribe(&self, thread_id: &ThreadId<str>, request: Sub) -> bool {
56        let (ack_tx, ack_rx) = oneshot::channel();
57        if self
58            .tx
59            .send((thread_id.to_owned(), request, ack_tx))
60            .is_err()
61        {
62            return false;
63        }
64        // An error means the ack sender was dropped mid-shutdown — the
65        // subscriber was not installed.
66        ack_rx.await.is_ok()
67    }
68}
69
70/// A running local agent system: the router task plus handles for feeding it
71/// input, attaching subscribers, and observing thread lifecycle.
72///
73/// The system runs until [`shutdown`](Self::shutdown) consumes it: individual
74/// threads idle out (and respawn on demand) while the router keeps running, so
75/// sending to a thread never races a teardown.
76///
77/// `Sub` is the observer's
78/// [`SubscribeRequest`](ThreadObserver::SubscribeRequest) type.
79pub struct RunningSystem<Sub: Send + 'static> {
80    sender: ChannelSender,
81    subscribe_tx: mpsc::UnboundedSender<SubscribeMessage<Sub>>,
82    active_threads: ActiveThreads,
83    lifecycle_rx: mpsc::UnboundedReceiver<ThreadLifecycleEvent>,
84    shutdown: CancellationToken,
85    task: tokio::task::JoinHandle<()>,
86}
87
88impl<Sub: Send + 'static> RunningSystem<Sub> {
89    /// The system's [`InputSender`](crate::traits::InputSender), for callback
90    /// servers and any other external message injectors.
91    pub fn sender(&self) -> ChannelSender {
92        self.sender.clone()
93    }
94
95    /// Deliver an input message to its thread (`message.group_id`).
96    /// `dedup_id` should be stable across redeliveries of the same message.
97    pub async fn send(&self, message: InputMessage, dedup_id: &str) {
98        self.sender
99            .send_to_input_queue(message, dedup_id)
100            .await
101            .expect("bug: router exited while the system was alive");
102    }
103
104    /// Convenience: send plain user text to a thread.
105    pub async fn send_user_text(&self, thread_id: &ThreadId<str>, text: impl Into<String>) {
106        let msg = InputMessage::user_text(thread_id.to_owned(), text);
107        self.send(msg, &uuid::Uuid::new_v4().to_string()).await
108    }
109
110    /// Attach a subscriber to a thread; resolves once the subscriber is
111    /// installed. See [`SubscribeHandle::subscribe`].
112    pub async fn subscribe(&self, thread_id: &ThreadId<str>, request: Sub) {
113        assert!(
114            self.subscribe_handle().subscribe(thread_id, request).await,
115            "bug: router exited while the system was alive"
116        );
117    }
118
119    /// A clonable handle for attaching subscribers (see
120    /// [`SubscribeHandle::subscribe`]).
121    pub fn subscribe_handle(&self) -> SubscribeHandle<Sub> {
122        SubscribeHandle {
123            tx: self.subscribe_tx.clone(),
124        }
125    }
126
127    /// Thread IDs with a live driver.
128    #[cfg(test)]
129    pub(crate) fn active_threads(&self) -> ActiveThreads {
130        self.active_threads.clone()
131    }
132
133    /// Whether no thread driver is currently live. Threads with active
134    /// subscriptions but no pending work do not count as live; their events
135    /// respawn a driver when they arrive.
136    pub fn is_idle(&self) -> bool {
137        self.active_threads
138            .lock()
139            .expect("bug: mutex poisoned")
140            .is_empty()
141    }
142
143    /// Wait for the next thread-driver lifecycle transition. Returns `None`
144    /// after the router shuts down and all lifecycle senders are dropped.
145    pub async fn next_lifecycle_event(&mut self) -> Option<ThreadLifecycleEvent> {
146        self.lifecycle_rx.recv().await
147    }
148
149    /// Return the next queued lifecycle transition without waiting.
150    pub fn try_next_lifecycle_event(
151        &mut self,
152    ) -> Result<ThreadLifecycleEvent, mpsc::error::TryRecvError> {
153        self.lifecycle_rx.try_recv()
154    }
155
156    /// Wind the whole system down (process exit): every driver interrupts
157    /// its in-flight completion (flushing pending history to the store) and
158    /// exits; resolves when the wind-down is complete.
159    pub async fn shutdown(self) {
160        self.shutdown.cancel();
161        if let Err(e) = self.task.await {
162            tracing::error!("router task failed during shutdown: {e}");
163        }
164    }
165}
166
167impl<C, S, H> LocalAgentSystem<C, S, H>
168where
169    C: ConversationStore + 'static,
170    S: StateStore + 'static,
171    H: HttpClient + 'static,
172{
173    /// Run the system with a custom [`ThreadObserver`]. Use this lower-level
174    /// path when the embedding owns event fan-out and thread identity.
175    ///
176    /// `make_observer` creates the observer for each thread driver as it
177    /// spawns. Drivers are spawned on demand when a thread receives its
178    /// first message and respawned when an idle thread receives another; the
179    /// same thread never has two drivers.
180    pub fn start_with_observer<O, F>(self, make_observer: F) -> RunningSystem<O::SubscribeRequest>
181    where
182        O: ThreadObserver + 'static,
183        F: Fn(&ThreadId<str>) -> O + 'static,
184    {
185        self.start_inner(make_observer)
186    }
187}
188
189impl<C, S, H> LocalAgentSystem<C, S, H>
190where
191    C: ConversationStore + 'static,
192    S: StateStore + 'static,
193    H: HttpClient + 'static,
194{
195    /// Spawn the router on the current
196    /// [`LocalSet`](tokio::task::LocalSet).
197    pub(crate) fn start_inner<O, F>(self, make_observer: F) -> RunningSystem<O::SubscribeRequest>
198    where
199        O: ThreadObserver + 'static,
200        F: Fn(&ThreadId<str>) -> O + 'static,
201    {
202        let sender = self.system.inner.sender.clone();
203        let (subscribe_tx, subscribe_rx) = mpsc::unbounded_channel();
204        let (lifecycle_tx, thread_lifecycle) = mpsc::unbounded_channel();
205        let active_threads: ActiveThreads = Default::default();
206        let shutdown = CancellationToken::new();
207
208        let task = tokio::task::spawn_local(route_loop(
209            self.system.inner,
210            self.input_rx,
211            subscribe_rx,
212            make_observer,
213            active_threads.clone(),
214            lifecycle_tx,
215            shutdown.clone(),
216        ));
217
218        RunningSystem {
219            sender,
220            subscribe_tx,
221            active_threads,
222            lifecycle_rx: thread_lifecycle,
223            shutdown,
224            task,
225        }
226    }
227}
228
229struct WorkerChannels<Sub> {
230    input_tx: mpsc::UnboundedSender<(InputMessage, String)>,
231    subscribe_tx: mpsc::UnboundedSender<(Sub, oneshot::Sender<()>)>,
232}
233
234enum RoutedMessage<Sub> {
235    Input(Box<InputMessage>, String),
236    Subscribe(ThreadId, Sub, oneshot::Sender<()>),
237}
238
239async fn route_loop<C, S, H, O, F>(
240    inner: Rc<SystemInner<C, S, ChannelSender, H>>,
241    mut input_rx: mpsc::UnboundedReceiver<(InputMessage, String)>,
242    mut subscribe_rx: mpsc::UnboundedReceiver<SubscribeMessage<O::SubscribeRequest>>,
243    make_observer: F,
244    active_threads: ActiveThreads,
245    lifecycle_tx: mpsc::UnboundedSender<ThreadLifecycleEvent>,
246    shutdown: CancellationToken,
247) where
248    C: ConversationStore + 'static,
249    S: StateStore + 'static,
250    H: HttpClient + 'static,
251    O: ThreadObserver + 'static,
252    F: Fn(&ThreadId<str>) -> O + 'static,
253{
254    let mut workers: HashMap<ThreadId, WorkerChannels<O::SubscribeRequest>> = HashMap::new();
255    let mut subscribe_closed = false;
256    // The router owns the driver futures directly (instead of spawning each
257    // as its own task): a completed driver yields its thread ID and its
258    // memory — the future itself and its worker entry — is released
259    // immediately. With per-thread tasks, both would be retained until the
260    // thread's next message, which adds up across many idle threads.
261    let mut drivers = FuturesUnordered::new();
262
263    loop {
264        let msg: Option<RoutedMessage<O::SubscribeRequest>> = tokio::select! {
265            biased;
266            _ = shutdown.cancelled() => None,
267            exited = drivers.next(), if !drivers.is_empty() => {
268                let exited: ThreadId = exited.expect("bug: drivers is empty");
269                // Only remove the exited driver's own entry: if the thread
270                // already respawned, the new entry's channel is still open.
271                if workers.get(&exited).is_some_and(|w| w.input_tx.is_closed()) {
272                    workers.remove(&exited);
273                }
274                continue;
275            }
276            msg = input_rx.recv() => msg.map(|(m, id)| RoutedMessage::Input(Box::new(m), id)),
277            req = subscribe_rx.recv(), if !subscribe_closed => {
278                match req {
279                    Some((thread_id, req, ack)) => Some(RoutedMessage::Subscribe(thread_id, req, ack)),
280                    None => {
281                        subscribe_closed = true;
282                        continue;
283                    }
284                }
285            }
286        };
287        let Some(msg) = msg else { break };
288
289        let thread_id = match &msg {
290            RoutedMessage::Input(input, _) => input.group_id.clone(),
291            RoutedMessage::Subscribe(thread_id, _, _) => thread_id.clone(),
292        };
293
294        // Reuse a live driver if one exists.
295        if let Some(w) = workers.get(&thread_id) {
296            if !w.input_tx.is_closed() {
297                match msg {
298                    RoutedMessage::Input(input, id) => {
299                        let _ = w.input_tx.send((*input, id));
300                    }
301                    RoutedMessage::Subscribe(_, req, ack) => {
302                        let _ = w.subscribe_tx.send((req, ack));
303                    }
304                }
305                continue;
306            }
307            workers.remove(&thread_id);
308        }
309
310        // Admission: event-style input cannot create or resume a thread. User
311        // text bypasses both checks because it is how threads are created and
312        // stopped threads are resumed. Subscribe requests are also ungated so
313        // a client can attach before a thread's first activity.
314        if let RoutedMessage::Input(input, dedup_id) = &msg
315            && !is_user_text_input(input)
316        {
317            match inner.conversation_store.thread_exists(&thread_id).await {
318                Ok(true) => {}
319                Ok(false) => {
320                    tracing::info!(
321                        %thread_id,
322                        %dedup_id,
323                        "dropping event for unknown thread",
324                    );
325                    continue;
326                }
327                Err(e) => {
328                    // Fail open: a flaky store must not drop a real tool
329                    // result; the driver's own preparation still absorbs
330                    // anything stale.
331                    tracing::warn!(
332                        %thread_id,
333                        "thread existence check failed, processing event anyway: {e}",
334                    );
335                }
336            }
337
338            match inner.state_store.is_thread_stopped(&thread_id).await {
339                Ok(false) => {}
340                Ok(true) => {
341                    tracing::info!(
342                        %thread_id,
343                        %dedup_id,
344                        "dropping event for stopped thread",
345                    );
346                    continue;
347                }
348                Err(e) => {
349                    // Fail open for the same reason as the existence check.
350                    tracing::warn!(
351                        %thread_id,
352                        "stopped-thread check failed, processing event anyway: {e}",
353                    );
354                }
355            }
356        }
357
358        // Spawn a new driver.
359        let (input_tx, input_rx_worker) = mpsc::unbounded_channel();
360        let (worker_subscribe_tx, worker_subscribe_rx) = mpsc::unbounded_channel();
361        let observer = make_observer(&thread_id);
362
363        drivers.push({
364            let thread_id = thread_id.clone();
365            let driver = drive_thread(
366                inner.clone(),
367                thread_id.clone(),
368                input_rx_worker,
369                worker_subscribe_rx,
370                observer,
371                active_threads.clone(),
372                lifecycle_tx.clone(),
373            );
374            async move {
375                // Contain a panicking driver to its own thread (the panic is
376                // logged); the router and the other drivers keep running.
377                rap_protocol::log_panic("thread_driver", driver).await;
378                thread_id
379            }
380        });
381
382        match msg {
383            RoutedMessage::Input(input, id) => {
384                let _ = input_tx.send((*input, id));
385            }
386            RoutedMessage::Subscribe(_, req, ack) => {
387                let _ = worker_subscribe_tx.send((req, ack));
388            }
389        }
390        workers.insert(
391            thread_id,
392            WorkerChannels {
393                input_tx,
394                subscribe_tx: worker_subscribe_tx,
395            },
396        );
397    }
398
399    // Wind down: dropping each driver's channels signals it to interrupt any
400    // in-flight completion (which flushes pending history items to the store)
401    // and exit. Drive every remaining driver to completion so the embedding
402    // is not torn down underneath them.
403    drop(workers);
404    while drivers.next().await.is_some() {}
405}
406#[cfg(test)]
407mod tests {
408    use super::ThreadLifecycleEvent;
409    use crate::system::events::AgentEvent;
410    use crate::system::local::ThreadLifecycleState;
411    use crate::system::test_support::*;
412
413    /// A stale event addressed to a thread that was never created must not wake
414    /// a phantom driver: no store records appear and no thread exit is reported.
415    #[tokio::test(flavor = "current_thread")]
416    async fn stale_event_to_unknown_thread_wakes_nothing() {
417        let local = tokio::task::LocalSet::new();
418        local
419            .run_until(async {
420                let (mut running, mut rx, mut ctrl, conv) = start_system(vec![], None);
421
422                // Create a real thread so the system is not trivially empty.
423                running
424                    .send_user_text(rap_protocol::ThreadId::from_ref("t1"), "hello")
425                    .await;
426                let _req = ctrl.next_request().await;
427                ctrl.send_text("hi");
428                ctrl.finish();
429                collect_until_finished(&mut rx).await;
430                wait_idle(&mut running).await;
431
432                // A stale subscription event and a stale tool result for a
433                // thread that does not exist.
434                running
435                    .send(
436                        subscription_event_input("ghost", "tc-ghost", "stale event").0,
437                        "ghost-sub",
438                    )
439                    .await;
440                running
441                    .send(
442                        tool_result_input("ghost", "tc-ghost", "stale result").0,
443                        "ghost-res",
444                    )
445                    .await;
446                for _ in 0..8 {
447                    tokio::task::yield_now().await;
448                }
449
450                assert!(
451                    running.is_idle(),
452                    "no driver may wake for an unknown thread"
453                );
454                // Lifecycle notifications for t1 may still be queued; what must
455                // never appear is any transition for the ghost thread.
456                while let Ok(event) = running.try_next_lifecycle_event() {
457                    assert_ne!(
458                        event.thread_id.as_str(),
459                        "ghost",
460                        "no lifecycle transition may be reported for a dropped event"
461                    );
462                }
463                assert!(
464                    conv.thread_info(rap_protocol::ThreadId::from_ref("ghost"))
465                        .is_none(),
466                    "a dropped event must not create thread records"
467                );
468            })
469            .await;
470    }
471
472    /// User text to a brand-new thread ID must still create the thread: the wake
473    /// policy gates only events.
474    #[tokio::test(flavor = "current_thread")]
475    async fn user_text_still_creates_new_threads() {
476        let local = tokio::task::LocalSet::new();
477        local
478            .run_until(async {
479                let (running, mut rx, mut ctrl, conv) = start_system(vec![], None);
480                running
481                    .send_user_text(rap_protocol::ThreadId::from_ref("brand-new"), "hello")
482                    .await;
483                let _req = ctrl.next_request().await;
484                ctrl.send_text("created");
485                ctrl.finish();
486                collect_until_finished(&mut rx).await;
487                assert!(
488                    conv.thread_info(rap_protocol::ThreadId::from_ref("brand-new"))
489                        .is_some()
490                );
491            })
492            .await;
493    }
494
495    /// An event for a real, idled-out thread must still respawn its driver: the
496    /// existence check refuses unknown threads, not idle ones. The stale result is
497    /// absorbed during preparation, but the driver wakes to do it, observable as
498    /// a `Live` transition followed by another `Idle`.
499    #[tokio::test(flavor = "current_thread")]
500    async fn event_respawns_idle_existing_thread() {
501        let local = tokio::task::LocalSet::new();
502        local
503            .run_until(async {
504                let (mut running, mut rx, mut ctrl, _conv) = start_system(vec![], None);
505
506                running
507                    .send_user_text(rap_protocol::ThreadId::from_ref("t1"), "hello")
508                    .await;
509                let _req = ctrl.next_request().await;
510                ctrl.send_text("hi");
511                ctrl.finish();
512                collect_until_finished(&mut rx).await;
513                wait_idle(&mut running).await;
514                // Drain transitions from the first driver so the assertions below
515                // observe only the respawned one.
516                while running.try_next_lifecycle_event().is_ok() {}
517
518                running
519                    .send(
520                        tool_result_input("t1", "tc-old", "late result").0,
521                        "res-late",
522                    )
523                    .await;
524                let live = tokio::time::timeout(
525                    std::time::Duration::from_secs(5),
526                    running.next_lifecycle_event(),
527                )
528                .await
529                .expect("timed out waiting for the driver to respawn")
530                .expect("thread lifecycle channel closed");
531                assert_eq!(
532                    live,
533                    ThreadLifecycleEvent {
534                        thread_id: "t1".into(),
535                        state: ThreadLifecycleState::Live,
536                    },
537                    "the existing thread's driver must wake"
538                );
539                let idle = tokio::time::timeout(
540                    std::time::Duration::from_secs(5),
541                    running.next_lifecycle_event(),
542                )
543                .await
544                .expect("timed out waiting for the respawned driver to exit")
545                .expect("thread lifecycle channel closed");
546                assert_eq!(
547                    idle,
548                    ThreadLifecycleEvent {
549                        thread_id: "t1".into(),
550                        state: ThreadLifecycleState::Idle,
551                    },
552                    "the respawned driver must idle back out"
553                );
554            })
555            .await;
556    }
557
558    #[tokio::test(flavor = "current_thread")]
559    async fn shutdown_flushes_in_flight_turn() {
560        let local = tokio::task::LocalSet::new();
561        local
562            .run_until(async {
563                let (running, mut rx, mut ctrl, conv) = start_system(vec![], None);
564                running
565                    .send_user_text(rap_protocol::ThreadId::from_ref("t1"), "hello")
566                    .await;
567                let _req = ctrl.next_request().await;
568                ctrl.send_text("partial answer");
569                loop {
570                    if let Evt::E(AgentEvent::TextChunk { .. }) = next_evt(&mut rx).await {
571                        break;
572                    }
573                }
574
575                // Shut down mid-completion: the driver cancels, flushes, syncs.
576                running.shutdown().await;
577
578                use crate::traits::ConversationStore;
579                let history = conv
580                    .load_history_up_to(rap_protocol::ThreadId::from_ref("t1"), None, None)
581                    .await
582                    .expect("load history");
583                assert!(
584                    history
585                        .iter()
586                        .any(|m| matches!(m, crate::message::InfinityMessage::Assistant { .. })),
587                    "partial assistant text must be persisted on shutdown, got {history:?}"
588                );
589            })
590            .await;
591    }
592}