Skip to main content

infinity_agent_core/system/local/
driver.rs

1//! The per-thread driver loop used by local (resident) agent systems.
2//!
3//! One driver owns one thread: it batches inputs from the thread's
4//! channel, runs [`Thread::step`]s, interrupts an in-flight completion when
5//! user text arrives, defers synthetic events while a tool call is pending,
6//! triggers auto-compaction, and idles out when there is nothing left to
7//! wait for. The [router](super::router) respawns a driver when new input
8//! arrives for an idle thread.
9
10use rap_protocol::ThreadId;
11use std::collections::HashSet;
12use std::future::Future;
13use std::pin::Pin;
14use std::rc::Rc;
15use std::sync::{Arc, Mutex};
16
17use tokio::sync::{mpsc, oneshot};
18
19use crate::message::{InputMessage, InputMessageContent, SyntheticKind, TaggedSyntheticKind};
20use crate::traits::{ConversationStore, StateStore};
21use infinity_provider_protocol::message::UserContent;
22use rap_client::http::HttpClient;
23
24use super::sender::ChannelSender;
25use crate::system::builder::SystemInner;
26use crate::system::defer::InMemoryDeferQueue;
27use crate::system::events::AgentEvent;
28use crate::system::observer::ThreadObserver;
29use crate::system::thread::{StepOutcome, Thread, is_user_text_input};
30
31type BoxError = Box<dyn std::error::Error + Send + Sync>;
32
33type StepFuture<'a> = Pin<Box<dyn Future<Output = Result<StepOutcome, BoxError>> + 'a>>;
34
35/// An in-flight step paired with its cancellation handle. Keeping the two in
36/// one value makes it impossible for them to desynchronize: interrupting the
37/// step consumes both, and dropping the pair (e.g. on driver teardown) drops
38/// the sender, which the step observes as a cancellation.
39struct InFlightStep<'a> {
40    fut: StepFuture<'a>,
41    cancel_tx: oneshot::Sender<()>,
42}
43
44impl InFlightStep<'_> {
45    /// Interrupt the step and wait for it to wind down. The cancellation
46    /// path flushes whatever streamed so far to the store before returning,
47    /// so no partial turn is lost. Inputs the model produced no output for
48    /// stay in memory (unpersisted) and are re-sent on the next round.
49    async fn cancel(self) -> Result<StepOutcome, BoxError> {
50        let _ = self.cancel_tx.send(());
51        self.fut.await
52    }
53}
54
55/// Set of thread IDs with a live driver.
56pub type ActiveThreads = Arc<Mutex<HashSet<ThreadId>>>;
57
58/// A transition in one thread driver's liveness.
59///
60/// For one driver, [`ThreadLifecycleState::Live`] is always reported before
61/// its matching [`ThreadLifecycleState::Idle`]. Because drivers respawn on
62/// demand, one thread can report many live/idle pairs over its lifetime.
63#[derive(Debug, Clone, PartialEq, Eq)]
64pub struct ThreadLifecycleEvent {
65    /// The thread whose driver changed state.
66    pub thread_id: ThreadId,
67    /// The driver's new lifecycle state.
68    pub state: ThreadLifecycleState,
69}
70
71/// One thread driver's current liveness.
72#[derive(Debug, Clone, Copy, PartialEq, Eq)]
73pub enum ThreadLifecycleState {
74    /// The driver spawned and is processing input or waiting on in-flight work.
75    Live,
76    /// The driver exited because no work is queued or expected. Active
77    /// subscriptions may still respawn it when an event arrives.
78    Idle,
79}
80
81fn is_compaction_complete(msg: &InputMessage) -> bool {
82    msg.synthetic
83        .as_ref()
84        .is_some_and(SyntheticKind::is_compaction_complete)
85}
86
87pub(crate) async fn drive_thread<C, S, H, O>(
88    inner: Rc<SystemInner<C, S, ChannelSender, H>>,
89    thread_id: ThreadId,
90    mut rx: mpsc::UnboundedReceiver<(InputMessage, String)>,
91    mut subscribe_rx: mpsc::UnboundedReceiver<(O::SubscribeRequest, oneshot::Sender<()>)>,
92    observer: O,
93    active_threads: ActiveThreads,
94    lifecycle_tx: mpsc::UnboundedSender<ThreadLifecycleEvent>,
95) where
96    C: ConversationStore + 'static,
97    S: StateStore + 'static,
98    H: HttpClient + 'static,
99    O: ThreadObserver + 'static,
100{
101    active_threads
102        .lock()
103        .expect("bug: mutex poisoned")
104        .insert(thread_id.to_owned());
105    // Report liveness transitions at the same two points where
106    // `active_threads` changes, so the channel and the set can never
107    // disagree about a driver's state.
108    let _ = lifecycle_tx.send(ThreadLifecycleEvent {
109        thread_id: thread_id.clone(),
110        state: ThreadLifecycleState::Live,
111    });
112    let _guard = DriverGuard {
113        thread_id: thread_id.clone(),
114        active_threads,
115        lifecycle_tx,
116    };
117
118    let thread = match Thread::load(inner, thread_id.clone()).await {
119        Ok(t) => t,
120        Err(e) => {
121            observer.on_event(
122                &thread_id,
123                &AgentEvent::Info {
124                    text: format!("Error: {e}"),
125                },
126            );
127            return;
128        }
129    };
130
131    let mut defer = InMemoryDeferQueue::new();
132    // Inputs that queued up while a step was in flight (or were generated
133    // internally, like the auto-compaction trigger).
134    let mut pending_batch: Vec<(InputMessage, String)> = Vec::new();
135    let mut compaction_triggered = false;
136    let mut total_tokens: u64 = 0;
137    // Set when subscribe_rx is closed, so selects stop polling it (a closed
138    // channel is always ready and would otherwise spin).
139    let mut subscribe_closed = false;
140
141    let mut in_flight: Option<InFlightStep<'_>> = None;
142
143    loop {
144        let new_inputs: Vec<(InputMessage, String)> = if let Some(flight) = in_flight.as_mut() {
145            tokio::select! {
146                biased;
147
148                res = &mut flight.fut => {
149                    in_flight = None;
150                    handle_step_result(
151                        res,
152                        &observer,
153                        &thread_id,
154                        &mut total_tokens,
155                        &mut compaction_triggered,
156                        &mut pending_batch,
157                    );
158                    continue;
159                },
160                first = rx.recv() => {
161                    let Some(first) = first else {
162                        // Input channel closed — the system is shutting down.
163                        // Interrupt the in-flight step and wait for it to wind
164                        // down so pending history items are synced to the
165                        // store.
166                        let _ = in_flight
167                            .take()
168                            .expect("bug: in-flight step missing during shutdown")
169                            .cancel()
170                            .await;
171                        return;
172                    };
173                    let mut batch = vec![first];
174                    while let Ok(item) = rx.try_recv() {
175                        batch.push(item);
176                    }
177
178                    if batch.iter().any(|(msg, _)| is_user_text_input(msg)) {
179                        // User text interrupts the in-flight completion.
180                        let res = in_flight
181                            .take()
182                            .expect("bug: in-flight step missing during interrupt")
183                            .cancel()
184                            .await;
185                        handle_step_result(
186                            res,
187                            &observer,
188                            &thread_id,
189                            &mut total_tokens,
190                            &mut compaction_triggered,
191                            &mut pending_batch,
192                        );
193
194                        let (mut user_inputs, non_user_inputs): (Vec<_>, Vec<_>) = batch
195                            .into_iter()
196                            .partition(|(msg, _)| is_user_text_input(msg));
197
198                        if let InputMessageContent::User(UserContent::Text(text)) =
199                            &mut user_inputs[0].0.content
200                        {
201                            text.text = format!("<interrupt>{}", text.text);
202                        } else {
203                            panic!("bug: user_inputs should only contain user text");
204                        }
205
206                        pending_batch.extend(non_user_inputs);
207                        user_inputs
208                    } else {
209                        pending_batch.extend(batch);
210                        continue;
211                    }
212                },
213                req = subscribe_rx.recv(), if !subscribe_closed => {
214                    match req {
215                        Some((req, ack)) => {
216                            observer.on_subscribe(&thread_id, req, thread.replay_snapshot());
217                            let _ = ack.send(());
218                        }
219                        None => subscribe_closed = true,
220                    }
221                    continue;
222                }
223            }
224        } else {
225            let mut batch: Vec<(InputMessage, String)> = std::mem::take(&mut pending_batch);
226            while let Ok(item) = rx.try_recv() {
227                batch.push(item);
228            }
229
230            // Deferred events whose blocking tool call has been resolved must
231            // be processed even if nothing else arrives.
232            let defer_ready = !defer.is_empty() && thread.pending_active_tool_call().is_none();
233
234            if batch.is_empty() && !defer_ready {
235                match check_before_idle(
236                    &mut batch,
237                    &mut rx,
238                    &mut subscribe_rx,
239                    &observer,
240                    &thread,
241                    &thread_id,
242                ) {
243                    IdleDecision::Continue => {
244                        // Input arrived after the initial drain but before the
245                        // synchronous idle check. Keep this driver alive and
246                        // process it instead of publishing a transient idle.
247                    }
248                    IdleDecision::Exit => {
249                        tracing::info!("Thread {} going idle", thread_id);
250                        return;
251                    }
252                    IdleDecision::AwaitInput => {
253                        // Park until something arrives.
254                        loop {
255                            tokio::select! {
256                                biased;
257                                msg = rx.recv() => {
258                                    match msg {
259                                        Some(m) => {
260                                            batch.push(m);
261                                            break;
262                                        }
263                                        None => return,
264                                    }
265                                }
266                                req = subscribe_rx.recv(), if !subscribe_closed => {
267                                    match req {
268                                        Some((req, ack)) => {
269                                            observer.on_subscribe(&thread_id, req, thread.replay_snapshot());
270                                            let _ = ack.send(());
271                                        }
272                                        None => subscribe_closed = true,
273                                    }
274                                }
275                            }
276                        }
277                        while let Ok(item) = rx.try_recv() {
278                            batch.push(item);
279                        }
280                    }
281                }
282            }
283
284            batch
285        };
286
287        // Reset the compaction trigger (and the stale token count) once a
288        // compaction round-trips: the next completion reports fresh usage.
289        for (msg, _) in &new_inputs {
290            if is_compaction_complete(msg) {
291                compaction_triggered = false;
292                total_tokens = 0;
293            }
294        }
295
296        let batch = match thread.filter_deferrable(new_inputs, &mut defer).await {
297            Ok(batch) => batch,
298            Err(e) => {
299                observer.on_event(
300                    &thread_id,
301                    &AgentEvent::Info {
302                        text: format!("Error: {e}"),
303                    },
304                );
305                continue;
306            }
307        };
308        if batch.is_empty() {
309            // Everything was deferred (or nothing arrived); wait for more input.
310            continue;
311        }
312
313        let (cancel_tx, cancel_rx) = oneshot::channel::<()>();
314        in_flight = Some(InFlightStep {
315            fut: Box::pin(thread.step_no_defer(batch, &observer, cancel_rx)),
316            cancel_tx,
317        });
318    }
319}
320
321enum IdleDecision {
322    Continue,
323    Exit,
324    AwaitInput,
325}
326
327/// Perform the final idle check without yielding to the router task. Since the
328/// router shares this `LocalSet`, an input either appears in `rx` here or is
329/// sent after the receivers are dropped, causing the router to spawn a new
330/// driver.
331fn check_before_idle<C, S, H, O>(
332    batch: &mut Vec<(InputMessage, String)>,
333    rx: &mut mpsc::UnboundedReceiver<(InputMessage, String)>,
334    subscribe_rx: &mut mpsc::UnboundedReceiver<(O::SubscribeRequest, oneshot::Sender<()>)>,
335    observer: &O,
336    thread: &Thread<C, S, ChannelSender, H>,
337    thread_id: &ThreadId<str>,
338) -> IdleDecision
339where
340    C: ConversationStore + 'static,
341    S: StateStore + 'static,
342    H: HttpClient + 'static,
343    O: ThreadObserver,
344{
345    while let Ok((request, ack)) = subscribe_rx.try_recv() {
346        observer.on_subscribe(thread_id, request, thread.replay_snapshot());
347        let _ = ack.send(());
348    }
349
350    if let Ok(item) = rx.try_recv() {
351        batch.push(item);
352        while let Ok(item) = rx.try_recv() {
353            batch.push(item);
354        }
355        IdleDecision::Continue
356    } else if thread.awaiting_tool_result() {
357        IdleDecision::AwaitInput
358    } else {
359        // Active subscriptions do not keep the driver resident. Their events
360        // respawn it; embeddings can keep associated resources alive based on
361        // subscription state when they observe this idle transition.
362        IdleDecision::Exit
363    }
364}
365
366fn handle_step_result(
367    res: Result<StepOutcome, BoxError>,
368    observer: &impl ThreadObserver,
369    thread_id: &ThreadId<str>,
370    total_tokens: &mut u64,
371    compaction_triggered: &mut bool,
372    pending_batch: &mut Vec<(InputMessage, String)>,
373) {
374    match res {
375        Ok(StepOutcome::Completed {
376            usage,
377            context_window,
378        }) => {
379            if let Some(u) = usage {
380                // Use total_tokens which includes cached input. When prompt
381                // caching is active, `input_tokens` alone undercounts usage.
382                *total_tokens = u.total_tokens;
383            }
384            // Background compaction: trigger when the context is > 75% full.
385            if !*compaction_triggered
386                && context_window > 0
387                && *total_tokens as usize > context_window * 3 / 4
388            {
389                *compaction_triggered = true;
390                tracing::info!(
391                    "Auto-compaction for thread {}: {} total tokens > 75% of {} context window",
392                    thread_id,
393                    total_tokens,
394                    context_window
395                );
396                observer.on_event(
397                    thread_id,
398                    &AgentEvent::Info {
399                        text: "✦ Auto-compaction triggered (context > 75%)".to_owned(),
400                    },
401                );
402                pending_batch.push((
403                    InputMessage {
404                        content: InputMessageContent::User(UserContent::text("")),
405                        group_id: thread_id.to_owned(),
406                        metadata: None,
407                        synthetic: Some(SyntheticKind::Tagged(TaggedSyntheticKind::Compaction)),
408                        display_as: None,
409                        subscription: false,
410                    },
411                    uuid::Uuid::new_v4().to_string(),
412                ));
413            }
414        }
415        Ok(StepOutcome::Skipped) => {}
416        Err(e) => {
417            observer.on_event(
418                thread_id,
419                &AgentEvent::Info {
420                    text: format!("Error: {e}"),
421                },
422            );
423        }
424    }
425}
426
427struct DriverGuard {
428    thread_id: ThreadId,
429    active_threads: ActiveThreads,
430    lifecycle_tx: mpsc::UnboundedSender<ThreadLifecycleEvent>,
431}
432
433impl Drop for DriverGuard {
434    fn drop(&mut self) {
435        self.active_threads
436            .lock()
437            .expect("bug: mutex poisoned")
438            .remove(&self.thread_id);
439        // Notify the embedding that this thread's driver exited, so it can
440        // release per-conversation resources once nothing else is live.
441        let _ = self.lifecycle_tx.send(ThreadLifecycleEvent {
442            thread_id: self.thread_id.clone(),
443            state: ThreadLifecycleState::Idle,
444        });
445    }
446}
447#[cfg(test)]
448mod tests {
449    use crate::system::events::AgentEvent;
450    use crate::system::test_support::*;
451
452    /// An active subscription does not keep a driver resident: once the
453    /// subscription tool's result settles the call, the driver idles out, and a
454    /// later subscription event respawns it.
455    #[tokio::test(flavor = "current_thread")]
456    async fn subscribed_thread_idles_and_event_respawns_it() {
457        let local = tokio::task::LocalSet::new();
458        local
459            .run_until(async {
460                let (mut running, mut rx, mut ctrl, _conv) =
461                    start_system(vec![Box::new(SubscribeTool)], None);
462                running
463                    .send_user_text(rap_protocol::ThreadId::from_ref("t1"), "subscribe")
464                    .await;
465                let _req = ctrl.next_request().await;
466                ctrl.send_tool_call("tc-sub", "subscribe_tool", serde_json::json!({}));
467                ctrl.finish();
468                collect_until_finished(&mut rx).await;
469
470                // The subscription result triggers one more round; after it the
471                // thread holds an active subscription and nothing else.
472                let _req2 = ctrl.next_request().await;
473                ctrl.send_text("subscribed");
474                ctrl.finish();
475                collect_until_finished(&mut rx).await;
476
477                wait_idle(&mut running).await;
478                assert!(
479                    running.is_idle(),
480                    "an active subscription must not keep the driver resident"
481                );
482
483                // A subscription event arrives later: the router respawns a
484                // driver and the event is processed in a child thread, which
485                // reports a request to the model.
486                running
487                    .send(
488                        subscription_event_input("t1", "tc-sub", "late event").0,
489                        "ev-late",
490                    )
491                    .await;
492                let child_req = ctrl.next_request().await;
493                assert!(
494                    tool_result_texts(&child_req)
495                        .iter()
496                        .any(|t| t.contains("late event")),
497                    "the event must reach a respawned driver"
498                );
499                ctrl.send_text("handled");
500                ctrl.finish();
501            })
502            .await;
503    }
504
505    #[tokio::test(flavor = "current_thread")]
506    async fn auto_compaction_triggers_and_applies() {
507        let local = tokio::task::LocalSet::new();
508        local
509            .run_until(async {
510                // Tiny context window so the usage report crosses the threshold.
511                let (mut running, mut rx, mut ctrl, _conv) =
512                    start_system(vec![], Some(small_context_entry()));
513                running
514                    .send_user_text(rap_protocol::ThreadId::from_ref("t1"), "hello")
515                    .await;
516                let _req = ctrl.next_request().await;
517                ctrl.send_text("hi there");
518                ctrl.finish_with_usage(high_usage());
519                collect_until_finished(&mut rx).await;
520
521                // The driver spawns a compaction child; close it with a summary.
522                let creq = ctrl.next_request().await;
523                assert!(is_compaction_req(&creq));
524                handle_compaction_child(&mut ctrl, &creq, "summary of everything");
525
526                // The parent eventually applies the compaction.
527                loop {
528                    if let Evt::E(AgentEvent::CompactionApplied) = next_evt(&mut rx).await {
529                        break;
530                    }
531                }
532                wait_idle(&mut running).await;
533            })
534            .await;
535    }
536}