vtcode_core/llm/
request_gap.rs

1//! Shared tracker for detecting idle gaps between dispatched LLM requests.
2//!
3//! Both the interactive session state (`SessionStats` in
4//! `src/agent/runloop/unified/state.rs`, in the `vtcode` binary crate) and
5//! the headless session state (`AgentSessionState` in
6//! [`crate::core::agent::session`]) need to answer the same two questions:
7//! "has a request been dispatched yet this session" and "how long has it
8//! been since the last one". Both use that signal to warn when the
9//! provider's server-side prompt cache has likely expired, so the next
10//! request may unexpectedly re-pay full input cost. This tracker holds the
11//! single piece of state (`last_request_at`) and the logic derived from it,
12//! so each embedding site only has to delegate its existing public methods
13//! rather than reimplement the same `Option<Instant>` bookkeeping.
14
15use std::time::{Duration, Instant};
16
17/// Tracks the wall-clock time of the most recently dispatched LLM request.
18#[derive(Debug, Default, Clone, Copy)]
19pub struct RequestGapTracker {
20    last_request_at: Option<Instant>,
21}
22
23impl RequestGapTracker {
24    /// Records that an LLM request was just dispatched, so the next call to
25    /// [`Self::cache_gap_exceeds`] can measure the idle gap since this request.
26    pub fn note_request_sent(&mut self) {
27        self.last_request_at = Some(Instant::now());
28    }
29
30    /// Returns the elapsed time since the last dispatched request when it
31    /// exceeds `threshold`, or `None` if there was no prior request or the gap
32    /// is still within the threshold. Used to warn that the provider prompt
33    /// cache has likely expired before the next request re-pays full input
34    /// cost.
35    pub fn cache_gap_exceeds(&self, threshold: Duration) -> Option<Duration> {
36        let last_request_at = self.last_request_at?;
37        let elapsed = last_request_at.elapsed();
38        (elapsed > threshold).then_some(elapsed)
39    }
40
41    /// Returns whether an LLM request has been dispatched at any point this
42    /// session. More precise than inferring from accumulated token usage
43    /// (which can be zero even after a request, e.g. an error response).
44    pub fn has_sent_request(&self) -> bool {
45        self.last_request_at.is_some()
46    }
47}
48
49/// Formats an elapsed cache gap for the advisory message. Sub-minute gaps
50/// are shown in seconds (e.g. "45s") rather than rounding down to "0 min",
51/// which would otherwise happen for any threshold under 60 seconds.
52pub fn format_gap(elapsed: Duration) -> String {
53    let secs = elapsed.as_secs();
54    if secs < 60 {
55        format!("{secs}s")
56    } else {
57        format!("{} min", secs / 60)
58    }
59}
60
61#[cfg(test)]
62mod tests {
63    use super::*;
64
65    #[test]
66    fn cache_gap_exceeds_is_none_without_a_prior_request() {
67        let tracker = RequestGapTracker::default();
68        assert_eq!(tracker.cache_gap_exceeds(Duration::from_millis(1)), None);
69    }
70
71    #[test]
72    fn cache_gap_exceeds_is_none_below_threshold() {
73        let mut tracker = RequestGapTracker::default();
74        tracker.note_request_sent();
75        assert_eq!(tracker.cache_gap_exceeds(Duration::from_secs(60)), None);
76    }
77
78    #[test]
79    fn cache_gap_exceeds_is_some_above_threshold() {
80        let mut tracker = RequestGapTracker::default();
81        tracker.note_request_sent();
82        std::thread::sleep(Duration::from_millis(10));
83        let gap = tracker.cache_gap_exceeds(Duration::from_millis(5));
84        assert!(gap.is_some());
85    }
86
87    #[test]
88    fn has_sent_request_is_false_until_note_request_sent() {
89        let mut tracker = RequestGapTracker::default();
90        assert!(!tracker.has_sent_request());
91        tracker.note_request_sent();
92        assert!(tracker.has_sent_request());
93    }
94}
vtcode_core/llm/request_gap.rs

vtcode_core/llm/
request_gap.rs
