Skip to main content

firstpass_proxy/
condense.rs

1//! Last-resort context condensing: drop the middle of a conversation that will not fit anywhere.
2//!
3//! ## Why this is narrow on purpose
4//!
5//! Condensing routinely — trimming every long conversation to save tokens — would mean the gate
6//! verifies an answer produced from a prompt the **client never sent**. The receipt would attest to
7//! a decision about a different question than the one asked, which is the one thing this product
8//! cannot afford to get wrong.
9//!
10//! So it fires in exactly one situation: the prompt has already overflowed the context window of
11//! **every rung on the ladder**, and the alternative is not a cheaper answer, it is *no answer*.
12//! At that point the trade is no longer "faithful vs. degraded" but "degraded vs. failed", and a
13//! degraded answer the receipt openly labels is the better of the two.
14//!
15//! ## What it keeps
16//!
17//! The head and the tail. The opening turns usually carry the task definition and the tail carries
18//! the live thread; the middle is where a long agent conversation accumulates tool output nobody
19//! refers to again. A marker turn is inserted where the elision happened, so the model is told the
20//! history is incomplete rather than being left to infer it from a discontinuity.
21
22use crate::provider::{ChatMessage, ModelRequest};
23
24/// Outcome of a condensing attempt.
25#[derive(Debug, Clone)]
26pub struct Condensed {
27    /// The request with its middle removed.
28    pub request: ModelRequest,
29    /// How many turns were dropped — recorded on the receipt, never inferred.
30    pub dropped: usize,
31}
32
33/// Drop the middle of `req`'s conversation, keeping `keep_head` opening turns and `keep_tail`
34/// closing ones.
35///
36/// Returns `None` when there is nothing useful to drop, so a caller cannot loop forever retrying a
37/// request that is not getting any smaller — a prompt that overflows on a single enormous turn
38/// cannot be fixed by removing turns, and pretending otherwise would just spend another call.
39#[must_use]
40pub fn condense(req: &ModelRequest, keep_head: usize, keep_tail: usize) -> Option<Condensed> {
41    let n = req.messages.len();
42    // Need at least one turn strictly between the head and tail slices to be worth a retry.
43    let kept = keep_head.saturating_add(keep_tail);
44    if n <= kept.saturating_add(1) {
45        return None;
46    }
47    let dropped = n - kept;
48
49    let mut messages: Vec<ChatMessage> = Vec::with_capacity(kept + 1);
50    messages.extend(req.messages.iter().take(keep_head).cloned());
51    // Tell the model its history is incomplete. Without this it sees a discontinuity and has to
52    // guess whether it misremembers — which is how condensing turns into confabulation.
53    messages.push(ChatMessage::text(
54        "user",
55        format!(
56            "[{dropped} earlier turns omitted: this conversation exceeded the context window of \
57             every available model. The opening and the most recent turns are shown. If answering \
58             depends on the omitted portion, say so rather than guessing.]"
59        ),
60    ));
61    messages.extend(req.messages.iter().skip(n - keep_tail).cloned());
62
63    let mut out = req.clone();
64    out.messages = messages;
65    // The raw body no longer describes what is being sent, and a provider that forwards `raw`
66    // verbatim would re-send the full conversation and overflow again. Clearing it forces the
67    // normalized-field path, which is built from the condensed `messages`.
68    out.raw = serde_json::Value::Null;
69    Some(Condensed {
70        request: out,
71        dropped,
72    })
73}
74
75#[cfg(test)]
76mod tests {
77    use super::*;
78
79    fn req(n: usize) -> ModelRequest {
80        ModelRequest {
81            model: "anthropic/claude-haiku-4-5".to_owned(),
82            system: Some("sys".to_owned()),
83            messages: (0..n)
84                .map(|i| ChatMessage::text("user", format!("turn {i}")))
85                .collect(),
86            max_tokens: 1024,
87            tools: serde_json::Value::Null,
88            raw: serde_json::json!({ "messages": "the full original" }),
89            cache_prefix: false,
90        }
91    }
92
93    #[test]
94    fn the_head_and_tail_survive_and_the_middle_goes() {
95        let c = condense(&req(20), 2, 3).expect("20 turns can be condensed");
96
97        assert_eq!(c.dropped, 15);
98        // 2 head + 1 marker + 3 tail
99        assert_eq!(c.request.messages.len(), 6);
100        assert_eq!(c.request.messages[0].text_view(), "turn 0");
101        assert_eq!(c.request.messages[1].text_view(), "turn 1");
102        assert!(
103            c.request.messages[2]
104                .text_view()
105                .contains("15 earlier turns omitted")
106        );
107        assert_eq!(c.request.messages[5].text_view(), "turn 19");
108    }
109
110    #[test]
111    fn the_model_is_told_the_history_is_incomplete() {
112        // Without the marker the model sees a discontinuity and has to guess whether it
113        // misremembers, which is how condensing turns into confabulation.
114        let c = condense(&req(30), 1, 1).expect("condensable");
115        let marker = c.request.messages[1].text_view();
116        assert!(marker.contains("omitted"), "{marker}");
117        assert!(
118            marker.contains("say so rather than guessing"),
119            "the model needs an instruction, not just a notice: {marker}"
120        );
121    }
122
123    #[test]
124    fn the_raw_body_is_cleared_or_the_retry_overflows_again() {
125        // Anthropic-dialect providers forward `raw` verbatim. Left in place, the retry would send
126        // the FULL conversation again and overflow identically — an extra call for nothing.
127        let c = condense(&req(20), 2, 2).expect("condensable");
128        assert!(
129            c.request.raw.is_null(),
130            "raw must not survive condensing: {:?}",
131            c.request.raw
132        );
133    }
134
135    #[test]
136    fn a_conversation_with_no_droppable_middle_returns_none() {
137        // The caller retries on Some. A request that is not getting smaller must return None, or
138        // the retry loop spends another call on the same overflow.
139        assert!(
140            condense(&req(5), 2, 3).is_none(),
141            "nothing between head and tail"
142        );
143        assert!(
144            condense(&req(6), 2, 3).is_none(),
145            "one turn is not worth a retry"
146        );
147        assert!(condense(&req(1), 2, 3).is_none());
148        assert!(condense(&req(7), 2, 3).is_some(), "two spare turns is");
149    }
150
151    #[test]
152    fn the_system_prompt_and_limits_are_untouched() {
153        // Condensing drops history, not the task definition or the caller's ceilings.
154        let c = condense(&req(20), 2, 2).expect("condensable");
155        assert_eq!(c.request.system.as_deref(), Some("sys"));
156        assert_eq!(c.request.max_tokens, 1024);
157        assert_eq!(c.request.model, "anthropic/claude-haiku-4-5");
158    }
159}