firstpass_proxy/condense.rs
1//! Last-resort context condensing: drop the middle of a conversation that will not fit anywhere.
2//!
3//! ## Why this is narrow on purpose
4//!
5//! Condensing routinely — trimming every long conversation to save tokens — would mean the gate
6//! verifies an answer produced from a prompt the **client never sent**. The receipt would attest to
7//! a decision about a different question than the one asked, which is the one thing this product
8//! cannot afford to get wrong.
9//!
10//! So it fires in exactly one situation: the prompt has already overflowed the context window of
11//! **every rung on the ladder**, and the alternative is not a cheaper answer, it is *no answer*.
12//! At that point the trade is no longer "faithful vs. degraded" but "degraded vs. failed", and a
13//! degraded answer the receipt openly labels is the better of the two.
14//!
15//! ## What it keeps
16//!
17//! The head and the tail. The opening turns usually carry the task definition and the tail carries
18//! the live thread; the middle is where a long agent conversation accumulates tool output nobody
19//! refers to again. A marker turn is inserted where the elision happened, so the model is told the
20//! history is incomplete rather than being left to infer it from a discontinuity.
21
22use crate::provider::{ChatMessage, ModelRequest};
23
24/// Outcome of a condensing attempt.
25#[derive(Debug, Clone)]
26pub struct Condensed {
27 /// The request with its middle removed.
28 pub request: ModelRequest,
29 /// How many turns were dropped — recorded on the receipt, never inferred.
30 pub dropped: usize,
31}
32
33/// Drop the middle of `req`'s conversation, keeping `keep_head` opening turns and `keep_tail`
34/// closing ones.
35///
36/// Returns `None` when there is nothing useful to drop, so a caller cannot loop forever retrying a
37/// request that is not getting any smaller — a prompt that overflows on a single enormous turn
38/// cannot be fixed by removing turns, and pretending otherwise would just spend another call.
39#[must_use]
40pub fn condense(req: &ModelRequest, keep_head: usize, keep_tail: usize) -> Option<Condensed> {
41 let n = req.messages.len();
42 // Need at least one turn strictly between the head and tail slices to be worth a retry.
43 let kept = keep_head.saturating_add(keep_tail);
44 if n <= kept.saturating_add(1) {
45 return None;
46 }
47 let dropped = n - kept;
48
49 let mut messages: Vec<ChatMessage> = Vec::with_capacity(kept + 1);
50 messages.extend(req.messages.iter().take(keep_head).cloned());
51 // Tell the model its history is incomplete. Without this it sees a discontinuity and has to
52 // guess whether it misremembers — which is how condensing turns into confabulation.
53 messages.push(ChatMessage::text(
54 "user",
55 format!(
56 "[{dropped} earlier turns omitted: this conversation exceeded the context window of \
57 every available model. The opening and the most recent turns are shown. If answering \
58 depends on the omitted portion, say so rather than guessing.]"
59 ),
60 ));
61 messages.extend(req.messages.iter().skip(n - keep_tail).cloned());
62
63 let mut out = req.clone();
64 out.messages = messages;
65 // The raw body no longer describes what is being sent, and a provider that forwards `raw`
66 // verbatim would re-send the full conversation and overflow again. Clearing it forces the
67 // normalized-field path, which is built from the condensed `messages`.
68 out.raw = serde_json::Value::Null;
69 Some(Condensed {
70 request: out,
71 dropped,
72 })
73}
74
75#[cfg(test)]
76mod tests {
77 use super::*;
78
79 fn req(n: usize) -> ModelRequest {
80 ModelRequest {
81 model: "anthropic/claude-haiku-4-5".to_owned(),
82 system: Some("sys".to_owned()),
83 messages: (0..n)
84 .map(|i| ChatMessage::text("user", format!("turn {i}")))
85 .collect(),
86 max_tokens: 1024,
87 tools: serde_json::Value::Null,
88 raw: serde_json::json!({ "messages": "the full original" }),
89 cache_prefix: false,
90 }
91 }
92
93 #[test]
94 fn the_head_and_tail_survive_and_the_middle_goes() {
95 let c = condense(&req(20), 2, 3).expect("20 turns can be condensed");
96
97 assert_eq!(c.dropped, 15);
98 // 2 head + 1 marker + 3 tail
99 assert_eq!(c.request.messages.len(), 6);
100 assert_eq!(c.request.messages[0].text_view(), "turn 0");
101 assert_eq!(c.request.messages[1].text_view(), "turn 1");
102 assert!(
103 c.request.messages[2]
104 .text_view()
105 .contains("15 earlier turns omitted")
106 );
107 assert_eq!(c.request.messages[5].text_view(), "turn 19");
108 }
109
110 #[test]
111 fn the_model_is_told_the_history_is_incomplete() {
112 // Without the marker the model sees a discontinuity and has to guess whether it
113 // misremembers, which is how condensing turns into confabulation.
114 let c = condense(&req(30), 1, 1).expect("condensable");
115 let marker = c.request.messages[1].text_view();
116 assert!(marker.contains("omitted"), "{marker}");
117 assert!(
118 marker.contains("say so rather than guessing"),
119 "the model needs an instruction, not just a notice: {marker}"
120 );
121 }
122
123 #[test]
124 fn the_raw_body_is_cleared_or_the_retry_overflows_again() {
125 // Anthropic-dialect providers forward `raw` verbatim. Left in place, the retry would send
126 // the FULL conversation again and overflow identically — an extra call for nothing.
127 let c = condense(&req(20), 2, 2).expect("condensable");
128 assert!(
129 c.request.raw.is_null(),
130 "raw must not survive condensing: {:?}",
131 c.request.raw
132 );
133 }
134
135 #[test]
136 fn a_conversation_with_no_droppable_middle_returns_none() {
137 // The caller retries on Some. A request that is not getting smaller must return None, or
138 // the retry loop spends another call on the same overflow.
139 assert!(
140 condense(&req(5), 2, 3).is_none(),
141 "nothing between head and tail"
142 );
143 assert!(
144 condense(&req(6), 2, 3).is_none(),
145 "one turn is not worth a retry"
146 );
147 assert!(condense(&req(1), 2, 3).is_none());
148 assert!(condense(&req(7), 2, 3).is_some(), "two spare turns is");
149 }
150
151 #[test]
152 fn the_system_prompt_and_limits_are_untouched() {
153 // Condensing drops history, not the task definition or the caller's ceilings.
154 let c = condense(&req(20), 2, 2).expect("condensable");
155 assert_eq!(c.request.system.as_deref(), Some("sys"));
156 assert_eq!(c.request.max_tokens, 1024);
157 assert_eq!(c.request.model, "anthropic/claude-haiku-4-5");
158 }
159}