lean_ctx_proxy/
verbosity.rs

1// SPDX-License-Identifier: Apache-2.0
2//! Cache-safe wire verbosity steer (#895 Track B).
3//!
4//! Opt-in (`proxy.verbosity_steer`). When on, the proxy appends a single
5//! **constant** "be concise" instruction to the *last user turn* of each
6//! request. Output-shaping for non-rules-aware clients: lean-ctx already steers
7//! verbosity through rules injection for editors that load rules, but raw API
8//! clients (and the holdout's treatment arm) need it on the wire.
9//!
10//! Cache-safety, by construction:
11//! - The steer is **constant** (identical bytes every turn), so it never adds
12//!   per-turn entropy.
13//! - It is appended **after** all existing content of the last user turn — i.e.
14//!   strictly after the last `cache_control` breakpoint — and never modifies a
15//!   `cache_control`-anchored block. The provider's cached prefix (everything up
16//!   to the last breakpoint) stays byte-identical across turns (#448/#498); only
17//!   the always-reprocessed tail grows by the constant suffix.
18//! - For array content the suffix is a **new** trailing text element; existing
19//!   blocks (including cache anchors) are left untouched. For plain-string
20//!   content (OpenAI prefix-cached; Anthropic strings carry no `cache_control`)
21//!   the suffix is concatenated.
22//! - Idempotent: if the steer is already present it is not appended again.
23
24use serde_json::{Map, Value};
25
26/// The constant verbosity instruction. Kept short and directive; its byte
27/// stability is what makes the steer prompt-cache-safe.
28pub const STEER: &str =
29    "Be concise: answer directly and do not restate the question or surrounding context.";
30
31/// Suffix form for plain-string content (leading separation from prior text).
32fn string_suffix() -> String {
33    format!("\n\n{STEER}")
34}
35
36/// True when `text` already ends with the steer, so we never double-append
37/// (keeps the rewrite idempotent and deterministic).
38fn already_steered(text: &str) -> bool {
39    text.trim_end().ends_with(STEER)
40}
41
42/// A `{ "type": "text", "text": STEER }` block for array content.
43fn steer_text_block() -> Value {
44    let mut m = Map::new();
45    m.insert("type".to_string(), Value::String("text".to_string()));
46    m.insert("text".to_string(), Value::String(STEER.to_string()));
47    Value::Object(m)
48}
49
50/// A `{ "type": "input_text", "text": STEER }` block for OpenAI Responses API
51/// content arrays (which require `input_text`, not `text`).
52fn steer_input_text_block() -> Value {
53    let mut m = Map::new();
54    m.insert("type".to_string(), Value::String("input_text".to_string()));
55    m.insert("text".to_string(), Value::String(STEER.to_string()));
56    Value::Object(m)
57}
58
59/// A `{ "text": STEER }` part for Gemini `parts` arrays.
60fn steer_gemini_part() -> Value {
61    let mut m = Map::new();
62    m.insert("text".to_string(), Value::String(STEER.to_string()));
63    Value::Object(m)
64}
65
66/// Append the steer to a content value that is either a plain string or an array
67/// of text blocks. `block` builds the trailing element for the array case.
68/// Returns whether the value changed.
69fn append_to_content(content: &mut Value, block: impl FnOnce() -> Value) -> bool {
70    match content {
71        Value::String(s) => {
72            if already_steered(s) {
73                return false;
74            }
75            s.push_str(&string_suffix());
76            true
77        }
78        Value::Array(parts) => {
79            // Already steered if the last text element is the steer.
80            let last_text = parts
81                .iter()
82                .rev()
83                .find_map(|p| p.get("text").and_then(Value::as_str));
84            if last_text.is_some_and(already_steered) {
85                return false;
86            }
87            parts.push(block());
88            true
89        }
90        _ => false,
91    }
92}
93
94/// Index of the last message in `messages` whose `role` matches.
95fn last_role_index(messages: &[Value], role: &str) -> Option<usize> {
96    messages
97        .iter()
98        .rposition(|m| m.get("role").and_then(Value::as_str) == Some(role))
99}
100
101/// Append the steer to the last user message of an Anthropic `/v1/messages`
102/// body. No-op (returns false) when there is no user turn.
103pub fn apply_anthropic(doc: &mut Value) -> bool {
104    let Some(messages) = doc.get_mut("messages").and_then(Value::as_array_mut) else {
105        return false;
106    };
107    let Some(idx) = last_role_index(messages, "user") else {
108        return false;
109    };
110    let Some(content) = messages[idx].get_mut("content") else {
111        return false;
112    };
113    let changed = append_to_content(content, steer_text_block);
114    if changed {
115        record();
116    }
117    changed
118}
119
120/// Append the steer to the last user message of an OpenAI Chat Completions body.
121/// OpenAI prefix-caches automatically, so appending to the newest turn never
122/// disturbs a previously cached prefix.
123pub fn apply_openai_chat(doc: &mut Value) -> bool {
124    let Some(messages) = doc.get_mut("messages").and_then(Value::as_array_mut) else {
125        return false;
126    };
127    let Some(idx) = last_role_index(messages, "user") else {
128        return false;
129    };
130    let Some(content) = messages[idx].get_mut("content") else {
131        return false;
132    };
133    let changed = append_to_content(content, steer_text_block);
134    if changed {
135        record();
136    }
137    changed
138}
139
140/// Append the steer to the last user item of an OpenAI Responses body. `input`
141/// is either a plain string or an array of role/content items.
142pub fn apply_openai_responses(doc: &mut Value) -> bool {
143    let changed = match doc.get_mut("input") {
144        Some(Value::String(s)) => {
145            if already_steered(s) {
146                false
147            } else {
148                s.push_str(&string_suffix());
149                true
150            }
151        }
152        Some(Value::Array(items)) => {
153            let Some(idx) = last_role_index(items, "user") else {
154                return false;
155            };
156            match items[idx].get_mut("content") {
157                Some(content) => append_to_content(content, steer_input_text_block),
158                None => false,
159            }
160        }
161        _ => false,
162    };
163    if changed {
164        record();
165    }
166    changed
167}
168
169/// Append the steer to the last user turn of a Gemini `generateContent` body
170/// (`contents[].parts[]`).
171pub fn apply_google(doc: &mut Value) -> bool {
172    let Some(contents) = doc.get_mut("contents").and_then(Value::as_array_mut) else {
173        return false;
174    };
175    let Some(idx) = last_role_index(contents, "user") else {
176        return false;
177    };
178    let Some(parts) = contents[idx].get_mut("parts").and_then(Value::as_array_mut) else {
179        return false;
180    };
181    let last_text = parts
182        .iter()
183        .rev()
184        .find_map(|p| p.get("text").and_then(Value::as_str));
185    if last_text.is_some_and(already_steered) {
186        return false;
187    }
188    parts.push(steer_gemini_part());
189    record();
190    true
191}
192
193use std::sync::atomic::{AtomicU64, Ordering};
194
195/// Count of requests steered, surfaced via [`steered_count`] for `/status`.
196static STEERED: AtomicU64 = AtomicU64::new(0);
197
198fn record() {
199    STEERED.fetch_add(1, Ordering::Relaxed);
200}
201
202/// Total requests that received a wire verbosity steer (telemetry).
203#[must_use]
204pub fn steered_count() -> u64 {
205    STEERED.load(Ordering::Relaxed)
206}
207
208#[cfg(test)]
209mod tests {
210    use super::*;
211    use serde_json::json;
212
213    #[test]
214    fn anthropic_appends_block_to_last_user() {
215        let mut doc = json!({
216            "messages": [
217                {"role": "user", "content": "first"},
218                {"role": "assistant", "content": "ok"},
219                {"role": "user", "content": [{"type": "text", "text": "second"}]}
220            ]
221        });
222        assert!(apply_anthropic(&mut doc));
223        let last = &doc["messages"][2]["content"];
224        let arr = last.as_array().unwrap();
225        assert_eq!(arr.len(), 2, "new steer block appended");
226        assert_eq!(arr[1]["text"], STEER);
227        // The earlier user turn is untouched.
228        assert_eq!(doc["messages"][0]["content"], "first");
229    }
230
231    #[test]
232    fn anthropic_string_content_concatenates() {
233        let mut doc = json!({"messages": [{"role": "user", "content": "hello"}]});
234        assert!(apply_anthropic(&mut doc));
235        let s = doc["messages"][0]["content"].as_str().unwrap();
236        assert!(s.starts_with("hello"));
237        assert!(s.ends_with(STEER));
238    }
239
240    #[test]
241    fn never_modifies_cache_control_blocks() {
242        let mut doc = json!({
243            "messages": [{
244                "role": "user",
245                "content": [
246                    {"type": "text", "text": "cached", "cache_control": {"type": "ephemeral"}}
247                ]
248            }]
249        });
250        assert!(apply_anthropic(&mut doc));
251        let arr = doc["messages"][0]["content"].as_array().unwrap();
252        // The cache-anchored block survives verbatim; the steer is a NEW block
253        // appended strictly after it.
254        assert_eq!(arr.len(), 2);
255        assert!(arr[0].get("cache_control").is_some());
256        assert_eq!(arr[0]["text"], "cached");
257        assert_eq!(arr[1]["text"], STEER);
258        assert!(arr[1].get("cache_control").is_none());
259    }
260
261    #[test]
262    fn idempotent_no_double_append() {
263        let mut doc = json!({"messages": [{"role": "user", "content": "hi"}]});
264        assert!(apply_anthropic(&mut doc));
265        let once = doc.clone();
266        assert!(!apply_anthropic(&mut doc), "second pass is a no-op");
267        assert_eq!(doc, once, "byte-identical after a redundant pass");
268    }
269
270    #[test]
271    fn deterministic_constant_suffix() {
272        let mk = || json!({"messages": [{"role": "user", "content": "ask"}]});
273        let mut a = mk();
274        let mut b = mk();
275        apply_anthropic(&mut a);
276        apply_anthropic(&mut b);
277        assert_eq!(a, b, "constant steer ⇒ byte-identical rewrite");
278    }
279
280    #[test]
281    fn openai_chat_appends_to_last_user() {
282        let mut doc = json!({
283            "messages": [
284                {"role": "system", "content": "sys"},
285                {"role": "user", "content": "q"}
286            ]
287        });
288        assert!(apply_openai_chat(&mut doc));
289        let s = doc["messages"][1]["content"].as_str().unwrap();
290        assert!(s.ends_with(STEER));
291        assert_eq!(doc["messages"][0]["content"], "sys", "system untouched");
292    }
293
294    #[test]
295    fn responses_string_and_array_input() {
296        let mut s = json!({"input": "plain"});
297        assert!(apply_openai_responses(&mut s));
298        assert!(s["input"].as_str().unwrap().ends_with(STEER));
299
300        let mut a =
301            json!({"input": [{"role": "user", "content": [{"type": "input_text", "text": "q"}]}]});
302        assert!(apply_openai_responses(&mut a));
303        let parts = a["input"][0]["content"].as_array().unwrap();
304        assert_eq!(parts.last().unwrap()["type"], "input_text");
305        assert_eq!(parts.last().unwrap()["text"], STEER);
306    }
307
308    #[test]
309    fn google_appends_part_to_last_user() {
310        let mut doc = json!({
311            "contents": [
312                {"role": "user", "parts": [{"text": "q1"}]},
313                {"role": "model", "parts": [{"text": "a1"}]},
314                {"role": "user", "parts": [{"text": "q2"}]}
315            ]
316        });
317        assert!(apply_google(&mut doc));
318        let parts = doc["contents"][2]["parts"].as_array().unwrap();
319        assert_eq!(parts.len(), 2);
320        assert_eq!(parts[1]["text"], STEER);
321    }
322
323    #[test]
324    fn no_user_turn_is_noop() {
325        let mut doc = json!({"messages": [{"role": "assistant", "content": "x"}]});
326        assert!(!apply_anthropic(&mut doc));
327    }
328}
lean_ctx_proxy/verbosity.rs

lean_ctx_proxy/
verbosity.rs
