Skip to main content

firstpass_proxy/
affinity.rs

1//! Session promotion (SPEC §8.4): start a continuing session on the rung it already needed.
2//!
3//! `[escalation.session_promotion]` has been parseable — and exported from `firstpass_core` — since
4//! before this module existed, but nothing ever read it. A config that parses and does nothing is
5//! the same class of defect as a receipt that records `$0.00`: the file states one thing and the
6//! system does another, with no error to notice. This makes it real.
7//!
8//! ## What it buys
9//!
10//! The escalation tax, measured at 18% of first-pass spend on MBPP: when turn 1 of a conversation
11//! escalates haiku → sonnet, turn 2 pays for the doomed haiku call all over again, and so does
12//! turn 3. A session that has demonstrated it needs the stronger rung should start there.
13//!
14//! ## What it must not become
15//!
16//! A ratchet. Promotion on its own is one-way — a session that escalates once would stay expensive
17//! for its whole life, including the trivial turns at the end ("thanks, that worked"). So every
18//! `probe_every` turns the promotion is deliberately ignored and the request starts one rung lower.
19//! If that cheaper rung passes its gate, the promotion is released. That probe is the only way a
20//! promotion ever comes down, which is why it is not optional.
21//!
22//! ## What it cannot do
23//!
24//! Lower quality. Like the bandit, this chooses only where the ladder **starts**; the gate still
25//! verifies whatever is served, and escalation still runs from wherever it started. A wrong
26//! promotion costs money, never correctness — and the probe bounds even that.
27
28use std::collections::HashMap;
29use std::sync::Mutex;
30use std::time::{Duration, Instant};
31
32use firstpass_core::config::SessionPromotion;
33
34/// Per-session state. Small and `Copy`-cheap; there is one of these per live session.
35#[derive(Debug, Clone, Copy, PartialEq, Eq)]
36struct Pin {
37    /// Rung this session is promoted to, once `failures` has crossed the threshold.
38    rung: u32,
39    /// Gate failures observed inside the window.
40    failures: u32,
41    /// Turns served since the last downward probe.
42    since_probe: u32,
43    /// Last touch, for both TTL expiry and eviction order.
44    seen: Instant,
45}
46
47/// What the router should do for this request.
48#[derive(Debug, Clone, Copy, PartialEq, Eq)]
49pub enum Decision {
50    /// Start at rung 0, as if promotion were off.
51    Cold,
52    /// Start at this rung — the session has earned the promotion.
53    Promoted(u32),
54    /// Start one rung below the promotion, to test whether it is still needed.
55    Probe(u32),
56}
57
58impl Decision {
59    /// The rung to actually start on.
60    #[must_use]
61    pub const fn start_rung(self) -> u32 {
62        match self {
63            Self::Cold => 0,
64            Self::Promoted(r) | Self::Probe(r) => r,
65        }
66    }
67
68    /// Label for the audit trace, so a receipt says *why* it did not start at rung 0.
69    #[must_use]
70    pub const fn reason(self) -> &'static str {
71        match self {
72            Self::Cold => "cold",
73            Self::Promoted(_) => "session-promoted",
74            Self::Probe(_) => "session-probe",
75        }
76    }
77}
78
79/// Decide from a pin alone. Pure — every rule below is testable without a clock or a map.
80fn decide(pin: Option<&Pin>, cfg: &SessionPromotion) -> Decision {
81    let Some(pin) = pin else {
82        return Decision::Cold;
83    };
84    if pin.failures < cfg.after_failures || pin.rung == 0 {
85        return Decision::Cold;
86    }
87    // Time to check whether the promotion is still earned. `probe_every` of 0 would divide by
88    // zero and also means "never probe", which is the ratchet this must not have — treat it as 1.
89    if pin.since_probe >= cfg.probe_every.max(1) {
90        return Decision::Probe(pin.rung.saturating_sub(1));
91    }
92    Decision::Promoted(pin.rung)
93}
94
95/// Tracks which sessions have earned a higher starting rung.
96///
97/// Keyed by `(tenant, session)`: a session id is only unique inside a tenant, and a promotion must
98/// never cross that boundary — one tenant's traffic pattern is not a fact about another's.
99#[derive(Debug)]
100pub struct SessionPromoter {
101    cfg: SessionPromotion,
102    ttl: Duration,
103    // ponytail: one mutex over the whole map. Each critical section is a hash lookup and a few
104    // integer writes, so contention is far below what a router's provider I/O costs. Shard by
105    // session hash if this ever shows up in a profile.
106    pins: Mutex<HashMap<(String, String), Pin>>,
107}
108
109impl SessionPromoter {
110    /// Build from config. `window` doubles as the idle TTL.
111    ///
112    /// # Errors
113    /// [`firstpass_core::Error::BadDuration`] when `window` is not `<int><unit>`.
114    pub fn new(cfg: SessionPromotion) -> firstpass_core::Result<Self> {
115        let ttl = cfg.window_duration()?;
116        Ok(Self {
117            cfg,
118            ttl,
119            pins: Mutex::new(HashMap::new()),
120        })
121    }
122
123    /// Where this request should start.
124    #[must_use]
125    pub fn decide(&self, tenant: &str, session: &str, now: Instant) -> Decision {
126        let key = (tenant.to_owned(), session.to_owned());
127        let Ok(pins) = self.pins.lock() else {
128            // A poisoned lock must not take routing down; falling back to cold is always safe,
129            // because cold is exactly the behaviour with promotion disabled.
130            return Decision::Cold;
131        };
132        match pins.get(&key) {
133            // Expired pins are treated as absent here and reaped on the next write, so a read
134            // never has to take a write lock.
135            Some(p) if now.duration_since(p.seen) < self.ttl => decide(Some(p), &self.cfg),
136            _ => Decision::Cold,
137        }
138    }
139
140    /// Record how a request ended: which rung actually served it, and whether the ladder had to
141    /// climb to get there.
142    pub fn record(
143        &self,
144        tenant: &str,
145        session: &str,
146        served_rung: u32,
147        escalated: bool,
148        now: Instant,
149    ) {
150        let key = (tenant.to_owned(), session.to_owned());
151        let Ok(mut pins) = self.pins.lock() else {
152            return;
153        };
154        self.evict_if_needed(&mut pins, now);
155
156        let expired = pins
157            .get(&key)
158            .is_some_and(|p| now.duration_since(p.seen) >= self.ttl);
159        if expired {
160            pins.remove(&key);
161        }
162
163        let entry = pins.entry(key).or_insert(Pin {
164            rung: 0,
165            failures: 0,
166            since_probe: 0,
167            seen: now,
168        });
169        entry.seen = now;
170
171        if escalated {
172            // The session needed a higher rung than it started on: that is the evidence promotion
173            // is for. Promote to where it actually landed.
174            entry.failures = entry.failures.saturating_add(1);
175            entry.rung = entry.rung.max(served_rung);
176            entry.since_probe = 0;
177        } else if served_rung < entry.rung {
178            // A probe (or any request) served below the promotion without escalating — the
179            // promotion is no longer earned. Release it to what actually worked. This is the path
180            // that stops promotion being one-way.
181            entry.rung = served_rung;
182            entry.failures = 0;
183            entry.since_probe = 0;
184        } else {
185            entry.since_probe = entry.since_probe.saturating_add(1);
186        }
187    }
188
189    /// Drop expired entries, then oldest-first until under the cap.
190    fn evict_if_needed(&self, pins: &mut HashMap<(String, String), Pin>, now: Instant) {
191        if pins.len() < self.cfg.max_sessions {
192            return;
193        }
194        pins.retain(|_, p| now.duration_since(p.seen) < self.ttl);
195        // ponytail: full sort only on the rare over-cap write, not per request. A heap would be
196        // asymptotically better and materially more code for a path that runs approximately never.
197        if pins.len() >= self.cfg.max_sessions {
198            let mut by_age: Vec<((String, String), Instant)> =
199                pins.iter().map(|(k, p)| (k.clone(), p.seen)).collect();
200            by_age.sort_by_key(|(_, seen)| *seen);
201            for (k, _) in by_age.into_iter().take(self.cfg.max_sessions.max(2) / 2) {
202                pins.remove(&k);
203            }
204        }
205    }
206}
207
208#[cfg(test)]
209mod tests {
210    use super::*;
211
212    fn cfg() -> SessionPromotion {
213        SessionPromotion {
214            after_failures: 1,
215            window: "30m".to_owned(),
216            probe_every: 3,
217            max_sessions: 100,
218        }
219    }
220
221    fn promoter() -> SessionPromoter {
222        SessionPromoter::new(cfg()).expect("30m parses")
223    }
224
225    #[test]
226    fn an_unknown_session_starts_cold() {
227        let p = promoter();
228        assert_eq!(p.decide("t", "never-seen", Instant::now()), Decision::Cold);
229    }
230
231    #[test]
232    fn a_session_that_escalated_starts_where_it_landed() {
233        // The escalation tax in one test: turn 1 climbs 0 → 1, so turn 2 must not pay for rung 0
234        // again.
235        let p = promoter();
236        let now = Instant::now();
237        p.record("t", "s", 1, true, now);
238        assert_eq!(p.decide("t", "s", now), Decision::Promoted(1));
239        assert_eq!(p.decide("t", "s", now).start_rung(), 1);
240    }
241
242    #[test]
243    fn one_escalation_is_not_enough_when_the_threshold_is_higher() {
244        let mut c = cfg();
245        c.after_failures = 3;
246        let p = SessionPromoter::new(c).expect("parses");
247        let now = Instant::now();
248        p.record("t", "s", 1, true, now);
249        p.record("t", "s", 1, true, now);
250        assert_eq!(p.decide("t", "s", now), Decision::Cold, "2 < 3");
251        p.record("t", "s", 1, true, now);
252        assert_eq!(p.decide("t", "s", now), Decision::Promoted(1));
253    }
254
255    #[test]
256    fn promotion_is_not_a_ratchet_it_probes_downward() {
257        // Without this the feature is a cost regression on any long conversation that gets easier.
258        let p = promoter();
259        let now = Instant::now();
260        p.record("t", "s", 2, true, now);
261        assert_eq!(p.decide("t", "s", now), Decision::Promoted(2));
262
263        // Serve at the promoted rung without escalating, probe_every(3) times.
264        for _ in 0..3 {
265            p.record("t", "s", 2, false, now);
266        }
267        assert_eq!(
268            p.decide("t", "s", now),
269            Decision::Probe(1),
270            "must periodically test one rung lower"
271        );
272    }
273
274    #[test]
275    fn a_successful_probe_releases_the_promotion() {
276        let p = promoter();
277        let now = Instant::now();
278        p.record("t", "s", 2, true, now);
279        for _ in 0..3 {
280            p.record("t", "s", 2, false, now);
281        }
282        assert_eq!(p.decide("t", "s", now), Decision::Probe(1));
283
284        // The probe served at rung 1 with no escalation — rung 2 is no longer earned.
285        p.record("t", "s", 1, false, now);
286        assert_eq!(
287            p.decide("t", "s", now),
288            Decision::Cold,
289            "released to rung 1"
290        );
291    }
292
293    #[test]
294    fn a_failed_probe_restores_the_promotion() {
295        let p = promoter();
296        let now = Instant::now();
297        p.record("t", "s", 2, true, now);
298        for _ in 0..3 {
299            p.record("t", "s", 2, false, now);
300        }
301        assert_eq!(p.decide("t", "s", now), Decision::Probe(1));
302
303        // Probe started at 1, still had to climb to 2 → promotion re-earned.
304        p.record("t", "s", 2, true, now);
305        assert_eq!(p.decide("t", "s", now), Decision::Promoted(2));
306    }
307
308    #[test]
309    fn a_promotion_never_crosses_a_tenant_boundary() {
310        // One tenant's traffic pattern is not a fact about another's, and a session id is only
311        // unique within a tenant.
312        let p = promoter();
313        let now = Instant::now();
314        p.record("tenant-a", "shared-id", 2, true, now);
315        assert_eq!(
316            p.decide("tenant-a", "shared-id", now),
317            Decision::Promoted(2)
318        );
319        assert_eq!(p.decide("tenant-b", "shared-id", now), Decision::Cold);
320    }
321
322    #[test]
323    fn an_idle_session_forgets_its_promotion() {
324        let mut c = cfg();
325        c.window = "1s".to_owned();
326        let p = SessionPromoter::new(c).expect("parses");
327        let t0 = Instant::now();
328        p.record("t", "s", 2, true, t0);
329        assert_eq!(p.decide("t", "s", t0), Decision::Promoted(2));
330
331        let later = t0 + Duration::from_secs(2);
332        assert_eq!(
333            p.decide("t", "s", later),
334            Decision::Cold,
335            "a reused session id must not inherit a stale rung"
336        );
337    }
338
339    #[test]
340    fn the_map_cannot_grow_without_bound() {
341        let mut c = cfg();
342        c.max_sessions = 10;
343        let p = SessionPromoter::new(c).expect("parses");
344        let now = Instant::now();
345        for i in 0..100 {
346            p.record("t", &format!("s{i}"), 1, true, now);
347        }
348        let len = p.pins.lock().expect("not poisoned").len();
349        assert!(len <= 10, "expected <= 10 tracked sessions, got {len}");
350    }
351
352    #[test]
353    fn probe_every_zero_is_rejected_at_parse_not_silently_coerced() {
354        // A permanent pin is a cost regression an operator would only discover in a bill, so it
355        // fails loudly at startup instead. `.max(1)` in `decide` is defence in depth for a config
356        // built in code rather than parsed.
357        let err = firstpass_core::config::Config::parse(
358            r#"
359            [escalation.session_promotion]
360            after_failures = 2
361            window = "30m"
362            probe_every = 0
363
364            [[route]]
365            mode = "enforce"
366            ladder = ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
367            "#,
368        )
369        .expect_err("probe_every = 0 must not parse");
370        let msg = err.to_string();
371        assert!(msg.contains("probe_every"), "unhelpful message: {msg}");
372
373        // Defence in depth: constructed directly, 0 behaves as 1 rather than dividing or pinning.
374        let mut c = cfg();
375        c.probe_every = 0;
376        let pin = Pin {
377            rung: 2,
378            failures: 5,
379            since_probe: 1,
380            seen: Instant::now(),
381        };
382        assert_eq!(decide(Some(&pin), &c), Decision::Probe(1));
383    }
384
385    #[test]
386    fn a_valid_promotion_block_parses_with_defaults() {
387        let cfg = firstpass_core::config::Config::parse(
388            r#"
389            [escalation.session_promotion]
390            after_failures = 2
391            window = "30m"
392
393            [[route]]
394            mode = "enforce"
395            ladder = ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
396            "#,
397        )
398        .expect("valid promotion config parses");
399        let p = cfg
400            .escalation
401            .session_promotion
402            .expect("promotion block present");
403        assert_eq!(p.after_failures, 2);
404        assert_eq!(p.probe_every, 5, "default must not be a ratchet");
405        assert_eq!(p.max_sessions, 10_000);
406    }
407
408    #[test]
409    fn the_decision_carries_its_reason_into_the_receipt() {
410        assert_eq!(Decision::Cold.reason(), "cold");
411        assert_eq!(Decision::Promoted(2).reason(), "session-promoted");
412        assert_eq!(Decision::Probe(1).reason(), "session-probe");
413    }
414}