use std::collections::{HashMap, HashSet};
use std::hash::{DefaultHasher, Hash, Hasher};
use parking_lot::Mutex;
use switchyard_protocol::{Request, Role};
use super::BENCH_SESSION_HEADER;
const MAX_FAILED_CONSULTS: u32 = 3;
const MAX_TRACKED_SCOPES: usize = 1_024;
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
pub(super) enum ScopeKey {
Instance,
Client(String),
Session(String),
}
#[derive(Default)]
struct ScopeState {
reviews: u32,
failed_consults: u32,
exhaustion_logged: bool,
}
#[derive(Default)]
struct GateState {
scopes: HashMap<ScopeKey, ScopeState>,
stall_fired: HashSet<u64>,
}
pub(super) struct ReviewBudget {
max_reviews: u32,
state: Mutex<GateState>,
}
impl ReviewBudget {
pub(super) fn new(max_reviews: u32) -> Self {
Self {
max_reviews,
state: Mutex::new(GateState::default()),
}
}
pub(super) fn check_exhausted(&self, scope: &ScopeKey) -> bool {
let mut state = self.state.lock();
let Some(entry) = state.scopes.get_mut(scope) else {
return false;
};
let exhausted =
entry.reviews >= self.max_reviews || entry.failed_consults >= MAX_FAILED_CONSULTS;
if exhausted && !entry.exhaustion_logged {
entry.exhaustion_logged = true;
tracing::info!(
target: "libsy",
scope = ?scope,
"advisor gate: review budget spent; passing through"
);
}
exhausted
}
pub(super) fn try_reserve(&self, scope: &ScopeKey) -> bool {
let mut state = self.state.lock();
if state.scopes.len() >= MAX_TRACKED_SCOPES && !state.scopes.contains_key(scope) {
let evict = state
.scopes
.keys()
.find(|key| **key != ScopeKey::Instance)
.cloned();
if let Some(key) = evict {
state.scopes.remove(&key);
}
}
let max_reviews = self.max_reviews;
let entry = state.scopes.entry(scope.clone()).or_default();
if entry.reviews >= max_reviews || entry.failed_consults >= MAX_FAILED_CONSULTS {
return false;
}
entry.reviews += 1;
true
}
pub(super) fn refund_failure(&self, scope: &ScopeKey) {
let mut state = self.state.lock();
let entry = state.scopes.entry(scope.clone()).or_default();
entry.reviews = entry.reviews.saturating_sub(1);
entry.failed_consults += 1;
}
pub(super) fn evict_scope(&self, scope: &ScopeKey) {
if *scope == ScopeKey::Instance {
return;
}
self.state.lock().scopes.remove(scope);
}
pub(super) fn try_mark_stall_fired(&self, key: u64) -> bool {
let mut state = self.state.lock();
if state.stall_fired.contains(&key) {
return false;
}
if state.stall_fired.len() >= MAX_TRACKED_SCOPES {
let drop = state.stall_fired.iter().next().copied();
if let Some(key) = drop {
state.stall_fired.remove(&key);
}
}
state.stall_fired.insert(key)
}
pub(super) fn clear_stall_fired(&self, key: u64) {
self.state.lock().stall_fired.remove(&key);
}
}
pub(super) fn budget_scope(request: &Request) -> ScopeKey {
let metadata = request.metadata.as_ref();
if let Some(value) = metadata
.and_then(|metadata| metadata.http_headers.as_ref())
.and_then(|headers| headers.get(BENCH_SESSION_HEADER))
.and_then(|value| value.to_str().ok())
&& !value.is_empty()
{
return ScopeKey::Client(value.to_string());
}
if let Some(id) = metadata.and_then(|metadata| metadata.session_id.as_deref())
&& !id.is_empty()
{
return ScopeKey::Session(id.to_string());
}
ScopeKey::Instance
}
pub(super) fn stall_key(request: &Request) -> u64 {
let text = request
.llm_request
.messages
.iter()
.find(|message| message.role == Role::User)
.and_then(|message| message.text_content("\n"))
.unwrap_or_default();
let mut hasher = DefaultHasher::new();
text.hash(&mut hasher);
hasher.finish()
}
#[cfg(test)]
mod tests {
use super::*;
fn scope() -> ScopeKey {
ScopeKey::Session("s1".to_string())
}
#[test]
fn reserving_spends_the_budget() {
let budget = ReviewBudget::new(1);
assert!(!budget.check_exhausted(&scope()));
assert!(budget.try_reserve(&scope()));
assert!(budget.check_exhausted(&scope()));
assert!(!budget.try_reserve(&scope()));
}
#[test]
fn a_refund_reopens_the_budget_until_the_failure_cap() {
let budget = ReviewBudget::new(1);
for _ in 0..MAX_FAILED_CONSULTS {
assert!(budget.try_reserve(&scope()));
budget.refund_failure(&scope());
}
assert!(budget.check_exhausted(&scope()));
assert!(!budget.try_reserve(&scope()));
}
#[test]
fn eviction_rearms_a_session_scope_but_never_the_instance() {
let budget = ReviewBudget::new(1);
assert!(budget.try_reserve(&scope()));
budget.evict_scope(&scope());
assert!(budget.try_reserve(&scope()));
assert!(budget.try_reserve(&ScopeKey::Instance));
budget.evict_scope(&ScopeKey::Instance);
assert!(!budget.try_reserve(&ScopeKey::Instance));
}
#[test]
fn the_stall_latch_admits_one_caller_per_key() {
let budget = ReviewBudget::new(1);
assert!(budget.try_mark_stall_fired(7));
assert!(!budget.try_mark_stall_fired(7));
assert!(budget.try_mark_stall_fired(8));
}
#[test]
fn clearing_the_stall_latch_readmits_the_key() {
let budget = ReviewBudget::new(1);
assert!(budget.try_mark_stall_fired(7));
budget.clear_stall_fired(7);
assert!(budget.try_mark_stall_fired(7));
assert!(!budget.try_mark_stall_fired(7));
}
}