use std::{
collections::HashMap,
sync::{Arc, Once, Weak},
time::{Duration, Instant},
};
use async_trait::async_trait;
use parking_lot::Mutex;
use tokio::sync::Mutex as AsyncMutex;
use crate::core::algorithm::{self, Algorithm, Driver};
use crate::core::classifier::{Classifier, Score};
use crate::core::processor::{Event, Processor};
use crate::{LibsyError, Result, RoutingOutcome};
use switchyard_protocol::{Category, ModelId, Request, Response};
struct SessionState<S> {
state: Arc<AsyncMutex<S>>,
last_accessed: Instant,
}
type SessionStates<S> = Mutex<HashMap<String, SessionState<S>>>;
const SESSION_STATE_TTL: Duration = Duration::from_secs(60 * 60);
const SESSION_CLEANUP_INTERVAL: Duration = Duration::from_secs(60 * 60);
pub struct FallThrough<S = ()> {
name: String,
processors: Vec<Arc<dyn Processor<S>>>,
classifiers: Vec<Arc<dyn Classifier<S>>>,
session_states: Option<Arc<SessionStates<S>>>,
cleanup_started: Once,
}
impl FallThrough<()> {
pub fn new() -> Self {
Self {
name: "fall_through".to_string(),
processors: Vec::new(),
classifiers: Vec::new(),
session_states: None,
cleanup_started: Once::new(),
}
}
}
impl<S> FallThrough<S>
where
S: Default + Send + 'static,
{
pub fn new_with_state() -> Self {
Self {
name: "fall_through".to_string(),
processors: Vec::new(),
classifiers: Vec::new(),
session_states: Some(Arc::new(Mutex::new(HashMap::new()))),
cleanup_started: Once::new(),
}
}
pub fn with_name(mut self, name: impl Into<String>) -> Self {
self.name = name.into();
self
}
pub fn with_processor(mut self, processor: Arc<dyn Processor<S>>) -> Self {
self.processors.push(processor);
self
}
pub fn with_classifier(mut self, classifier: Arc<dyn Classifier<S>>) -> Self {
self.classifiers.push(classifier);
self
}
pub(crate) async fn execute(&self, driver: Driver, request: Request) -> Result<RoutingOutcome> {
self.start_cleanup_task();
let session = session_id(&request);
let session_final = request
.metadata
.as_ref()
.and_then(|metadata| metadata.session_final)
== Some(true);
let result = self.execute_session(driver, request).await;
if session_final && let Some(session) = session.as_deref() {
self.remove_session(session);
}
result
}
fn start_cleanup_task(&self) {
let Some(states) = &self.session_states else {
return;
};
let states = Arc::downgrade(states);
self.cleanup_started.call_once(move || {
drop(tokio::spawn(cleanup_inactive_sessions(states)));
});
}
async fn execute_session(&self, driver: Driver, request: Request) -> Result<RoutingOutcome> {
let mut request = request;
let session_state = self.session_state(&request);
let (score, served) = match session_state {
Some(state) => {
let mut state = state.lock().await;
self.route(&mut state, &driver, &mut request).await?
}
None => {
let mut state = S::default();
self.route(&mut state, &driver, &mut request).await?
}
};
let target = score.target;
match served {
Some(response) => Ok(RoutingOutcome::answered(target, request, response)),
None => {
let chosen = driver.models_for(score.category.as_ref().unwrap_or(&Category::Any));
let mut fallback_models: Vec<ModelId> = Vec::new();
for candidate in chosen.iter().chain(driver.models_for(&Category::Any)) {
if *candidate != target && !fallback_models.contains(candidate) {
fallback_models.push(candidate.clone());
}
}
Ok(RoutingOutcome::route_to(target, fallback_models, request))
}
}
}
fn remove_session(&self, session: &str) {
if let Some(states) = &self.session_states {
states.lock().remove(session);
}
}
fn session_state(&self, request: &Request) -> Option<Arc<AsyncMutex<S>>> {
let states = self.session_states.as_ref()?;
let session_id = session_id(request)?;
let mut states = states.lock();
let now = Instant::now();
let session = states.entry(session_id).or_insert_with(|| SessionState {
state: Arc::new(AsyncMutex::new(S::default())),
last_accessed: now,
});
session.last_accessed = now;
Some(Arc::clone(&session.state))
}
async fn route(
&self,
state: &mut S,
driver: &Driver,
request: &mut Request,
) -> Result<(Score, Option<Response>)> {
for processor in &self.processors {
let event = Event::Request { request, driver };
processor.process(state, event).await?;
}
let mut routed = None;
for classifier in &self.classifiers {
let (scores, response) = classifier.score(state, request, driver).await?;
if let Some(score) = scores.argmax(false)? {
routed = Some((score, response));
break;
}
}
let Some((score, served)) = routed else {
return Err(LibsyError::AlgorithmError {
message: "every classifier abstained".to_string(),
});
};
algorithm::ensure_model_is_target(driver.models_for(&Category::Any), &score.target)?;
let target = score.target.clone();
tracing::info!(algorithm=self.name, target=%score.target, confidence=score.confidence, "Model selected");
for processor in &self.processors {
let event = Event::Decision {
request,
selected_model_id: &target,
category: score.category.clone(),
driver,
};
processor.process(state, event).await?;
}
Ok((score, served))
}
}
async fn cleanup_inactive_sessions<S>(states: Weak<SessionStates<S>>)
where
S: Send + 'static,
{
let start = tokio::time::Instant::now() + SESSION_CLEANUP_INTERVAL;
let mut interval = tokio::time::interval_at(start, SESSION_CLEANUP_INTERVAL);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
interval.tick().await;
let Some(states) = states.upgrade() else {
break;
};
remove_inactive_sessions(&states, Instant::now(), SESSION_STATE_TTL);
}
}
fn remove_inactive_sessions<S>(states: &SessionStates<S>, now: Instant, ttl: Duration) {
states.lock().retain(|_, session| {
Arc::strong_count(&session.state) > 1
|| now.saturating_duration_since(session.last_accessed) < ttl
});
}
fn session_id(request: &Request) -> Option<String> {
request
.metadata
.as_ref()?
.session_id
.as_deref()
.filter(|id| !id.is_empty())
.map(str::to_string)
}
#[async_trait]
impl<S> Algorithm for FallThrough<S>
where
S: Default + Send + 'static,
{
fn name(&self) -> &str {
&self.name
}
async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<RoutingOutcome> {
self.execute(driver, request).await
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::Classification;
use crate::algorithms::llm_class::DefaultCategoryClassifier;
use crate::algorithms::util::prompts;
use crate::core::testing::{Serve, category_models, echo, reply, test_drive_with_models};
use switchyard_protocol::{LlmRequest, Message, Metadata, Role, completion_text, text_request};
#[derive(Debug, thiserror::Error)]
#[error("{0}")]
struct TestError(&'static str);
fn test_error(message: &'static str) -> LibsyError {
LibsyError::external("test", TestError(message))
}
fn capturing(into: Arc<Mutex<Option<Request>>>) -> impl Serve {
move |target: ModelId, request: Request| {
let into = Arc::clone(&into);
async move {
*into.lock() = Some(request);
Ok(reply(target))
}
}
}
const NOTE: &str = "the previous model was stalling";
fn target_set(names: &[&str]) -> Vec<ModelId> {
names.iter().map(|name| ModelId::from(*name)).collect()
}
struct FixedClassifier(Vec<Score>);
#[async_trait]
impl Classifier for FixedClassifier {
async fn score(
&self,
_state: &mut (),
_request: &mut Request,
_driver: &Driver,
) -> Result<(Classification, Option<Response>)> {
Ok((
Classification::Scores(
self.0
.iter()
.map(|s| Score {
confidence: s.confidence,
target: s.target.clone(),
category: None,
})
.collect(),
),
None,
))
}
}
fn score(target: &str, confidence: f64) -> Score {
Score {
confidence,
target: ModelId::from(target),
category: None,
}
}
fn fixed(scores: Vec<Score>) -> Arc<dyn Classifier> {
Arc::new(FixedClassifier(scores))
}
fn request() -> Request {
Request {
llm_request: LlmRequest {
model: Some("auto".to_string()),
messages: vec![Message::text(Role::User, "hi")],
..LlmRequest::default()
},
raw_request: None,
metadata: Some(Metadata {
session_id: Some("session-1".to_string()),
..Metadata::default()
}),
}
}
async fn run_request<S>(
router: &Arc<FallThrough<S>>,
request: Request,
serve: impl Serve,
) -> Result<(String, ModelId)>
where
S: Default + Send + 'static,
{
let (selected_model, response) = test_drive_with_models(
router.clone(),
request,
category_models(Category::Any, &["strong", "weak"]),
serve,
)
.await?;
let text = response
.llm_response
.into_agg()
.await
.map(|agg| completion_text(&agg))
.map_err(|error| LibsyError::external("aggregating fall-through response", error))?;
Ok((text, selected_model))
}
async fn run_turn<S>(
router: &Arc<FallThrough<S>>,
serve: impl Serve,
) -> Result<(String, ModelId)>
where
S: Default + Send + 'static,
{
run_request(router, request(), serve).await
}
async fn run_with(router: FallThrough, serve: impl Serve) -> Result<(String, ModelId)> {
run_turn(&Arc::new(router), serve).await
}
#[tokio::test]
async fn a_note_reaches_the_model_in_the_conversation() -> Result<()> {
struct Noting;
#[async_trait]
impl Processor for Noting {
async fn process(&self, _state: &mut (), event: Event<'_>) -> Result<()> {
if let Event::Decision { request, .. } = event {
prompts::append_note(request, NOTE);
}
Ok(())
}
}
let captured = Arc::new(Mutex::new(None));
let router = FallThrough::new()
.with_processor(Arc::new(Noting))
.with_classifier(Arc::new(DefaultCategoryClassifier(Category::Any)));
test_drive_with_models(
Arc::new(router),
Request {
llm_request: text_request(Some("auto".to_string()), "fix the build"),
raw_request: None,
metadata: None,
},
category_models(Category::Any, &["capable", "efficient"]),
capturing(Arc::clone(&captured)),
)
.await?;
let request = captured
.lock()
.take()
.ok_or_else(|| LibsyError::external("test", TestError("the model was never called")))?;
let messages: Vec<String> = request
.llm_request
.messages
.iter()
.filter_map(|message| message.text_content("|"))
.collect();
assert_eq!(messages, vec![format!("fix the build|{NOTE}")]);
assert!(
request.llm_request.instructions.is_empty(),
"a note is not an instruction"
);
Ok(())
}
#[tokio::test]
async fn the_selected_category_leads_the_candidate_list() -> Result<()> {
use futures::StreamExt;
let router = Arc::new(
FallThrough::<()>::new()
.with_classifier(Arc::new(DefaultCategoryClassifier(Category::Capable))),
);
let models = crate::RuntimeModels::new(
[
(Category::Capable, target_set(&["premium", "reasoning"])),
(Category::Any, target_set(&["fast", "premium", "reasoning"])),
]
.into(),
);
let stream = router.run_stream(request(), Arc::new(models));
tokio::pin!(stream);
while let Some(step) = stream.next().await {
if let crate::Step::Done(outcome) = step? {
assert_eq!(
outcome.selected_model_ids,
target_set(&["premium", "reasoning", "fast"])
);
return Ok(());
}
}
Err(test_error("expected a Done step"))
}
#[tokio::test]
async fn selected_target_leads_the_ordered_candidate_list() -> Result<()> {
use futures::StreamExt;
let router =
Arc::new(FallThrough::<()>::new().with_classifier(fixed(vec![score("mid", 0.9)])));
let models = category_models(Category::Any, &["weak", "mid", "strong"]);
let stream = router.run_stream(request(), Arc::new(models.into()));
tokio::pin!(stream);
while let Some(step) = stream.next().await {
if let crate::Step::Done(outcome) = step? {
assert_eq!(
outcome.selected_model_ids,
target_set(&["mid", "weak", "strong"])
);
assert_eq!(outcome.request.llm_request.model.as_deref(), Some("mid"));
assert!(outcome.response.is_none());
return Ok(());
}
}
Err(test_error("expected a Done step"))
}
#[tokio::test]
async fn argmax_picks_the_highest_confidence_target() -> Result<()> {
let router = FallThrough::<()>::new()
.with_classifier(fixed(vec![score("weak", 0.2), score("strong", 0.9)]));
let (model, selected_model) = run_with(router, echo()).await?;
assert_eq!(model, "strong");
assert_eq!(selected_model, "strong");
Ok(())
}
#[tokio::test]
async fn falls_through_the_first_abstaining_classifier() -> Result<()> {
let router = FallThrough::<()>::new()
.with_classifier(fixed(vec![]))
.with_classifier(fixed(vec![score("weak", 1.0)]));
let (model, _) = run_with(router, echo()).await?;
assert_eq!(model, "weak");
Ok(())
}
#[tokio::test]
async fn first_deciding_classifier_wins_the_cascade() -> Result<()> {
let router = FallThrough::<()>::new()
.with_classifier(fixed(vec![score("strong", 0.6)]))
.with_classifier(fixed(vec![score("weak", 1.0)]));
let (model, _) = run_with(router, echo()).await?;
assert_eq!(model, "strong");
Ok(())
}
#[tokio::test]
async fn all_abstaining_is_an_error() -> Result<()> {
let router = FallThrough::<()>::new().with_classifier(fixed(vec![]));
let error = run_with(router, echo())
.await
.err()
.ok_or_else(|| test_error("expected classifiers to abstain"))?;
assert!(matches!(
error,
LibsyError::AlgorithmError { message } if message == "every classifier abstained"
));
Ok(())
}
#[tokio::test]
async fn processor_observes_request_then_decision() -> Result<()> {
use parking_lot::Mutex;
struct RecordingProcessor(Arc<Mutex<Vec<&'static str>>>);
#[async_trait]
impl Processor for RecordingProcessor {
async fn process(&self, _state: &mut (), event: Event<'_>) -> Result<()> {
let kind = match event {
Event::Request { .. } => "request",
Event::Decision { .. } => "decision",
_ => "other",
};
self.0.lock().push(kind);
Ok(())
}
}
let seen = Arc::new(Mutex::new(Vec::new()));
let router = FallThrough::<()>::new()
.with_processor(Arc::new(RecordingProcessor(seen.clone())))
.with_classifier(fixed(vec![score("strong", 1.0)]));
run_with(router, echo()).await?;
assert_eq!(*seen.lock(), vec!["request", "decision"]);
Ok(())
}
#[tokio::test]
async fn a_rewrite_propagates_down_the_chain_and_into_the_model_call() -> Result<()> {
use parking_lot::Mutex;
struct Appender(&'static str);
#[async_trait]
impl Processor for Appender {
async fn process(&self, _state: &mut (), event: Event<'_>) -> Result<()> {
if let Event::Request { request, .. } = event {
request
.llm_request
.messages
.push(Message::text(Role::User, self.0));
}
Ok(())
}
}
struct TrailClassifier(Arc<Mutex<Vec<String>>>);
#[async_trait]
impl Classifier for TrailClassifier {
async fn score(
&self,
_state: &mut (),
request: &mut Request,
_driver: &Driver,
) -> Result<(Classification, Option<Response>)> {
*self.0.lock() = request
.llm_request
.messages
.iter()
.filter_map(|message| message.text_content(""))
.collect();
request
.llm_request
.messages
.push(Message::text(Role::User, "classifier"));
Ok((Classification::Scores(vec![score("strong", 1.0)]), None))
}
}
let seen_by_classifier = Arc::new(Mutex::new(Vec::new()));
let seen_by_model = Arc::new(Mutex::new(None));
let router = FallThrough::new()
.with_processor(Arc::new(Appender("first")))
.with_processor(Arc::new(Appender("second")))
.with_classifier(Arc::new(TrailClassifier(seen_by_classifier.clone())));
run_turn(&Arc::new(router), capturing(seen_by_model.clone())).await?;
assert_eq!(*seen_by_classifier.lock(), vec!["hi", "first", "second"]);
let routed = seen_by_model
.lock()
.take()
.ok_or_else(|| test_error("the model was never called"))?;
let trail: Vec<String> = routed
.llm_request
.messages
.iter()
.filter_map(|message| message.text_content(""))
.collect();
assert_eq!(trail, vec!["hi", "first", "second", "classifier"]);
Ok(())
}
#[tokio::test]
async fn state_is_shared_within_a_session_and_isolated_between_sessions() -> Result<()> {
#[derive(Default)]
struct TurnState {
count: u32,
}
struct CountingProcessor;
#[async_trait]
impl Processor<TurnState> for CountingProcessor {
async fn process(&self, state: &mut TurnState, event: Event<'_>) -> Result<()> {
if let Event::Request { .. } = event {
state.count += 1;
}
Ok(())
}
}
struct ThresholdClassifier;
#[async_trait]
impl Classifier<TurnState> for ThresholdClassifier {
async fn score(
&self,
state: &mut TurnState,
_request: &mut Request,
_driver: &Driver,
) -> Result<(Classification, Option<Response>)> {
let target = if state.count >= 2 { "strong" } else { "weak" };
Ok((Classification::Scores(vec![score(target, 1.0)]), None))
}
}
let router = Arc::new(
FallThrough::<TurnState>::new_with_state()
.with_processor(Arc::new(CountingProcessor))
.with_classifier(Arc::new(ThresholdClassifier)),
);
let (turn1, _) = run_turn(&router, echo()).await?;
let (turn2, _) = run_turn(&router, echo()).await?;
let final_request = Request {
metadata: Some(Metadata {
session_id: Some("session-1".to_string()),
session_final: Some(true),
..Metadata::default()
}),
..request()
};
let (final_turn, _) = run_request(&router, final_request, echo()).await?;
let (restarted_session, _) = run_turn(&router, echo()).await?;
let (second_session, _) = run_request(
&router,
Request {
metadata: Some(Metadata {
session_id: Some("session-2".to_string()),
..Metadata::default()
}),
..request()
},
echo(),
)
.await?;
let anonymous = Request {
metadata: None,
..request()
};
let (anonymous1, _) = run_request(&router, anonymous.clone(), echo()).await?;
let (anonymous2, _) = run_request(&router, anonymous, echo()).await?;
assert_eq!(turn1, "weak");
assert_eq!(turn2, "strong");
assert_eq!(final_turn, "strong");
assert_eq!(restarted_session, "weak");
assert_eq!(second_session, "weak");
assert_eq!(anonymous1, "weak");
assert_eq!(anonymous2, "weak");
Ok(())
}
#[tokio::test]
async fn final_session_is_removed_when_routing_fails() {
let router = Arc::new(FallThrough::<u32>::new_with_state());
let final_request = Request {
metadata: Some(Metadata {
session_id: Some("session-1".to_string()),
session_final: Some(true),
..Metadata::default()
}),
..request()
};
let result = test_drive_with_models(
router.clone(),
final_request,
category_models(Category::Any, &["strong"]),
echo(),
)
.await;
assert!(matches!(result, Err(LibsyError::AlgorithmError { .. })));
let states = router
.session_states
.as_ref()
.expect("stateful router has a session registry")
.lock();
assert!(!states.contains_key("session-1"));
}
#[test]
fn cleanup_removes_only_inactive_idle_sessions() {
let router = FallThrough::<u32>::new_with_state();
let _active_state = router
.session_state(&request()) .expect("session state was inserted");
let inactive_request = Request {
metadata: Some(Metadata {
session_id: Some("session-2".to_string()),
..Metadata::default()
}),
..request()
};
drop(router.session_state(&inactive_request));
let states = router
.session_states
.as_ref()
.expect("stateful router has a session registry");
let now = Instant::now() + SESSION_STATE_TTL + Duration::from_secs(1);
remove_inactive_sessions(states, now, SESSION_STATE_TTL);
let states = states.lock();
assert!(states.contains_key("session-1"));
assert!(!states.contains_key("session-2"));
}
}