use std::{
collections::HashMap, future::Future, panic::AssertUnwindSafe, pin::Pin, sync::Arc,
time::Instant,
};
use async_trait::async_trait;
use futures::{FutureExt, Stream, StreamExt};
use parking_lot::Mutex;
use serde_json::Value;
use tokio::sync::{mpsc, oneshot};
use tokio_stream::wrappers::ReceiverStream;
use tracing::Instrument;
use switchyard_protocol::{Category, ModelId, Request, Response};
use crate::{DriverError, LibsyError, Result, observability};
pub type StepStream = Pin<Box<dyn Stream<Item = Result<Step>> + Send>>;
#[derive(Clone, Debug, Default)]
pub struct RuntimeModels {
by_category: HashMap<Category, Vec<ModelId>>,
subagent: Option<HashMap<Category, Vec<ModelId>>>,
}
impl RuntimeModels {
pub fn new(by_category: HashMap<Category, Vec<ModelId>>) -> Self {
Self {
by_category,
subagent: None,
}
}
pub fn with_subagent(mut self, models: HashMap<Category, Vec<ModelId>>) -> Self {
self.subagent = Some(models);
self
}
pub fn models_for(&self, category: &Category) -> &[ModelId] {
self.by_category.get(category).map_or(&[], Vec::as_slice)
}
pub fn subagent_models_for(&self, category: &Category) -> &[ModelId] {
self.subagent
.as_ref()
.and_then(|models| models.get(category))
.map_or(&[], Vec::as_slice)
}
}
impl From<HashMap<Category, Vec<ModelId>>> for RuntimeModels {
fn from(by_category: HashMap<Category, Vec<ModelId>>) -> Self {
Self::new(by_category)
}
}
#[derive(Clone, Copy)]
enum Scope {
Parent,
Subagent,
}
pub struct CallModel {
pub algorithm: String,
pub request: Request,
pub models: Vec<ModelId>,
reply: oneshot::Sender<Result<Response>>,
}
impl CallModel {
pub fn respond(self, result: Result<Response>) -> Result<()> {
self.reply
.send(result)
.map_err(|_| DriverError::ResponseDropped.into())
}
}
pub struct RoutingOutcome {
pub selected_model_ids: Vec<ModelId>,
pub request: Request,
pub response: Option<Response>,
pub metadata: Option<crate::OutcomeMetadata>,
}
impl RoutingOutcome {
pub fn selected_model_id(&self) -> Result<&ModelId> {
self.selected_model_ids.first().ok_or(LibsyError::NoTargets)
}
pub fn route_to(
selected_model_id: ModelId,
fallback_models: Vec<ModelId>,
mut request: Request,
) -> Self {
request.llm_request.model = Some(selected_model_id.to_string());
let mut selected_model_ids = Vec::with_capacity(1 + fallback_models.len());
selected_model_ids.push(selected_model_id);
selected_model_ids.extend(fallback_models);
Self {
selected_model_ids,
request,
response: None,
metadata: None,
}
}
pub fn answered(selected_model_id: ModelId, mut request: Request, response: Response) -> Self {
request.llm_request.model = Some(selected_model_id.to_string());
Self {
selected_model_ids: vec![selected_model_id],
request,
response: Some(response),
metadata: None,
}
}
}
#[derive(Clone)]
pub struct Driver {
step_tx: mpsc::Sender<Result<Step>>,
algorithm: String,
evidence: Arc<Mutex<Option<Value>>>,
models: Arc<RuntimeModels>,
scope: Scope,
}
impl Driver {
pub(crate) fn new(
algorithm: &str,
models: Arc<RuntimeModels>,
) -> (Self, mpsc::Receiver<Result<Step>>) {
let (step_tx, step_rx) = mpsc::channel(1);
(
Self {
step_tx,
algorithm: algorithm.to_string(),
evidence: Arc::new(Mutex::new(None)),
models,
scope: Scope::Parent,
},
step_rx,
)
}
pub(crate) fn set_evidence(&self, evidence: Value) {
*self.evidence.lock() = Some(evidence);
}
pub(crate) fn set_evidence_if_empty(&self, evidence: Value) {
let mut current = self.evidence.lock();
if current.is_none() {
*current = Some(evidence);
}
}
#[tracing::instrument(
target = "libsy",
name = "libsy.llm_call",
skip_all,
fields(
algorithm = self.algorithm,
selected_model = %models.first().map(ModelId::as_str).unwrap_or("NoTargets"),
openinference.span.kind = "CHAIN",
outcome = tracing::field::Empty,
input_tokens = tracing::field::Empty,
output_tokens = tracing::field::Empty,
total_tokens = tracing::field::Empty,
reasoning_tokens = tracing::field::Empty,
)
)]
pub async fn call_model(&self, mut request: Request, models: Vec<ModelId>) -> Result<Response> {
let Some(selected_model_id) = models.first().cloned() else {
return Err(LibsyError::NoTargets);
};
request.llm_request.model = Some(selected_model_id.to_string());
let started = Instant::now();
let (reply, response) = oneshot::channel::<Result<Response>>();
let call = CallModel {
algorithm: self.algorithm.clone(),
request,
models,
reply,
};
let result = async {
self.step_tx
.send(Ok(Step::CallModel(Box::new(call))))
.await
.map_err(|_| DriverError::StreamClosed)?;
response
.await
.map_err(|_| LibsyError::from(DriverError::ResponseDropped))?
}
.await;
let elapsed = started.elapsed();
observability::record_llm_call(
&self.algorithm,
selected_model_id.as_str(),
elapsed,
&result,
&tracing::Span::current(),
);
result
}
pub fn models_for(&self, category: &Category) -> &[ModelId] {
match self.scope {
Scope::Parent => self.models.models_for(category),
Scope::Subagent => self.models.subagent_models_for(category),
}
}
pub fn first_model_for(&self, category: &Category) -> Result<&ModelId> {
self.models_for(category)
.first()
.ok_or_else(|| LibsyError::AlgorithmError {
message: format!("no models available for category {}", category.as_str()),
})
}
pub fn for_subagent(&self) -> Result<Self> {
if self.models.subagent.is_none() {
return Err(LibsyError::AlgorithmError {
message: "delegated work has no sub-agent models".to_string(),
});
}
Ok(Self {
scope: Scope::Subagent,
..self.clone()
})
}
pub(crate) async fn finish(&self, result: Result<RoutingOutcome>) -> Result<()> {
let result = result.map(|mut outcome| {
let metadata = outcome.metadata.get_or_insert_with(|| {
crate::OutcomeMetadata::new(self.algorithm.clone(), self.evidence.lock().take())
});
observability::record_outcome(metadata, &outcome.selected_model_ids);
outcome
});
let selected_model = result
.as_ref()
.ok()
.and_then(|outcome| outcome.selected_model_id().ok().cloned());
let step = result.map(|outcome| Step::Done(Box::new(outcome)));
self.step_tx
.send(step)
.await
.map_err(|_| DriverError::StreamClosed)?;
if let Some(selected_model) = selected_model {
observability::record_decision(&self.algorithm, &selected_model);
}
Ok(())
}
}
pub enum Step {
CallModel(Box<CallModel>),
Done(Box<RoutingOutcome>),
}
pub async fn drive<F, Fut>(
algorithm: Arc<dyn Algorithm>,
request: Request,
models: Arc<RuntimeModels>,
serve: F,
) -> Result<RoutingOutcome>
where
F: Fn(CallModel) -> Fut,
Fut: Future<Output = Result<()>>,
{
let stream = algorithm.run_stream(request, models);
tokio::pin!(stream);
let mut in_flight = futures::stream::FuturesUnordered::new();
let mut final_outcome: Option<RoutingOutcome> = None;
loop {
tokio::select! {
Some(result) = in_flight.next() => match result {
Ok(()) => {}, Err(err) => return Err(err), },
step = stream.next() => {
match step {
None => break, Some(item) => match item? {
Step::CallModel(call) => in_flight.push(serve(*call)),
Step::Done(outcome) => {
final_outcome = Some(*outcome);
break;
}
}
}
},
}
}
final_outcome.ok_or(LibsyError::MissingFinalResponse)
}
fn panic_message(payload: &(dyn std::any::Any + Send)) -> String {
payload
.downcast_ref::<&'static str>()
.map(|message| (*message).to_string())
.or_else(|| payload.downcast_ref::<String>().cloned())
.unwrap_or_else(|| "unknown panic payload".to_string())
}
struct AbortOnDrop(tokio::task::AbortHandle);
impl Drop for AbortOnDrop {
fn drop(&mut self) {
self.0.abort();
}
}
pub(crate) fn ensure_model_is_target(targets: &[ModelId], name: &ModelId) -> Result<()> {
targets
.iter()
.any(|target| target == name)
.then_some(())
.ok_or_else(|| LibsyError::TargetNotFound {
target: name.clone(),
})
}
#[derive(Clone, Hash, PartialEq, Eq)]
pub(crate) enum RoutingIdentity {
Session(String),
Subagent { session: String, agent: String },
}
impl RoutingIdentity {
pub(crate) fn from_request(request: &Request) -> Option<Self> {
let metadata = request.metadata.as_ref()?;
let session = metadata.session_id.as_deref().filter(|id| !id.is_empty())?;
if metadata.is_subagent {
let agent = metadata.agent_id.as_deref().filter(|id| !id.is_empty())?;
Some(Self::Subagent {
session: session.to_string(),
agent: agent.to_string(),
})
} else {
Some(Self::Session(session.to_string()))
}
}
}
#[async_trait]
pub trait Algorithm: Send + Sync + 'static {
fn name(&self) -> &str;
async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<RoutingOutcome>;
fn run_stream(self: Arc<Self>, request: Request, models: Arc<RuntimeModels>) -> StepStream {
let (driver, step_rx) = Driver::new(self.name(), models);
let span = observability::run_span(self.name(), &request);
let handle = tokio::spawn(
async move {
let algorithm = self.name().to_string();
let route = AssertUnwindSafe(self.route(driver.clone(), request)).catch_unwind();
let result = observability::observe_run(&algorithm, async move {
route.await.unwrap_or_else(|payload| {
Err(LibsyError::AlgorithmError {
message: format!(
"algorithm task panicked: {}",
panic_message(payload.as_ref())
),
})
})
})
.await;
let _ = driver.finish(result).await;
}
.instrument(span),
);
let abort_guard = AbortOnDrop(handle.abort_handle());
Box::pin(ReceiverStream::new(step_rx).map(move |step| {
let _keep_alive = &abort_guard;
step
}))
}
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use super::*;
use crate::core::testing::{Serve, ServeResult, echo, reply, test_drive};
use futures::StreamExt;
use switchyard_protocol::{
LlmResponse, LlmResponseChunk, completion_text, text_request, text_response,
};
#[derive(Debug, thiserror::Error)]
#[error("{0}")]
struct TestError(&'static str);
fn test_error(message: &'static str) -> LibsyError {
LibsyError::external("test", TestError(message))
}
struct TestAlgo {
target_set: Vec<ModelId>,
}
#[async_trait]
impl Algorithm for TestAlgo {
fn name(&self) -> &str {
"test"
}
async fn route(
self: Arc<Self>,
driver: Driver,
request: Request,
) -> Result<RoutingOutcome> {
let target = self
.target_set
.first()
.ok_or(LibsyError::NoTargets)?
.clone();
let response = driver
.call_model(request.clone(), vec![target.clone()])
.await?;
driver.set_evidence(serde_json::json!({"source": "test"}));
driver.set_evidence_if_empty(serde_json::json!({"source": "ignored"}));
Ok(RoutingOutcome::answered(target, request, response))
}
}
fn orch(target_set: Vec<ModelId>) -> Arc<dyn Algorithm> {
Arc::new(TestAlgo { target_set })
}
fn request() -> Request {
Request {
llm_request: text_request(Some("auto".to_string()), "hi".to_string()),
raw_request: None,
metadata: None,
}
}
#[test]
fn routing_outcome_constructors_stamp_selection_and_preserve_payloads() {
let outcome = RoutingOutcome::route_to(
"selected".into(),
target_set(&["fallback-one", "fallback-two"]),
request(),
);
assert_eq!(
outcome.selected_model_ids,
target_set(&["selected", "fallback-one", "fallback-two"])
);
assert_eq!(outcome.request.model_id().as_deref(), Some("selected"));
assert!(outcome.response.is_none());
assert!(outcome.metadata.is_none());
let outcome = RoutingOutcome::route_to("only".into(), Vec::new(), request());
assert_eq!(outcome.selected_model_ids, target_set(&["only"]));
let outcome = RoutingOutcome::answered(
"answered".into(),
request(),
Response {
llm_response: LlmResponse::Agg(text_response(None, "existing")),
metadata: None,
upstream_headers: http::HeaderMap::new(),
},
);
assert_eq!(outcome.selected_model_ids, target_set(&["answered"]));
assert_eq!(outcome.request.model_id().as_deref(), Some("answered"));
assert_eq!(
outcome
.response
.as_ref()
.and_then(|response| response.llm_response.as_agg())
.map(completion_text),
Some("existing".to_string())
);
}
fn target_set(names: &[&str]) -> Vec<ModelId> {
names.iter().map(|name| ModelId::from(*name)).collect()
}
#[tokio::test]
async fn typed_driver_preserves_call_and_stream_boundaries() -> Result<()> {
tokio::time::timeout(std::time::Duration::from_secs(1), async {
let (driver, mut step_rx) = Driver::new("test", Arc::new(RuntimeModels::default()));
let first_driver = driver.clone();
let mut first = tokio::spawn(async move {
first_driver
.call_model(request(), vec![ModelId::from("first")])
.await
});
let second = tokio::spawn(async move {
driver
.call_model(request(), vec![ModelId::from("second")])
.await
});
let mut calls = HashMap::new();
for _ in 0..2 {
let step = step_rx.recv().await.ok_or(DriverError::StreamClosed)??;
let Step::CallModel(call) = step else {
return Err(test_error("expected a CallModel step"));
};
let selected_model = call
.models
.first()
.ok_or_else(|| test_error("model call has no candidates"))?
.to_string();
calls.insert(selected_model, call);
}
assert!(
tokio::time::timeout(std::time::Duration::from_millis(20), &mut first)
.await
.is_err(),
"call completed before the host responded"
);
calls
.remove("second")
.ok_or_else(|| test_error("missing second call"))?
.respond(Ok(reply("second response")))?;
calls
.remove("first")
.ok_or_else(|| test_error("missing first call"))?
.respond(Ok(reply("first response")))?;
let first_response = first
.await
.map_err(|source| LibsyError::external("joining a test task", source))??;
let second_response = second
.await
.map_err(|source| LibsyError::external("joining a test task", source))??;
assert_eq!(
first_response.llm_response.as_agg().map(completion_text),
Some("first response".to_string())
);
assert_eq!(
second_response.llm_response.as_agg().map(completion_text),
Some("second response".to_string())
);
let (driver, mut step_rx) = Driver::new("test", Arc::new(RuntimeModels::default()));
let producer = tokio::spawn(async move {
driver
.call_model(request(), vec![ModelId::from("dropped")])
.await
});
let step = step_rx.recv().await.ok_or(DriverError::StreamClosed)??;
let Step::CallModel(call) = step else {
return Err(test_error("expected a CallModel step"));
};
drop(call);
let result = producer
.await
.map_err(|source| LibsyError::external("joining a test task", source))?;
assert!(matches!(
result,
Err(LibsyError::Driver(DriverError::ResponseDropped))
));
let (driver, step_rx) = Driver::new("test", Arc::new(RuntimeModels::default()));
drop(step_rx);
let result = driver
.call_model(request(), vec![ModelId::from("closed")])
.await;
assert!(matches!(
result,
Err(LibsyError::Driver(DriverError::StreamClosed))
));
Ok(())
})
.await
.map_err(|error| LibsyError::external("waiting for typed driver boundaries", error))?
}
#[test]
fn target_lookup_returns_the_missing_target() {
let error = ensure_model_is_target(&target_set(&[]), &ModelId::from("missing")).err();
assert!(matches!(
error,
Some(LibsyError::TargetNotFound { target }) if target == "missing"
));
}
fn streaming_orch(chunks: Vec<LlmResponseChunk>) -> (Arc<dyn Algorithm>, impl Serve) {
let algo = orch(target_set(&["stream/model"]));
let serve = move |_target: ModelId, _request: Request| {
let chunks = chunks.clone();
async move {
let stream =
futures::stream::iter(chunks.into_iter().map(|chunk| Ok(chunk.into()))).boxed();
Ok(Response {
llm_response: LlmResponse::Stream(stream),
metadata: None,
upstream_headers: http::HeaderMap::new(),
})
}
};
(algo, serve)
}
#[tokio::test]
async fn run_returns_a_streamed_response_the_caller_aggregates() -> Result<()> {
let (orch, serve) = streaming_orch(vec![
LlmResponseChunk::MessageStart {
id: Some("m1".to_string()),
model: Some("stream/model".to_string()),
},
LlmResponseChunk::TextDelta {
index: 0,
text: "hel".to_string(),
},
LlmResponseChunk::TextDelta {
index: 0,
text: "lo".to_string(),
},
LlmResponseChunk::MessageStop {
reason: Some("stop".to_string()),
},
]);
let (selected_model, response) = test_drive(orch, request(), serve).await?;
let agg = response
.llm_response
.into_agg()
.await
.map_err(|error| LibsyError::external("aggregating response stream", error))?;
assert_eq!(completion_text(&agg), "hello");
assert_eq!(agg.model.as_deref(), Some("stream/model"));
assert_eq!(selected_model, "stream/model");
Ok(())
}
#[tokio::test]
async fn aggregating_a_streamed_response_propagates_a_mid_stream_error() -> Result<()> {
let (orch, serve) = streaming_orch(vec![
LlmResponseChunk::TextDelta {
index: 0,
text: "partial".to_string(),
},
LlmResponseChunk::StreamError {
message: "upstream exploded".to_string(),
},
]);
let (_, response) = test_drive(orch, request(), serve).await?;
match response.llm_response.into_agg().await {
Ok(_) => panic!("expected a mid-stream error, got an aggregate"),
Err(err) => {
assert!(err.to_string().contains("upstream exploded"));
Ok(())
}
}
}
#[tokio::test]
async fn run_offloads_via_promise_then_finishes() -> Result<()> {
let stream = orch(target_set(&["offload/model"]))
.run_stream(request(), Arc::new(RuntimeModels::default()));
tokio::pin!(stream);
let mut saw_call = false;
let mut final_completion = None;
while let Some(step) = stream.next().await {
match step? {
Step::CallModel(call) => {
saw_call = true;
assert_eq!(call.models, vec![ModelId::from("offload/model")]);
call.respond(Ok(Response {
llm_response: LlmResponse::Agg(text_response(
None,
"fulfilled".to_string(),
)),
metadata: None,
upstream_headers: http::HeaderMap::new(),
}))?;
}
Step::Done(outcome) => {
let metadata = outcome
.metadata
.as_ref()
.expect("run_stream should attach outcome metadata");
assert_eq!(metadata.algorithm, "test");
assert_eq!(
uuid::Uuid::parse_str(metadata.outcome_id())
.expect("outcome id should be a UUID")
.get_version_num(),
7
);
assert_eq!(
metadata.evidence,
Some(serde_json::json!({"source": "test"}))
);
let response = outcome
.response
.ok_or_else(|| test_error("expected an answered outcome"))?;
final_completion = Some(
response
.llm_response
.as_agg()
.map(completion_text)
.unwrap_or_default(),
);
}
}
}
assert!(saw_call, "expected a CallModel step before Done");
assert_eq!(
final_completion.ok_or_else(|| test_error("no Done step"))?,
"fulfilled"
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 12)]
async fn requests_are_processed_in_parallel() -> Result<()> {
use std::time::Duration;
use tokio::sync::Barrier;
const N: usize = 12;
let barrier = Arc::new(Barrier::new(N));
let algo = orch(target_set(&["m"]));
let mut handles = Vec::new();
for _ in 0..N {
let algo = algo.clone();
let barrier = barrier.clone();
let serve = move |target: ModelId, _request: Request| {
let barrier = barrier.clone();
async move {
barrier.wait().await;
Ok(reply(target))
}
};
handles.push(tokio::spawn(async move {
test_drive(algo, request(), serve)
.await
.map(|(_, response)| {
response
.llm_response
.as_agg()
.map(completion_text)
.unwrap_or_default()
})
}));
}
for handle in handles {
let completion = tokio::time::timeout(Duration::from_secs(5), handle)
.await
.map_err(|error| LibsyError::external("waiting for test task", error))?
.map_err(|source| LibsyError::external("joining a test task", source))??;
assert_eq!(completion, "m");
}
Ok(())
}
#[tokio::test]
async fn offload_error_propagates_back_to_the_algorithm() -> Result<()> {
let stream = orch(target_set(&["offload/model"]))
.run_stream(request(), Arc::new(RuntimeModels::default()));
tokio::pin!(stream);
let mut saw_error = false;
while let Some(step) = stream.next().await {
match step {
Ok(Step::CallModel(call)) => {
call.respond(Err(test_error("upstream model call failed")))?;
}
Ok(Step::Done(..)) => {
return Err(test_error(
"expected the offload error to propagate, got a response",
));
}
Err(err) => {
assert!(err.to_string().contains("upstream model call failed"));
saw_error = true;
}
}
}
assert!(saw_error, "expected an error step");
Ok(())
}
#[tokio::test]
async fn dropping_the_stream_cancels_the_algorithm_task() -> Result<()> {
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use tokio::sync::mpsc;
struct DropGuard(Arc<AtomicBool>);
impl Drop for DropGuard {
fn drop(&mut self) {
self.0.store(true, Ordering::SeqCst);
}
}
struct StuckAlgo {
started: mpsc::UnboundedSender<()>,
dropped: Arc<AtomicBool>,
}
#[async_trait]
impl Algorithm for StuckAlgo {
fn name(&self) -> &str {
"stuck"
}
async fn route(
self: Arc<Self>,
_driver: Driver,
_request: Request,
) -> Result<RoutingOutcome> {
let _guard = DropGuard(self.dropped.clone());
let _ = self.started.send(());
std::future::pending::<()>().await;
unreachable!()
}
}
let (started_tx, mut started_rx) = mpsc::unbounded_channel();
let dropped = Arc::new(AtomicBool::new(false));
let algo: Arc<dyn Algorithm> = Arc::new(StuckAlgo {
started: started_tx,
dropped: dropped.clone(),
});
let stream = algo.run_stream(request(), Arc::new(RuntimeModels::default()));
started_rx
.recv()
.await
.ok_or_else(|| test_error("task never started"))?;
drop(stream);
tokio::time::sleep(Duration::from_millis(100)).await;
assert!(
dropped.load(Ordering::SeqCst),
"algorithm task was NOT cancelled after dropping the stream"
);
Ok(())
}
#[tokio::test]
async fn route_panic_surfaces_as_a_stream_error() -> Result<()> {
struct Panicky;
#[async_trait]
impl Algorithm for Panicky {
fn name(&self) -> &str {
"panicky"
}
async fn route(
self: Arc<Self>,
_driver: Driver,
_request: Request,
) -> Result<RoutingOutcome> {
panic!("boom");
}
}
let algo: Arc<dyn Algorithm> = Arc::new(Panicky);
let stream = algo.run_stream(request(), Arc::new(RuntimeModels::default()));
tokio::pin!(stream);
let mut saw_error = false;
while let Some(step) = stream.next().await {
match step {
Err(err) => {
assert!(err.to_string().contains("algorithm task panicked: boom"));
saw_error = true;
}
Ok(_) => return Err(test_error("expected the panic to surface as an error step")),
}
}
assert!(saw_error, "expected an error step from the panicked task");
Ok(())
}
#[tokio::test]
async fn a_panic_with_a_leaked_driver_clone_still_terminates_the_run() -> Result<()> {
struct LeakyPanic;
#[async_trait]
impl Algorithm for LeakyPanic {
fn name(&self) -> &str {
"leaky_panic"
}
async fn route(
self: Arc<Self>,
driver: Driver,
_request: Request,
) -> Result<RoutingOutcome> {
tokio::spawn(async move {
let _keep_alive = driver;
std::future::pending::<()>().await;
});
tokio::task::yield_now().await;
panic!("boom");
}
}
let algo: Arc<dyn Algorithm> = Arc::new(LeakyPanic);
let result = tokio::time::timeout(
std::time::Duration::from_secs(1),
test_drive(algo, request(), echo()),
)
.await
.map_err(|error| LibsyError::external("waiting for the panicked run to end", error))?;
match result {
Ok(_) => Err(test_error(
"expected the panic to end the run with an error",
)),
Err(err) => {
assert!(err.to_string().contains("algorithm task panicked: boom"));
Ok(())
}
}
}
#[tokio::test]
async fn cancelling_run_cancels_the_algorithm_task() -> Result<()> {
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use tokio::sync::mpsc;
struct DropGuard(Arc<AtomicBool>);
impl Drop for DropGuard {
fn drop(&mut self) {
self.0.store(true, Ordering::SeqCst);
}
}
struct StuckAlgo {
started: mpsc::UnboundedSender<()>,
dropped: Arc<AtomicBool>,
}
#[async_trait]
impl Algorithm for StuckAlgo {
fn name(&self) -> &str {
"stuck"
}
async fn route(
self: Arc<Self>,
_driver: Driver,
_request: Request,
) -> Result<RoutingOutcome> {
let _guard = DropGuard(self.dropped.clone());
let _ = self.started.send(());
std::future::pending::<()>().await;
unreachable!()
}
}
let (started_tx, mut started_rx) = mpsc::unbounded_channel();
let dropped = Arc::new(AtomicBool::new(false));
let algo: Arc<dyn Algorithm> = Arc::new(StuckAlgo {
started: started_tx,
dropped: dropped.clone(),
});
let run_task = tokio::spawn(async move { test_drive(algo, request(), echo()).await });
started_rx
.recv()
.await
.ok_or_else(|| test_error("task never started"))?;
run_task.abort();
tokio::time::sleep(Duration::from_millis(100)).await;
assert!(
dropped.load(Ordering::SeqCst),
"algorithm task was NOT cancelled after cancelling run"
);
Ok(())
}
struct Hedge {
winner: String,
loser: String,
}
#[async_trait]
impl Algorithm for Hedge {
fn name(&self) -> &str {
"hedge"
}
async fn route(
self: Arc<Self>,
driver: Driver,
request: Request,
) -> Result<RoutingOutcome> {
let outcome_request = request.clone();
let win = driver.call_model(request.clone(), vec![self.winner.clone().into()]);
let lose = driver.call_model(request, vec![self.loser.clone().into()]);
tokio::select! {
res = win => Ok(RoutingOutcome::answered(
self.winner.clone().into(),
outcome_request,
res?,
)),
res = lose => Ok(RoutingOutcome::answered(
self.loser.clone().into(),
outcome_request,
res?,
)),
}
}
}
fn hedge(loser_delay: Option<std::time::Duration>) -> (Arc<dyn Algorithm>, impl Serve) {
let started = Arc::new(tokio::sync::Notify::new());
let algo = Arc::new(Hedge {
winner: "winner".to_string(),
loser: "loser".to_string(),
});
let serve = move |target: ModelId, _request: Request| {
let started = started.clone();
async move {
if target == "loser" {
started.notify_one();
match loser_delay {
Some(delay) => tokio::time::sleep(delay).await,
None => std::future::pending::<()>().await,
}
} else {
started.notified().await;
}
Ok(reply(target))
}
};
(algo, serve)
}
#[tokio::test]
async fn run_returns_the_winner_without_a_late_loser_overwriting_it() -> Result<()> {
let (algo, serve) = hedge(Some(std::time::Duration::from_millis(50)));
let (_, response) = test_drive(algo, request(), serve).await?;
assert_eq!(
response
.llm_response
.as_agg()
.map(completion_text)
.unwrap_or_default(),
"winner"
);
Ok(())
}
#[tokio::test]
async fn run_returns_the_winner_without_hanging_on_a_pending_loser() -> Result<()> {
let (algo, serve) = hedge(None);
let run = test_drive(algo, request(), serve);
let (_, response) = tokio::time::timeout(std::time::Duration::from_secs(1), run)
.await
.map_err(|error| LibsyError::external("waiting for pending loser", error))??;
assert_eq!(
response
.llm_response
.as_agg()
.map(completion_text)
.unwrap_or_default(),
"winner"
);
Ok(())
}
#[tokio::test]
async fn run_surfaces_a_terminal_error_with_many_calls_in_flight() -> Result<()> {
use std::sync::atomic::{AtomicUsize, Ordering};
const N: usize = 10;
struct FanOutThenError {
all_started: Arc<tokio::sync::Notify>,
n: usize,
}
#[async_trait]
impl Algorithm for FanOutThenError {
fn name(&self) -> &str {
"fan_out_then_error"
}
async fn route(
self: Arc<Self>,
driver: Driver,
request: Request,
) -> Result<RoutingOutcome> {
let offloads = futures::future::join_all(
(0..self.n)
.map(|i| driver.call_model(request.clone(), vec![format!("m{i}").into()])),
);
tokio::select! {
_ = offloads => Err(test_error("offloads unexpectedly completed")),
_ = self.all_started.notified() => {
Err(test_error("terminal error while calls pending"))
}
}
}
}
let all_started = Arc::new(tokio::sync::Notify::new());
let algo: Arc<dyn Algorithm> = Arc::new(FanOutThenError {
all_started: all_started.clone(),
n: N,
});
let started = Arc::new(AtomicUsize::new(0));
let serve = move |_target: ModelId, _request: Request| {
let started = started.clone();
let all_started = all_started.clone();
async move {
if started.fetch_add(1, Ordering::SeqCst) + 1 == N {
all_started.notify_one();
}
std::future::pending::<ServeResult>().await
}
};
let run = test_drive(algo, request(), serve);
let result = tokio::time::timeout(std::time::Duration::from_millis(500), run)
.await
.map_err(|error| {
LibsyError::external("waiting for terminal error with full call cap", error)
})?;
match result {
Ok(_) => Err(test_error("expected the terminal error, got a response")),
Err(err) => {
assert!(
err.to_string()
.contains("terminal error while calls pending")
);
Ok(())
}
}
}
}