use std::collections::HashMap;
use std::future::Future;
use std::sync::Arc;
use futures::future::BoxFuture;
use switchyard_protocol::{
Category, LlmClientError, LlmResponse, ModelId, Request, Response, text_response,
};
use crate::core::algorithm::{Algorithm, CallModel, Driver, RuntimeModels};
use crate::{LibsyError, Result};
pub(crate) fn category_models(
category: Category,
names: &[&str],
) -> HashMap<Category, Vec<ModelId>> {
[(
category,
names.iter().map(|name| ModelId::from(*name)).collect(),
)]
.into()
}
pub(crate) fn empty_driver() -> Driver {
Driver::new("test", Arc::new(RuntimeModels::default())).0
}
pub(crate) type ServeResult = std::result::Result<Response, LlmClientError>;
pub(crate) trait Serve: Send + Sync + 'static {
fn serve(&self, target: ModelId, request: Request) -> BoxFuture<'static, ServeResult>;
}
impl<F, Fut> Serve for F
where
F: Fn(ModelId, Request) -> Fut + Send + Sync + 'static,
Fut: Future<Output = ServeResult> + Send + 'static,
{
fn serve(&self, target: ModelId, request: Request) -> BoxFuture<'static, ServeResult> {
Box::pin(self(target, request))
}
}
pub(crate) async fn test_drive(
algorithm: Arc<dyn Algorithm>,
request: Request,
serve: impl Serve,
) -> Result<(ModelId, Response)> {
test_drive_with_models(algorithm, request, RuntimeModels::default(), serve).await
}
pub(crate) async fn test_drive_with_models(
algorithm: Arc<dyn Algorithm>,
request: Request,
models: impl Into<RuntimeModels>,
serve: impl Serve,
) -> Result<(ModelId, Response)> {
let serve = Arc::new(serve);
let routing_serve = Arc::clone(&serve);
let outcome = crate::drive(algorithm, request, Arc::new(models.into()), move |call| {
fulfill(Arc::clone(&routing_serve), call)
})
.await?;
let selected_model = outcome.selected_model_id()?.clone();
let response = match outcome.response {
Some(response) => response,
None => serve
.serve(selected_model.clone(), outcome.request)
.await
.map_err(|source| LibsyError::client_call(selected_model.clone(), source))?,
};
Ok((selected_model, response))
}
async fn fulfill(serve: Arc<impl Serve>, call: CallModel) -> Result<()> {
let request = call.request.clone();
let target = call.models.first().cloned().ok_or(LibsyError::NoTargets)?;
let result = serve
.serve(target.clone(), request)
.await
.map_err(|source| LibsyError::client_call(target, source));
call.respond(result)
}
pub(crate) fn echo() -> impl Serve {
|target: ModelId, _request: Request| async move { Ok(reply(target)) }
}
pub(crate) fn reply(completion: impl Into<String>) -> Response {
Response {
llm_response: LlmResponse::Agg(text_response(None, completion.into())),
metadata: None,
upstream_headers: http::HeaderMap::new(),
}
}