#![forbid(unsafe_code)]
use std::{
collections::{BTreeMap, VecDeque},
marker::PhantomData,
sync::{
Arc, PoisonError, RwLock,
atomic::{AtomicU64, Ordering},
},
task::Poll,
};
use futures::StreamExt;
use serde::de::DeserializeOwned;
use tracing::Instrument;
use rig_core::{
completion::{CompletionRequest, ModelRef, ProviderCapabilities},
effect::{
Answered, CustomEffect, EffectFamily, EffectId, EffectKind, Family, FamilyDescriptor,
HandlerDescriptor, HandlerKey, Key, RetrieveQuery, RetrievedDocuments, family,
},
error::{ErrorKind, ErrorReport},
serve::{Dispatch, DispatchContext, ErasedHandler, Recorder, Serve},
streaming::{CompletionStream, Relayed, StreamEvents},
tool::{PublishedContext, ToolContext, ToolResult},
vector_store::{
VectorSearchIdResult, VectorSearchResult,
request::{Filter, VectorSearchRequest},
},
wasm_compat::WasmCompatSend,
};
#[derive(Clone, Default)]
pub struct Handlers {
shared: Arc<Shared>,
parent: Option<EffectId>,
}
#[derive(Default)]
struct Shared {
table: RwLock<Table>,
next_id: AtomicU64,
}
#[derive(Default)]
struct Table {
handlers: BTreeMap<HandlerKey, (ErasedHandler, HandlerDescriptor)>,
recorder: Option<Arc<dyn Recorder + Send + Sync>>,
}
impl std::fmt::Debug for Handlers {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Handlers")
.field("keys", &self.keys())
.field("parent", &self.parent)
.finish_non_exhaustive()
}
}
impl Handlers {
fn read(&self) -> std::sync::RwLockReadGuard<'_, Table> {
self.shared
.table
.read()
.unwrap_or_else(PoisonError::into_inner)
}
fn write(&self) -> std::sync::RwLockWriteGuard<'_, Table> {
self.shared
.table
.write()
.unwrap_or_else(PoisonError::into_inner)
}
pub fn register(
&self,
key: impl Into<HandlerKey>,
handler: impl Serve + 'static,
) -> Result<(), ErrorReport> {
self.register_erased(key, ErasedHandler::new(handler))
}
pub fn register_erased(
&self,
key: impl Into<HandlerKey>,
handler: ErasedHandler,
) -> Result<(), ErrorReport> {
let key = key.into();
let described = handler.descriptor();
let descriptor = HandlerDescriptor {
key: key.clone(),
family: described.family,
layers: described.layers,
};
let mut table = self.write();
if let Some((_, current)) = table.handlers.get(&key) {
let (current, family) = (current.family.family(), descriptor.family.family());
if current != family {
return Err(ErrorReport::new(
ErrorKind::HandlerUnavailable,
format!(
"key `{key}` serves the {current:?} family; a {family:?} handler cannot replace it"
),
)
.with_retryable(false));
}
}
if let Some(recorder) = &table.recorder {
recorder.handlers(vec![descriptor.clone()]);
}
let displaced = table.handlers.insert(key, (handler, descriptor));
drop(table);
drop(displaced);
Ok(())
}
pub fn deregister(&self, key: &HandlerKey) -> bool {
let removed = self.write().handlers.remove(key);
removed.is_some()
}
pub fn record_to(&self, recorder: impl Recorder + Send + Sync) {
self.record_with(Arc::new(recorder));
}
pub(crate) fn record_with(&self, recorder: Arc<dyn Recorder + Send + Sync>) {
let mut table = self.write();
recorder.handlers(table.handlers.values().map(|(_, d)| d.clone()).collect());
table.recorder = Some(recorder);
}
pub fn descriptor(&self, key: &HandlerKey) -> Option<HandlerDescriptor> {
self.read().handlers.get(key).map(|(_, d)| d.clone())
}
pub fn descriptors(&self) -> Vec<HandlerDescriptor> {
self.read()
.handlers
.values()
.map(|(_, d)| d.clone())
.collect()
}
pub fn keys(&self) -> Vec<HandlerKey> {
self.read().handlers.keys().cloned().collect()
}
pub const fn parent(&self) -> Option<EffectId> {
self.parent
}
pub(crate) fn downgrade(&self) -> WeakHandlers {
WeakHandlers(Arc::downgrade(&self.shared))
}
#[cfg(not(target_family = "wasm"))]
pub fn serving(dispatch: &Dispatch) -> Option<Self> {
dispatch.scope::<Self>().map(|handlers| (*handlers).clone())
}
pub fn resume_ids_after(&self, last: EffectId) {
self.shared
.next_id
.fetch_max(last.as_u64(), Ordering::SeqCst);
}
pub(crate) fn mint_id(&self) -> EffectId {
EffectId::from_raw(self.shared.next_id.fetch_add(1, Ordering::SeqCst) + 1)
}
fn lookup(
&self,
key: &HandlerKey,
) -> Result<(ErasedHandler, Option<Arc<dyn Recorder + Send + Sync>>), ErrorReport> {
let table = self.read();
let (handler, _) = table
.handlers
.get(key)
.ok_or_else(|| handler_unavailable(key))?;
Ok((handler.clone(), table.recorder.clone()))
}
fn open(&self, id: EffectId, key: &HandlerKey, kind: &EffectKind, beside: Beside) -> Opened {
let (handler, recorder) = match &beside.handler {
Some(handler) => (handler.clone(), self.read().recorder.clone()),
None => self.lookup(key)?,
};
let mut dispatch = Dispatch::new(id, kind.streams());
#[cfg(not(target_family = "wasm"))]
{
let child = Self {
shared: Arc::clone(&self.shared),
parent: Some(id),
};
dispatch = dispatch.with_scope(Arc::new(child));
}
if let Some(context) = beside.context {
dispatch = dispatch.with_scope(Arc::new(context));
}
if let Some(published) = beside.published {
dispatch = dispatch.with_scope(published);
}
if let Some(recorder) = recorder {
let origin = DispatchContext {
parent: self.parent,
..DispatchContext::default()
};
recorder.begin(id, key.clone(), kind.clone(), origin);
dispatch = dispatch.recorded_by(recorder);
}
Ok((handler, dispatch))
}
pub fn dispatch(
&self,
key: &HandlerKey,
kind: EffectKind,
) -> impl Future<Output = Result<Answered, ErrorReport>> + WasmCompatSend + 'static {
let beside = Beside::for_kind(&kind);
self.serve(self.mint_id(), key.clone(), kind, beside)
}
pub fn dispatch_stream(&self, key: &HandlerKey, kind: EffectKind) -> StreamEvents {
self.serve_stream(self.mint_id(), key.clone(), kind, None)
}
pub(crate) fn serve(
&self,
id: EffectId,
key: HandlerKey,
kind: EffectKind,
beside: Beside,
) -> impl Future<Output = Result<Answered, ErrorReport>> + WasmCompatSend + 'static {
let this = self.clone();
async move {
yield_once().await;
let (handler, dispatch) = this.open(id, &key, &kind, beside)?;
handler.handle(kind, dispatch).await.into_outcome().await
}
.instrument(tracing::Span::current())
}
pub(crate) fn serve_stream(
&self,
id: EffectId,
key: HandlerKey,
kind: EffectKind,
handler: Option<ErasedHandler>,
) -> StreamEvents {
if !kind.streams() {
let report = ErrorReport::new(
ErrorKind::Request,
format!(
"invalid dispatch: `{}` is a unary effect and cannot be dispatched as a stream",
kind.name()
),
);
return Box::pin(futures::stream::iter([Err(report)]));
}
let this = self.clone();
let beside = Beside {
handler,
..Beside::default()
};
let events = async_stream::stream! {
yield_once().await;
let (handler, dispatch) = match this.open(id, &key, &kind, beside) {
Ok(opened) => opened,
Err(report) => {
yield Err(report);
return;
}
};
let mut events = handler.handle(kind, dispatch).await.into_stream();
let (mut ahead, mut drained, mut ended) = (VecDeque::new(), false, false);
loop {
let next = std::future::poll_fn(|cx| {
while !drained && ahead.len() < READ_AHEAD {
match events.poll_next_unpin(cx) {
Poll::Ready(Some(item)) => ahead.push_back(item),
Poll::Ready(None) => drained = true,
Poll::Pending => break,
}
}
match ahead.pop_front() {
Some(item) => Poll::Ready(Some(item)),
None if drained => Poll::Ready(None),
None => Poll::Pending,
}
})
.await;
let Some(item) = next else { break };
ended |= matches!(item, Ok(Relayed::Done(_)) | Err(_));
yield item;
}
if !ended {
yield Err(rig_core::serve::stream_truncated());
}
};
Box::pin(tracing_futures::Instrument::instrument(
events,
tracing::Span::current(),
))
}
pub fn handle<F: Family>(&self, key: &HandlerKey) -> Result<Handle<F>, ErrorReport> {
let descriptor = self
.descriptor(key)
.ok_or_else(|| handler_unavailable(key))?;
let found = descriptor.family.family();
if found != F::FAMILY {
return Err(family_mismatch(key, F::FAMILY, found));
}
Ok(Handle {
handlers: self.clone(),
descriptor,
_family: PhantomData,
})
}
pub fn bind<F: Family>(&self, key: &Key<F>) -> Result<Handle<F>, ErrorReport> {
self.handle(key.raw())
}
pub fn custom<E: CustomEffect>(
&self,
key: &HandlerKey,
) -> Result<Handle<family::Custom<E>>, ErrorReport> {
let handle = self.handle::<family::Custom<E>>(key)?;
match &handle.descriptor.family {
FamilyDescriptor::Custom { kind } if kind == E::KIND => Ok(handle),
FamilyDescriptor::Custom { kind } => Err(ErrorReport::new(
ErrorKind::HandlerUnavailable,
format!(
"handler `{key}` serves the custom kind `{kind}`, not `{}`",
E::KIND
),
)),
other => Err(family_mismatch(key, EffectFamily::Custom, other.family())),
}
}
}
pub(crate) struct WeakHandlers(std::sync::Weak<Shared>);
impl WeakHandlers {
pub(crate) fn upgrade(&self) -> Option<Handlers> {
self.0.upgrade().map(|shared| Handlers {
shared,
parent: None,
})
}
}
type Opened = Result<(ErasedHandler, Dispatch), ErrorReport>;
const READ_AHEAD: usize = 64;
async fn yield_once() {
let mut yielded = false;
std::future::poll_fn(|cx| {
if std::mem::replace(&mut yielded, true) {
return Poll::Ready(());
}
cx.waker().wake_by_ref();
Poll::Pending
})
.await;
}
#[derive(Default)]
pub(crate) struct Beside {
pub(crate) handler: Option<ErasedHandler>,
pub(crate) context: Option<ToolContext>,
pub(crate) published: Option<Arc<PublishedContext>>,
}
impl Beside {
pub(crate) fn for_kind(kind: &EffectKind) -> Self {
Self {
published: matches!(kind, EffectKind::ToolCall { .. }).then(PublishedContext::new),
..Self::default()
}
}
}
fn handler_unavailable(key: &HandlerKey) -> ErrorReport {
ErrorReport::new(
ErrorKind::HandlerUnavailable,
format!("no handler serves key `{key}`"),
)
.with_retryable(false)
}
fn family_mismatch(key: &HandlerKey, wanted: EffectFamily, found: EffectFamily) -> ErrorReport {
ErrorReport::new(
ErrorKind::HandlerUnavailable,
format!("handler `{key}` serves the {found} family, not {wanted}"),
)
}
#[derive(Clone)]
pub struct Handle<F: Family> {
handlers: Handlers,
descriptor: HandlerDescriptor,
_family: PhantomData<fn() -> F>,
}
pub type ModelHandle = Handle<family::Completion>;
pub type ToolHandle = Handle<family::Tool>;
pub type IndexHandle = Handle<family::Retrieve>;
impl<F: Family> std::fmt::Debug for Handle<F> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Handle")
.field("family", &F::FAMILY)
.field("key", &self.descriptor.key)
.finish_non_exhaustive()
}
}
impl<F: Family> Handle<F> {
pub fn key(&self) -> &HandlerKey {
&self.descriptor.key
}
pub fn descriptor(&self) -> HandlerDescriptor {
self.handlers
.descriptor(&self.descriptor.key)
.unwrap_or_else(|| self.descriptor.clone())
}
pub fn dispatch(
&self,
request: F::Request,
) -> impl Future<Output = Result<F::Answer, ErrorReport>> + WasmCompatSend + 'static {
self.dispatch_mapped(request, Ok)
}
fn dispatch_mapped<T: 'static>(
&self,
request: F::Request,
map: fn(F::Answer) -> Result<T, ErrorReport>,
) -> impl Future<Output = Result<T, ErrorReport>> + WasmCompatSend + 'static {
let id = self.handlers.mint_id();
let served = F::wrap(request).map(|kind| {
let beside = Beside::for_kind(&kind);
self.handlers
.serve(id, self.descriptor.key.clone(), kind, beside)
});
async move { F::unwrap(served?.await?).and_then(map) }
}
}
impl ModelHandle {
fn family(&self) -> Option<(ModelRef, ProviderCapabilities)> {
match self.descriptor().family {
FamilyDescriptor::Completion {
model,
capabilities,
} => Some((model, capabilities)),
_ => None,
}
}
pub fn capabilities(&self) -> ProviderCapabilities {
self.family()
.map(|(_, capabilities)| capabilities)
.unwrap_or_default()
}
pub fn label(&self) -> ModelRef {
self.family()
.map_or_else(|| ModelRef::new(self.key().as_str()), |(model, _)| model)
}
pub fn call(
&self,
request: CompletionRequest,
) -> impl Future<Output = Result<<family::Completion as Family>::Answer, ErrorReport>>
+ WasmCompatSend
+ 'static {
self.dispatch(request)
}
pub fn stream(&self, request: CompletionRequest) -> CompletionStream {
let kind = EffectKind::Completion {
request: request.into(),
stream: true,
};
let events = self.handlers.serve_stream(
self.handlers.mint_id(),
self.descriptor.key.clone(),
kind,
None,
);
CompletionStream::relay(self.label().to_string(), events)
}
}
#[derive(Debug, Clone)]
pub struct ToolAnswer {
pub result: ToolResult,
pub context: ToolContext,
}
impl ToolHandle {
pub fn call(
&self,
name: impl Into<String>,
args: impl Into<String>,
context: ToolContext,
) -> impl Future<Output = Result<ToolAnswer, ErrorReport>> + WasmCompatSend + 'static {
let kind = EffectKind::ToolCall {
name: name.into(),
args: args.into(),
};
let mut inbound = context.for_dispatch();
let published = PublishedContext::new();
let beside = Beside {
handler: None,
context: Some(inbound.clone()),
published: Some(Arc::clone(&published)),
};
let served = self.handlers.serve(
self.handlers.mint_id(),
self.descriptor.key.clone(),
kind,
beside,
);
async move {
let result = family::Tool::unwrap(served.await?)?;
let context = published.take().unwrap_or_else(|| {
inbound.clear_scope();
inbound
});
Ok(ToolAnswer { result, context })
}
}
}
impl IndexHandle {
pub fn top_n<T: DeserializeOwned + WasmCompatSend + 'static>(
&self,
req: VectorSearchRequest<Filter<serde_json::Value>>,
) -> impl Future<Output = Result<Vec<VectorSearchResult<T>>, ErrorReport>> + WasmCompatSend + 'static
{
self.dispatch_mapped(RetrieveQuery::TopN { req }, |documents| match documents {
RetrievedDocuments::Scored(results) => results
.into_iter()
.map(
|(score, id, document)| match serde_json::from_value(document) {
Ok(document) => Ok(VectorSearchResult {
score,
id,
document,
}),
Err(error) => Err(ErrorReport::new(
ErrorKind::Json,
format!("retrieved document `{id}` did not deserialize: {error}"),
)),
},
)
.collect(),
RetrievedDocuments::Ids(_) => Err(wrong_shape("scored documents")),
})
}
pub fn top_n_ids(
&self,
req: VectorSearchRequest<Filter<serde_json::Value>>,
) -> impl Future<Output = Result<Vec<VectorSearchIdResult>, ErrorReport>> + WasmCompatSend + 'static
{
self.dispatch_mapped(
RetrieveQuery::TopNIds { req },
|documents| match documents {
RetrievedDocuments::Ids(results) => Ok(results
.into_iter()
.map(|(score, id)| VectorSearchIdResult { score, id })
.collect()),
RetrievedDocuments::Scored(_) => Err(wrong_shape("scored ids")),
},
)
}
}
fn wrong_shape(expected: &'static str) -> ErrorReport {
ErrorReport::new(
ErrorKind::Internal,
format!("expected {expected}, the retrieve handler answered another shape"),
)
}
#[cfg(test)]
mod tests;