use std::{
fmt,
marker::PhantomData,
pin::Pin,
task::{Context, Poll},
};
use serde::de::DeserializeOwned;
use rig_core::{
completion::{CompletionRequest, ProviderCapabilities},
effect::{
CustomEffect, EffectId, EffectKind, EmbedInputs, EmbedModality, EmbedOutputs, Family,
FamilyDescriptor, HandlerDescriptor, HandlerKey, MemoryOp, MemoryOutcome, RetrieveQuery,
RetrievedDocuments, family,
},
embeddings::{Embedding, EmbeddingResponse},
error::{ErrorKind, ErrorReport},
id::ConversationId,
message::Message,
operation::RerankRequest,
streaming::CompletionStream,
tool::ToolContext,
vector_store::{
VectorSearchIdResult, VectorSearchResult,
request::{Filter, VectorSearchRequest},
},
};
use super::{DispatchOptions, Dispatcher, EffectStream, Pending};
use rig_core::effect::Key;
#[derive(Clone)]
pub struct Handle<F: Family> {
dispatcher: Dispatcher,
descriptor: HandlerDescriptor,
_family: PhantomData<fn() -> F>,
}
pub type ModelHandle = Handle<family::Completion>;
pub type ToolHandle = Handle<family::Tool>;
pub type MemoryHandle = Handle<family::Memory>;
pub type IndexHandle = Handle<family::Retrieve>;
pub type EmbedHandle = Handle<family::Embed>;
pub type RerankHandle = Handle<family::Rerank>;
impl<F: Family> fmt::Debug for Handle<F> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Handle")
.field("family", &F::FAMILY)
.field("key", &self.descriptor.key)
.finish_non_exhaustive()
}
}
impl<F: Family> Handle<F> {
pub fn key(&self) -> &HandlerKey {
&self.descriptor.key
}
pub fn descriptor(&self) -> HandlerDescriptor {
self.dispatcher
.descriptor(&self.descriptor.key)
.unwrap_or_else(|| self.descriptor.clone())
}
pub fn is_closed(&self) -> bool {
self.dispatcher.is_closed()
}
fn dispatch_kind(&self, kind: EffectKind) -> Pending {
self.dispatcher.dispatch(&self.descriptor.key, kind)
}
fn dispatch_wrapped(&self, kind: Result<EffectKind, ErrorReport>) -> Pending {
match kind {
Ok(kind) => self.dispatch_kind(kind),
Err(report) => self.dispatcher.refused(report),
}
}
}
fn family_mismatch(
key: &HandlerKey,
wanted: rig_core::effect::EffectFamily,
found: &FamilyDescriptor,
) -> ErrorReport {
ErrorReport::new(
ErrorKind::HandlerUnavailable,
format!(
"handler `{key}` serves the {} family, not {wanted}",
found.family()
),
)
}
impl Dispatcher {
pub fn handle<F: Family>(&self, key: &HandlerKey) -> Result<Handle<F>, ErrorReport> {
if self.is_closed() {
return Err(super::dispatcher::bus_closed());
}
let descriptor = self
.descriptor(key)
.ok_or_else(|| super::dispatcher::handler_unavailable(key))?;
if descriptor.family.family() != F::FAMILY {
return Err(family_mismatch(key, F::FAMILY, &descriptor.family));
}
Ok(Handle {
dispatcher: self.clone(),
descriptor,
_family: PhantomData,
})
}
pub fn bind<F: Family>(&self, key: &Key<F>) -> Result<Handle<F>, ErrorReport> {
self.handle(key.raw())
}
pub fn custom<E: CustomEffect>(
&self,
key: &HandlerKey,
) -> Result<Handle<family::Custom<E>>, ErrorReport> {
let handle = self.handle::<family::Custom<E>>(key)?;
match &handle.descriptor.family {
FamilyDescriptor::Custom { kind } if kind == E::KIND => Ok(handle),
FamilyDescriptor::Custom { kind } => Err(ErrorReport::new(
ErrorKind::HandlerUnavailable,
format!(
"handler `{key}` serves the custom kind `{kind}`, not `{}`",
E::KIND
),
)),
other => Err(family_mismatch(key, F_CUSTOM, other)),
}
}
}
const F_CUSTOM: rig_core::effect::EffectFamily = rig_core::effect::EffectFamily::Custom;
#[must_use = "a dispatch does nothing until polled"]
pub struct Typed<F: Family, T = <F as Family>::Answer> {
pending: Pending,
map: fn(F::Answer) -> Result<T, ErrorReport>,
}
impl<F: Family, T> Typed<F, T> {
pub const fn parent(&self) -> Option<EffectId> {
self.pending.parent()
}
pub const fn id(&self) -> EffectId {
self.pending.id()
}
fn narrow(pending: Pending, map: fn(F::Answer) -> Result<T, ErrorReport>) -> Self {
Self { pending, map }
}
}
impl<F: Family, T> fmt::Debug for Typed<F, T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Typed")
.field("family", &F::FAMILY)
.field("id", &self.id())
.finish()
}
}
impl<F: Family, T> Unpin for Typed<F, T> {}
impl<F: Family, T> Future for Typed<F, T> {
type Output = Result<T, ErrorReport>;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
let this = self.get_mut();
match Pin::new(&mut this.pending).poll(cx) {
Poll::Pending => Poll::Pending,
Poll::Ready(Err(report)) => Poll::Ready(Err(report)),
Poll::Ready(Ok(outcome)) => Poll::Ready(F::unwrap(outcome).and_then(this.map)),
}
}
}
pub type Completion = Typed<family::Completion>;
#[derive(Debug, Clone)]
pub struct ToolAnswer {
pub result: rig_core::tool::ToolResult,
pub context: ToolContext,
}
#[must_use = "a dispatch does nothing until polled"]
pub struct ToolCall {
pending: Pending,
published: Option<std::sync::Arc<rig_core::tool::PublishedContext>>,
inbound: ToolContext,
}
impl ToolCall {
pub const fn parent(&self) -> Option<EffectId> {
self.pending.parent()
}
pub const fn id(&self) -> EffectId {
self.pending.id()
}
}
impl fmt::Debug for ToolCall {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ToolCall").field("id", &self.id()).finish()
}
}
impl Unpin for ToolCall {}
impl Future for ToolCall {
type Output = Result<ToolAnswer, ErrorReport>;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
let this = self.get_mut();
match Pin::new(&mut this.pending).poll(cx) {
Poll::Pending => Poll::Pending,
Poll::Ready(Err(report)) => Poll::Ready(Err(report)),
Poll::Ready(Ok(outcome)) => Poll::Ready(family::Tool::unwrap(outcome).map(|result| {
let context = this
.published
.as_ref()
.and_then(|published| published.take())
.unwrap_or_else(|| {
let mut inbound = std::mem::take(&mut this.inbound);
inbound.clear_scope();
inbound
});
ToolAnswer { result, context }
})),
}
}
}
pub type Retrieval<T> = Typed<family::Retrieve, Vec<VectorSearchResult<T>>>;
impl<F: Family> Handle<F> {
pub fn dispatch(&self, request: F::Request) -> Typed<F> {
Typed::narrow(self.dispatch_wrapped(F::wrap(request)), Ok)
}
}
fn wrong_shape(expected: &'static str, family: rig_core::effect::EffectFamily) -> ErrorReport {
ErrorReport::new(
ErrorKind::Internal,
format!("expected {expected}, the {family} handler answered another shape"),
)
}
impl ModelHandle {
pub fn capabilities(&self) -> ProviderCapabilities {
match self.descriptor().family {
FamilyDescriptor::Completion { capabilities, .. } => capabilities,
FamilyDescriptor::Tool { .. }
| FamilyDescriptor::Embed { .. }
| FamilyDescriptor::Memory {}
| FamilyDescriptor::Retrieve {}
| FamilyDescriptor::Rerank { .. }
| FamilyDescriptor::Custom { .. } => ProviderCapabilities::default(),
}
}
pub fn label(&self) -> rig_core::completion::ModelRef {
match self.descriptor().family {
FamilyDescriptor::Completion { model, .. } => model,
FamilyDescriptor::Tool { .. }
| FamilyDescriptor::Embed { .. }
| FamilyDescriptor::Memory {}
| FamilyDescriptor::Retrieve {}
| FamilyDescriptor::Rerank { .. }
| FamilyDescriptor::Custom { .. } => {
rig_core::completion::ModelRef::new(self.key().as_str())
}
}
}
pub fn call(&self, request: CompletionRequest) -> Completion {
self.call_with(request, DispatchOptions::default())
}
pub fn call_observed(
&self,
request: CompletionRequest,
context: rig_core::observe::AdapterContext,
) -> Completion {
self.call_with(
request,
DispatchOptions::default().with_adapter_context(context),
)
}
fn call_with(&self, request: CompletionRequest, options: DispatchOptions) -> Completion {
Typed::narrow(
self.dispatcher.dispatch_with(
&self.descriptor.key,
EffectKind::Completion {
request,
stream: false,
},
options,
),
Ok,
)
}
pub fn stream(&self, request: CompletionRequest) -> CompletionStream {
self.stream_with(request, DispatchOptions::default())
}
pub fn stream_observed(
&self,
request: CompletionRequest,
context: rig_core::observe::AdapterContext,
) -> CompletionStream {
self.stream_with(
request,
DispatchOptions::default().with_adapter_context(context),
)
}
fn stream_with(
&self,
request: CompletionRequest,
options: DispatchOptions,
) -> CompletionStream {
let provider = self.label().to_string();
let stream: EffectStream = self.dispatcher.dispatch_stream_with(
&self.descriptor.key,
EffectKind::Completion {
request,
stream: true,
},
options,
);
wrap_stream(provider, stream)
}
}
impl ToolHandle {
pub fn name(&self) -> String {
match self.descriptor().family {
FamilyDescriptor::Tool { name, .. } => name,
FamilyDescriptor::Completion { .. }
| FamilyDescriptor::Embed { .. }
| FamilyDescriptor::Memory {}
| FamilyDescriptor::Retrieve {}
| FamilyDescriptor::Rerank { .. }
| FamilyDescriptor::Custom { .. } => self.key().to_string(),
}
}
pub fn call(
&self,
name: impl Into<String>,
args: impl Into<String>,
context: ToolContext,
) -> ToolCall {
let kind = EffectKind::ToolCall {
name: name.into(),
args: args.into(),
};
let inbound = context.for_dispatch();
drop(context);
let pending = self.dispatcher.dispatch_with(
&self.descriptor.key,
kind,
DispatchOptions::default().with_tool_context(inbound.clone()),
);
let published = pending.published_context();
ToolCall {
pending,
published,
inbound,
}
}
}
impl MemoryHandle {
pub fn load(&self, conversation: ConversationId) -> Typed<family::Memory, Vec<Message>> {
Typed::narrow(
self.dispatch_wrapped(family::Memory::wrap(MemoryOp::Load { conversation })),
|answer| match answer {
MemoryOutcome::Loaded { messages } => Ok(messages),
MemoryOutcome::Appended | MemoryOutcome::Cleared => {
Err(wrong_shape("loaded messages", family::Memory::FAMILY))
}
},
)
}
pub fn append(
&self,
conversation: ConversationId,
messages: Vec<Message>,
) -> Typed<family::Memory, ()> {
Typed::narrow(
self.dispatch_wrapped(family::Memory::wrap(MemoryOp::Append {
conversation,
messages,
})),
|answer| match answer {
MemoryOutcome::Appended => Ok(()),
MemoryOutcome::Loaded { .. } | MemoryOutcome::Cleared => {
Err(wrong_shape("an append", family::Memory::FAMILY))
}
},
)
}
pub fn clear(&self, conversation: ConversationId) -> Typed<family::Memory, ()> {
Typed::narrow(
self.dispatch_wrapped(family::Memory::wrap(MemoryOp::Clear { conversation })),
|answer| match answer {
MemoryOutcome::Cleared => Ok(()),
MemoryOutcome::Loaded { .. } | MemoryOutcome::Appended => {
Err(wrong_shape("a clear", family::Memory::FAMILY))
}
},
)
}
}
fn deserialize_scored<T: DeserializeOwned>(
documents: RetrievedDocuments,
) -> Result<Vec<VectorSearchResult<T>>, ErrorReport> {
match documents {
RetrievedDocuments::Scored(results) => results
.into_iter()
.map(
|(score, id, document)| match serde_json::from_value::<T>(document) {
Ok(document) => Ok(VectorSearchResult {
score,
id,
document,
}),
Err(error) => Err(ErrorReport::new(
ErrorKind::Json,
format!("retrieved document `{id}` did not deserialize: {error}"),
)),
},
)
.collect(),
RetrievedDocuments::Ids(_) => {
Err(wrong_shape("scored documents", family::Retrieve::FAMILY))
}
}
}
impl IndexHandle {
pub fn top_n<T: DeserializeOwned>(
&self,
req: VectorSearchRequest<Filter<serde_json::Value>>,
) -> Retrieval<T> {
Typed::narrow(
self.dispatch_wrapped(family::Retrieve::wrap(RetrieveQuery::TopN { req })),
deserialize_scored::<T>,
)
}
pub fn top_n_ids(
&self,
req: VectorSearchRequest<Filter<serde_json::Value>>,
) -> Typed<family::Retrieve, Vec<VectorSearchIdResult>> {
Typed::narrow(
self.dispatch_wrapped(family::Retrieve::wrap(RetrieveQuery::TopNIds { req })),
|documents| match documents {
RetrievedDocuments::Ids(results) => Ok(results
.into_iter()
.map(|(score, id)| VectorSearchIdResult { score, id })
.collect()),
RetrievedDocuments::Scored(_) => {
Err(wrong_shape("scored ids", family::Retrieve::FAMILY))
}
},
)
}
}
impl EmbedHandle {
pub fn modality(&self) -> Option<EmbedModality> {
match self.descriptor().family {
FamilyDescriptor::Embed { modality, .. } => Some(modality),
FamilyDescriptor::Completion { .. }
| FamilyDescriptor::Tool { .. }
| FamilyDescriptor::Memory {}
| FamilyDescriptor::Retrieve {}
| FamilyDescriptor::Rerank { .. }
| FamilyDescriptor::Custom { .. } => None,
}
}
pub fn ndims(&self) -> Option<usize> {
match self.descriptor().family {
FamilyDescriptor::Embed { dims, .. } => dims,
FamilyDescriptor::Completion { .. }
| FamilyDescriptor::Tool { .. }
| FamilyDescriptor::Memory {}
| FamilyDescriptor::Retrieve {}
| FamilyDescriptor::Rerank { .. }
| FamilyDescriptor::Custom { .. } => None,
}
}
pub fn max_documents(&self) -> Option<usize> {
match self.descriptor().family {
FamilyDescriptor::Embed { max_documents, .. } => Some(max_documents),
FamilyDescriptor::Completion { .. }
| FamilyDescriptor::Tool { .. }
| FamilyDescriptor::Memory {}
| FamilyDescriptor::Retrieve {}
| FamilyDescriptor::Rerank { .. }
| FamilyDescriptor::Custom { .. } => None,
}
}
pub fn embed_texts(&self, texts: Vec<String>) -> Typed<family::Embed, EmbeddingResponse> {
Typed::narrow(
self.dispatch_wrapped(family::Embed::wrap(EmbedInputs::Texts(texts))),
|outputs| match outputs {
EmbedOutputs::Texts(response) => Ok(response),
EmbedOutputs::Images(_) => {
Err(wrong_shape("text embeddings", family::Embed::FAMILY))
}
},
)
}
pub fn embed_text(&self, text: &str) -> Typed<family::Embed, Embedding> {
fn first(outputs: EmbedOutputs) -> Result<Embedding, ErrorReport> {
match outputs {
EmbedOutputs::Texts(mut response) => response.embeddings.pop().ok_or_else(|| {
ErrorReport::new(
ErrorKind::Response,
"embedding handler returned an empty response for embed_text",
)
}),
EmbedOutputs::Images(_) => {
Err(wrong_shape("text embeddings", family::Embed::FAMILY))
}
}
}
Typed::narrow(
self.dispatch_wrapped(family::Embed::wrap(EmbedInputs::Texts(vec![
text.to_owned(),
]))),
first,
)
}
pub fn embed_images(&self, images: Vec<Vec<u8>>) -> Typed<family::Embed, EmbeddingResponse> {
Typed::narrow(
self.dispatch_wrapped(family::Embed::wrap(EmbedInputs::Images(images))),
|outputs| match outputs {
EmbedOutputs::Images(response) => Ok(response),
EmbedOutputs::Texts(_) => {
Err(wrong_shape("image embeddings", family::Embed::FAMILY))
}
},
)
}
}
impl RerankHandle {
pub fn label(&self) -> String {
match self.descriptor().family {
FamilyDescriptor::Rerank { model, .. } => model,
FamilyDescriptor::Completion { .. }
| FamilyDescriptor::Tool { .. }
| FamilyDescriptor::Embed { .. }
| FamilyDescriptor::Memory {}
| FamilyDescriptor::Retrieve {}
| FamilyDescriptor::Custom { .. } => self.key().to_string(),
}
}
pub fn max_documents(&self) -> Option<usize> {
match self.descriptor().family {
FamilyDescriptor::Rerank { max_documents, .. } => Some(max_documents),
FamilyDescriptor::Completion { .. }
| FamilyDescriptor::Tool { .. }
| FamilyDescriptor::Embed { .. }
| FamilyDescriptor::Memory {}
| FamilyDescriptor::Retrieve {}
| FamilyDescriptor::Custom { .. } => None,
}
}
pub fn rerank(
&self,
query: impl Into<String>,
documents: Vec<String>,
) -> Typed<family::Rerank> {
self.dispatch(RerankRequest {
query: query.into(),
documents,
})
}
}
pub(crate) fn wrap_stream(provider: impl Into<String>, stream: EffectStream) -> CompletionStream {
CompletionStream::relay(provider, Box::pin(stream))
}
const _: () = {
const fn assert_view<T: Clone + Send + Sync + 'static>() {}
assert_view::<ModelHandle>();
assert_view::<ToolHandle>();
assert_view::<MemoryHandle>();
assert_view::<IndexHandle>();
assert_view::<EmbedHandle>();
const fn assert_unpin<T: Unpin>() {}
assert_unpin::<Completion>();
assert_unpin::<ToolCall>();
};
#[cfg(test)]
mod tests;
pub trait DispatchScope {
fn dispatcher(&self) -> Option<Dispatcher>;
}
impl DispatchScope for rig_core::serve::Dispatch {
fn dispatcher(&self) -> Option<Dispatcher> {
self.scope::<Dispatcher>().map(|scoped| (*scoped).clone())
}
}