1#![forbid(unsafe_code)]
2
3use std::{
21 collections::{BTreeMap, VecDeque},
22 marker::PhantomData,
23 sync::{
24 Arc, PoisonError, RwLock,
25 atomic::{AtomicU64, Ordering},
26 },
27 task::Poll,
28};
29
30use futures::StreamExt;
31use serde::de::DeserializeOwned;
32use tracing::Instrument;
33
34use rig_core::{
35 completion::{CompletionRequest, ModelRef, ProviderCapabilities},
36 effect::{
37 Answered, CustomEffect, EffectFamily, EffectId, EffectKind, Family, FamilyDescriptor,
38 HandlerDescriptor, HandlerKey, Key, RetrieveQuery, RetrievedDocuments, family,
39 },
40 error::{ErrorKind, ErrorReport},
41 serve::{Dispatch, DispatchContext, ErasedHandler, Recorder, Serve, catch_panics},
42 streaming::{CompletionStream, Relayed, StreamEvents},
43 tool::{PublishedContext, ToolContext, ToolResult},
44 vector_store::{
45 VectorSearchIdResult, VectorSearchResult,
46 request::{Filter, VectorSearchRequest},
47 },
48 wasm_compat::WasmCompatSend,
49};
50
51#[derive(Clone, Default)]
133pub struct Handlers {
134 shared: Arc<Shared>,
135 parent: Option<EffectId>,
139}
140
141#[derive(Default)]
142struct Shared {
143 table: RwLock<Table>,
144 next_id: AtomicU64,
145}
146
147#[derive(Default)]
148struct Table {
149 handlers: BTreeMap<HandlerKey, (ErasedHandler, HandlerDescriptor)>,
150 recorder: Option<Arc<dyn Recorder + Send + Sync>>,
151}
152
153impl std::fmt::Debug for Handlers {
154 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
155 f.debug_struct("Handlers")
156 .field("keys", &self.keys())
157 .field("parent", &self.parent)
158 .finish_non_exhaustive()
159 }
160}
161
162impl Handlers {
163 fn read(&self) -> std::sync::RwLockReadGuard<'_, Table> {
164 self.shared
165 .table
166 .read()
167 .unwrap_or_else(PoisonError::into_inner)
168 }
169
170 fn write(&self) -> std::sync::RwLockWriteGuard<'_, Table> {
171 self.shared
172 .table
173 .write()
174 .unwrap_or_else(PoisonError::into_inner)
175 }
176
177 pub fn register(
181 &self,
182 key: impl Into<HandlerKey>,
183 handler: impl Serve + 'static,
184 ) -> Result<(), ErrorReport> {
185 self.register_erased(key, ErasedHandler::new(handler))
186 }
187
188 pub fn register_erased(
190 &self,
191 key: impl Into<HandlerKey>,
192 handler: ErasedHandler,
193 ) -> Result<(), ErrorReport> {
194 let key = key.into();
195 let described = handler.descriptor();
197 let descriptor = HandlerDescriptor {
198 key: key.clone(),
199 family: described.family,
200 layers: described.layers,
201 };
202 let mut table = self.write();
203 if let Some((_, current)) = table.handlers.get(&key) {
204 let (current, family) = (current.family.family(), descriptor.family.family());
205 if current != family {
206 return Err(ErrorReport::new(
207 ErrorKind::HandlerUnavailable,
208 format!(
209 "key `{key}` serves the {current:?} family; a {family:?} handler cannot replace it"
210 ),
211 )
212 .with_retryable(false));
213 }
214 }
215 if let Some(recorder) = &table.recorder {
216 recorder.handlers(vec![descriptor.clone()]);
217 }
218 let displaced = table.handlers.insert(key, (handler, descriptor));
219 drop(table);
220 drop(displaced);
222 Ok(())
223 }
224
225 pub fn deregister(&self, key: &HandlerKey) -> bool {
228 let removed = self.write().handlers.remove(key);
229 removed.is_some()
230 }
231
232 pub fn record_to(&self, recorder: impl Recorder + Send + Sync) {
243 self.record_with(Arc::new(recorder));
244 }
245
246 pub(crate) fn record_with(&self, recorder: Arc<dyn Recorder + Send + Sync>) {
247 let mut table = self.write();
248 recorder.handlers(table.handlers.values().map(|(_, d)| d.clone()).collect());
249 table.recorder = Some(recorder);
250 }
251
252 pub fn descriptor(&self, key: &HandlerKey) -> Option<HandlerDescriptor> {
254 self.read().handlers.get(key).map(|(_, d)| d.clone())
255 }
256
257 pub fn descriptors(&self) -> Vec<HandlerDescriptor> {
259 self.read()
260 .handlers
261 .values()
262 .map(|(_, d)| d.clone())
263 .collect()
264 }
265
266 pub fn keys(&self) -> Vec<HandlerKey> {
268 self.read().handlers.keys().cloned().collect()
269 }
270
271 pub const fn parent(&self) -> Option<EffectId> {
274 self.parent
275 }
276
277 pub(crate) fn downgrade(&self) -> WeakHandlers {
279 WeakHandlers(Arc::downgrade(&self.shared))
280 }
281
282 #[cfg(not(target_family = "wasm"))]
287 pub fn serving(dispatch: &Dispatch) -> Option<Self> {
288 dispatch.scope::<Self>().map(|handlers| (*handlers).clone())
289 }
290
291 pub fn resume_ids_after(&self, last: EffectId) {
297 self.shared
298 .next_id
299 .fetch_max(last.as_u64(), Ordering::SeqCst);
300 }
301
302 pub(crate) fn mint_id(&self) -> EffectId {
304 EffectId::from_raw(self.shared.next_id.fetch_add(1, Ordering::SeqCst) + 1)
305 }
306
307 fn lookup(
309 &self,
310 key: &HandlerKey,
311 ) -> Result<(ErasedHandler, Option<Arc<dyn Recorder + Send + Sync>>), ErrorReport> {
312 let table = self.read();
313 let (handler, _) = table
314 .handlers
315 .get(key)
316 .ok_or_else(|| handler_unavailable(key))?;
317 Ok((handler.clone(), table.recorder.clone()))
318 }
319
320 fn open(&self, id: EffectId, key: &HandlerKey, kind: &EffectKind, beside: Beside) -> Opened {
323 let (handler, recorder) = match &beside.handler {
324 Some(handler) => (handler.clone(), self.read().recorder.clone()),
325 None => self.lookup(key)?,
326 };
327 let mut dispatch = Dispatch::new(id, kind.streams());
328 #[cfg(not(target_family = "wasm"))]
329 {
330 let child = Self {
331 shared: Arc::clone(&self.shared),
332 parent: Some(id),
333 };
334 dispatch = dispatch.with_scope(Arc::new(child));
335 }
336 if let Some(context) = beside.context {
337 dispatch = dispatch.with_scope(Arc::new(context));
338 }
339 if let Some(published) = beside.published {
340 dispatch = dispatch.with_scope(published);
341 }
342 if let Some(recorder) = &recorder {
343 let origin = DispatchContext {
344 parent: self.parent,
345 ..DispatchContext::default()
346 };
347 recorder.begin(id, key.clone(), kind.clone(), origin);
348 dispatch = dispatch.recorded_by(Arc::clone(recorder));
349 }
350 Ok((handler, dispatch, recorder))
351 }
352
353 pub fn dispatch(
357 &self,
358 key: &HandlerKey,
359 kind: EffectKind,
360 ) -> impl Future<Output = Result<Answered, ErrorReport>> + WasmCompatSend + 'static {
361 let beside = Beside::for_kind(&kind);
362 self.serve(self.mint_id(), key.clone(), kind, beside)
363 }
364
365 pub fn dispatch_stream(&self, key: &HandlerKey, kind: EffectKind) -> StreamEvents {
368 self.serve_stream(self.mint_id(), key.clone(), kind, None)
369 }
370
371 pub(crate) fn serve(
375 &self,
376 id: EffectId,
377 key: HandlerKey,
378 kind: EffectKind,
379 beside: Beside,
380 ) -> impl Future<Output = Result<Answered, ErrorReport>> + WasmCompatSend + 'static {
381 let this = self.clone();
382 async move {
383 yield_once().await;
384 let (handler, dispatch, recorder) = this.open(id, &key, &kind, beside)?;
385 let work = async move { handler.handle(kind, dispatch).await.into_outcome().await };
386 catch_panics(recorder, id, work).await
387 }
388 .instrument(tracing::Span::current())
390 }
391
392 pub(crate) fn serve_stream(
396 &self,
397 id: EffectId,
398 key: HandlerKey,
399 kind: EffectKind,
400 handler: Option<ErasedHandler>,
401 ) -> StreamEvents {
402 if !kind.streams() {
403 let report = ErrorReport::new(
404 ErrorKind::Request,
405 format!(
406 "invalid dispatch: `{}` is a unary effect and cannot be dispatched as a stream",
407 kind.name()
408 ),
409 );
410 return Box::pin(futures::stream::iter([Err(report)]));
411 }
412 let this = self.clone();
413 let beside = Beside {
414 handler,
415 ..Beside::default()
416 };
417 let events = async_stream::stream! {
418 yield_once().await;
419 let (handler, dispatch, _) = match this.open(id, &key, &kind, beside) {
420 Ok(opened) => opened,
421 Err(report) => {
422 yield Err(report);
423 return;
424 }
425 };
426 let mut events = handler.handle(kind, dispatch).await.into_stream();
427 let (mut ahead, mut drained, mut ended) = (VecDeque::new(), false, false);
428 loop {
429 let next = std::future::poll_fn(|cx| {
432 while !drained && ahead.len() < READ_AHEAD {
433 match events.poll_next_unpin(cx) {
434 Poll::Ready(Some(item)) => ahead.push_back(item),
435 Poll::Ready(None) => drained = true,
436 Poll::Pending => break,
437 }
438 }
439 match ahead.pop_front() {
440 Some(item) => Poll::Ready(Some(item)),
441 None if drained => Poll::Ready(None),
442 None => Poll::Pending,
443 }
444 })
445 .await;
446 let Some(item) = next else { break };
447 ended |= matches!(item, Ok(Relayed::Done(_)) | Err(_));
448 yield item;
449 }
450 if !ended {
451 yield Err(rig_core::serve::stream_truncated());
452 }
453 };
454 Box::pin(tracing_futures::Instrument::instrument(
455 events,
456 tracing::Span::current(),
457 ))
458 }
459
460 pub fn handle<F: Family>(&self, key: &HandlerKey) -> Result<Handle<F>, ErrorReport> {
463 let descriptor = self
464 .descriptor(key)
465 .ok_or_else(|| handler_unavailable(key))?;
466 let found = descriptor.family.family();
467 if found != F::FAMILY {
468 return Err(family_mismatch(key, F::FAMILY, found));
469 }
470 Ok(Handle {
471 handlers: self.clone(),
472 descriptor,
473 _family: PhantomData,
474 })
475 }
476
477 pub fn bind<F: Family>(&self, key: &Key<F>) -> Result<Handle<F>, ErrorReport> {
479 self.handle(key.raw())
480 }
481
482 pub fn custom<E: CustomEffect>(
485 &self,
486 key: &HandlerKey,
487 ) -> Result<Handle<family::Custom<E>>, ErrorReport> {
488 let handle = self.handle::<family::Custom<E>>(key)?;
489 match &handle.descriptor.family {
490 FamilyDescriptor::Custom { kind } if kind == E::KIND => Ok(handle),
491 FamilyDescriptor::Custom { kind } => Err(ErrorReport::new(
492 ErrorKind::HandlerUnavailable,
493 format!(
494 "handler `{key}` serves the custom kind `{kind}`, not `{}`",
495 E::KIND
496 ),
497 )),
498 other => Err(family_mismatch(key, EffectFamily::Custom, other.family())),
499 }
500 }
501}
502
503pub(crate) struct WeakHandlers(std::sync::Weak<Shared>);
505
506impl WeakHandlers {
507 pub(crate) fn upgrade(&self) -> Option<Handlers> {
508 self.0.upgrade().map(|shared| Handlers {
509 shared,
510 parent: None,
511 })
512 }
513}
514
515type Opened = Result<
516 (
517 ErasedHandler,
518 Dispatch,
519 Option<Arc<dyn Recorder + Send + Sync>>,
520 ),
521 ErrorReport,
522>;
523
524const READ_AHEAD: usize = 64;
526
527async fn yield_once() {
530 let mut yielded = false;
531 std::future::poll_fn(|cx| {
532 if std::mem::replace(&mut yielded, true) {
533 return Poll::Ready(());
534 }
535 cx.waker().wake_by_ref();
536 Poll::Pending
537 })
538 .await;
539}
540
541#[derive(Default)]
543pub(crate) struct Beside {
544 pub(crate) handler: Option<ErasedHandler>,
547 pub(crate) context: Option<ToolContext>,
549 pub(crate) published: Option<Arc<PublishedContext>>,
551}
552
553impl Beside {
554 pub(crate) fn for_kind(kind: &EffectKind) -> Self {
557 Self {
558 published: matches!(kind, EffectKind::ToolCall { .. }).then(PublishedContext::new),
559 ..Self::default()
560 }
561 }
562}
563
564fn handler_unavailable(key: &HandlerKey) -> ErrorReport {
565 ErrorReport::new(
566 ErrorKind::HandlerUnavailable,
567 format!("no handler serves key `{key}`"),
568 )
569 .with_retryable(false)
570}
571
572fn family_mismatch(key: &HandlerKey, wanted: EffectFamily, found: EffectFamily) -> ErrorReport {
573 ErrorReport::new(
574 ErrorKind::HandlerUnavailable,
575 format!("handler `{key}` serves the {found} family, not {wanted}"),
576 )
577}
578
579#[derive(Clone)]
581pub struct Handle<F: Family> {
582 handlers: Handlers,
583 descriptor: HandlerDescriptor,
584 _family: PhantomData<fn() -> F>,
585}
586
587pub type ModelHandle = Handle<family::Completion>;
589pub type ToolHandle = Handle<family::Tool>;
591pub type IndexHandle = Handle<family::Retrieve>;
593
594impl<F: Family> std::fmt::Debug for Handle<F> {
595 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
596 f.debug_struct("Handle")
597 .field("family", &F::FAMILY)
598 .field("key", &self.descriptor.key)
599 .finish_non_exhaustive()
600 }
601}
602
603impl<F: Family> Handle<F> {
604 pub fn key(&self) -> &HandlerKey {
606 &self.descriptor.key
607 }
608
609 pub fn descriptor(&self) -> HandlerDescriptor {
613 self.handlers
614 .descriptor(&self.descriptor.key)
615 .unwrap_or_else(|| self.descriptor.clone())
616 }
617
618 pub fn dispatch(
621 &self,
622 request: F::Request,
623 ) -> impl Future<Output = Result<F::Answer, ErrorReport>> + WasmCompatSend + 'static {
624 self.dispatch_mapped(request, Ok)
625 }
626
627 fn dispatch_mapped<T: 'static>(
628 &self,
629 request: F::Request,
630 map: fn(F::Answer) -> Result<T, ErrorReport>,
631 ) -> impl Future<Output = Result<T, ErrorReport>> + WasmCompatSend + 'static {
632 let id = self.handlers.mint_id();
633 let served = F::wrap(request).map(|kind| {
634 let beside = Beside::for_kind(&kind);
635 self.handlers
636 .serve(id, self.descriptor.key.clone(), kind, beside)
637 });
638 async move { F::unwrap(served?.await?).and_then(map) }
639 }
640}
641
642impl ModelHandle {
643 fn family(&self) -> Option<(ModelRef, ProviderCapabilities)> {
644 match self.descriptor().family {
645 FamilyDescriptor::Completion {
646 model,
647 capabilities,
648 } => Some((model, capabilities)),
649 _ => None,
650 }
651 }
652
653 pub fn capabilities(&self) -> ProviderCapabilities {
655 self.family()
656 .map(|(_, capabilities)| capabilities)
657 .unwrap_or_default()
658 }
659
660 pub fn label(&self) -> ModelRef {
662 self.family()
663 .map_or_else(|| ModelRef::new(self.key().as_str()), |(model, _)| model)
664 }
665
666 pub fn call(
668 &self,
669 request: CompletionRequest,
670 ) -> impl Future<Output = Result<<family::Completion as Family>::Answer, ErrorReport>>
671 + WasmCompatSend
672 + 'static {
673 self.dispatch(request)
674 }
675
676 pub fn stream(&self, request: CompletionRequest) -> CompletionStream {
678 let kind = EffectKind::Completion {
679 request: request.into(),
680 stream: true,
681 };
682 let events = self.handlers.serve_stream(
683 self.handlers.mint_id(),
684 self.descriptor.key.clone(),
685 kind,
686 None,
687 );
688 CompletionStream::relay(self.label().to_string(), events)
689 }
690}
691
692#[derive(Debug, Clone)]
694pub struct ToolAnswer {
695 pub result: ToolResult,
697 pub context: ToolContext,
700}
701
702impl ToolHandle {
703 pub fn call(
707 &self,
708 name: impl Into<String>,
709 args: impl Into<String>,
710 context: ToolContext,
711 ) -> impl Future<Output = Result<ToolAnswer, ErrorReport>> + WasmCompatSend + 'static {
712 let kind = EffectKind::ToolCall {
713 name: name.into(),
714 args: args.into(),
715 };
716 let mut inbound = context.for_dispatch();
719 let published = PublishedContext::new();
720 let beside = Beside {
721 handler: None,
722 context: Some(inbound.clone()),
723 published: Some(Arc::clone(&published)),
724 };
725 let served = self.handlers.serve(
726 self.handlers.mint_id(),
727 self.descriptor.key.clone(),
728 kind,
729 beside,
730 );
731 async move {
732 let result = family::Tool::unwrap(served.await?)?;
733 let context = published.take().unwrap_or_else(|| {
734 inbound.clear_scope();
735 inbound
736 });
737 Ok(ToolAnswer { result, context })
738 }
739 }
740}
741
742impl IndexHandle {
743 pub fn top_n<T: DeserializeOwned + WasmCompatSend + 'static>(
745 &self,
746 req: VectorSearchRequest<Filter<serde_json::Value>>,
747 ) -> impl Future<Output = Result<Vec<VectorSearchResult<T>>, ErrorReport>> + WasmCompatSend + 'static
748 {
749 self.dispatch_mapped(RetrieveQuery::TopN { req }, |documents| match documents {
750 RetrievedDocuments::Scored(results) => results
751 .into_iter()
752 .map(
753 |(score, id, document)| match rig_core::json_utils::from_value(&document) {
754 Ok(document) => Ok(VectorSearchResult {
755 score,
756 id,
757 document,
758 }),
759 Err(error) => Err(ErrorReport::new(
760 ErrorKind::Json,
761 format!("retrieved document `{id}` did not deserialize: {error}"),
762 )),
763 },
764 )
765 .collect(),
766 RetrievedDocuments::Ids(_) => Err(wrong_shape("scored documents")),
767 })
768 }
769
770 pub fn top_n_ids(
772 &self,
773 req: VectorSearchRequest<Filter<serde_json::Value>>,
774 ) -> impl Future<Output = Result<Vec<VectorSearchIdResult>, ErrorReport>> + WasmCompatSend + 'static
775 {
776 self.dispatch_mapped(
777 RetrieveQuery::TopNIds { req },
778 |documents| match documents {
779 RetrievedDocuments::Ids(results) => Ok(results
780 .into_iter()
781 .map(|(score, id)| VectorSearchIdResult { score, id })
782 .collect()),
783 RetrievedDocuments::Scored(_) => Err(wrong_shape("scored ids")),
784 },
785 )
786 }
787}
788
789fn wrong_shape(expected: &'static str) -> ErrorReport {
790 ErrorReport::new(
791 ErrorKind::Internal,
792 format!("expected {expected}, the retrieve handler answered another shape"),
793 )
794}
795
796#[cfg(test)]
797mod tests;