1use std::sync::{Arc, Mutex};
25
26use crate::{
27 completion::{CompletionRequest, ModelRef, UnsupportedOption},
28 driver::{DynModel, Sending, without_messages},
29 effect::{
30 Answered, EffectFamily, EffectId, EffectKind, EmbedInputs, EmbedModality, EmbedOutputs,
31 FamilyDescriptor, HandlerDescriptor, HandlerKey, RetrieveQuery, RetrievedDocuments,
32 ToolEmbeddingDescriptor,
33 },
34 error::{ErrorKind, ErrorReport},
35 operation::{Completion, Embedding, Rerank},
36 providers::registry::ProviderId,
37 tool::{Tool, ToolEmbedding, ToolExecutionError},
38 vector_store::{VectorStoreError, VectorStoreIndex, request::DynamicSearchFilter},
39 wasm_compat::{WasmBoxedFuture, WasmCompatSend, WasmCompatSync},
40 wire::Operation,
41};
42
43use super::{Dispatch, Reply, Serve};
44use crate::effect::family;
45
46fn wrong_family(handler: EffectFamily, kind: &EffectKind) -> ErrorReport {
47 ErrorReport::new(
48 ErrorKind::HandlerUnavailable,
49 format!(
50 "a {handler} handler cannot serve a `{}` effect",
51 kind.name()
52 ),
53 )
54}
55
56pub struct ModelAdapter<Op: Operation> {
60 label: ModelRef,
61 model: DynModel<Op>,
62 checked: Mutex<Option<(CompletionRequest, Vec<UnsupportedOption>)>>,
65}
66
67impl<Op: Operation> ModelAdapter<Op> {
68 pub fn new(label: impl Into<ModelRef>, model: impl Into<DynModel<Op>>) -> Self {
70 Self {
71 label: label.into(),
72 model: model.into(),
73 checked: Mutex::new(None),
74 }
75 }
76
77 fn sending(
80 &self,
81 handler: HandlerKey,
82 dispatch: &Dispatch,
83 observation: Option<crate::observe::AdapterContext>,
84 ) -> Sending {
85 let mut extensions = http::Extensions::new();
86 extensions.insert(Serving {
87 effect: dispatch.id(),
88 handler,
89 provider: self.model.spec().map(|spec| spec.provider),
90 });
91 Sending {
92 observation,
93 extensions,
94 }
95 }
96}
97
98#[derive(Clone, Debug, PartialEq, Eq)]
119#[non_exhaustive]
120pub struct Serving {
121 pub effect: EffectId,
123 pub handler: HandlerKey,
125 pub provider: Option<ProviderId>,
128}
129
130impl Serve for ModelAdapter<Completion> {
133 type Family = family::Completion;
134
135 fn descriptor(&self) -> HandlerDescriptor {
136 HandlerDescriptor {
137 key: crate::effect::model_key(self.label.as_str()),
138 family: FamilyDescriptor::Completion {
139 model: self.label.clone(),
140 capabilities: self.model.capabilities().completion,
141 },
142 layers: Vec::new(),
143 }
144 }
145
146 fn check(&self, kind: &EffectKind) -> Vec<UnsupportedOption> {
154 let EffectKind::Completion { request, .. } = kind else {
155 return Vec::new();
156 };
157 let memo = request.provider_options.is_empty();
158 let checked = without_messages(request);
159 if memo
160 && let Ok(last) = self.checked.lock()
161 && let Some((last, refused)) = last.as_ref()
162 && *last == checked
163 {
164 return refused.clone();
165 }
166 let refused = match self.model.check(request) {
167 Err(crate::completion::CheckError::Unsupported(refused)) => refused,
168 Ok(()) | Err(crate::completion::CheckError::Invalid(_)) => Vec::new(),
169 };
170 if memo && let Ok(mut last) = self.checked.lock() {
171 *last = Some((checked, refused.clone()));
172 }
173 refused
174 }
175
176 async fn serve(&self, kind: EffectKind, dispatch: Dispatch) -> Reply {
177 let model = &self.model;
178 let sending = self.sending(
179 crate::effect::model_key(self.label.as_str()),
180 &dispatch,
181 dispatch.adapter_context(),
182 );
183 match kind {
184 EffectKind::Completion {
185 request,
186 stream: false,
187 } => {
188 let result = model.finished(Arc::unwrap_or_clone(request), sending).await;
189 let result = result.map(Arc::new);
190 Reply::Outcome(result.map(Answered::Completion).map_err(ErrorReport::from))
191 }
192 EffectKind::Completion {
193 request,
194 stream: true,
195 } => match model.opened(Arc::unwrap_or_clone(request), sending) {
196 Ok(stream) => Reply::Stream(stream.into_relay()),
197 Err(error) => Reply::Outcome(Err(ErrorReport::from(error))),
198 },
199 other @ (EffectKind::ToolCall { .. }
200 | EffectKind::Embed { .. }
201 | EffectKind::Retrieve { .. }
202 | EffectKind::Rerank { .. }
203 | EffectKind::Custom { .. }) => {
204 Reply::Outcome(Err(wrong_family(EffectFamily::Completion, &other)))
205 }
206 }
207 }
208}
209
210impl Serve for ModelAdapter<Embedding> {
212 type Family = family::Embed;
213
214 fn descriptor(&self) -> HandlerDescriptor {
215 let capabilities = self.model.capabilities();
216 HandlerDescriptor {
217 key: crate::effect::embed_key(self.label.as_str()),
218 family: FamilyDescriptor::Embed {
219 model: self.label.to_string(),
220 dims: Some(capabilities.ndims),
221 max_documents: capabilities.max_documents,
222 modality: EmbedModality::Text,
223 },
224 layers: Vec::new(),
225 }
226 }
227
228 async fn serve(&self, kind: EffectKind, dispatch: Dispatch) -> Reply {
229 let model = &self.model;
230 let sending = self.sending(
231 crate::effect::embed_key(self.label.as_str()),
232 &dispatch,
233 None,
234 );
235 match kind {
236 EffectKind::Embed {
237 inputs: EmbedInputs::Texts(texts),
238 } => Reply::Outcome(
239 model
240 .finished(texts, sending)
241 .await
242 .map(|response| Answered::Embeddings(EmbedOutputs::Texts(response)))
243 .map_err(ErrorReport::from),
244 ),
245 EffectKind::Embed {
246 inputs: EmbedInputs::Images(_),
247 } => Reply::Outcome(Err(ErrorReport::new(
248 ErrorKind::HandlerUnavailable,
249 "a text embedding handler cannot embed images",
250 ))),
251 other @ (EffectKind::Completion { .. }
252 | EffectKind::ToolCall { .. }
253 | EffectKind::Retrieve { .. }
254 | EffectKind::Rerank { .. }
255 | EffectKind::Custom { .. }) => {
256 Reply::Outcome(Err(wrong_family(EffectFamily::Embed, &other)))
257 }
258 }
259 }
260}
261
262impl Serve for ModelAdapter<Rerank> {
264 type Family = family::Rerank;
265
266 fn descriptor(&self) -> HandlerDescriptor {
267 HandlerDescriptor {
268 key: crate::effect::rerank_key(self.label.as_str()),
269 family: FamilyDescriptor::Rerank {
270 model: self.label.to_string(),
271 max_documents: self.model.capabilities().max_documents,
272 },
273 layers: Vec::new(),
274 }
275 }
276
277 async fn serve(&self, kind: EffectKind, dispatch: Dispatch) -> Reply {
278 let model = &self.model;
279 let sending = self.sending(
280 crate::effect::rerank_key(self.label.as_str()),
281 &dispatch,
282 None,
283 );
284 match kind {
285 EffectKind::Rerank { request } => Reply::Outcome(
286 model
287 .finished(request, sending)
288 .await
289 .map(Answered::Reranked)
290 .map_err(ErrorReport::from),
291 ),
292 other @ (EffectKind::Completion { .. }
293 | EffectKind::ToolCall { .. }
294 | EffectKind::Embed { .. }
295 | EffectKind::Retrieve { .. }
296 | EffectKind::Custom { .. }) => {
297 Reply::Outcome(Err(wrong_family(EffectFamily::Rerank, &other)))
298 }
299 }
300 }
301}
302
303pub(crate) async fn call_tool<A, O>(
313 kind: EffectKind,
314 dispatch: Dispatch,
315 bound: Option<&str>,
316 call: impl AsyncFnOnce(&mut crate::tool::ToolContext, A) -> Result<O, ToolExecutionError>,
317) -> Reply
318where
319 A: serde::de::DeserializeOwned,
320 O: crate::tool::IntoToolOutput,
321{
322 let args = match (kind, bound) {
323 (EffectKind::ToolCall { name, .. }, Some(bound)) if bound != name => {
324 return Reply::Outcome(Err(ErrorReport::new(
325 ErrorKind::Internal,
326 format!("tool handler `{bound}` asked to run `{name}`"),
327 )));
328 }
329 (EffectKind::ToolCall { args, .. }, _) => args,
330 (
331 other @ (EffectKind::Completion { .. }
332 | EffectKind::Embed { .. }
333 | EffectKind::Retrieve { .. }
334 | EffectKind::Rerank { .. }
335 | EffectKind::Custom { .. }),
336 _,
337 ) => return Reply::Outcome(Err(wrong_family(EffectFamily::Tool, &other))),
338 };
339 let result = run_tool(&args, &dispatch, call).await;
340 Reply::Outcome(Ok(Answered::ToolResult { result }))
341}
342
343pub(crate) async fn run_tool<A, O>(
347 args: &str,
348 dispatch: &Dispatch,
349 call: impl AsyncFnOnce(&mut crate::tool::ToolContext, A) -> Result<O, ToolExecutionError>,
350) -> crate::tool::ToolResult
351where
352 A: serde::de::DeserializeOwned,
353 O: crate::tool::IntoToolOutput,
354{
355 let mut context = dispatch
356 .scope::<crate::tool::ToolContext>()
357 .map(|inbound| inbound.for_dispatch())
358 .unwrap_or_default()
359 .with_scopes(dispatch.scopes());
360 let result = run_in(args, &mut context, call).await;
361 if let Some(published) = dispatch.scope::<crate::tool::PublishedContext>() {
362 published.publish(context);
363 }
364 result
365}
366
367pub(crate) async fn run_direct<A, O>(
371 context: &mut crate::tool::ToolContext,
372 args: serde_json::Value,
373 call: impl AsyncFnOnce(&mut crate::tool::ToolContext, A) -> Result<O, ToolExecutionError>,
374) -> Result<crate::tool::ToolOutput, ToolExecutionError>
375where
376 A: serde::de::DeserializeOwned,
377 O: crate::tool::IntoToolOutput,
378{
379 let mut dispatched = context.for_dispatch();
380 let result = run_in(&args.to_string(), &mut dispatched, call).await;
381 context.accept_dispatch_result(dispatched);
382 result.into_result()
383}
384
385async fn run_in<A, O>(
388 args: &str,
389 context: &mut crate::tool::ToolContext,
390 call: impl AsyncFnOnce(&mut crate::tool::ToolContext, A) -> Result<O, ToolExecutionError>,
391) -> crate::tool::ToolResult
392where
393 A: serde::de::DeserializeOwned,
394 O: crate::tool::IntoToolOutput,
395{
396 match crate::tool::contextual::parse_tool_args::<A>(args) {
397 Ok(args) => crate::tool::contextual::tool_result_from(call(context, args).await),
398 Err(error) => crate::tool::ToolResult::failed(error),
399 }
400}
401
402pub struct ToolAdapter<T> {
404 tool: T,
405 embedding: Option<ToolEmbeddingDescriptor>,
406}
407
408impl<T: Tool> ToolAdapter<T> {
409 pub fn new(tool: T) -> Self {
411 Self {
412 tool,
413 embedding: None,
414 }
415 }
416
417 pub fn retrievable(tool: T) -> Result<Self, serde_json::Error>
420 where
421 T: ToolEmbedding,
422 {
423 let embedding = ToolEmbeddingDescriptor {
424 context: serde_json::to_value(tool.context())?,
425 embedding_docs: tool.embedding_docs(),
426 };
427 Ok(Self {
428 tool,
429 embedding: Some(embedding),
430 })
431 }
432
433 pub fn tool(&self) -> &T {
435 &self.tool
436 }
437
438 pub fn embedding(&self) -> Option<&ToolEmbeddingDescriptor> {
440 self.embedding.as_ref()
441 }
442
443 pub async fn run(
448 &self,
449 args: serde_json::Value,
450 ) -> Result<crate::tool::ToolOutput, ToolExecutionError> {
451 self.run_with(&mut crate::tool::ToolContext::new(), args)
452 .await
453 }
454
455 pub async fn run_with(
500 &self,
501 context: &mut crate::tool::ToolContext,
502 args: serde_json::Value,
503 ) -> Result<crate::tool::ToolOutput, ToolExecutionError> {
504 run_direct(context, args, async |context, args| {
505 self.call(context, args).await
506 })
507 .await
508 }
509
510 async fn call(
512 &self,
513 context: &mut crate::tool::ToolContext,
514 args: T::Args,
515 ) -> Result<T::Output, ToolExecutionError> {
516 self.tool
517 .call(context, args)
518 .await
519 .map_err(|error| self.tool.map_error(error))
520 }
521}
522
523impl<T> Serve for ToolAdapter<T>
524where
525 T: Tool + 'static,
526{
527 type Family = family::Tool;
528
529 fn descriptor(&self) -> HandlerDescriptor {
530 HandlerDescriptor {
531 key: crate::effect::tool_key(T::NAME),
532 family: FamilyDescriptor::Tool {
533 name: T::NAME.to_owned(),
534 description: self.tool.description(),
535 parameters: self.tool.parameters(),
536 embedding: self.embedding.clone(),
537 },
538 layers: Vec::new(),
539 }
540 }
541
542 fn key(&self) -> HandlerKey {
543 crate::effect::tool_key(T::NAME)
544 }
545
546 fn footprint(&self, kind: &EffectKind) -> Option<crate::tool::Touches> {
547 match kind {
548 EffectKind::ToolCall { name, args } if name == T::NAME => {
549 let args = crate::tool::contextual::parse_tool_args::<T::Args>(args).ok()?;
550 Some(self.tool.footprint(&args))
551 }
552 _ => None,
553 }
554 }
555
556 async fn serve(&self, kind: EffectKind, dispatch: Dispatch) -> Reply {
557 call_tool(kind, dispatch, Some(T::NAME), async |context, args| {
558 self.call(context, args).await
559 })
560 .await
561 }
562}
563
564pub trait ToolCallback:
567 for<'a> Fn(
568 &'a mut crate::tool::ToolContext,
569 serde_json::Value,
570 ) -> WasmBoxedFuture<
571 'a,
572 Result<crate::tool::ToolOutput, crate::tool::ToolExecutionError>,
573 > + WasmCompatSend
574 + WasmCompatSync
575{
576}
577
578impl<F> ToolCallback for F where
579 F: for<'a> Fn(
580 &'a mut crate::tool::ToolContext,
581 serde_json::Value,
582 ) -> WasmBoxedFuture<
583 'a,
584 Result<crate::tool::ToolOutput, crate::tool::ToolExecutionError>,
585 > + WasmCompatSend
586 + WasmCompatSync
587{
588}
589
590pub struct RetrieveAdapter<I> {
594 index: I,
595 label: Option<String>,
596}
597
598impl<I> RetrieveAdapter<I> {
599 pub fn new(index: I) -> Self {
601 Self { index, label: None }
602 }
603
604 pub fn labelled(label: impl Into<String>, index: I) -> Self {
608 Self {
609 index,
610 label: Some(label.into()),
611 }
612 }
613
614 pub fn index(&self) -> &I {
616 &self.index
617 }
618}
619
620impl<I, F> Serve for RetrieveAdapter<I>
621where
622 I: VectorStoreIndex<Filter = F> + 'static,
623 F: DynamicSearchFilter + WasmCompatSend + WasmCompatSync + 'static,
624{
625 type Family = family::Retrieve;
626
627 fn descriptor(&self) -> HandlerDescriptor {
628 HandlerDescriptor {
629 key: self
630 .label
631 .as_deref()
632 .map_or_else(|| HandlerKey::from("retrieve"), crate::effect::retrieve_key),
633 family: FamilyDescriptor::Retrieve {},
634 layers: Vec::new(),
635 }
636 }
637
638 async fn serve(&self, kind: EffectKind, _dispatch: Dispatch) -> Reply {
639 match kind {
640 EffectKind::Retrieve { query } => {
641 let outcome = match query {
642 RetrieveQuery::TopN { req } => {
643 match req.try_map_filter(F::from_dynamic_filter) {
644 Ok(req) => self
645 .index
646 .top_n::<serde_json::Value>(req)
647 .await
648 .map(|results| {
649 Answered::Documents(RetrievedDocuments::Scored(
650 results
651 .into_iter()
652 .map(|result| {
653 (
654 result.score,
655 result.id,
656 F::normalize_dynamic_document(result.document),
657 )
658 })
659 .collect(),
660 ))
661 })
662 .map_err(ErrorReport::from),
663 Err(error) => Err(ErrorReport::from(VectorStoreError::from(error))),
664 }
665 }
666 RetrieveQuery::TopNIds { req } => {
667 match req.try_map_filter(F::from_dynamic_filter) {
668 Ok(req) => self
669 .index
670 .top_n_ids(req)
671 .await
672 .map(|results| {
673 Answered::Documents(RetrievedDocuments::Ids(
674 results
675 .into_iter()
676 .map(|result| (result.score, result.id))
677 .collect(),
678 ))
679 })
680 .map_err(ErrorReport::from),
681 Err(error) => Err(ErrorReport::from(VectorStoreError::from(error))),
682 }
683 }
684 };
685 Reply::Outcome(outcome)
686 }
687 other @ (EffectKind::Completion { .. }
688 | EffectKind::ToolCall { .. }
689 | EffectKind::Embed { .. }
690 | EffectKind::Rerank { .. }
691 | EffectKind::Custom { .. }) => {
692 Reply::Outcome(Err(wrong_family(EffectFamily::Retrieve, &other)))
693 }
694 }
695 }
696}