1use std::collections::{BTreeMap, HashMap, HashSet};
21
22use crate::completion::{CompletionRequest, CompletionResponse, FinishReason, Usage};
23use crate::error::ProviderError;
24use crate::message::citation;
25use crate::message::{
26 Api, AssistantContent, CallId, Image, LocalCallId, Opaque, Origin, Reasoning, Text, ToolCall,
27 ToolFunction, ToolName,
28};
29use crate::streaming::{Item, Part, PartKind, StreamEvent};
30use crate::telemetry::{GenAiOperation, SpanBuilder, SpanCombinator};
31use crate::wire::{
32 Assembled, Call, Descriptor, Emit, Fold, Mode, Operation, Out, Reply, Shared, WireCitation,
33};
34
35#[derive(Debug, Clone, Copy, PartialEq, Eq)]
37pub struct Completion;
38
39impl Operation for Completion {
40 type Request = CompletionRequest;
41 type Event = StreamEvent;
42 type End = Finish;
43 type Response = CompletionResponse;
44 type Fold = ReplyFold;
45 type Emit = Assembled;
46
47 fn fold(request: &Self::Request, call: &mut Call<'_>) -> Self::Fold {
50 let telemetry = call
51 .wire
52 .telemetry
53 .map_or(GenAiOperation::Chat, |telemetry| telemetry(call.mode));
54 debug_assert!(telemetry.is_completion());
55 let replay = call.wire.replay;
56 let model = request
57 .model
58 .as_deref()
59 .or(call.wire.model)
60 .or_else(|| replay.map(|target| target.model()))
61 .unwrap_or_default();
62 let span = SpanBuilder::new(call.wire.name, model, telemetry)
63 .streaming(matches!(call.mode, Mode::Streaming))
64 .system_instructions(
65 request.system_instructions(),
66 request.record_telemetry_content,
67 )
68 .build();
69 call.instrument(span.clone());
70 let api = replay.map_or_else(
71 || Api::from(call.wire.name.to_owned()),
72 |target| target.api(),
73 );
74 let mut origin = Origin::new(api, call.wire.name, model);
75 if replay.is_some_and(|target| target.binds_context(model)) {
77 origin.context =
78 replay.map(|target| crate::completion::history::context_of(request, target, model));
79 }
80 ReplyFold {
81 span,
82 facts: call.wire.facts.cloned().unwrap_or_default(),
83 wire: replay.is_some_and(|target| target.states_finish_reason()),
84 call_id_slot: replay.and_then(|target| target.call_id_slot()),
85 catalog_priced: replay.map(|target| target.catalog_priced()),
86 blob_slots: replay.map_or(&[], |target| target.blob_slots()),
87 accept_unknown_finish: request.accept_unknown_finish_reasons,
88 ..ReplyFold::new(origin)
89 }
90 }
91
92 fn prepare(
101 mut request: Self::Request,
102 wire: &Descriptor<'_>,
103 ) -> Result<Self::Request, ProviderError> {
104 let Some(target) = wire.replay else {
105 return Err(ProviderError::request(format!(
106 "completion wire `{}` names no replay target",
107 wire.name
108 )));
109 };
110 request.validate_message_content()?;
113 request.model = request
116 .model
117 .take()
118 .filter(|model| !model.is_empty())
119 .or_else(|| Some(target.model().to_owned()).filter(|model| !model.is_empty()));
120 crate::completion::options::check(target, &mut request)?;
123 let written = request.cache_points.iter().any(|point| {
126 matches!(
127 point,
128 crate::completion::CachePoint::Messages(_)
129 | crate::completion::CachePoint::Within { .. }
130 )
131 });
132 let written = written.then(|| {
133 let documents = (!target.takes_documents())
134 .then(|| request.normalized_documents())
135 .flatten();
136 (request.chat_history.clone(), documents)
137 });
138 let stored = target.continues_stored(&request);
139 let model = request.model.as_deref().unwrap_or(target.model());
140 let binds = target.binds_context(model) && !target.drops_unbound_items(&request);
142 let shape = crate::completion::history::Request {
143 model: request.model.as_deref(),
144 stored,
145 foreign_reasoning: request.foreign_reasoning,
146 tools: stored || target.declares_tools(&request),
149 context: binds.then(|| crate::completion::history::context_of(&request, target, model)),
150 };
151 let mut history = std::mem::take(&mut request.chat_history);
155 if !target.takes_documents() {
156 if let Some(documents) = request.normalized_documents() {
157 crate::completion::request::insert_after_leading_system(&mut history, documents);
158 }
159 request.documents.clear();
160 }
161 let mut kept = wire.prepared.and_then(|prepared| prepared.0.lock().ok());
163 let mut fresh = Default::default();
164 let memo = kept.as_deref_mut().unwrap_or(&mut fresh);
165 let adapted = crate::completion::history::adapt_for(history, target, &shape, memo);
166 drop(kept);
167 if let Some((written, documents)) = written {
168 request.cache_points = crate::completion::history::cache_points(
169 &request.cache_points,
170 &written,
171 documents.as_ref(),
172 &adapted,
173 target,
174 &shape,
175 );
176 }
177 request.chat_history = adapted;
178 request.validate_message_content()?;
179 Ok(request)
180 }
181}
182
183impl crate::wire::reply::Closing<Completion> for Assembled {
184 fn close(shared: &mut Shared<Completion>) {
185 let Shared {
186 fold, items, end, ..
187 } = shared;
188 let cut = end
189 .as_ref()
190 .and_then(|end| end.reason.as_ref())
191 .is_some_and(crate::completion::FinishReason::truncated_output);
192 fold.close_open(items, cut);
193 }
194}
195
196impl Emit<Completion> for Assembled {}
197
198#[derive(Debug, Clone, Default, PartialEq, serde::Serialize, serde::Deserialize)]
200pub struct Finish {
201 pub usage: Usage,
207 #[serde(default, skip_serializing_if = "Option::is_none")]
209 pub reason: Option<FinishReason>,
210 pub response_id: Option<String>,
212 pub model: Option<String>,
214 #[serde(default, skip_serializing_if = "Option::is_none")]
217 pub error: Option<String>,
218}
219
220#[derive(Debug, Clone, PartialEq)]
222pub enum Block {
223 Text,
225 Reasoning {
227 redacted: bool,
229 },
230 Image(Image),
233 Call {
235 id: CallId,
237 name: ToolName,
239 },
240 Opaque {
242 replay: bool,
244 },
245}
246
247pub struct ReplyFold {
254 span: tracing::Span,
255 origin: Origin,
257 facts: crate::catalog::ModelFacts,
259 open: BTreeMap<usize, Draft>,
261 held: Vec<(usize, Draft)>,
266 ended: HashMap<usize, usize>,
269 next_part: u32,
270 call_ids: HashSet<CallId>,
271 wire: bool,
274 call_id_slot: Option<&'static str>,
278 catalog_priced: Option<bool>,
284 blob_slots: &'static [&'static str],
288 accept_unknown_finish: bool,
293 first_incomplete: Option<usize>,
297 unfinished: Vec<CallId>,
301 last_call: Option<usize>,
303 run: Option<usize>,
305 next_auto: usize,
306 by_index: Option<BTreeMap<usize, usize>>,
309 lead: Option<usize>,
311 choice: Vec<Option<AssistantContent>>,
313 open_text: BTreeMap<usize, AssistantContent>,
316}
317
318struct Draft {
320 part: Part,
321 started: bool,
323 item: serde_json::Value,
325 body: Body,
326 citations: Vec<WireCitation>,
328}
329
330#[derive(Clone, Copy, PartialEq, Eq)]
332enum Closing {
333 Complete,
335 Incomplete,
338}
339
340enum Body {
341 Text(String),
342 Image(Image),
343 Reasoning {
344 text: String,
345 redacted: bool,
346 },
347 Call {
348 id: Option<CallId>,
349 name: String,
350 arguments: Arguments,
351 },
352 Opaque {
353 replay: bool,
354 },
355}
356
357impl Body {
358 fn kind(&self) -> PartKind {
359 match self {
360 Self::Text(_) => PartKind::Text,
361 Self::Reasoning { .. } => PartKind::Reasoning,
362 Self::Image(_) => PartKind::Image,
363 Self::Call { .. } => PartKind::ToolCall,
364 Self::Opaque { .. } => PartKind::Opaque,
365 }
366 }
367}
368
369#[derive(Default)]
371struct Arguments {
372 text: String,
373 streamed: usize,
375 overflowed: bool,
376 substantive: bool,
378 announced: Option<serde_json::Value>,
381}
382
383const MAX_TOOL_INPUT_BYTES: usize = 32 * 1024 * 1024;
385
386impl Arguments {
387 fn push(&mut self, fragment: &str, name: &str) {
388 self.substantive |= !fragment.trim().is_empty();
389 if self.text.trim() == "null" && !fragment.trim().is_empty() {
392 self.text.clear();
393 }
394 if self.text.len().saturating_add(fragment.len()) > MAX_TOOL_INPUT_BYTES {
395 if !self.overflowed {
396 self.overflowed = true;
397 tracing::warn!(
398 tool = name,
399 "streamed tool-call input exceeded the accumulation bound; truncating"
400 );
401 }
402 } else {
403 self.text.push_str(fragment);
404 }
405 }
406
407 fn function(&self, name: ToolName) -> ToolFunction {
410 if self.text.is_empty()
411 && let Some(announced) = &self.announced
412 {
413 return ToolFunction::new(name, announced.clone());
414 }
415 let mut function = ToolFunction::parse(name, &self.text);
416 if self.overflowed {
417 function.invalid_arguments = Some(self.text.clone());
418 }
419 function
420 }
421
422 fn unstreamed(&mut self) -> Option<String> {
426 if "null".starts_with(self.text.trim()) {
427 return None;
428 }
429 let fragment = self.text.get(self.streamed..)?;
430 if fragment.is_empty() {
431 return None;
432 }
433 let fragment = fragment.to_owned();
434 self.streamed = self.text.len();
435 Some(fragment)
436 }
437
438 fn complete(&self) -> bool {
440 !self.overflowed && crate::json_utils::is_whole_object(&self.text)
441 }
442}
443
444fn stated(id: &str) -> bool {
447 !id.is_empty() && id != "null"
448}
449
450#[derive(Debug, Clone, Copy, Default)]
452pub struct CallFragment<'a> {
453 pub id: Option<&'a str>,
455 pub name: Option<&'a str>,
457 pub arguments: Option<&'a str>,
459}
460
461pub const AUTO_INDEX: usize = 1 << 30;
464
465fn not_open(index: usize) -> ProviderError {
466 ProviderError::Response(format!(
467 "the reply wrote to item {index}, which is not open"
468 ))
469}
470
471fn stream_call(items: &mut Items, draft: &mut Draft) {
475 let Body::Call {
476 name, arguments, ..
477 } = &mut draft.body
478 else {
479 return;
480 };
481 if !draft.started {
482 let Ok(name) = ToolName::new(name.clone()) else {
483 return;
484 };
485 draft.started = true;
486 emit(
487 items,
488 StreamEvent::Start {
489 part: draft.part,
490 kind: PartKind::ToolCall,
491 name: Some(name),
492 },
493 );
494 }
495 if let Some(json) = arguments.unstreamed() {
496 emit(
497 items,
498 StreamEvent::Arguments {
499 part: draft.part,
500 json,
501 },
502 );
503 }
504}
505
506impl ReplyFold {
507 pub(crate) fn new(origin: Origin) -> Self {
509 Self {
510 span: tracing::Span::none(),
511 origin,
512 facts: crate::catalog::ModelFacts::default(),
513 open: BTreeMap::new(),
514 held: Vec::new(),
515 ended: HashMap::new(),
516 next_part: 0,
517 call_ids: HashSet::new(),
518 wire: false,
519 call_id_slot: None,
520 catalog_priced: None,
521 blob_slots: &[],
522 accept_unknown_finish: false,
523 first_incomplete: None,
524 unfinished: Vec::new(),
525 last_call: None,
526 run: None,
527 next_auto: AUTO_INDEX,
528 by_index: None,
529 lead: None,
530 choice: Vec::new(),
531 open_text: BTreeMap::new(),
532 }
533 }
534
535 pub(crate) fn relayed(label: impl Into<String>) -> Self {
539 let label = label.into();
540 Self::new(Origin::new(label.clone(), label.clone(), label))
541 }
542
543 pub(crate) fn set_origin(&mut self, origin: Origin) {
545 self.origin = origin;
546 }
547
548 fn pricing(&self, origin: &Origin) -> Option<crate::catalog::Pricing> {
552 let priced = self.catalog_priced.unwrap_or_else(|| {
553 crate::providers::registry::ProviderId::catalog(&origin.provider)
554 .is_none_or(|id| id.catalog_priced())
555 });
556 if !priced {
557 return None;
558 }
559 self.facts
560 .for_model(&origin.provider, &origin.model)?
561 .pricing
562 }
563
564 fn next(&mut self) -> Part {
566 let part = Part::new(self.next_part);
567 self.next_part += 1;
568 part
569 }
570
571 fn fresh_index(&mut self) -> usize {
572 let index = self.next_auto;
573 self.next_auto += 1;
574 index
575 }
576
577 fn draft(&mut self, index: usize) -> Result<&mut Draft, ProviderError> {
578 self.open.get_mut(&index).ok_or_else(|| not_open(index))
579 }
580
581 pub(crate) fn open_item(
582 &mut self,
583 items: &mut Items,
584 index: usize,
585 block: Block,
586 item: serde_json::Value,
587 ) -> Result<(), ProviderError> {
588 if self.open.contains_key(&index) {
589 return Err(ProviderError::Response(format!(
590 "the reply opened item {index} twice"
591 )));
592 }
593 let body = match block {
594 Block::Text => Body::Text(String::new()),
595 Block::Image(image) => Body::Image(image),
596 Block::Reasoning { redacted } => Body::Reasoning {
597 text: String::new(),
598 redacted,
599 },
600 Block::Call { id, name } => Body::Call {
601 id: Some(id),
602 name: name.into(),
603 arguments: Arguments::default(),
604 },
605 Block::Opaque { replay } => Body::Opaque { replay },
606 };
607 self.insert(index, body, item);
608 stream_call(items, self.draft(index)?);
609 Ok(())
610 }
611
612 fn insert(&mut self, index: usize, body: Body, item: serde_json::Value) {
613 self.ended.remove(&index);
614 let part = self.next();
615 if let Some(by_index) = &mut self.by_index {
616 by_index.insert(part.index(), index);
617 }
618 let started = false;
619 self.open.insert(
620 index,
621 Draft {
622 part,
623 started,
624 item,
625 body,
626 citations: Vec::new(),
627 },
628 );
629 }
630
631 pub(crate) fn push_item(
632 &mut self,
633 items: &mut Items,
634 index: usize,
635 fragment: &str,
636 ) -> Result<(), ProviderError> {
637 if fragment.is_empty() {
638 return Ok(());
639 }
640 let draft = self.draft(index)?;
641 let event = match &mut draft.body {
642 Body::Text(text) => {
643 text.push_str(fragment);
644 StreamEvent::Text {
645 part: draft.part,
646 text: fragment.to_owned(),
647 }
648 }
649 Body::Reasoning { text, .. } => {
650 text.push_str(fragment);
651 StreamEvent::Reasoning {
652 part: draft.part,
653 text: fragment.to_owned(),
654 }
655 }
656 Body::Call {
657 name, arguments, ..
658 } => {
659 arguments.push(fragment, name);
660 stream_call(items, draft);
661 return Ok(());
662 }
663 Body::Image(image) => {
664 if let crate::message::DocumentSourceKind::Base64(data) = &mut image.data {
665 data.push_str(fragment);
666 return Ok(());
667 }
668 return Err(ProviderError::Response(format!(
669 "the reply wrote data to the image item {index}, which holds no base64 data"
670 )));
671 }
672 Body::Opaque { .. } => {
673 return Err(ProviderError::Response(format!(
674 "the reply wrote text to the opaque item {index}"
675 )));
676 }
677 };
678 if !draft.started {
679 draft.started = true;
680 emit(
681 items,
682 StreamEvent::Start {
683 part: draft.part,
684 kind: draft.body.kind(),
685 name: None,
686 },
687 );
688 }
689 emit(items, event);
690 Ok(())
691 }
692
693 fn distinct_call_id(
700 &mut self,
701 id: Option<CallId>,
702 item: serde_json::Value,
703 ) -> (CallId, serde_json::Value) {
704 let (id, item) = match id {
705 Some(id) if !self.call_ids.contains(&id) => (id, item),
706 Some(id) => {
707 tracing::warn!(%id, "the provider named two tool calls with one id; renaming the second");
708 (CallId::Local(LocalCallId::new()), serde_json::Value::Null)
709 }
710 None if self.call_id_slot.is_some() => (CallId::Local(LocalCallId::new()), item),
711 None => (CallId::Local(LocalCallId::new()), serde_json::Value::Null),
712 };
713 self.call_ids.insert(id.clone());
714 (id, item)
715 }
716
717 fn close_item(
725 &mut self,
726 items: &mut Items,
727 index: usize,
728 closing: Closing,
729 ) -> Result<(), ProviderError> {
730 let mut draft = self.open.remove(&index).ok_or_else(|| not_open(index))?;
731 stream_call(items, &mut draft);
734 if self.run == Some(index) {
735 self.run = None;
736 }
737 if closing == Closing::Incomplete && matches!(draft.body, Body::Call { .. }) {
738 self.held.push((index, draft));
739 return Ok(());
740 }
741 self.settle(items, index, draft, (closing, false))
742 }
743
744 fn settle(
746 &mut self,
747 items: &mut Items,
748 index: usize,
749 draft: Draft,
750 (closing, unfinished): (Closing, bool),
751 ) -> Result<(), ProviderError> {
752 self.ended.insert(index, draft.part.index());
753 let Draft {
754 part,
755 started,
756 item,
757 body,
758 citations,
759 } = draft;
760 let item = match (closing, &body) {
761 (Closing::Complete, _) | (Closing::Incomplete, Body::Opaque { .. }) => item,
762 (Closing::Incomplete, _) => {
763 self.cut_at(part.index());
764 serde_json::Value::Null
765 }
766 };
767 let content = match body {
768 Body::Text(text) => {
769 if text.is_empty() && item.is_null() {
770 return Ok(());
771 }
772 let mut text = Text::new(text);
775 citation::attach(
776 &mut text,
777 Vec::new(),
778 citations,
779 &self.origin.provider,
780 index,
781 );
782 self.with_item(AssistantContent::Text(text), item)
783 }
784 Body::Reasoning { text, redacted } => {
785 if text.is_empty() && !redacted && item.is_null() {
786 return Ok(());
787 }
788 let reasoning = Reasoning {
789 text,
790 redacted,
791 native: None,
792 };
793 self.with_item(AssistantContent::Reasoning(reasoning), item)
794 }
795 Body::Image(image) => self.with_item(AssistantContent::Image(image), item),
796 Body::Opaque { replay } => AssistantContent::Opaque(Opaque {
797 item,
798 replay: replay && closing == Closing::Complete,
799 }),
800 Body::Call {
801 id,
802 name,
803 arguments,
804 } => {
805 let Ok(name) = ToolName::new(name) else {
806 tracing::warn!(
807 index,
808 "the provider closed a tool call without a name; nothing can answer it"
809 );
810 return Ok(());
811 };
812 let function = arguments.function(name);
813 let (id, item) = self.distinct_call_id(id, item);
814 if unfinished {
815 self.unfinished.push(id.clone());
816 }
817 let call = ToolCall::new(id, function);
818 let content = self.with_item(AssistantContent::ToolCall(call), item);
819 if arguments.streamed == 0
823 && let Some(fragment) = StreamEvent::fragment(part, &content)
824 {
825 emit(items, fragment);
826 }
827 emit(items, StreamEvent::End { part, content });
828 return Ok(());
829 }
830 };
831 if !started {
832 emit(
833 items,
834 StreamEvent::Start {
835 part,
836 kind: PartKind::of(&content),
837 name: None,
838 },
839 );
840 }
841 emit(items, StreamEvent::End { part, content });
842 Ok(())
843 }
844
845 fn with_item(&self, block: AssistantContent, item: serde_json::Value) -> AssistantContent {
848 if item.is_null() {
849 return block;
850 }
851 let mut block = block.with_native(item);
852 if let Some(Some(native)) = block.native_slot() {
853 native.find_blob(self.blob_slots);
854 }
855 block
856 }
857
858 fn edit_ended(
861 &mut self,
862 items: &mut Items,
863 index: usize,
864 edit: impl FnOnce(&mut serde_json::Value),
865 ) -> Result<(), ProviderError> {
866 let position = *self.ended.get(&index).ok_or_else(|| not_open(index))?;
867 let slots = self.blob_slots;
868 let content = ended_content(&mut self.choice, items, position);
869 if let Some(content) = content {
870 if let Some(item) = item_of(content) {
871 edit(item);
872 }
873 if let Some(Some(native)) = content.native_slot() {
874 native.find_blob(slots);
875 }
876 }
877 Ok(())
878 }
879
880 fn cite_item(
886 &mut self,
887 items: &mut Items,
888 index: usize,
889 citations: Vec<WireCitation>,
890 replace: bool,
891 ) {
892 let provider = self.origin.provider.as_str();
893 if let Some(draft) = self.open.get_mut(&index) {
894 if !matches!(draft.body, Body::Text(_)) {
895 tracing::warn!(
896 provider,
897 index,
898 "dropped citations of an item that is not text"
899 );
900 } else if replace {
901 draft.citations = citations;
902 } else {
903 draft.citations.extend(citations);
904 }
905 return;
906 }
907 let Some(position) = self.ended.get(&index).copied() else {
908 tracing::warn!(
909 provider,
910 index,
911 "dropped citations of an item the reply never opened"
912 );
913 return;
914 };
915 let content = ended_content(&mut self.choice, items, position);
916 match content {
917 Some(AssistantContent::Text(text)) => {
918 let kept = if replace {
919 Vec::new()
920 } else {
921 text.citations().to_vec()
922 };
923 citation::attach(text, kept, citations, provider, index);
924 }
925 _ => tracing::warn!(
926 provider,
927 index,
928 "dropped citations of an item that is not text"
929 ),
930 }
931 }
932
933 pub(crate) fn run_item(
934 &mut self,
935 items: &mut Items,
936 block: Block,
937 fragment: &str,
938 ) -> Result<usize, ProviderError> {
939 let current = self.run.filter(|index| {
940 self.open.get(index).is_some_and(|draft| {
941 matches!(
942 (&draft.body, &block),
943 (Body::Text(_), Block::Text)
944 | (
945 Body::Reasoning {
946 redacted: false,
947 ..
948 },
949 Block::Reasoning { redacted: false }
950 )
951 )
952 })
953 });
954 let index = match current {
955 Some(index) => index,
956 None => {
957 self.end_run(items)?;
958 let index = self.fresh_index();
959 self.open_item(items, index, block, serde_json::Value::Null)?;
960 self.run = Some(index);
961 index
962 }
963 };
964 self.push_item(items, index, fragment)?;
965 Ok(index)
966 }
967
968 pub(crate) fn end_run(&mut self, items: &mut Items) -> Result<(), ProviderError> {
969 match self.run.take() {
970 Some(index) => self.close_item(items, index, Closing::Complete),
971 None => Ok(()),
972 }
973 }
974
975 pub(crate) fn write_content(
977 &mut self,
978 items: &mut Items,
979 content: AssistantContent,
980 ) -> Result<(), ProviderError> {
981 let content = match content {
982 AssistantContent::ToolCall(mut call) => {
983 let (id, _) = self.distinct_call_id(Some(call.id.clone()), serde_json::Value::Null);
984 if id != call.id {
985 call.native = None;
986 }
987 call.id = id;
988 AssistantContent::ToolCall(call)
989 }
990 content => content,
991 };
992 for event in StreamEvent::whole(self.next(), &content) {
993 emit(items, event);
994 }
995 Ok(())
996 }
997
998 pub(crate) fn close_open(&mut self, items: &mut Items, cut: bool) {
1007 let mut closing: Vec<(usize, Draft, (Closing, bool))> = std::mem::take(&mut self.held)
1008 .into_iter()
1009 .map(|(index, draft)| (index, draft, (Closing::Incomplete, cut)))
1010 .collect();
1011 for (index, draft) in std::mem::take(&mut self.open) {
1012 let complete = self.run.take_if(|run| *run == index).is_some();
1013 let how = if complete {
1014 (Closing::Complete, false)
1015 } else {
1016 (Closing::Incomplete, true)
1017 };
1018 closing.push((index, draft, how));
1019 }
1020 closing.sort_by_key(|(_, draft, _)| draft.part);
1021 for (index, mut draft, how) in closing {
1022 stream_call(items, &mut draft);
1024 if let Err(error) = self.settle(items, index, draft, how) {
1025 items.push_back(Err(error));
1026 }
1027 }
1028 }
1029
1030 fn cut_at(&mut self, position: usize) {
1032 self.first_incomplete = Some(
1033 self.first_incomplete
1034 .map_or(position, |first| first.min(position)),
1035 );
1036 }
1037
1038 pub fn origin(&self) -> &Origin {
1040 &self.origin
1041 }
1042
1043 pub(crate) fn partial(
1051 &self,
1052 end: Option<&Finish>,
1053 reply: &Reply,
1054 failure: Option<&ProviderError>,
1055 ) -> CompletionResponse {
1056 let mut response =
1057 self.response(end.cloned().unwrap_or_default(), reply.clone(), Vec::new());
1058 if end.is_none() {
1059 match failure {
1060 Some(error) => {
1061 response.error.get_or_insert_with(|| error.to_string());
1062 }
1063 None => {
1064 response.aborted = Some(
1065 "the caller stopped reading before the provider ended the reply".to_owned(),
1066 );
1067 }
1068 }
1069 }
1070 let unfinished =
1074 end.is_none() || !self.open.is_empty() || self.choice.iter().any(Option::is_none);
1075 let canonical_from = if unfinished {
1076 Some(0)
1077 } else {
1078 self.first_incomplete
1079 };
1080 let parts = self
1081 .choice
1082 .iter()
1083 .enumerate()
1084 .map(|(index, part)| {
1085 let part = part.clone().or_else(|| self.open_text.get(&index).cloned());
1086 (index, part)
1087 })
1088 .collect();
1089 response.choice = self.ordered(parts, canonical_from);
1090 response
1091 }
1092
1093 fn ordered(
1097 &self,
1098 parts: Vec<(usize, Option<AssistantContent>)>,
1099 canonical_from: Option<usize>,
1100 ) -> Vec<AssistantContent> {
1101 let mut parts: Vec<(usize, AssistantContent)> = parts
1102 .into_iter()
1103 .filter_map(|(position, part)| {
1104 let part = part?;
1105 Some(match canonical_from {
1106 Some(first) if position >= first => (position, canonical(part)),
1107 _ => (position, part),
1108 })
1109 })
1110 .collect();
1111 if let Some(by_index) = &self.by_index {
1112 parts.sort_by_key(|(position, _)| {
1113 (
1114 by_index.get(position).copied().unwrap_or(usize::MAX),
1115 *position,
1116 )
1117 });
1118 }
1119 if let Some(lead) = self.lead {
1120 parts.sort_by_key(|(position, _)| *position != lead);
1121 }
1122 parts.into_iter().map(|(_, part)| part).collect()
1123 }
1124
1125 fn response(
1127 &self,
1128 end: Finish,
1129 reply: Reply,
1130 choice: Vec<AssistantContent>,
1131 ) -> CompletionResponse {
1132 let Finish {
1133 usage,
1134 reason,
1135 response_id,
1136 model,
1137 error,
1138 } = end;
1139 use crate::provider_response::reported;
1140 let mut origin = self.origin.clone();
1141 let response_model = reported(model);
1142 named_by_reply(&mut origin, response_model.as_deref());
1143 origin.response_model = response_model;
1144 origin.response_id = reported(response_id);
1145 let usage = match usage.cost {
1146 Some(_) => usage,
1147 None => usage.cost(
1148 self.pricing(&origin)
1149 .and_then(|pricing| pricing.cost(&usage)),
1150 ),
1151 };
1152 let error = error.or_else(|| {
1153 if !self.wire {
1154 return None;
1155 }
1156 match &reason {
1157 None => Some("the provider ended the reply without a finish reason".to_owned()),
1158 Some(FinishReason::Length) => None,
1159 Some(_) if !self.unfinished.is_empty() => Some(
1160 "the provider ended the reply with a tool call it never finished".to_owned(),
1161 ),
1162 Some(_) => None,
1163 }
1164 });
1165 let mut response = CompletionResponse::new(choice, usage, origin, reply.raw)
1166 .with_optional_finish_reason(reason)
1167 .accept_unknown_finish_reasons(self.accept_unknown_finish)
1168 .with_unfinished_calls(self.unfinished.clone());
1169 response.error = error;
1170 response.provider_request_id = reported(reply.provider_request_id);
1171 response
1172 }
1173}
1174
1175fn named_by_reply(origin: &mut Origin, reported: Option<&str>) {
1178 if origin.model.is_empty()
1179 && let Some(model) = reported
1180 {
1181 model.clone_into(&mut origin.model);
1182 }
1183}
1184
1185pub(crate) fn canonical(block: AssistantContent) -> AssistantContent {
1187 match block {
1188 AssistantContent::Opaque(opaque) => AssistantContent::Opaque(Opaque {
1189 replay: false,
1190 ..opaque
1191 }),
1192 block => block.into_canonical(),
1193 }
1194}
1195
1196fn ended_content<'a>(
1199 choice: &'a mut [Option<AssistantContent>],
1200 items: &'a mut Items,
1201 position: usize,
1202) -> Option<&'a mut AssistantContent> {
1203 items
1204 .iter_mut()
1205 .find_map(|item| match item {
1206 Ok(Item::Event(StreamEvent::End { part, content })) if part.index() == position => {
1207 Some(content)
1208 }
1209 _ => None,
1210 })
1211 .or_else(|| choice.get_mut(position).and_then(Option::as_mut))
1212}
1213
1214fn item_of(content: &mut AssistantContent) -> Option<&mut serde_json::Value> {
1216 let native = match content {
1217 AssistantContent::Text(text) => text.native.as_mut(),
1218 AssistantContent::ToolCall(call) => call.native.as_mut(),
1219 AssistantContent::Reasoning(reasoning) => reasoning.native.as_mut(),
1220 AssistantContent::Image(image) => image.native.as_mut(),
1221 AssistantContent::Opaque(opaque) => return Some(&mut opaque.item),
1222 };
1223 native.map(|native| &mut native.item)
1224}
1225
1226pub(crate) type Items = std::collections::VecDeque<Result<Item<StreamEvent>, ProviderError>>;
1227
1228fn emit(items: &mut Items, event: StreamEvent) {
1229 items.push_back(Ok(Item::Event(event)));
1230}
1231
1232pub fn merge(item: &mut serde_json::Value, delta: &serde_json::Map<String, serde_json::Value>) {
1237 use serde_json::Value;
1238 if !item.is_object() {
1239 *item = Value::Object(serde_json::Map::new());
1240 }
1241 let Value::Object(item) = item else {
1242 return;
1243 };
1244 for (key, value) in delta {
1245 if key == "type" {
1246 continue;
1247 }
1248 match (item.get_mut(key), value) {
1249 (Some(Value::String(existing)), Value::String(fragment)) => {
1250 existing.push_str(fragment);
1251 }
1252 (Some(Value::Array(existing)), Value::Array(more)) => {
1253 existing.extend(more.iter().cloned());
1254 }
1255 _ => {
1256 item.insert(key.clone(), value.clone());
1257 }
1258 }
1259 }
1260}
1261
1262impl Fold<Completion> for ReplyFold {
1263 fn absorb(&mut self, event: &StreamEvent) -> Result<(), ProviderError> {
1264 match event {
1265 StreamEvent::Start { part, .. } => {
1266 if self.choice.len() <= part.index() {
1267 self.choice.resize(part.index() + 1, None);
1268 }
1269 }
1270 StreamEvent::End { part, content } => {
1271 if self.choice.len() <= part.index() {
1272 self.choice.resize(part.index() + 1, None);
1273 }
1274 if let Some(slot) = self.choice.get_mut(part.index()) {
1275 *slot = Some(content.clone());
1276 }
1277 self.open_text.remove(&part.index());
1278 }
1279 StreamEvent::Text { part, text } => {
1280 if let AssistantContent::Text(open) = self
1281 .open_text
1282 .entry(part.index())
1283 .or_insert_with(|| AssistantContent::text(""))
1284 {
1285 open.text.push_str(text);
1286 }
1287 }
1288 StreamEvent::Reasoning { part, text } => {
1289 if let AssistantContent::Reasoning(open) = self
1290 .open_text
1291 .entry(part.index())
1292 .or_insert_with(|| AssistantContent::reasoning(""))
1293 {
1294 open.text.push_str(text);
1295 }
1296 }
1297 StreamEvent::Arguments { .. } => {}
1298 }
1299 Ok(())
1300 }
1301
1302 fn finish(mut self, end: Finish, reply: Reply) -> Result<CompletionResponse, ProviderError> {
1303 if let Some(FinishReason::Other(reason)) = &end.reason {
1304 tracing::warn!(
1305 provider = %self.origin.provider,
1306 reason = %reason,
1307 accepted = self.accept_unknown_finish,
1308 "the provider ended the reply with an unknown finish reason"
1309 );
1310 }
1311 let parts = std::mem::take(&mut self.choice).into_iter().enumerate();
1315 let choice = self.ordered(parts.collect(), self.first_incomplete);
1316 let response = self.response(end, reply, choice);
1317 self.span
1318 .record_response(response.response_id(), response.model(), &response.usage);
1319 Ok(response)
1320 }
1321}
1322
1323impl<'id> Out<'id, Completion> {
1324 pub(crate) fn catalog_pricing(
1329 &self,
1330 reported: Option<&str>,
1331 ) -> Option<crate::catalog::Pricing> {
1332 let shared = self.lock();
1333 let mut origin = shared.fold.origin.clone();
1334 named_by_reply(&mut origin, reported);
1335 shared.fold.pricing(&origin)
1336 }
1337
1338 pub fn open(
1343 &mut self,
1344 index: usize,
1345 block: Block,
1346 item: serde_json::Value,
1347 ) -> Result<(), ProviderError> {
1348 let mut shared = self.lock();
1349 let Shared { fold, items, .. } = &mut *shared;
1350 fold.open_item(items, index, block, item)
1351 }
1352
1353 pub fn push(&mut self, index: usize, fragment: &str) -> Result<(), ProviderError> {
1356 let mut shared = self.lock();
1357 let Shared { fold, items, .. } = &mut *shared;
1358 fold.push_item(items, index, fragment)
1359 }
1360
1361 pub fn edit(
1367 &mut self,
1368 index: usize,
1369 edit: impl FnOnce(&mut serde_json::Value),
1370 ) -> Result<(), ProviderError> {
1371 let mut shared = self.lock();
1372 let Shared { fold, items, .. } = &mut *shared;
1373 let held = fold.held.iter_mut().rev().find(|(at, _)| *at == index);
1375 match fold.open.get_mut(&index).or(held.map(|(_, draft)| draft)) {
1376 Some(draft) => {
1377 edit(&mut draft.item);
1378 Ok(())
1379 }
1380 None => fold.edit_ended(items, index, edit),
1381 }
1382 }
1383
1384 pub fn close(&mut self, index: usize) -> Result<(), ProviderError> {
1392 let mut shared = self.lock();
1393 let Shared { fold, items, .. } = &mut *shared;
1394 fold.close_item(items, index, Closing::Incomplete)
1395 }
1396
1397 pub fn finish(&mut self, index: usize) -> Result<(), ProviderError> {
1400 let mut shared = self.lock();
1401 let Shared { fold, items, .. } = &mut *shared;
1402 fold.close_item(items, index, Closing::Complete)
1403 }
1404
1405 pub fn finish_open(&mut self) -> Result<(), ProviderError> {
1408 let mut shared = self.lock();
1409 let Shared { fold, items, .. } = &mut *shared;
1410 let mut open: Vec<(Part, usize)> = fold
1411 .open
1412 .iter()
1413 .map(|(index, draft)| (draft.part, *index))
1414 .collect();
1415 open.sort();
1416 for (_, index) in open {
1417 fold.close_item(items, index, Closing::Complete)?;
1418 }
1419 Ok(())
1420 }
1421
1422 pub fn whole(
1426 &mut self,
1427 index: usize,
1428 block: Block,
1429 item: serde_json::Value,
1430 text: &str,
1431 ) -> Result<(), ProviderError> {
1432 self.open(index, block, item)?;
1433 self.push(index, text)?;
1434 self.finish(index)
1435 }
1436
1437 pub fn order_by_index(&mut self) {
1442 let mut shared = self.lock();
1443 if shared.fold.by_index.is_none() {
1444 shared.fold.by_index = Some(BTreeMap::new());
1445 }
1446 }
1447
1448 pub fn lead(&mut self, index: usize) -> Result<(), ProviderError> {
1453 let mut shared = self.lock();
1454 let position = shared.fold.draft(index)?.part.index();
1455 shared.fold.lead.get_or_insert(position);
1456 Ok(())
1457 }
1458
1459 pub fn restate(&mut self, index: usize, text: &str) -> Result<(), ProviderError> {
1464 let mut shared = self.lock();
1465 let draft = shared.fold.draft(index)?;
1466 match &mut draft.body {
1467 Body::Text(body) | Body::Reasoning { text: body, .. } => text.clone_into(body),
1468 Body::Call { arguments, .. } => {
1469 text.clone_into(&mut arguments.text);
1470 arguments.streamed = if draft.started { text.len() } else { 0 };
1472 }
1473 Body::Image(_) | Body::Opaque { .. } => {}
1474 }
1475 Ok(())
1476 }
1477
1478 pub fn fragment(
1491 &mut self,
1492 index: Option<usize>,
1493 fragment: CallFragment<'_>,
1494 ) -> Result<(), ProviderError> {
1495 let mut shared = self.lock();
1496 let Shared {
1497 fold: turn, items, ..
1498 } = &mut *shared;
1499 let new_id = fragment.id.filter(|id| stated(id));
1500 let index = match index {
1501 Some(index) => {
1502 let names = fragment.name.is_some_and(|name| !name.is_empty());
1503 let held = turn.open.get(&index).and_then(|draft| match &draft.body {
1504 Body::Call {
1505 id: Some(id),
1506 arguments,
1507 ..
1508 } if new_id.is_some_and(|new| id.wire() != new) => Some(arguments.complete()),
1509 _ => None,
1510 });
1511 if let Some(complete) = held.filter(|complete| *complete || names) {
1512 let moved = turn.fresh_index();
1514 if let Some(draft) = turn.open.remove(&index) {
1515 turn.open.insert(moved, draft);
1516 }
1517 let closing = if complete {
1518 Closing::Complete
1519 } else {
1520 Closing::Incomplete
1521 };
1522 turn.close_item(items, moved, closing)?;
1523 }
1524 index
1525 }
1526 None => {
1527 let owner = new_id.and_then(|new| {
1528 turn.open
1529 .iter()
1530 .find_map(|(index, draft)| match &draft.body {
1531 Body::Call { id: Some(id), .. } if id.wire() == new => Some(*index),
1532 _ => None,
1533 })
1534 });
1535 let continues = |last: &usize| {
1536 turn.open.get(last).is_some_and(|draft| {
1537 matches!(&draft.body, Body::Call { arguments, .. } if !arguments.complete())
1538 })
1539 };
1540 match (owner, new_id, turn.last_call) {
1541 (Some(index), _, _) => index,
1542 (None, None, Some(last)) if continues(&last) => last,
1543 _ => turn.fresh_index(),
1544 }
1545 }
1546 };
1547 turn.last_call = Some(index);
1548 if !turn.open.contains_key(&index) {
1549 let body = Body::Call {
1550 id: None,
1551 name: String::new(),
1552 arguments: Arguments::default(),
1553 };
1554 turn.insert(index, body, serde_json::Value::Null);
1555 }
1556 let Body::Call {
1557 id,
1558 name,
1559 arguments,
1560 } = &mut turn.draft(index)?.body
1561 else {
1562 return Err(ProviderError::Response(format!(
1563 "the reply sent a call fragment for item {index}, which is not a call"
1564 )));
1565 };
1566 if let Some(call_id) = fragment.id.filter(|id| stated(id)) {
1567 *id = Some(CallId::from_wire(call_id));
1568 }
1569 if name.is_empty()
1572 && let Some(fragment) = fragment.name.filter(|name| !name.is_empty())
1573 {
1574 fragment.clone_into(name);
1575 }
1576 if let Some(fragment) = fragment.arguments {
1577 arguments.push(fragment, name);
1578 }
1579 stream_call(items, turn.draft(index)?);
1580 Ok(())
1581 }
1582
1583 pub fn announce(
1586 &mut self,
1587 index: usize,
1588 announced: serde_json::Value,
1589 ) -> Result<(), ProviderError> {
1590 if let Body::Call { arguments, .. } = &mut self.lock().fold.draft(index)?.body {
1591 arguments.announced = Some(announced);
1592 }
1593 Ok(())
1594 }
1595
1596 pub fn fresh_index(&mut self) -> usize {
1598 self.lock().fold.fresh_index()
1599 }
1600
1601 pub fn run(&mut self, block: Block, fragment: &str) -> Result<usize, ProviderError> {
1606 let mut shared = self.lock();
1607 let Shared { fold, items, .. } = &mut *shared;
1608 fold.run_item(items, block, fragment)
1609 }
1610
1611 pub fn end_run(&mut self) -> Result<(), ProviderError> {
1614 let mut shared = self.lock();
1615 let Shared { fold, items, .. } = &mut *shared;
1616 fold.end_run(items)
1617 }
1618
1619 pub fn cite(&mut self, index: usize, citation: WireCitation) {
1626 let mut shared = self.lock();
1627 let Shared { fold, items, .. } = &mut *shared;
1628 fold.cite_item(items, index, vec![citation], false);
1629 }
1630
1631 pub fn set_citations(&mut self, index: usize, citations: Vec<WireCitation>) {
1634 let mut shared = self.lock();
1635 let Shared { fold, items, .. } = &mut *shared;
1636 fold.cite_item(items, index, citations, true);
1637 }
1638
1639 #[cfg(any(test, feature = "test-utils"))]
1642 pub(crate) fn content(&mut self, content: AssistantContent) -> Result<(), ProviderError> {
1643 let mut shared = self.lock();
1644 let Shared { fold, items, .. } = &mut *shared;
1645 fold.write_content(items, content)
1646 }
1647}
1648
1649pub(crate) fn events_of(response: &CompletionResponse) -> Vec<Item<StreamEvent>> {
1652 let parts = (0..).zip(&response.choice);
1653 let events = parts.flat_map(|(at, content)| StreamEvent::whole(Part::new(at), content));
1654 events.map(Item::Event).collect()
1655}
1656
1657#[cfg(test)]
1658mod tests;