Skip to main content

rig_core/providers/cohere/
streaming.rs

1//! The native chat reply decoder, for a whole reply and for the event
2//! stream alike. Each content part, the tool plan and each tool call
3//! becomes one block in the order it arrives. A block's provider item is
4//! the part as Cohere stated it, with the citations that point at it, so
5//! citations and the tool plan round-trip through history.
6//!
7//! ```
8//! use rig_core::providers::cohere::streaming::ChatDecoder;
9//!
10//! let decoder = ChatDecoder::default();
11//! # let _ = decoder;
12//! ```
13
14use std::collections::BTreeMap;
15
16use serde_json::{Map, Value, json};
17
18use super::chat::PLAN;
19use crate::completion::{FinishReason, Usage, token_sum};
20use crate::error::ProviderError;
21use crate::json_utils::Lenient;
22use crate::message::{CallId, Source, SourceLocation, ToolName};
23use crate::operation::{Block, CallFragment, Completion, Finish};
24use crate::providers::internal::wire;
25use crate::wire::{Decoder, Flow, Out, SpanUnit, WireCitation, WireEvent, WireFrame, WireSpan};
26
27/// The stream's event tags; any other tag classifies as unknown.
28const KNOWN_EVENT_TYPES: &[&str] = &[
29    "message-start",
30    "content-start",
31    "content-delta",
32    "content-end",
33    "tool-plan-delta",
34    "tool-call-start",
35    "tool-call-delta",
36    "tool-call-end",
37    "citation-start",
38    "citation-end",
39    "message-end",
40    "debug",
41];
42
43/// The wire index of the tool plan. Content parts keep their own indices,
44/// and calls theirs from [`CALLS`] on.
45const PLAN_INDEX: usize = 1 << 21;
46
47/// The wire index of the first tool call. A content or call index Cohere
48/// states must stay below it.
49const CALLS: usize = 1 << 20;
50
51/// `index` as Cohere stated it for a content part or call, failing when it
52/// would reach the indices the decoder keeps for calls and the plan.
53fn checked(index: usize) -> Result<usize, ProviderError> {
54    if index < CALLS {
55        Ok(index)
56    } else {
57        Err(ProviderError::Response(format!(
58            "Cohere stated index {index}, past the {CALLS} parts or calls a reply may hold"
59        )))
60    }
61}
62
63/// One stream event, or the whole reply a unary call answers (`kind` is
64/// then empty), as Cohere sent it. Cohere's error body is kept as its text,
65/// a JSON string.
66#[derive(Debug, Clone, PartialEq)]
67pub struct ChatEvent {
68    /// The event as sent.
69    pub fields: Value,
70}
71
72impl ChatEvent {
73    fn kind(&self) -> &str {
74        self.fields.str("type").unwrap_or_default()
75    }
76
77    /// The part or call the event addresses.
78    fn index(&self) -> Result<usize, ProviderError> {
79        self.fields
80            .u64("index")
81            .and_then(|index| usize::try_from(index).ok())
82            .ok_or_else(|| {
83                ProviderError::Response(format!("Cohere `{}` names no index", self.kind()))
84            })
85            .and_then(checked)
86    }
87
88    /// The object at `pointer` under the event's `delta.message`.
89    fn message(&self, key: &str) -> Option<&Map<String, Value>> {
90        self.fields
91            .get("delta")?
92            .get("message")?
93            .get(key)?
94            .as_object()
95    }
96}
97
98/// What an open block is.
99#[derive(Debug, Clone, Copy, PartialEq, Eq)]
100enum Kind {
101    Text,
102    Thinking,
103    Plan,
104    Call,
105    Opaque,
106}
107
108/// Decodes native chat replies, a whole reply or a stream of events. An
109/// `*-end` event states its block complete, and `message-end` states every
110/// block still open complete, but a call whose arguments are not JSON.
111#[derive(Debug, Default)]
112pub struct ChatDecoder {
113    /// Each open block's kind and, for a call, the argument text streamed.
114    open: BTreeMap<usize, (Kind, String)>,
115    /// Every index a block opened at and its kind, so a citation finds its
116    /// block.
117    started: BTreeMap<usize, Kind>,
118    message_id: Option<String>,
119}
120
121impl ChatDecoder {
122    /// Open the content part at `index` as Cohere states it.
123    fn content(
124        &mut self,
125        index: usize,
126        part: Map<String, Value>,
127        out: &mut Out<'_, Completion>,
128    ) -> Result<(), ProviderError> {
129        let index = checked(index)?;
130        let part = Value::Object(part);
131        let (kind, block, key) = match part.str("type") {
132            Some("text") | None => (Kind::Text, Block::Text, "text"),
133            Some("thinking") => (
134                Kind::Thinking,
135                Block::Reasoning { redacted: false },
136                "thinking",
137            ),
138            // A part rig does not know goes back as it came.
139            Some(_) => (Kind::Opaque, Block::Opaque { replay: true }, ""),
140        };
141        let text = part.str(key).unwrap_or_default().to_owned();
142        self.open.insert(index, (kind, String::new()));
143        self.started.insert(index, kind);
144        out.open(index, block, part)?;
145        out.push(index, &text)
146    }
147
148    /// Grow the open part at `index`, and its item, by a delta's text.
149    fn grow(
150        &mut self,
151        index: usize,
152        delta: &Map<String, Value>,
153        out: &mut Out<'_, Completion>,
154    ) -> Result<(), ProviderError> {
155        let Some((kind, _)) = self.open.get(&index) else {
156            return Err(ProviderError::Response(format!(
157                "Cohere streamed content to part {index}, which is not open"
158            )));
159        };
160        let key = match kind {
161            Kind::Thinking => "thinking",
162            Kind::Plan => PLAN,
163            Kind::Text | Kind::Call => "text",
164            // An unknown part's deltas merge into its item.
165            Kind::Opaque => {
166                return out.edit(index, |item| {
167                    crate::operation::completion::merge(item, delta)
168                });
169            }
170        };
171        let Some(text) = delta.get(key).and_then(Value::as_str) else {
172            return Ok(());
173        };
174        out.push(index, text)?;
175        let delta = Map::from_iter([(key.to_owned(), Value::from(text))]);
176        out.edit(index, |item| {
177            crate::operation::completion::merge(item, &delta)
178        })
179    }
180
181    /// Append a fragment of the tool plan, opening it at its first.
182    fn plan(&mut self, text: &str, out: &mut Out<'_, Completion>) -> Result<(), ProviderError> {
183        if !self.started.contains_key(&PLAN_INDEX) {
184            self.open.insert(PLAN_INDEX, (Kind::Plan, String::new()));
185            self.started.insert(PLAN_INDEX, Kind::Plan);
186            let item = json!({"type": PLAN, PLAN: ""});
187            out.open(PLAN_INDEX, Block::Reasoning { redacted: false }, item)?;
188        }
189        let delta = Map::from_iter([(PLAN.to_owned(), Value::from(text))]);
190        self.grow(PLAN_INDEX, &delta, out)
191    }
192
193    /// Open the tool call at `index` from its stated id, name and the
194    /// arguments it opens with.
195    fn call(
196        &mut self,
197        index: usize,
198        call: Map<String, Value>,
199        out: &mut Out<'_, Completion>,
200    ) -> Result<(), ProviderError> {
201        // The plan is whole once the calls it plans begin.
202        if self.open.contains_key(&PLAN_INDEX) {
203            self.stop(PLAN_INDEX, out)?;
204        }
205        let index = CALLS + checked(index)?;
206        let item = Value::Object(call);
207        let id = item.str("id").unwrap_or_default().to_owned();
208        let arguments = item
209            .at("/function/arguments")
210            .and_then(Value::as_str)
211            .unwrap_or_default()
212            .to_owned();
213        self.open.insert(index, (Kind::Call, String::new()));
214        self.started.insert(index, Kind::Call);
215        match ToolName::new(
216            item.at("/function/name")
217                .and_then(Value::as_str)
218                .unwrap_or_default(),
219        ) {
220            Ok(name) => {
221                let id = CallId::from_wire(&id);
222                out.open(index, Block::Call { id, name }, item)?;
223            }
224            // A nameless call: the writer drops it with a warning.
225            Err(_) => out.fragment(
226                Some(index),
227                CallFragment {
228                    id: Some(&id),
229                    ..CallFragment::default()
230                },
231            )?,
232        }
233        self.arguments(index, &arguments, out)
234    }
235
236    /// Append argument text to the open call at `index`.
237    fn arguments(
238        &mut self,
239        index: usize,
240        fragment: &str,
241        out: &mut Out<'_, Completion>,
242    ) -> Result<(), ProviderError> {
243        let Some((Kind::Call, json)) = self.open.get_mut(&index) else {
244            return Err(ProviderError::Response(format!(
245                "Cohere streamed arguments to call {}, which is not open",
246                index.saturating_sub(CALLS)
247            )));
248        };
249        json.push_str(fragment);
250        out.push(index, fragment)
251    }
252
253    /// Attach `citation` to the item of the block it cites: the tool plan,
254    /// or the content part at its `content_index` (the first by default).
255    /// A citation of a text part is also cited on its block. A citation of
256    /// a block that never opened has nowhere to go, and one whose
257    /// `content_index` reaches [`CALLS`] fails the reply.
258    fn cite(&self, citation: Value, out: &mut Out<'_, Completion>) -> Result<(), ProviderError> {
259        let index = if citation.str("type") == Some("PLAN") {
260            PLAN_INDEX
261        } else {
262            checked(
263                citation
264                    .u64("content_index")
265                    .map_or(Ok(0), usize::try_from)
266                    .unwrap_or(usize::MAX),
267            )?
268        };
269        let Some(kind) = self.started.get(&index) else {
270            tracing::warn!(
271                index,
272                "Cohere cited a block the reply never opened; dropping it"
273            );
274            return Ok(());
275        };
276        if *kind == Kind::Text
277            && let Some(cited) = citation_of(&citation)
278        {
279            out.cite(index, cited);
280        }
281        out.edit(index, |item| {
282            if let Some(item) = item.as_object_mut() {
283                match item.get_mut("citations") {
284                    Some(Value::Array(citations)) => citations.push(citation),
285                    _ => {
286                        item.insert("citations".to_owned(), Value::Array(vec![citation]));
287                    }
288                }
289            }
290        })
291    }
292
293    /// End the block at `index` as stated complete. Its item becomes the
294    /// block's native unless it would replay nothing: blank text, or a call
295    /// whose arguments are not a JSON object.
296    fn stop(&mut self, index: usize, out: &mut Out<'_, Completion>) -> Result<(), ProviderError> {
297        let Some((kind, json)) = self.open.remove(&index) else {
298            return Err(ProviderError::Response(format!(
299                "Cohere ended block {index}, which is not open"
300            )));
301        };
302        out.edit(index, |item| {
303            let kept = match kind {
304                Kind::Text => !item.str("text").unwrap_or_default().trim().is_empty(),
305                Kind::Thinking | Kind::Plan | Kind::Opaque => true,
306                Kind::Call => {
307                    let arguments = if json.trim().is_empty() {
308                        "{}"
309                    } else {
310                        json.as_str()
311                    };
312                    let object = crate::json_utils::parse_tool_arguments(arguments)
313                        .is_ok_and(|parsed| parsed.is_object());
314                    if let Some(function) = item.get_mut("function").and_then(Value::as_object_mut)
315                    {
316                        function.insert("arguments".to_owned(), Value::from(arguments));
317                    }
318                    object
319                }
320            };
321            if !kept {
322                *item = Value::Null;
323            }
324        })?;
325        out.finish(index)
326    }
327
328    /// End the reply: every block still open is complete, but a call whose
329    /// arguments are not JSON yet, which the end closes unfinished. Its
330    /// cost is the `billed_units` priced at the catalog, since the `tokens`
331    /// it reports as usage include tokens Cohere does not bill.
332    fn end(
333        &mut self,
334        usage_value: Option<&Value>,
335        reason: Option<&str>,
336        error: Option<&str>,
337        mut out: Out<'_, Completion>,
338    ) -> Result<Flow, ProviderError> {
339        let open: Vec<usize> = self
340            .open
341            .iter()
342            .filter(|(_, (kind, json))| {
343                *kind != Kind::Call
344                    || json.trim().is_empty()
345                    || crate::json_utils::parse_tool_arguments(json).is_ok_and(|v| v.is_object())
346            })
347            .map(|(index, _)| *index)
348            .collect();
349        for index in open {
350            self.stop(index, &mut out)?;
351        }
352        let error = error
353            .filter(|error| !error.is_empty())
354            .map(str::to_owned)
355            .or_else(|| {
356                (reason == Some("ERROR")).then(|| "Cohere ended the reply with an error".to_owned())
357            });
358        let mut usage = usage_of(usage_value)?;
359        if let Some(billed) = billed_of(usage_value) {
360            usage.cost = out
361                .catalog_pricing(None)
362                .and_then(|pricing| pricing.cost(&billed));
363        }
364        Ok(out.end(Finish {
365            usage,
366            reason: reason.map(finish_of),
367            response_id: self.message_id.clone(),
368            model: None,
369            error,
370        }))
371    }
372
373    /// A whole reply, written block by block through the calls a stream
374    /// makes, then ended.
375    fn whole(
376        &mut self,
377        reply: &Value,
378        mut out: Out<'_, Completion>,
379    ) -> Result<Flow, ProviderError> {
380        self.message_id = reply.str("id").map(str::to_owned);
381        // Cohere's error body, sent with a success status, as `classify` kept it.
382        if let Value::String(body) = reply {
383            return Err(ProviderError::from_provider_body(body.clone()));
384        }
385        let message = reply.get("message").unwrap_or(&Value::Null);
386        // The plan leads, as it streams first.
387        if let Some(plan) = message.str(PLAN).filter(|plan| !plan.is_empty()) {
388            self.plan(plan, &mut out)?;
389            self.stop(PLAN_INDEX, &mut out)?;
390        }
391        // A part or call that is not an object states nothing to keep.
392        for (index, part) in message.arr("content").iter().enumerate() {
393            if let Some(part) = part.as_object() {
394                self.content(index, part.clone(), &mut out)?;
395                self.stop(index, &mut out)?;
396            }
397        }
398        for (index, call) in message.arr("tool_calls").iter().enumerate() {
399            if let Some(call) = call.as_object() {
400                self.call(index, call.clone(), &mut out)?;
401                self.stop(CALLS + index, &mut out)?;
402            }
403        }
404        for citation in message.arr("citations") {
405            self.cite(citation.clone(), &mut out)?;
406        }
407        self.end(reply.get("usage"), reply.str("finish_reason"), None, out)
408    }
409}
410
411/// How a native `finish_reason` ends the turn. `ERROR`, `TIMEOUT` and any
412/// reason Cohere does not document are [`FinishReason::Other`], which fails
413/// the turn.
414fn finish_of(reason: &str) -> FinishReason {
415    match reason {
416        "COMPLETE" | "STOP_SEQUENCE" => FinishReason::Stop,
417        "MAX_TOKENS" => FinishReason::Length,
418        "TOOL_CALL" => FinishReason::ToolCalls,
419        other => FinishReason::Other(other.to_owned()),
420    }
421}
422
423/// One native citation as a [`WireCitation`]: a span of its content part
424/// in characters, checked against the `text` Cohere quotes, cited by each
425/// document or tool output it names. A citation without both offsets is
426/// `None` and stays only in the item.
427fn citation_of(citation: &Value) -> Option<WireCitation> {
428    let mut span = WireSpan::new(
429        citation.u64("start")?,
430        citation.u64("end")?,
431        SpanUnit::Chars,
432    );
433    if let Some(text) = citation.str("text") {
434        span = span.quoted(text);
435    }
436    let sources = citation
437        .arr("sources")
438        .iter()
439        .filter_map(|source| {
440            let id = source.str("id")?.to_owned();
441            match source.str("type") {
442                Some("document") => {
443                    let cited = Source::new(SourceLocation::Document {
444                        index: None,
445                        id: Some(id),
446                        within: None,
447                    });
448                    Some(match source.at("/document/title").and_then(Value::as_str) {
449                        Some(title) => cited.title(title),
450                        None => cited,
451                    })
452                }
453                Some("tool") => Some(Source::new(SourceLocation::ToolOutput { id })),
454                _ => None,
455            }
456        })
457        .collect();
458    Some(WireCitation::new(Some(span), sources))
459}
460
461/// The `billed_units` Cohere charges for, as usage to price: input and
462/// output, which leave out the tokens Cohere adds and does not bill.
463/// `None` unless both are reported.
464fn billed_of(usage: Option<&Value>) -> Option<Usage> {
465    let count = |pointer: &str| usage?.at(pointer)?.as_u64_lenient();
466    let input = count("/billed_units/input_tokens")?;
467    let output = count("/billed_units/output_tokens")?;
468    Some(Usage::new().input_tokens(input).output_tokens(output))
469}
470
471/// Rig's usage from Cohere's: the `tokens` the model read and wrote,
472/// else the `billed_units`, with `cached_tokens` among the input; an error
473/// when their sum overflows.
474fn usage_of(usage: Option<&Value>) -> Result<Usage, ProviderError> {
475    let count = |pointer: &str| usage?.at(pointer)?.as_u64_lenient();
476    let input = count("/tokens/input_tokens").or_else(|| count("/billed_units/input_tokens"));
477    let output = count("/tokens/output_tokens").or_else(|| count("/billed_units/output_tokens"));
478    let total = input
479        .zip(output)
480        .map(|(input, output)| token_sum([input, output]))
481        .transpose()?;
482    Ok(Usage {
483        input_tokens: input,
484        output_tokens: output,
485        cached_input_tokens: count("/cached_tokens"),
486        cache_creation_input_tokens: None,
487        reasoning_tokens: count("/tokens/reasoning_tokens"),
488        total_tokens: total,
489        tool_use_prompt_tokens: None,
490        ..Usage::default()
491    })
492}
493
494impl<'id> Decoder<'id, Completion> for ChatDecoder {
495    type Event = ChatEvent;
496
497    /// A stream event by its `type`; a frame without one is the whole
498    /// reply, recognized by its `message`.
499    fn classify(&self, frame: WireFrame) -> WireEvent<ChatEvent> {
500        let data = frame.as_str();
501        wire::classify_or_untagged(
502            &data,
503            "type",
504            |tag| KNOWN_EVENT_TYPES.contains(&tag),
505            |fields| ChatEvent { fields },
506            |data| {
507                wire::classify_marker_keyed_frame(data, &["message"])
508                    .map(|fields| ChatEvent { fields })
509            },
510        )
511        // An untagged frame with a `message` string is Cohere's error body,
512        // reported as sent.
513        .map(|event| match event.fields.get("message") {
514            Some(Value::String(_)) if event.kind().is_empty() => ChatEvent {
515                fields: Value::String(data.into_owned()),
516            },
517            _ => event,
518        })
519    }
520
521    fn decode(
522        &mut self,
523        event: ChatEvent,
524        mut out: Out<'id, Completion>,
525    ) -> Result<Flow, ProviderError> {
526        match event.kind() {
527            "" => return self.whole(&event.fields, out),
528            "message-start" => self.message_id = event.fields.str("id").map(str::to_owned),
529            "content-start" => {
530                let part = event.message("content").cloned().unwrap_or_default();
531                self.content(event.index()?, part, &mut out)?;
532            }
533            "content-delta" => {
534                let empty = Map::new();
535                let delta = event.message("content").unwrap_or(&empty);
536                self.grow(event.index()?, delta, &mut out)?;
537            }
538            "content-end" => self.stop(event.index()?, &mut out)?,
539            "tool-plan-delta" => {
540                if let Some(text) = event
541                    .fields
542                    .at("/delta/message/tool_plan")
543                    .and_then(Value::as_str)
544                {
545                    self.plan(text, &mut out)?;
546                }
547            }
548            "tool-call-start" => {
549                let call = event.message("tool_calls").cloned().unwrap_or_default();
550                self.call(event.index()?, call, &mut out)?;
551            }
552            "tool-call-delta" => {
553                let fragment = event
554                    .fields
555                    .at("/delta/message/tool_calls/function/arguments")
556                    .and_then(Value::as_str)
557                    .unwrap_or_default();
558                self.arguments(CALLS + event.index()?, fragment, &mut out)?;
559            }
560            "tool-call-end" => self.stop(CALLS + event.index()?, &mut out)?,
561            "citation-start" => {
562                if let Some(citation) = event.message("citations") {
563                    self.cite(Value::Object(citation.clone()), &mut out)?;
564                }
565            }
566            "message-end" => {
567                let delta = event.fields.get("delta");
568                return self.end(
569                    delta.and_then(|delta| delta.get("usage")),
570                    delta.and_then(|delta| delta.str("finish_reason")),
571                    delta.and_then(|delta| delta.str("error")),
572                    out,
573                );
574            }
575            // `citation-end` and `debug`: the classifier passes only the
576            // listed tags.
577            _ => {}
578        }
579        Ok(Flow::More)
580    }
581}
582
583pub(crate) mod document;
584
585#[cfg(test)]
586mod tests;