Skip to main content

rig_core/providers/ollama/
streaming.rs

1//! The decoder of native Ollama chat replies: a whole `/api/chat` body, or a
2//! stream of NDJSON records of the same shape, the last one `done`.
3//!
4//! A record's `thinking` is reasoning, its `content` text and its
5//! `tool_calls` whole calls. A reply with no `thinking` may carry its
6//! reasoning as a leading `<think>` block in `content`; that block is split
7//! out as reasoning (see [`ChatDecoder`]).
8//!
9//! ```
10//! use rig_core::providers::ollama::streaming::ChatDecoder;
11//!
12//! let decoder = ChatDecoder::default();
13//! # let _ = decoder;
14//! ```
15
16use serde::Deserialize;
17use serde_json::{Map, Value};
18
19use crate::completion::{FinishReason, Usage, token_sum};
20use crate::error::ProviderError;
21use crate::json_utils::Lenient;
22use crate::message::{CallId, ToolName};
23use crate::operation::{Block, Completion, Finish};
24use crate::providers::internal::wire;
25use crate::wire::{Decoder, Flow, Out, WireEvent, WireFrame};
26
27/// The keys that make a JSON line a chat record: every record carries
28/// `message` or `done`, and an in-band failure carries `error`.
29const RECORD_KEYS: &[&str] = &["message", "done", "error"];
30
31/// The tag that opens an inline reasoning block.
32const THINK_OPEN: &str = "<think>";
33/// The tag that closes it.
34const THINK_CLOSE: &str = "</think>";
35
36/// One `/api/chat` record, as the daemon sent it.
37#[derive(Debug, Default, Deserialize)]
38#[serde(transparent)]
39pub struct ChatRecord(pub Map<String, Value>);
40
41/// Decode native Ollama chat replies.
42///
43/// `thinking` and `content` each continue the block of the record before
44/// while the kind stays the same, and each call is a whole block. The
45/// reply ends at the record that says `done`; its `done_reason` is the
46/// finish reason, `tool_calls` when a `stop` reply called a tool, and its
47/// `prompt_eval_count`, `prompt_eval_cached_count` (the part of the prompt
48/// read from the daemon's cache) and `eval_count` are the usage.
49///
50/// A reply with no `thinking` whose `content`, after leading whitespace,
51/// starts with `<think>` has that block split out as reasoning, by its tags
52/// alone. Content is held only while its start could still become
53/// `<think>`; content that cannot is text at once. Inside the block,
54/// reasoning streams as it arrives, holding back only trailing whitespace
55/// and a partial `</think>`. After `</think>` the rest is text. A block
56/// that never closes is reasoning to the end of the reply.
57#[derive(Debug, Default)]
58pub struct ChatDecoder {
59    /// Where the inline split stands.
60    split: Split,
61    /// Whether the reply called a tool.
62    called: bool,
63    /// The model the records name.
64    model: Option<String>,
65}
66
67/// Where a reply's inline reasoning split stands.
68#[derive(Debug)]
69enum Split {
70    /// Content held while its start, after whitespace, could still become
71    /// `<think>`.
72    Opening(String),
73    /// Inside a leading `<think>` block. `held` is trailing whitespace and a
74    /// partial `</think>` not yet written; `started` is whether any
75    /// reasoning was, so leading whitespace is dropped until then.
76    Inside { held: String, started: bool },
77    /// Content is text. `trim` drops leading whitespace until visible text
78    /// arrives after a split block.
79    Text { trim: bool },
80}
81
82impl Default for Split {
83    fn default() -> Self {
84        Self::Opening(String::new())
85    }
86}
87
88/// The length of the longest proper prefix of `tag` that `text` ends with.
89/// The tags are ASCII, so the cut is a character boundary.
90fn partial_suffix(text: &str, tag: &str) -> usize {
91    (1..tag.len())
92        .rev()
93        .find(|&len| text.ends_with(&tag[..len]))
94        .unwrap_or(0)
95}
96
97impl<'id> Decoder<'id, Completion> for ChatDecoder {
98    type Event = ChatRecord;
99
100    fn classify(&self, frame: WireFrame) -> WireEvent<ChatRecord> {
101        wire::classify_marker_keyed_frame(&frame.as_str(), RECORD_KEYS)
102    }
103
104    /// Fields are read leniently: one the record leaves out or sends with
105    /// another type is absent, so only `error` and `done` steer the reply.
106    fn decode(
107        &mut self,
108        ChatRecord(fields): ChatRecord,
109        mut out: Out<'id, Completion>,
110    ) -> Result<Flow, ProviderError> {
111        let record = Value::Object(fields);
112        if let Some(error) = record.get("error").filter(|error| !error.is_null()) {
113            let body = serde_json::json!({ "error": error }).to_string();
114            return Err(ProviderError::from_provider_body(body));
115        }
116        if let Some(model) = record.str("model").filter(|model| !model.is_empty())
117            && self.model.as_deref() != Some(model)
118        {
119            tracing::Span::current().record("gen_ai.response.model", model);
120            self.model = Some(model.to_owned());
121        }
122        let message = record.get("message").unwrap_or(&Value::Null);
123        if let Some(thinking) = message
124            .str("thinking")
125            .filter(|thinking| !thinking.is_empty())
126        {
127            // A reply with native thinking splits nothing out of its content.
128            self.release(&mut out)?;
129            reason(thinking, &mut out)?;
130        }
131        if let Some(content) = message.str("content").filter(|content| !content.is_empty()) {
132            self.content(content, &mut out)?;
133        }
134        for call in message.arr("tool_calls") {
135            self.call(call, &mut out)?;
136        }
137        if record.bool("done") != Some(true) {
138            return Ok(Flow::More);
139        }
140        self.release(&mut out)?;
141        out.end_run()?;
142        let reason = record.str("done_reason").map(|reason| match reason {
143            "stop" if self.called => FinishReason::ToolCalls,
144            "stop" => FinishReason::Stop,
145            "length" => FinishReason::Length,
146            other => FinishReason::Other(other.to_owned()),
147        });
148        let (input, output) = (record.u64("prompt_eval_count"), record.u64("eval_count"));
149        let usage = Usage {
150            input_tokens: input,
151            output_tokens: output,
152            total_tokens: input
153                .zip(output)
154                .map(|(input, output)| token_sum([input, output]))
155                .transpose()?,
156            cached_input_tokens: record.u64("prompt_eval_cached_count"),
157            ..Usage::default()
158        };
159        Ok(out.end(Finish {
160            usage,
161            reason,
162            model: self.model.take(),
163            ..Finish::default()
164        }))
165    }
166}
167
168impl ChatDecoder {
169    /// Write a fragment of `content`, through the inline split.
170    fn content(
171        &mut self,
172        content: &str,
173        out: &mut Out<'_, Completion>,
174    ) -> Result<(), ProviderError> {
175        match std::mem::replace(&mut self.split, Split::Text { trim: false }) {
176            Split::Text { trim } => {
177                let text = if trim { content.trim_start() } else { content };
178                self.split = Split::Text {
179                    trim: trim && text.is_empty(),
180                };
181                if !text.is_empty() {
182                    out.run(Block::Text, text)?;
183                }
184                Ok(())
185            }
186            Split::Opening(mut held) => {
187                held.push_str(content);
188                let trimmed = held.trim_start();
189                if let Some(rest) = trimmed.strip_prefix(THINK_OPEN) {
190                    let rest = rest.to_owned();
191                    self.split = Split::Inside {
192                        held: String::new(),
193                        started: false,
194                    };
195                    self.content(&rest, out)
196                } else if THINK_OPEN.starts_with(trimmed) {
197                    self.split = Split::Opening(held);
198                    Ok(())
199                } else {
200                    out.run(Block::Text, &held)?;
201                    Ok(())
202                }
203            }
204            Split::Inside {
205                mut held,
206                mut started,
207            } => {
208                held.push_str(content);
209                if let Some((reasoning, rest)) = held.split_once(THINK_CLOSE) {
210                    write_reasoning(reasoning.trim_end(), started, out)?;
211                    self.split = Split::Text { trim: true };
212                    return self.content(rest, out);
213                }
214                let cut = held.len() - partial_suffix(&held, THINK_CLOSE);
215                let cut = held[..cut].trim_end().len();
216                started |= write_reasoning(&held[..cut], started, out)?;
217                self.split = Split::Inside {
218                    held: held.split_off(cut),
219                    started,
220                };
221                Ok(())
222            }
223        }
224    }
225
226    /// End the split: content held at the start is text as it arrived, and
227    /// what an open block held is reasoning.
228    fn release(&mut self, out: &mut Out<'_, Completion>) -> Result<(), ProviderError> {
229        match std::mem::replace(&mut self.split, Split::Text { trim: false }) {
230            Split::Opening(held) if !held.is_empty() => {
231                out.run(Block::Text, &held)?;
232            }
233            Split::Inside { held, started } => {
234                write_reasoning(held.trim_end(), started, out)?;
235            }
236            Split::Opening(_) | Split::Text { .. } => {}
237        }
238        Ok(())
239    }
240
241    /// Write one whole call. A call without a name is dropped: nothing can
242    /// answer it.
243    fn call(&mut self, call: &Value, out: &mut Out<'_, Completion>) -> Result<(), ProviderError> {
244        let Ok(name) = ToolName::new(
245            call.at("/function/name")
246                .and_then(Value::as_str)
247                .unwrap_or_default(),
248        ) else {
249            tracing::warn!("Ollama sent a tool call without a name; nothing can answer it");
250            return Ok(());
251        };
252        let arguments = match call.at("/function/arguments") {
253            None | Some(Value::Null) => "{}".to_owned(),
254            Some(Value::String(arguments)) => arguments.clone(),
255            Some(arguments) => arguments.to_string(),
256        };
257        self.release(out)?;
258        out.end_run()?;
259        self.called = true;
260        let id = CallId::from_wire(call.str("id").unwrap_or_default());
261        let index = out.fresh_index();
262        out.open(index, Block::Call { id, name }, call.clone())?;
263        out.push(index, &arguments)?;
264        out.finish(index)
265    }
266}
267
268/// Write a fragment of an inline block's reasoning, its leading whitespace
269/// dropped until the block has `started`. Returns whether it wrote any.
270fn write_reasoning(
271    text: &str,
272    started: bool,
273    out: &mut Out<'_, Completion>,
274) -> Result<bool, ProviderError> {
275    let text = if started { text } else { text.trim_start() };
276    if text.is_empty() {
277        return Ok(false);
278    }
279    reason(text, out)?;
280    Ok(true)
281}
282
283/// Write a fragment of reasoning. Its block's item is the `thinking` it
284/// joins, so a same-model replay sends it back.
285fn reason(text: &str, out: &mut Out<'_, Completion>) -> Result<(), ProviderError> {
286    let index = out.run(Block::Reasoning { redacted: false }, text)?;
287    out.edit(index, |item| match item {
288        Value::Object(fields) => {
289            if let Some(Value::String(thinking)) = fields.get_mut("thinking") {
290                thinking.push_str(text);
291            }
292        }
293        _ => *item = serde_json::json!({ "thinking": text }),
294    })
295}
296
297pub(crate) mod document;
298
299#[cfg(test)]
300mod tests;