rig_core/providers/ollama/
streaming.rs1use serde::Deserialize;
17use serde_json::{Map, Value};
18
19use crate::completion::{FinishReason, Usage, token_sum};
20use crate::error::ProviderError;
21use crate::json_utils::Lenient;
22use crate::message::{CallId, ToolName};
23use crate::operation::{Block, Completion, Finish};
24use crate::providers::internal::wire;
25use crate::wire::{Decoder, Flow, Out, WireEvent, WireFrame};
26
27const RECORD_KEYS: &[&str] = &["message", "done", "error"];
30
31const THINK_OPEN: &str = "<think>";
33const THINK_CLOSE: &str = "</think>";
35
36#[derive(Debug, Default, Deserialize)]
38#[serde(transparent)]
39pub struct ChatRecord(pub Map<String, Value>);
40
41#[derive(Debug, Default)]
58pub struct ChatDecoder {
59 split: Split,
61 called: bool,
63 model: Option<String>,
65}
66
67#[derive(Debug)]
69enum Split {
70 Opening(String),
73 Inside { held: String, started: bool },
77 Text { trim: bool },
80}
81
82impl Default for Split {
83 fn default() -> Self {
84 Self::Opening(String::new())
85 }
86}
87
88fn partial_suffix(text: &str, tag: &str) -> usize {
91 (1..tag.len())
92 .rev()
93 .find(|&len| text.ends_with(&tag[..len]))
94 .unwrap_or(0)
95}
96
97impl<'id> Decoder<'id, Completion> for ChatDecoder {
98 type Event = ChatRecord;
99
100 fn classify(&self, frame: WireFrame) -> WireEvent<ChatRecord> {
101 wire::classify_marker_keyed_frame(&frame.as_str(), RECORD_KEYS)
102 }
103
104 fn decode(
107 &mut self,
108 ChatRecord(fields): ChatRecord,
109 mut out: Out<'id, Completion>,
110 ) -> Result<Flow, ProviderError> {
111 let record = Value::Object(fields);
112 if let Some(error) = record.get("error").filter(|error| !error.is_null()) {
113 let body = serde_json::json!({ "error": error }).to_string();
114 return Err(ProviderError::from_provider_body(body));
115 }
116 if let Some(model) = record.str("model").filter(|model| !model.is_empty())
117 && self.model.as_deref() != Some(model)
118 {
119 tracing::Span::current().record("gen_ai.response.model", model);
120 self.model = Some(model.to_owned());
121 }
122 let message = record.get("message").unwrap_or(&Value::Null);
123 if let Some(thinking) = message
124 .str("thinking")
125 .filter(|thinking| !thinking.is_empty())
126 {
127 self.release(&mut out)?;
129 reason(thinking, &mut out)?;
130 }
131 if let Some(content) = message.str("content").filter(|content| !content.is_empty()) {
132 self.content(content, &mut out)?;
133 }
134 for call in message.arr("tool_calls") {
135 self.call(call, &mut out)?;
136 }
137 if record.bool("done") != Some(true) {
138 return Ok(Flow::More);
139 }
140 self.release(&mut out)?;
141 out.end_run()?;
142 let reason = record.str("done_reason").map(|reason| match reason {
143 "stop" if self.called => FinishReason::ToolCalls,
144 "stop" => FinishReason::Stop,
145 "length" => FinishReason::Length,
146 other => FinishReason::Other(other.to_owned()),
147 });
148 let (input, output) = (record.u64("prompt_eval_count"), record.u64("eval_count"));
149 let usage = Usage {
150 input_tokens: input,
151 output_tokens: output,
152 total_tokens: input
153 .zip(output)
154 .map(|(input, output)| token_sum([input, output]))
155 .transpose()?,
156 cached_input_tokens: record.u64("prompt_eval_cached_count"),
157 ..Usage::default()
158 };
159 Ok(out.end(Finish {
160 usage,
161 reason,
162 model: self.model.take(),
163 ..Finish::default()
164 }))
165 }
166}
167
168impl ChatDecoder {
169 fn content(
171 &mut self,
172 content: &str,
173 out: &mut Out<'_, Completion>,
174 ) -> Result<(), ProviderError> {
175 match std::mem::replace(&mut self.split, Split::Text { trim: false }) {
176 Split::Text { trim } => {
177 let text = if trim { content.trim_start() } else { content };
178 self.split = Split::Text {
179 trim: trim && text.is_empty(),
180 };
181 if !text.is_empty() {
182 out.run(Block::Text, text)?;
183 }
184 Ok(())
185 }
186 Split::Opening(mut held) => {
187 held.push_str(content);
188 let trimmed = held.trim_start();
189 if let Some(rest) = trimmed.strip_prefix(THINK_OPEN) {
190 let rest = rest.to_owned();
191 self.split = Split::Inside {
192 held: String::new(),
193 started: false,
194 };
195 self.content(&rest, out)
196 } else if THINK_OPEN.starts_with(trimmed) {
197 self.split = Split::Opening(held);
198 Ok(())
199 } else {
200 out.run(Block::Text, &held)?;
201 Ok(())
202 }
203 }
204 Split::Inside {
205 mut held,
206 mut started,
207 } => {
208 held.push_str(content);
209 if let Some((reasoning, rest)) = held.split_once(THINK_CLOSE) {
210 write_reasoning(reasoning.trim_end(), started, out)?;
211 self.split = Split::Text { trim: true };
212 return self.content(rest, out);
213 }
214 let cut = held.len() - partial_suffix(&held, THINK_CLOSE);
215 let cut = held[..cut].trim_end().len();
216 started |= write_reasoning(&held[..cut], started, out)?;
217 self.split = Split::Inside {
218 held: held.split_off(cut),
219 started,
220 };
221 Ok(())
222 }
223 }
224 }
225
226 fn release(&mut self, out: &mut Out<'_, Completion>) -> Result<(), ProviderError> {
229 match std::mem::replace(&mut self.split, Split::Text { trim: false }) {
230 Split::Opening(held) if !held.is_empty() => {
231 out.run(Block::Text, &held)?;
232 }
233 Split::Inside { held, started } => {
234 write_reasoning(held.trim_end(), started, out)?;
235 }
236 Split::Opening(_) | Split::Text { .. } => {}
237 }
238 Ok(())
239 }
240
241 fn call(&mut self, call: &Value, out: &mut Out<'_, Completion>) -> Result<(), ProviderError> {
244 let Ok(name) = ToolName::new(
245 call.at("/function/name")
246 .and_then(Value::as_str)
247 .unwrap_or_default(),
248 ) else {
249 tracing::warn!("Ollama sent a tool call without a name; nothing can answer it");
250 return Ok(());
251 };
252 let arguments = match call.at("/function/arguments") {
253 None | Some(Value::Null) => "{}".to_owned(),
254 Some(Value::String(arguments)) => arguments.clone(),
255 Some(arguments) => arguments.to_string(),
256 };
257 self.release(out)?;
258 out.end_run()?;
259 self.called = true;
260 let id = CallId::from_wire(call.str("id").unwrap_or_default());
261 let index = out.fresh_index();
262 out.open(index, Block::Call { id, name }, call.clone())?;
263 out.push(index, &arguments)?;
264 out.finish(index)
265 }
266}
267
268fn write_reasoning(
271 text: &str,
272 started: bool,
273 out: &mut Out<'_, Completion>,
274) -> Result<bool, ProviderError> {
275 let text = if started { text } else { text.trim_start() };
276 if text.is_empty() {
277 return Ok(false);
278 }
279 reason(text, out)?;
280 Ok(true)
281}
282
283fn reason(text: &str, out: &mut Out<'_, Completion>) -> Result<(), ProviderError> {
286 let index = out.run(Block::Reasoning { redacted: false }, text)?;
287 out.edit(index, |item| match item {
288 Value::Object(fields) => {
289 if let Some(Value::String(thinking)) = fields.get_mut("thinking") {
290 thinking.push_str(text);
291 }
292 }
293 _ => *item = serde_json::json!({ "thinking": text }),
294 })
295}
296
297pub(crate) mod document;
298
299#[cfg(test)]
300mod tests;