rig_core/providers/ollama/
chat.rs

1//! The native Ollama chat wire: `POST /api/chat`, a whole reply or a stream
2//! of NDJSON records, both read by [`ChatDecoder`].
3//!
4//! The request carries the daemon's own fields: `think` and `keep_alive` at
5//! the top level, model parameters such as `num_ctx` in `options`, and the
6//! output schema as `format`.
7//!
8//! ```
9//! use rig_core::providers::ollama::OllamaConfig;
10//!
11//! let wire = OllamaConfig::new().client().native_completion("qwen3:4b").wire;
12//! assert_eq!(wire.model, "qwen3:4b");
13//! ```
14
15use serde_json::{Map, Value, json};
16
17use crate::completion::options::{BaseInput, FinalBody, RawAt, request_params};
18use crate::completion::{
19    Accepts, CompletionRequest, Media, Place, ProviderCapabilities, Replay, ReplayTarget,
20};
21use crate::error::EncodeError;
22use crate::message::{
23    AssistantContent, AssistantMessage, DocumentSourceKind as Source, Message, ToolCall,
24    ToolResult, ToolResultContent, UserContent,
25};
26use crate::operation::Completion;
27use crate::providers::internal::wire_ids::WireIds;
28use crate::wire::{Capabilities, Descriptor, Encoded, Framing, Mode, Wire};
29
30use super::streaming::ChatDecoder;
31use super::{OllamaConfig, PROVIDER_NAME};
32
33/// The chat endpoint, relative to the daemon's address.
34const CHAT_PATH: &str = "/api/chat";
35
36/// The `additional_params` keys `/api/chat` reads at the top level of its
37/// request. Every other key is a model parameter and goes in `options`.
38const TOP_LEVEL: &[&str] = &[
39    "think",
40    "format",
41    "keep_alive",
42    "logprobs",
43    "top_logprobs",
44    "truncate",
45    "shift",
46];
47
48/// The native chat wire of an Ollama daemon.
49#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
50pub struct Chat {
51    /// The daemon's address and credential.
52    pub provider: OllamaConfig,
53    /// The model to address, e.g. [`QWEN3`](super::QWEN3).
54    pub model: String,
55}
56
57impl Chat {
58    /// The wire for `model` on `provider`.
59    pub fn new(provider: OllamaConfig, model: impl Into<String>) -> Self {
60        Self {
61            provider,
62            model: model.into(),
63        }
64    }
65
66    /// The `/api/chat` body `request` sends in `mode`: the wire's encoding,
67    /// then the mapped options, then `additional_params`, split by where the
68    /// daemon reads each key. `think` and the keys in [`TOP_LEVEL`] go at the
69    /// top level, `tools` join the request's tools, an `options` object
70    /// merges into `options`, and every other key is an `options` entry.
71    /// `temperature` and `max_tokens` (as `num_predict`) go in `options`,
72    /// where a caller's own entries win.
73    fn body(&self, request: &CompletionRequest, mode: Mode) -> Result<FinalBody, EncodeError> {
74        request_params(
75            self,
76            request,
77            |input| self.base(request, mode, input),
78            RawAt::Split {
79                top: TOP_LEVEL,
80                rest: "options",
81            },
82            &[],
83        )
84    }
85
86    /// The wire's own encoding of `request`.
87    fn base(
88        &self,
89        request: &CompletionRequest,
90        mode: Mode,
91        input: &mut BaseInput<'_>,
92    ) -> Result<Map<String, Value>, EncodeError> {
93        let model = request.model.clone().unwrap_or_else(|| self.model.clone());
94        if input
95            .param("keep_alive")
96            .is_some_and(|value| !(value.is_string() || value.is_number()))
97        {
98            return Err(EncodeError::request(
99                "Ollama `keep_alive` must be a duration string or a number of seconds",
100            ));
101        }
102        if input
103            .param("options")
104            .is_some_and(|value| !value.is_object())
105        {
106            return Err(EncodeError::request(
107                "Ollama `additional_params.options` must be an object",
108            ));
109        }
110        let messages = self.messages(&request.chat_history, &model)?;
111
112        let mut tools: Vec<Value> = request
113            .tools
114            .iter()
115            .map(|tool| {
116                json!({"type": "function", "function": {
117                    "name": tool.name,
118                    "description": tool.description,
119                    "parameters": tool.parameters,
120                }})
121            })
122            .collect();
123        tools.extend(input.raw_tools()?);
124        if request.tool_choice.is_some() {
125            tracing::warn!("Ollama has no tool choice; `tool_choice` is ignored");
126        }
127
128        let mut options = Map::new();
129        if let Some(temperature) = request.temperature {
130            options.insert("temperature".to_owned(), Value::from(temperature));
131        }
132        if let Some(max_tokens) = request.max_tokens {
133            options.insert("num_predict".to_owned(), Value::from(max_tokens));
134        }
135
136        // Defer the schema until a tool result exists: a constrained reply
137        // cannot call a tool.
138        let answered = messages
139            .iter()
140            .any(|message| message.get("role").and_then(Value::as_str) == Some("tool"));
141        let format = request
142            .output_schema
143            .clone()
144            .filter(|_| tools.is_empty() || answered)
145            .map(|schema| schema.to_value());
146
147        let fields = [
148            ("model", Some(Value::String(model))),
149            ("messages", Some(Value::Array(messages))),
150            ("tools", (!tools.is_empty()).then_some(Value::Array(tools))),
151            ("format", format),
152            (
153                "options",
154                (!options.is_empty()).then_some(Value::Object(options)),
155            ),
156            ("stream", Some(Value::Bool(mode == Mode::Streaming))),
157        ];
158        Ok(fields
159            .into_iter()
160            .filter_map(|(key, value)| Some((key.to_owned(), value?)))
161            .collect())
162    }
163
164    /// The history as `/api/chat` messages, each call and result spelled by
165    /// one [`WireIds`].
166    fn messages(&self, history: &[Message], model: &str) -> Result<Vec<Value>, EncodeError> {
167        let ids = WireIds::for_target(history, self, model);
168        let mut messages = Vec::new();
169        for message in history {
170            match message {
171                Message::System { content } => {
172                    messages.push(json!({"role": "system", "content": content}));
173                }
174                Message::User { content } => {
175                    let mut user = UserParts::default();
176                    for part in content {
177                        match part {
178                            UserContent::ToolResult(result) => {
179                                user.push_to(&mut messages);
180                                messages.push(tool_message(result, &ids)?);
181                            }
182                            part => user.add(part)?,
183                        }
184                    }
185                    user.push_to(&mut messages);
186                }
187                Message::Assistant(turn) => {
188                    messages.extend(self.assistant(turn, &ids));
189                }
190            }
191        }
192        if messages.is_empty() {
193            return Err(EncodeError::request(
194                "Ollama chat request has no messages after conversion",
195            ));
196        }
197        Ok(messages)
198    }
199
200    /// One assistant turn: its text joined as `content`, its reasoning as
201    /// `thinking`, and its calls. A turn with none of them is `None`.
202    fn assistant(&self, turn: &AssistantMessage, ids: &WireIds) -> Option<Value> {
203        let (mut text, mut thinking, mut calls) = (String::new(), Vec::new(), Vec::new());
204        for block in &turn.content {
205            match block {
206                AssistantContent::Text(block) => text.push_str(&block.text),
207                AssistantContent::Reasoning(block) if !block.text.is_empty() => {
208                    thinking.push(block.text.as_str());
209                }
210                AssistantContent::ToolCall(call) => {
211                    calls.push(call_item(call, block.replay(self, ids), ids));
212                }
213                AssistantContent::Reasoning(_)
214                | AssistantContent::Image(_)
215                | AssistantContent::Opaque(_) => {}
216            }
217        }
218        if text.is_empty() && thinking.is_empty() && calls.is_empty() {
219            return None;
220        }
221        let mut message = Map::from_iter([
222            ("role".to_owned(), Value::from("assistant")),
223            ("content".to_owned(), Value::String(text)),
224        ]);
225        if !thinking.is_empty() {
226            message.insert("thinking".to_owned(), Value::String(thinking.join("\n")));
227        }
228        if !calls.is_empty() {
229            message.insert("tool_calls".to_owned(), Value::Array(calls));
230        }
231        Some(Value::Object(message))
232    }
233}
234
235/// The text and images of one user message as they gather.
236#[derive(Default)]
237struct UserParts {
238    texts: Vec<String>,
239    images: Vec<String>,
240}
241
242impl UserParts {
243    /// Add a part: text, or an image as base64 data. The adapter has
244    /// replaced every other part ([`ReplayTarget::encodes`]).
245    fn add(&mut self, part: &UserContent) -> Result<(), EncodeError> {
246        match part {
247            UserContent::Text(text) => self.texts.push(text.text.clone()),
248            UserContent::Image(image) => match &image.data {
249                Source::Base64(data) => self.images.push(data.clone()),
250                _ => return Err(unsendable("an image that is not base64 data")),
251            },
252            UserContent::Document(document) => match &document.data {
253                Source::String(text) => self.texts.push(text.clone()),
254                _ => return Err(unsendable("a document")),
255            },
256            UserContent::Audio(_) => return Err(unsendable("audio")),
257            UserContent::Video(_) => return Err(unsendable("video")),
258            UserContent::ToolResult(_) => return Err(unsendable("a tool result as user content")),
259        }
260        Ok(())
261    }
262
263    /// Push the parts gathered so far as one user message, if there are any.
264    fn push_to(&mut self, messages: &mut Vec<Value>) {
265        if self.texts.is_empty() && self.images.is_empty() {
266            return;
267        }
268        let mut message = Map::from_iter([
269            ("role".to_owned(), Value::from("user")),
270            ("content".to_owned(), Value::String(self.texts.join("\n"))),
271        ]);
272        if !self.images.is_empty() {
273            message.insert(
274                "images".to_owned(),
275                Value::Array(self.images.drain(..).map(Value::String).collect()),
276            );
277        }
278        self.texts.clear();
279        messages.push(Value::Object(message));
280    }
281}
282
283/// The error for content `/api/chat` cannot carry, which the adapter
284/// replaces before a request is encoded.
285fn unsendable(what: &str) -> EncodeError {
286    EncodeError::request(format!("Ollama chat cannot carry {what}"))
287}
288
289/// A result as the `tool` message answering its call: its text joined into
290/// one string, with the tool's name and the call's id.
291fn tool_message(result: &ToolResult, ids: &WireIds) -> Result<Value, EncodeError> {
292    let texts = result
293        .content
294        .iter()
295        .map(|part| match part {
296            ToolResultContent::Text(text) => Ok(text.text.clone()),
297            ToolResultContent::Json { value } => Ok(value.to_string()),
298            ToolResultContent::Image(_) => Err(unsendable("an image in a tool result")),
299        })
300        .collect::<Result<Vec<_>, _>>()?;
301    Ok(json!({
302        "role": "tool",
303        "content": texts.join("\n"),
304        "tool_name": result.name,
305        "tool_call_id": ids.spell(&result.call),
306    }))
307}
308
309/// `call` as a `tool_calls` entry: its item as the daemon sent it when it is
310/// current, with the canonical name and arguments and the call's spelled id.
311fn call_item(call: &ToolCall, replay: Replay<'_>, ids: &WireIds) -> Value {
312    let mut item = match replay {
313        Replay::Item(item) => match item.into_owned() {
314            Value::Object(item) => item,
315            _ => Map::new(),
316        },
317        Replay::Identity(identity) => identity,
318        Replay::Rebuild => Map::new(),
319    };
320    item.insert("id".to_owned(), Value::String(ids.spell(&call.id)));
321    let function = item
322        .entry("function")
323        .or_insert_with(|| Value::Object(Map::new()));
324    if !function.is_object() {
325        *function = Value::Object(Map::new());
326    }
327    if let Value::Object(function) = function {
328        function.insert("name".to_owned(), Value::from(call.function.name.as_str()));
329        function.insert(
330            "arguments".to_owned(),
331            Value::Object(call.function.arguments.clone()),
332        );
333    }
334    Value::Object(item)
335}
336
337impl Wire for Chat {
338    type Op = Completion;
339    type Payload = Encoded;
340    type Frame = crate::wire::WireFrame;
341    type Decoder<'id> = ChatDecoder;
342    type Reassembler = super::streaming::document::ChatResponse;
343
344    /// The output schema is deferred while tools are unanswered, so it
345    /// composes with tools.
346    fn describe(&self) -> Descriptor<'_> {
347        Descriptor::new(PROVIDER_NAME)
348            .model(self.model.as_str())
349            .capabilities(Capabilities::completion(
350                ProviderCapabilities::default().with_native_output_tool_composition(true),
351            ))
352            .replay(self)
353    }
354
355    fn encode(&self, request: CompletionRequest, mode: Mode) -> Result<Encoded, EncodeError> {
356        let body = self.body(&request, mode)?;
357        let target = match mode {
358            Mode::Unary => crate::providers::internal::LogTarget::Completions,
359            Mode::Streaming => crate::providers::internal::LogTarget::Streaming,
360        };
361        crate::providers::internal::trace_json(target, "Ollama chat request", &body);
362        let request = self
363            .provider
364            .request(http::Method::POST, CHAT_PATH)
365            .body(body.into_body())?;
366        let framing = match mode {
367            Mode::Unary => Framing::Whole,
368            Mode::Streaming => Framing::Ndjson,
369        };
370        Ok(Encoded::new(request, framing)
371            .with_route(Some(CHAT_PATH))
372            .with_projection(ChatDecoder::project))
373    }
374
375    fn decoder<'id>(&self) -> Self::Decoder<'id> {
376        ChatDecoder::default()
377    }
378}
379
380impl ReplayTarget for Chat {
381    /// Section 6.5 of the typed-options design, for `/api/chat`.
382    fn map_options(
383        &self,
384        _request: &CompletionRequest,
385        fields: crate::completion::options::OptionFields<'_>,
386    ) -> crate::completion::options::OptionMap {
387        use crate::completion::options::{Mapping, OptionFields, OptionMap};
388        use crate::completion::{CacheRetention, Effort, Reasoning};
389        let OptionFields {
390            reasoning,
391            cache,
392            service_tier,
393            verbosity,
394            parallel_tool_calls,
395            top_p,
396            seed,
397            stop,
398        } = fields;
399        const NO_FIELD: &str = "Ollama's `/api/chat` has no such field";
400        OptionMap {
401            reasoning: Mapping::of(reasoning, |reasoning| match reasoning {
402                Reasoning::Off => Mapping::Send(json!({"think": false})),
403                Reasoning::Effort(
404                    effort @ (Effort::Low | Effort::Medium | Effort::High | Effort::Max),
405                ) => Mapping::Send(json!({"think": effort.as_str()})),
406                Reasoning::Effort(effort) => Mapping::unsupported(format!(
407                    "Ollama has no `{}` thinking level",
408                    effort.as_str()
409                )),
410                Reasoning::Budget { .. } => {
411                    Mapping::unsupported("Ollama takes a thinking level, not a budget")
412                }
413            }),
414            cache: Mapping::of(cache, |cache| match cache {
415                CacheRetention::None => Mapping::Omit("Ollama keeps no prompt cache to stop"),
416                CacheRetention::Short | CacheRetention::Long => Mapping::unsupported(
417                    "Ollama has no prompt cache retention; `keep_alive` keeps the model loaded",
418                ),
419            }),
420            service_tier: Mapping::of(service_tier, |_| Mapping::unsupported(NO_FIELD)),
421            verbosity: Mapping::of(verbosity, |_| Mapping::unsupported(NO_FIELD)),
422            parallel_tool_calls: Mapping::of(parallel_tool_calls, |_| {
423                Mapping::unsupported(NO_FIELD)
424            }),
425            top_p: Mapping::of(top_p, |top_p| {
426                Mapping::Send(json!({"options": {"top_p": top_p}}))
427            }),
428            seed: Mapping::of(seed, |seed| {
429                Mapping::Send(json!({"options": {"seed": seed}}))
430            }),
431            stop: Mapping::of_stop(stop, |stop| {
432                Mapping::Send(json!({"options": {"stop": stop}}))
433            }),
434        }
435    }
436
437    fn api(&self) -> crate::message::Api {
438        crate::message::Api::from_static("ollama.chat")
439    }
440
441    fn provider(&self) -> &str {
442        PROVIDER_NAME
443    }
444
445    fn model(&self) -> &str {
446        &self.model
447    }
448
449    /// Images in user messages, and none in assistant turns or tool results,
450    /// which the adapter moves to a user message.
451    fn accepts(&self, _model: &str) -> Accepts {
452        Accepts {
453            assistant_images: false,
454            tool_result_images: false,
455            ..Accepts::ALL
456        }
457    }
458
459    /// An image as base64 data in a user message. The daemon never fetches a
460    /// URL, and reads no audio, video or document; a text document's text
461    /// is sent by the adapter.
462    fn encodes(&self, _model: &str, media: Media<'_>) -> bool {
463        matches!(
464            media,
465            Media::Image(image, Place::User) if matches!(image.data, Source::Base64(_))
466        )
467    }
468
469    fn call_id_slot(&self) -> Option<&'static str> {
470        Some("/id")
471    }
472
473    /// A message needs content, thinking or calls.
474    fn sends_alone(&self, block: &AssistantContent) -> bool {
475        match block {
476            AssistantContent::ToolCall(_) => true,
477            AssistantContent::Text(text) => !text.text.is_empty(),
478            AssistantContent::Reasoning(reasoning) => !reasoning.text.is_empty(),
479            AssistantContent::Image(_) | AssistantContent::Opaque(_) => false,
480        }
481    }
482}
483
484#[cfg(test)]
485mod tests;
rig_core/providers/ollama/chat.rs

rig_core/providers/ollama/
chat.rs
