rig_core/providers/ollama/
chat.rs

1//! The native Ollama chat wire: `POST /api/chat`, a whole reply or a stream
2//! of NDJSON records, both read by [`ChatDecoder`].
3//!
4//! The request carries the daemon's own fields: `think` and `keep_alive` at
5//! the top level, model parameters such as `num_ctx` in `options`, and the
6//! output schema as `format`.
7//!
8//! ```
9//! use rig_core::providers::ollama::OllamaConfig;
10//!
11//! let wire = OllamaConfig::new().client().native_completion("qwen3:4b").wire;
12//! assert_eq!(wire.model, "qwen3:4b");
13//! ```
14
15use serde_json::{Map, Value, json};
16use std::borrow::Cow;
17use std::sync::Arc;
18
19use crate::completion::options::{BaseInput, FinalBody, RawAt, request_params};
20use crate::completion::{
21    Accepts, CompletionRequest, Media, Place, ProviderCapabilities, Replay, ReplayTarget,
22};
23use crate::error::EncodeError;
24use crate::message::{
25    AssistantContent, AssistantMessage, DocumentData, DocumentSourceKind as Source, Message,
26    ToolCall, ToolResult, ToolResultContent, UserContent,
27};
28use crate::operation::Completion;
29use crate::providers::internal::wire_ids::WireIds;
30use crate::wire::{Capabilities, Descriptor, Encoded, Framing, Mode, Wire};
31
32use super::streaming::ChatDecoder;
33use super::{OllamaConfig, PROVIDER_NAME};
34
35/// The chat endpoint, relative to the daemon's address.
36const CHAT_PATH: &str = "/api/chat";
37
38/// The `additional_params` keys `/api/chat` reads at the top level of its
39/// request. Every other key is a model parameter and goes in `options`.
40const TOP_LEVEL: &[&str] = &[
41    "think",
42    "format",
43    "keep_alive",
44    "logprobs",
45    "top_logprobs",
46    "truncate",
47    "shift",
48];
49
50/// The native chat wire of an Ollama daemon.
51#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
52pub struct Chat {
53    /// The daemon's address and credential.
54    pub provider: OllamaConfig,
55    /// The model to address, e.g. [`QWEN3`](super::QWEN3).
56    pub model: String,
57    /// The model facts the wire describes its model by.
58    #[serde(skip)]
59    pub facts: crate::catalog::ModelFacts,
60}
61
62impl Chat {
63    /// The wire for `model` on `provider`.
64    pub fn new(provider: OllamaConfig, model: impl Into<String>) -> Self {
65        Self {
66            provider,
67            model: model.into(),
68            facts: crate::catalog::ModelFacts::default(),
69        }
70    }
71
72    /// The same wire, describing its model by `facts`.
73    pub fn with_facts(mut self, facts: crate::catalog::ModelFacts) -> Self {
74        self.facts = facts;
75        self
76    }
77
78    /// The `/api/chat` body `request` sends in `mode`: the wire's encoding,
79    /// then the mapped options, then `additional_params`, split by where the
80    /// daemon reads each key. `think` and the keys in [`TOP_LEVEL`] go at the
81    /// top level, `tools` join the request's tools, an `options` object
82    /// merges into `options`, and every other key is an `options` entry.
83    /// `temperature` and `max_tokens` (as `num_predict`) go in `options`,
84    /// where a caller's own entries win.
85    fn body(&self, request: &CompletionRequest, mode: Mode) -> Result<FinalBody, EncodeError> {
86        request_params(
87            self,
88            request,
89            |input| self.base(request, mode, input),
90            RawAt::Split {
91                top: TOP_LEVEL,
92                rest: "options",
93            },
94            &[],
95        )
96    }
97
98    /// The wire's own encoding of `request`.
99    fn base(
100        &self,
101        request: &CompletionRequest,
102        mode: Mode,
103        input: &mut BaseInput<'_>,
104    ) -> Result<Map<String, Value>, EncodeError> {
105        let model = request.model.clone().unwrap_or_else(|| self.model.clone());
106        if input
107            .param("keep_alive")
108            .is_some_and(|value| !(value.is_string() || value.is_number()))
109        {
110            return Err(EncodeError::request(
111                "Ollama `keep_alive` must be a duration string or a number of seconds",
112            ));
113        }
114        if input
115            .param("options")
116            .is_some_and(|value| !value.is_object())
117        {
118            return Err(EncodeError::request(
119                "Ollama `additional_params.options` must be an object",
120            ));
121        }
122        let messages = self.messages(&request.chat_history, &model)?;
123
124        let mut tools: Vec<Value> = request
125            .tools
126            .iter()
127            .map(|tool| {
128                json!({"type": "function", "function": {
129                    "name": tool.name,
130                    "description": tool.description,
131                    "parameters": tool.parameters,
132                }})
133            })
134            .collect();
135        tools.extend(input.raw_tools()?);
136        if request.tool_choice.is_some() {
137            tracing::warn!("Ollama has no tool choice; `tool_choice` is ignored");
138        }
139
140        let mut options = Map::new();
141        if let Some(temperature) = request.temperature {
142            options.insert("temperature".to_owned(), Value::from(temperature));
143        }
144        if let Some(max_tokens) = request.max_tokens {
145            options.insert("num_predict".to_owned(), Value::from(max_tokens));
146        }
147
148        // Defer the schema until a tool result exists: a constrained reply
149        // cannot call a tool.
150        let answered = messages
151            .iter()
152            .any(|message| message.get("role").and_then(Value::as_str) == Some("tool"));
153        let format = request
154            .output_schema
155            .clone()
156            .filter(|_| tools.is_empty() || answered)
157            .map(|schema| schema.to_value());
158
159        Ok(crate::providers::internal::set_fields([
160            ("model", Some(Value::String(model))),
161            ("messages", Some(Value::Array(messages))),
162            ("tools", (!tools.is_empty()).then_some(Value::Array(tools))),
163            ("format", format),
164            (
165                "options",
166                (!options.is_empty()).then_some(Value::Object(options)),
167            ),
168            ("stream", Some(Value::Bool(mode == Mode::Streaming))),
169        ]))
170    }
171
172    /// The history as `/api/chat` messages, each call and result spelled by
173    /// one [`WireIds`].
174    fn messages(&self, history: &[Arc<Message>], model: &str) -> Result<Vec<Value>, EncodeError> {
175        let ids = WireIds::for_target(history, self, model);
176        let mut messages = Vec::new();
177        for message in history {
178            match &**message {
179                Message::System { content } => {
180                    messages.push(json!({"role": "system", "content": content}));
181                }
182                Message::User { content } => {
183                    let mut user = UserParts::default();
184                    for part in content {
185                        match part {
186                            UserContent::ToolResult(result) => {
187                                user.push_to(&mut messages);
188                                messages.push(tool_message(result, &ids)?);
189                            }
190                            part => user.add(part)?,
191                        }
192                    }
193                    user.push_to(&mut messages);
194                }
195                Message::Assistant(turn) => {
196                    messages.extend(self.assistant(turn, &ids));
197                }
198            }
199        }
200        if messages.is_empty() {
201            return Err(EncodeError::request(
202                "Ollama chat request has no messages after conversion",
203            ));
204        }
205        Ok(messages)
206    }
207
208    /// One assistant turn: its text joined as `content`, its reasoning as
209    /// `thinking`, and its calls. A turn with none of them is `None`.
210    fn assistant(&self, turn: &AssistantMessage, ids: &WireIds) -> Option<Value> {
211        let (mut text, mut thinking, mut calls) = (String::new(), Vec::new(), Vec::new());
212        for block in &turn.content {
213            match block {
214                AssistantContent::Text(block) => text.push_str(&block.text),
215                AssistantContent::Reasoning(block) if !block.text.is_empty() => {
216                    thinking.push(block.text.as_str());
217                }
218                AssistantContent::ToolCall(call) => {
219                    calls.push(call_item(call, block.replay(self, ids), ids));
220                }
221                AssistantContent::Reasoning(_)
222                | AssistantContent::Image(_)
223                | AssistantContent::Opaque(_) => {}
224            }
225        }
226        if text.is_empty() && thinking.is_empty() && calls.is_empty() {
227            return None;
228        }
229        let mut message = Map::from_iter([
230            ("role".to_owned(), Value::from("assistant")),
231            ("content".to_owned(), Value::String(text)),
232        ]);
233        if !thinking.is_empty() {
234            message.insert("thinking".to_owned(), Value::String(thinking.join("\n")));
235        }
236        if !calls.is_empty() {
237            message.insert("tool_calls".to_owned(), Value::Array(calls));
238        }
239        Some(Value::Object(message))
240    }
241}
242
243/// The text and images of one user message as they gather.
244#[derive(Default)]
245struct UserParts<'a> {
246    texts: Vec<&'a str>,
247    images: Vec<String>,
248}
249
250impl<'a> UserParts<'a> {
251    /// Add a part: text, or an image as base64 data. The adapter has
252    /// replaced every other part ([`ReplayTarget::encodes`]).
253    fn add(&mut self, part: &'a UserContent) -> Result<(), EncodeError> {
254        match part {
255            UserContent::Text(text) => self.texts.push(&text.text),
256            UserContent::Image(image) => match &image.data {
257                Source::Base64(data) => self.images.push(data.clone()),
258                _ => return Err(unsendable("an image that is not base64 data")),
259            },
260            UserContent::Document(document) => match &document.data {
261                DocumentData::Text(text) => self.texts.push(text),
262                DocumentData::File(_) => return Err(unsendable("a document")),
263            },
264            UserContent::Audio(_) => return Err(unsendable("audio")),
265            UserContent::Video(_) => return Err(unsendable("video")),
266            UserContent::ToolResult(_) => return Err(unsendable("a tool result as user content")),
267        }
268        Ok(())
269    }
270
271    /// Push the parts gathered so far as one user message, if there are any.
272    fn push_to(&mut self, messages: &mut Vec<Value>) {
273        if self.texts.is_empty() && self.images.is_empty() {
274            return;
275        }
276        let mut message = Map::from_iter([
277            ("role".to_owned(), Value::from("user")),
278            ("content".to_owned(), Value::String(self.texts.join("\n"))),
279        ]);
280        if !self.images.is_empty() {
281            message.insert(
282                "images".to_owned(),
283                Value::Array(self.images.drain(..).map(Value::String).collect()),
284            );
285        }
286        self.texts.clear();
287        messages.push(Value::Object(message));
288    }
289}
290
291/// The error for content `/api/chat` cannot carry, which the adapter
292/// replaces before a request is encoded.
293fn unsendable(what: &str) -> EncodeError {
294    EncodeError::request(format!("Ollama chat cannot carry {what}"))
295}
296
297/// A result as the `tool` message answering its call: its text joined into
298/// one string, with the tool's name and the call's id.
299fn tool_message(result: &ToolResult, ids: &WireIds) -> Result<Value, EncodeError> {
300    let texts = result
301        .content
302        .iter()
303        .map(|part| match part {
304            ToolResultContent::Text(text) => Ok(Cow::Borrowed(text.text.as_str())),
305            ToolResultContent::Json { value } => Ok(Cow::Owned(value.to_string())),
306            ToolResultContent::Image(_) => Err(unsendable("an image in a tool result")),
307        })
308        .collect::<Result<Vec<_>, _>>()?;
309    Ok(Value::Object(Map::from_iter([
310        ("role".to_owned(), Value::from("tool")),
311        ("content".to_owned(), Value::String(texts.join("\n"))),
312        ("tool_name".to_owned(), Value::from(result.name.as_str())),
313        (
314            "tool_call_id".to_owned(),
315            Value::String(ids.spell(&result.call)),
316        ),
317    ])))
318}
319
320/// `call` as a `tool_calls` entry: its item as the daemon sent it when it is
321/// current, with the canonical name and arguments and the call's spelled id.
322fn call_item(call: &ToolCall, replay: Replay<'_>, ids: &WireIds) -> Value {
323    let mut item = match replay {
324        Replay::Item(item) => match item.into_owned() {
325            Value::Object(item) => item,
326            _ => Map::new(),
327        },
328        Replay::Identity(identity) => identity,
329        Replay::Rebuild => Map::new(),
330    };
331    item.insert("id".to_owned(), Value::String(ids.spell(&call.id)));
332    let function = item
333        .entry("function")
334        .or_insert_with(|| Value::Object(Map::new()));
335    if !function.is_object() {
336        *function = Value::Object(Map::new());
337    }
338    if let Value::Object(function) = function {
339        function.insert("name".to_owned(), Value::from(call.function.name.as_str()));
340        function.insert(
341            "arguments".to_owned(),
342            Value::Object(call.function.arguments.clone()),
343        );
344    }
345    Value::Object(item)
346}
347
348impl Wire for Chat {
349    type Op = Completion;
350    type Payload = Encoded;
351    type Frame = crate::wire::WireFrame;
352    type Event = super::streaming::ChatRecord;
353    type Decoder<'id> = ChatDecoder;
354    type Reassembler = super::streaming::document::ChatResponse;
355
356    /// The output schema is deferred while tools are unanswered, so it
357    /// composes with tools.
358    fn describe(&self) -> Descriptor<'_> {
359        Descriptor::new(PROVIDER_NAME)
360            .model(self.model.as_str())
361            .facts(&self.facts)
362            .capabilities(Capabilities::completion(
363                ProviderCapabilities::default().with_native_output_tool_composition(true),
364            ))
365            .replay(self)
366    }
367
368    fn encode(&self, request: CompletionRequest, mode: Mode) -> Result<Encoded, EncodeError> {
369        let body = self.body(&request, mode)?;
370        let target = match mode {
371            Mode::Unary => crate::providers::internal::LogTarget::Completions,
372            Mode::Streaming => crate::providers::internal::LogTarget::Streaming,
373        };
374        crate::providers::internal::trace_json(target, "Ollama chat request", &body);
375        let request = self
376            .provider
377            .request(http::Method::POST, CHAT_PATH)
378            .body(body.into_body())?;
379        let framing = match mode {
380            Mode::Unary => Framing::Whole,
381            Mode::Streaming => Framing::Ndjson,
382        };
383        Ok(Encoded::new(request, framing)
384            .with_route(Some(CHAT_PATH))
385            .with_projection(ChatDecoder::project))
386    }
387
388    fn decoder<'id>(&self) -> Self::Decoder<'id> {
389        ChatDecoder::default()
390    }
391}
392
393impl ReplayTarget for Chat {
394    /// Section 6.5 of the typed-options design, for `/api/chat`.
395    fn map_options(
396        &self,
397        _request: &CompletionRequest,
398        fields: crate::completion::options::OptionFields<'_>,
399    ) -> crate::completion::options::OptionMap {
400        use crate::completion::options::{Mapping, OptionMap};
401        use crate::completion::{CacheRetention, Effort, Reasoning};
402        const NO_FIELD: &str = "Ollama's `/api/chat` has no such field";
403        OptionMap {
404            reasoning: Mapping::of(fields.reasoning, |reasoning| match reasoning {
405                Reasoning::Off => Mapping::Send(json!({"think": false})),
406                Reasoning::Effort(
407                    effort @ (Effort::Low | Effort::Medium | Effort::High | Effort::Max),
408                ) => Mapping::Send(json!({"think": effort.as_str()})),
409                Reasoning::Effort(effort) => Mapping::unsupported(format!(
410                    "Ollama has no `{}` thinking level",
411                    effort.as_str()
412                )),
413                Reasoning::Budget { .. } => {
414                    Mapping::unsupported("Ollama takes a thinking level, not a budget")
415                }
416            }),
417            cache: Mapping::of(fields.cache, |cache| match cache {
418                CacheRetention::None => Mapping::Omit("Ollama keeps no prompt cache to stop"),
419                CacheRetention::Short | CacheRetention::Long => Mapping::unsupported(
420                    "Ollama has no prompt cache retention; `keep_alive` keeps the model loaded",
421                ),
422            }),
423            service_tier: Mapping::of(fields.service_tier, |_| Mapping::unsupported(NO_FIELD)),
424            verbosity: Mapping::of(fields.verbosity, |_| Mapping::unsupported(NO_FIELD)),
425            parallel_tool_calls: Mapping::of(fields.parallel_tool_calls, |_| {
426                Mapping::unsupported(NO_FIELD)
427            }),
428            top_p: Mapping::of(fields.top_p, |top_p| {
429                Mapping::Send(json!({"options": {"top_p": top_p}}))
430            }),
431            seed: Mapping::of(fields.seed, |seed| {
432                Mapping::Send(json!({"options": {"seed": seed}}))
433            }),
434            stop: Mapping::of_stop(fields.stop, |stop| {
435                Mapping::Send(json!({"options": {"stop": stop}}))
436            }),
437            cache_key: Mapping::unrouted(fields.cache_key),
438        }
439    }
440
441    fn facts(&self) -> Option<&crate::catalog::ModelFacts> {
442        Some(&self.facts)
443    }
444
445    fn api(&self) -> crate::message::Api {
446        crate::message::Api::from_static("ollama.chat")
447    }
448
449    fn provider(&self) -> &str {
450        PROVIDER_NAME
451    }
452
453    fn model(&self) -> &str {
454        &self.model
455    }
456
457    /// Images in user messages, and none in assistant turns or tool results,
458    /// which the adapter moves to a user message. Another model's reasoning
459    /// goes in `thinking`, as the model's own does.
460    fn accepts(&self, _model: &str) -> Accepts {
461        Accepts {
462            assistant_images: false,
463            tool_result_images: false,
464            foreign_reasoning: true,
465            ..Accepts::ALL
466        }
467    }
468
469    /// An image as base64 data in a user message. The daemon never fetches a
470    /// URL, and reads no audio, video or document; a text document's text
471    /// is sent by the adapter.
472    fn encodes(&self, _model: &str, media: Media<'_>) -> bool {
473        matches!(
474            media,
475            Media::Image(image, Place::User) if matches!(image.data, Source::Base64(_))
476        )
477    }
478
479    fn call_id_slot(&self) -> Option<&'static str> {
480        Some("/id")
481    }
482
483    /// A message needs content, thinking or calls.
484    fn sends_alone(&self, block: &AssistantContent) -> bool {
485        match block {
486            AssistantContent::ToolCall(_) => true,
487            AssistantContent::Text(text) => !text.text.is_empty(),
488            AssistantContent::Reasoning(reasoning) => !reasoning.text.is_empty(),
489            AssistantContent::Image(_) | AssistantContent::Opaque(_) => false,
490        }
491    }
492}
493
494#[cfg(test)]
495mod tests;
rig_core/providers/ollama/chat.rs

rig_core/providers/ollama/
chat.rs
