Skip to main content

paper/core/models/
chat.rs

1//! A minimal client for OpenAI-compatible chat servers.
2//!
3//! The same wire format is spoken by Ollama, LM Studio, vLLM, llama.cpp's
4//! server and OpenRouter, so one client covers the local polish engine and the
5//! opt-in cloud fallback. Nothing here is required for search or for the
6//! extractive answer path: the client is only reachable through
7//! `paper answer --polish`.
8
9use anyhow::{bail, Context, Result};
10use serde::Deserialize;
11use std::time::Duration;
12
13/// Upper bound on a chat reply. A polished answer is a paragraph or two.
14const MAX_RESPONSE_BYTES: usize = 64 * 1024;
15
16/// A chat endpoint that speaks the OpenAI `/v1/chat/completions` format.
17#[derive(Debug, Clone)]
18pub struct ChatClient {
19    client: reqwest::Client,
20    base_url: String,
21    api_key: Option<String>,
22    model: String,
23}
24
25#[derive(Debug, Deserialize)]
26struct ChatResponse {
27    choices: Vec<Choice>,
28}
29
30#[derive(Debug, Deserialize)]
31struct Choice {
32    message: Message,
33}
34
35#[derive(Debug, Deserialize)]
36struct Message {
37    content: Content,
38}
39
40/// Servers return either a plain string or a list of content parts.
41#[derive(Debug, Deserialize)]
42#[serde(untagged)]
43enum Content {
44    Text(String),
45    Parts(Vec<Part>),
46}
47
48#[derive(Debug, Deserialize)]
49struct Part {
50    #[serde(default)]
51    text: String,
52}
53
54impl Content {
55    fn into_text(self) -> String {
56        match self {
57            Content::Text(text) => text,
58            Content::Parts(parts) => parts
59                .into_iter()
60                .map(|p| p.text)
61                .collect::<Vec<_>>()
62                .join(""),
63        }
64    }
65}
66
67impl ChatClient {
68    /// Build a client for `base_url`.
69    ///
70    /// `api_key` is optional: local servers usually need none, cloud servers
71    /// do. The URL may or may not already end in `/v1`.
72    pub fn new(
73        base_url: &str,
74        api_key: Option<String>,
75        model: &str,
76        timeout: Duration,
77    ) -> Result<Self> {
78        if base_url.trim().is_empty() {
79            bail!("no base URL configured for the chat engine");
80        }
81        if model.trim().is_empty() {
82            bail!("no model configured for the chat engine");
83        }
84        let client = reqwest::Client::builder()
85            .timeout(timeout)
86            .build()
87            .context("building the HTTP client")?;
88        Ok(Self {
89            client,
90            base_url: base_url.trim_end_matches('/').to_string(),
91            api_key,
92            model: model.to_string(),
93        })
94    }
95
96    fn endpoint(&self) -> String {
97        if self.base_url.ends_with("/v1") {
98            format!("{}/chat/completions", self.base_url)
99        } else {
100            format!("{}/v1/chat/completions", self.base_url)
101        }
102    }
103
104    pub fn model(&self) -> &str {
105        &self.model
106    }
107
108    /// Send one system + one user message and return the assistant's text.
109    pub async fn chat(&self, system: &str, user: &str) -> Result<String> {
110        let body = serde_json::json!({
111            "model": self.model,
112            "messages": [
113                {"role": "system", "content": system},
114                {"role": "user", "content": user},
115            ],
116            "temperature": 0,
117            "stream": false,
118        });
119
120        let mut request = self.client.post(self.endpoint()).json(&body);
121        if let Some(key) = &self.api_key {
122            request = request.bearer_auth(key);
123        }
124        let response = request
125            .send()
126            .await
127            .with_context(|| format!("calling {}", self.endpoint()))?;
128        let status = response.status();
129        if !status.is_success() {
130            let body = response.text().await.unwrap_or_default();
131            bail!("chat engine returned {status}: {}", truncate(&body, 400));
132        }
133        // A chat reply is a few kilobytes at most. Refuse anything larger
134        // rather than letting a broken or hostile server exhaust memory.
135        if let Some(len) = response.content_length() {
136            if len > MAX_RESPONSE_BYTES as u64 {
137                bail!("chat engine response is too large ({len} bytes)");
138            }
139        }
140        let body = response.text().await.context("reading the chat response")?;
141        if body.len() > MAX_RESPONSE_BYTES {
142            bail!("chat engine response is too large ({} bytes)", body.len());
143        }
144        let parsed: ChatResponse =
145            serde_json::from_str(&body).context("parsing the chat engine response")?;
146        let text = parsed
147            .choices
148            .into_iter()
149            .next()
150            .map(|c| c.message.content.into_text())
151            .unwrap_or_default();
152        if text.trim().is_empty() {
153            bail!("chat engine returned an empty answer");
154        }
155        Ok(text)
156    }
157}
158
159fn truncate(text: &str, max: usize) -> String {
160    if text.chars().count() <= max {
161        return text.to_string();
162    }
163    let head: String = text.chars().take(max).collect();
164    format!("{head}…")
165}