Skip to main content

tinyharness_lib/provider/
mod.rs

1#[cfg(feature = "test-util")]
2pub mod mock;
3pub mod model_variants;
4pub mod ollama;
5pub mod openai_compat;
6pub mod openai_compat_provider;
7pub mod sockudo;
8pub mod tool_pairs;
9
10use std::fmt::Display;
11use std::future::Future;
12
13use serde::{Deserialize, Serialize};
14
15use crate::SecretString;
16use crate::config::{OllamaThinkType, Settings};
17use crate::image::ImageAttachment;
18use crate::provider::ollama::OllamaProvider;
19use crate::provider::openai_compat_provider::OpenAiCompatProvider;
20use crate::provider::sockudo::SockudoProvider;
21
22/// Channel capacity for streaming chat responses from providers to the
23/// agent loop.
24///
25/// A small buffer applies backpressure to the provider's HTTP read loop
26/// when the UI consumes chunks slowly: once the buffer is full,
27/// `send().await` suspends the provider task instead of letting chunks
28/// accumulate without bound. 64 chunks is far more than a terminal renders
29/// per frame, so streaming throughput is unaffected in practice.
30pub const STREAM_CHANNEL_CAPACITY: usize = 64;
31
32#[derive(Debug, Clone, Serialize, Deserialize)]
33pub struct ToolDefinition {
34    pub name: String,
35    pub description: String,
36    pub parameters: schemars::Schema,
37}
38
39#[derive(Debug, Clone, Serialize, Deserialize)]
40pub struct ToolCall {
41    /// OpenAI tool call ID (e.g. `call_abc123`). Required by OpenAI-compatible
42    /// servers when echoing the call back in subsequent assistant messages
43    /// and when building matching `tool` result messages. Some servers generate
44    /// it for us; others (or local models) may not, in which case we fall
45    /// back to a synthetic id at serialization time.
46    #[serde(default, skip_serializing_if = "Option::is_none")]
47    pub id: Option<String>,
48    pub function: ToolCallFunction,
49}
50
51#[derive(Debug, Clone, Serialize, Deserialize)]
52pub struct ToolCallFunction {
53    pub name: String,
54    pub arguments: serde_json::Value,
55    /// Gemini `thought_signature` required for multi-turn tool calling.
56    /// Gemini returns this on tool calls and requires it back in subsequent
57    /// turns. Ollama Cloud doesn't preserve it, so we capture and re-inject it.
58    #[serde(skip_serializing_if = "Option::is_none", default)]
59    pub thought_signature: Option<String>,
60}
61
62#[derive(Debug, Clone, Serialize, Deserialize)]
63pub struct ChatMessage {
64    pub content: String,
65    #[serde(default)]
66    pub tool_calls: Vec<ToolCall>,
67    /// Thinking/reasoning content from the model, as a **delta** during
68    /// streaming. Ollama fills this from its `thinking` field; OpenAI-compatible
69    /// servers (OpenRouter, vLLM with a reasoning parser, DeepSeek) from
70    /// `delta.reasoning` / `delta.reasoning_content` / `delta.reasoning_details`.
71    /// Only populated when the model actually reasons.
72    #[serde(default)]
73    pub thinking: Option<String>,
74}
75
76#[derive(Debug, Clone, Deserialize)]
77pub struct ChatMessageResponse {
78    pub message: ChatMessage,
79    pub done: bool,
80    #[serde(default)]
81    pub is_error: bool,
82    #[serde(default)]
83    pub usage: Option<TokenUsage>,
84}
85
86/// Token usage information from the provider.
87#[derive(Debug, Clone, Default, Deserialize, Serialize)]
88pub struct TokenUsage {
89    pub prompt_tokens: u32,
90    pub completion_tokens: u32,
91    pub total_tokens: u32,
92}
93
94#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
95pub enum Role {
96    System,
97    User,
98    Assistant,
99    Tool,
100}
101
102impl Display for Role {
103    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
104        match self {
105            Role::System => f.write_str("System"),
106            Role::User => f.write_str("User"),
107            Role::Assistant => f.write_str("Assistant"),
108            Role::Tool => f.write_str("Tool"),
109        }
110    }
111}
112
113#[derive(Debug, Clone, Serialize, Deserialize)]
114pub struct Message {
115    pub role: Role,
116    pub content: String,
117    pub tool_calls: Vec<ToolCall>,
118    /// OpenAI `tool_call_id` for `Role::Tool` messages — links the result back
119    /// to the originating assistant tool call. Required by OpenAI-compatible
120    /// servers. `None` is allowed for round-trips with providers that don't
121    /// use it, but the OpenAI-compatible serialiser will substitute a
122    /// synthetic id when missing.
123    #[serde(default, skip_serializing_if = "Option::is_none")]
124    pub tool_call_id: Option<String>,
125    /// Optional images attached to the message (multimodal models).
126    /// Only meaningful for `User` role messages.
127    #[serde(default, skip_serializing_if = "Vec::is_empty")]
128    pub images: Vec<ImageAttachment>,
129    /// Thinking/reasoning chain from the model, captured during streaming
130    /// and persisted for debugging. Not sent to the provider in API requests
131    /// (skipped in serialization for request types; only stored in sessions).
132    #[serde(default, skip_serializing_if = "Option::is_none")]
133    pub thinking: Option<String>,
134}
135
136impl Default for Message {
137    fn default() -> Self {
138        Message::simple(Role::User, "")
139    }
140}
141
142impl Message {
143    /// Create a new message with the given role and content, no tool calls, no images.
144    pub fn simple(role: Role, content: impl Into<String>) -> Self {
145        Message {
146            role,
147            content: content.into(),
148            tool_calls: vec![],
149            tool_call_id: None,
150            images: vec![],
151            thinking: None,
152        }
153    }
154}
155
156/// Trait for LLM provider backends (Ollama, OpenAI-compatible, Sockudo).
157///
158/// Async methods return `impl Future<...> + Send` (RPITIT) rather than
159/// `Pin<Box<dyn Future>>`: no allocation, no extra crates. The trade-off is
160/// that the trait is not dyn-compatible — for dynamic dispatch over a
161/// runtime-selected backend use [`AnyProvider`], the concrete enum covering
162/// all built-in providers.
163///
164/// Implementations are expected to be cheap to clone where the provider
165/// spawns background streaming tasks (see [`Provider::chat`]).
166pub trait Provider: Send + Sync {
167    /// Check whether the backend is reachable and healthy.
168    ///
169    /// Called once at startup; a failure is a non-fatal warning — errors
170    /// surface on the first real request. Providers whose dedicated health
171    /// endpoint may be missing (OpenAI-compatible servers) fall back to the
172    /// model list: a non-empty list proves the backend is up.
173    fn health_check(&self) -> impl Future<Output = Result<(), String>> + Send;
174
175    /// List model names available on the backend.
176    ///
177    /// Returns an empty vec when the backend doesn't expose a model list or
178    /// the request fails; callers fall back to saved or first-known models.
179    fn list_models(&self) -> impl Future<Output = Vec<String>> + Send;
180
181    /// Select the model used for subsequent [`Provider::chat`] requests.
182    fn select_model(&mut self, name: String);
183
184    /// The currently selected model, if any.
185    fn current_model(&self) -> Option<String>;
186
187    /// Send a chat request and return a receiver for streaming response chunks.
188    ///
189    /// Returns `Err(String)` if the request cannot be started (e.g. no model selected).
190    /// On success, the provider spawns a background task that streams `ChatMessageResponse`
191    /// chunks through the returned receiver. The caller drains the receiver until it
192    /// receives a chunk with `done: true`; dropping the receiver stops the stream.
193    ///
194    /// Token usage, when available, is included in the final `ChatMessageResponse`
195    /// chunk (in the `usage` field). No separate method is needed to retrieve it.
196    fn chat(
197        &mut self,
198        messages: Vec<Message>,
199        tools: Vec<ToolDefinition>,
200    ) -> impl Future<Output = Result<tokio::sync::mpsc::Receiver<ChatMessageResponse>, String>> + Send;
201
202    /// Set the request timeout in seconds. Only meaningful for providers that use timeouts.
203    fn set_timeout(&mut self, _timeout_secs: u64) {}
204
205    /// Set the maximum number of retries. Only meaningful for providers that use retries.
206    fn set_retries(&mut self, _max_retries: u32) {}
207
208    /// Set the think/reasoning level. Only meaningful for Ollama.
209    fn set_think_type(&mut self, _think_type: OllamaThinkType) {}
210}
211
212/// Concrete wrapper over all built-in provider backends.
213///
214/// The [`Provider`] trait is not dyn-compatible (its async methods use
215/// RPITIT return types), so dynamic dispatch over a runtime-selected
216/// backend goes through this enum instead of `Arc<Mutex<dyn Provider>>`.
217/// Commands and the agent loop share an `Arc<Mutex<AnyProvider>>`.
218pub enum AnyProvider {
219    Ollama(crate::provider::ollama::OllamaProvider),
220    OpenAiCompat(crate::provider::openai_compat_provider::OpenAiCompatProvider),
221    Sockudo(crate::provider::sockudo::SockudoProvider),
222    /// Test-only variant. Constructed via `MockProviderHandle` (feature
223    /// `test-util`); never built in production code paths.
224    #[cfg(feature = "test-util")]
225    Mock(crate::provider::mock::MockProviderHandle),
226}
227
228impl AnyProvider {
229    /// Build the provider for `kind` from resolved settings values.
230    ///
231    /// `url` and credentials come from the caller (CLI flags / settings);
232    /// `timeout_secs`/`max_retries` are the effective per-provider values
233    /// from `Settings::effective_timeout_secs()`/`effective_max_retries()`.
234    #[allow(clippy::too_many_arguments)]
235    pub fn build(
236        kind: crate::config::ProviderKind,
237        base_url: String,
238        api_key: Option<SecretString>,
239        timeout_secs: u64,
240        max_retries: u32,
241        think_type: OllamaThinkType,
242        sockudo: SockudoCredentials,
243    ) -> Result<Self, String> {
244        Ok(match kind {
245            crate::config::ProviderKind::Ollama => AnyProvider::Ollama(OllamaProvider::new(
246                base_url,
247                timeout_secs,
248                max_retries,
249                think_type,
250            )?),
251            crate::config::ProviderKind::LlamaCpp => AnyProvider::OpenAiCompat(
252                OpenAiCompatProvider::with_options(base_url, None, timeout_secs, max_retries)
253                    .with_static_models(vec!["llama-cpp".to_string()]),
254            ),
255            crate::config::ProviderKind::Vllm => AnyProvider::OpenAiCompat(
256                OpenAiCompatProvider::with_options(base_url, None, timeout_secs, max_retries),
257            ),
258            crate::config::ProviderKind::OpenAiCompat => {
259                // Empty key = explicit no-auth (e.g. `--api-key ""`).
260                let key = api_key.filter(|k| !k.is_empty());
261                AnyProvider::OpenAiCompat(OpenAiCompatProvider::with_options(
262                    base_url,
263                    key,
264                    timeout_secs,
265                    max_retries,
266                ))
267            }
268            crate::config::ProviderKind::Sockudo => AnyProvider::Sockudo(SockudoProvider::new(
269                base_url,
270                sockudo.app_id,
271                sockudo.app_key,
272                sockudo.app_secret.unwrap_or_default(),
273            )),
274        })
275    }
276
277    /// The provider kind this instance wraps.
278    ///
279    /// Note: `LlamaCpp`/`Vllm` are both backed by `OpenAiCompatProvider`,
280    /// so `kind()` cannot distinguish them — it reports `OpenAiCompat` for
281    /// both. Use `build()`'s input `ProviderKind` where the exact kind
282    /// matters.
283    pub fn kind(&self) -> crate::config::ProviderKind {
284        match self {
285            AnyProvider::Ollama(_) => crate::config::ProviderKind::Ollama,
286            AnyProvider::OpenAiCompat(_) => crate::config::ProviderKind::OpenAiCompat,
287            AnyProvider::Sockudo(_) => crate::config::ProviderKind::Sockudo,
288            #[cfg(feature = "test-util")]
289            AnyProvider::Mock(_) => crate::config::ProviderKind::Ollama,
290        }
291    }
292}
293
294/// Credentials needed to construct the Sockudo provider.
295#[derive(Debug, Clone, Default)]
296pub struct SockudoCredentials {
297    pub app_id: String,
298    pub app_key: String,
299    pub app_secret: Option<SecretString>,
300}
301
302impl From<&Settings> for SockudoCredentials {
303    fn from(settings: &Settings) -> Self {
304        SockudoCredentials {
305            app_id: settings.sockudo_app_id.clone().unwrap_or_default(),
306            app_key: settings.sockudo_app_key.clone().unwrap_or_default(),
307            app_secret: settings.sockudo_app_secret.clone(),
308        }
309    }
310}
311
312impl Provider for AnyProvider {
313    async fn health_check(&self) -> Result<(), String> {
314        match self {
315            AnyProvider::Ollama(p) => p.health_check().await,
316            AnyProvider::OpenAiCompat(p) => p.health_check().await,
317            AnyProvider::Sockudo(p) => p.health_check().await,
318            #[cfg(feature = "test-util")]
319            AnyProvider::Mock(p) => p.health_check().await,
320        }
321    }
322
323    async fn list_models(&self) -> Vec<String> {
324        match self {
325            AnyProvider::Ollama(p) => p.list_models().await,
326            AnyProvider::OpenAiCompat(p) => p.list_models().await,
327            AnyProvider::Sockudo(p) => p.list_models().await,
328            #[cfg(feature = "test-util")]
329            AnyProvider::Mock(p) => p.list_models().await,
330        }
331    }
332
333    fn select_model(&mut self, name: String) {
334        match self {
335            AnyProvider::Ollama(p) => p.select_model(name),
336            AnyProvider::OpenAiCompat(p) => p.select_model(name),
337            AnyProvider::Sockudo(p) => p.select_model(name),
338            #[cfg(feature = "test-util")]
339            AnyProvider::Mock(p) => p.select_model(name),
340        }
341    }
342
343    fn current_model(&self) -> Option<String> {
344        match self {
345            AnyProvider::Ollama(p) => p.current_model(),
346            AnyProvider::OpenAiCompat(p) => p.current_model(),
347            AnyProvider::Sockudo(p) => p.current_model(),
348            #[cfg(feature = "test-util")]
349            AnyProvider::Mock(p) => p.current_model(),
350        }
351    }
352
353    async fn chat(
354        &mut self,
355        messages: Vec<Message>,
356        tools: Vec<ToolDefinition>,
357    ) -> Result<tokio::sync::mpsc::Receiver<ChatMessageResponse>, String> {
358        match self {
359            AnyProvider::Ollama(p) => p.chat(messages, tools).await,
360            AnyProvider::OpenAiCompat(p) => p.chat(messages, tools).await,
361            AnyProvider::Sockudo(p) => p.chat(messages, tools).await,
362            #[cfg(feature = "test-util")]
363            AnyProvider::Mock(p) => p.chat(messages, tools).await,
364        }
365    }
366
367    fn set_timeout(&mut self, timeout_secs: u64) {
368        match self {
369            AnyProvider::Ollama(p) => p.set_timeout(timeout_secs),
370            AnyProvider::OpenAiCompat(p) => p.set_timeout(timeout_secs),
371            AnyProvider::Sockudo(p) => p.set_timeout(timeout_secs),
372            #[cfg(feature = "test-util")]
373            AnyProvider::Mock(p) => p.set_timeout(timeout_secs),
374        }
375    }
376
377    fn set_retries(&mut self, max_retries: u32) {
378        match self {
379            AnyProvider::Ollama(p) => p.set_retries(max_retries),
380            AnyProvider::OpenAiCompat(p) => p.set_retries(max_retries),
381            AnyProvider::Sockudo(p) => p.set_retries(max_retries),
382            #[cfg(feature = "test-util")]
383            AnyProvider::Mock(p) => p.set_retries(max_retries),
384        }
385    }
386
387    fn set_think_type(&mut self, think_type: OllamaThinkType) {
388        match self {
389            AnyProvider::Ollama(p) => p.set_think_type(think_type),
390            AnyProvider::OpenAiCompat(p) => p.set_think_type(think_type),
391            AnyProvider::Sockudo(p) => p.set_think_type(think_type),
392            #[cfg(feature = "test-util")]
393            AnyProvider::Mock(p) => p.set_think_type(think_type),
394        }
395    }
396}
397
398#[cfg(test)]
399mod tests {
400    use super::*;
401    use crate::provider::mock::{MockProvider, MockProviderHandle};
402    use std::sync::{Arc, Mutex as StdMutex};
403
404    #[test]
405    fn build_returns_provider_for_each_kind() {
406        let creds = SockudoCredentials::default();
407        for kind in [
408            crate::config::ProviderKind::Ollama,
409            crate::config::ProviderKind::LlamaCpp,
410            crate::config::ProviderKind::Vllm,
411            crate::config::ProviderKind::OpenAiCompat,
412            crate::config::ProviderKind::Sockudo,
413        ] {
414            let provider = AnyProvider::build(
415                kind,
416                "http://localhost:1234".to_string(),
417                Some(SecretString::new("k")),
418                30,
419                0,
420                OllamaThinkType::Medium,
421                creds.clone(),
422            )
423            .unwrap_or_else(|e| panic!("build({kind}) failed: {e}"));
424            let _ = provider.current_model();
425        }
426    }
427
428    #[test]
429    fn build_ollama_invalid_url_is_an_error() {
430        let creds = SockudoCredentials::default();
431        let result = AnyProvider::build(
432            crate::config::ProviderKind::Ollama,
433            "::not a url::".to_string(),
434            None,
435            5,
436            0,
437            OllamaThinkType::Medium,
438            creds,
439        );
440        assert!(result.is_err(), "invalid Ollama URL should be rejected");
441    }
442
443    #[tokio::test]
444    async fn any_provider_delegates_to_mock() {
445        let mock = Arc::new(StdMutex::new(MockProvider::new()));
446        mock.lock().unwrap().enqueue_text("via any");
447        let mut provider = AnyProvider::Mock(MockProviderHandle::new(mock));
448
449        assert_eq!(provider.current_model().as_deref(), Some("mock-model"));
450        provider.select_model("m2".to_string());
451        assert_eq!(provider.current_model().as_deref(), Some("m2"));
452
453        let mut rx = provider
454            .chat(vec![Message::simple(Role::User, "hi")], vec![])
455            .await
456            .expect("chat via AnyProvider should work");
457        let chunk = rx.recv().await.expect("chunk");
458        assert!(chunk.done);
459        assert_eq!(chunk.message.content, "via any");
460    }
461
462    #[tokio::test]
463    async fn any_provider_health_check_delegates() {
464        let mut mock = MockProvider::new();
465        assert!(mock.health_check().await.is_ok());
466        mock.set_health_ok(false);
467        assert!(mock.health_check().await.is_err());
468    }
469}