Skip to main content

tinyharness_lib/provider/
mod.rs

1#[cfg(feature = "test-util")]
2pub mod mock;
3pub mod model_variants;
4pub mod openai_compat;
5pub mod openai_compat_provider;
6pub mod sockudo;
7pub mod tool_pairs;
8
9use std::fmt::Display;
10use std::future::Future;
11
12use serde::{Deserialize, Serialize};
13
14use crate::SecretString;
15use crate::config::Settings;
16use crate::image::ImageAttachment;
17use crate::provider::openai_compat_provider::OpenAiCompatProvider;
18use crate::provider::sockudo::SockudoProvider;
19
20/// Channel capacity for streaming chat responses from providers to the
21/// agent loop.
22///
23/// A small buffer applies backpressure to the provider's HTTP read loop
24/// when the UI consumes chunks slowly: once the buffer is full,
25/// `send().await` suspends the provider task instead of letting chunks
26/// accumulate without bound. 64 chunks is far more than a terminal renders
27/// per frame, so streaming throughput is unaffected in practice.
28pub const STREAM_CHANNEL_CAPACITY: usize = 64;
29
30#[derive(Debug, Clone, Serialize, Deserialize)]
31pub struct ToolDefinition {
32    pub name: String,
33    pub description: String,
34    pub parameters: schemars::Schema,
35}
36
37#[derive(Debug, Clone, Serialize, Deserialize)]
38pub struct ToolCall {
39    /// OpenAI tool call ID (e.g. `call_abc123`). Required by OpenAI-compatible
40    /// servers when echoing the call back in subsequent assistant messages
41    /// and when building matching `tool` result messages. Some servers generate
42    /// it for us; others (or local models) may not, in which case we fall
43    /// back to a synthetic id at serialization time.
44    #[serde(default, skip_serializing_if = "Option::is_none")]
45    pub id: Option<String>,
46    pub function: ToolCallFunction,
47}
48
49#[derive(Debug, Clone, Serialize, Deserialize)]
50pub struct ToolCallFunction {
51    pub name: String,
52    pub arguments: serde_json::Value,
53    /// Gemini `thought_signature` required for multi-turn tool calling.
54    /// Gemini returns this on tool calls and requires it back in subsequent
55    /// turns. Ollama Cloud doesn't preserve it, so we capture and re-inject it.
56    #[serde(skip_serializing_if = "Option::is_none", default)]
57    pub thought_signature: Option<String>,
58}
59
60#[derive(Debug, Clone, Serialize, Deserialize)]
61pub struct ChatMessage {
62    pub content: String,
63    #[serde(default)]
64    pub tool_calls: Vec<ToolCall>,
65    /// Thinking/reasoning content from the model, as a **delta** during
66    /// streaming. Ollama fills this from its `thinking` field; OpenAI-compatible
67    /// servers (OpenRouter, vLLM with a reasoning parser, DeepSeek) from
68    /// `delta.reasoning` / `delta.reasoning_content` / `delta.reasoning_details`.
69    /// Only populated when the model actually reasons.
70    #[serde(default)]
71    pub thinking: Option<String>,
72}
73
74#[derive(Debug, Clone, Deserialize)]
75pub struct ChatMessageResponse {
76    pub message: ChatMessage,
77    pub done: bool,
78    #[serde(default)]
79    pub is_error: bool,
80    #[serde(default)]
81    pub usage: Option<TokenUsage>,
82}
83
84/// Token usage information from the provider.
85#[derive(Debug, Clone, Default, Deserialize, Serialize)]
86pub struct TokenUsage {
87    pub prompt_tokens: u32,
88    pub completion_tokens: u32,
89    pub total_tokens: u32,
90}
91
92#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
93pub enum Role {
94    System,
95    User,
96    Assistant,
97    Tool,
98}
99
100impl Display for Role {
101    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
102        match self {
103            Role::System => f.write_str("System"),
104            Role::User => f.write_str("User"),
105            Role::Assistant => f.write_str("Assistant"),
106            Role::Tool => f.write_str("Tool"),
107        }
108    }
109}
110
111#[derive(Debug, Clone, Serialize, Deserialize)]
112pub struct Message {
113    pub role: Role,
114    pub content: String,
115    pub tool_calls: Vec<ToolCall>,
116    /// OpenAI `tool_call_id` for `Role::Tool` messages — links the result back
117    /// to the originating assistant tool call. Required by OpenAI-compatible
118    /// servers. `None` is allowed for round-trips with providers that don't
119    /// use it, but the OpenAI-compatible serialiser will substitute a
120    /// synthetic id when missing.
121    #[serde(default, skip_serializing_if = "Option::is_none")]
122    pub tool_call_id: Option<String>,
123    /// Optional images attached to the message (multimodal models).
124    /// Only meaningful for `User` role messages.
125    #[serde(default, skip_serializing_if = "Vec::is_empty")]
126    pub images: Vec<ImageAttachment>,
127    /// Thinking/reasoning chain from the model, captured during streaming
128    /// and persisted for debugging. Not sent to the provider in API requests
129    /// (skipped in serialization for request types; only stored in sessions).
130    #[serde(default, skip_serializing_if = "Option::is_none")]
131    pub thinking: Option<String>,
132}
133
134impl Default for Message {
135    fn default() -> Self {
136        Message::simple(Role::User, "")
137    }
138}
139
140impl Message {
141    /// Create a new message with the given role and content, no tool calls, no images.
142    pub fn simple(role: Role, content: impl Into<String>) -> Self {
143        Message {
144            role,
145            content: content.into(),
146            tool_calls: vec![],
147            tool_call_id: None,
148            images: vec![],
149            thinking: None,
150        }
151    }
152}
153
154/// Trait for LLM provider backends (OpenAI-compatible endpoints, Sockudo).
155///
156/// Async methods return `impl Future<...> + Send` (RPITIT) rather than
157/// `Pin<Box<dyn Future>>`: no allocation, no extra crates. The trade-off is
158/// that the trait is not dyn-compatible — for dynamic dispatch over a
159/// runtime-selected backend use [`AnyProvider`], the concrete enum covering
160/// all built-in providers.
161///
162/// Implementations are expected to be cheap to clone where the provider
163/// spawns background streaming tasks (see [`Provider::chat`]).
164pub trait Provider: Send + Sync {
165    /// Check whether the backend is reachable and healthy.
166    ///
167    /// Called once at startup; a failure is a non-fatal warning — errors
168    /// surface on the first real request. Providers whose dedicated health
169    /// endpoint may be missing (OpenAI-compatible servers) fall back to the
170    /// model list: a non-empty list proves the backend is up.
171    fn health_check(&self) -> impl Future<Output = Result<(), String>> + Send;
172
173    /// List model names available on the backend.
174    ///
175    /// Returns an empty vec when the backend doesn't expose a model list or
176    /// the request fails; callers fall back to saved or first-known models.
177    fn list_models(&self) -> impl Future<Output = Vec<String>> + Send;
178
179    /// Select the model used for subsequent [`Provider::chat`] requests.
180    fn select_model(&mut self, name: String);
181
182    /// The currently selected model, if any.
183    fn current_model(&self) -> Option<String>;
184
185    /// Send a chat request and return a receiver for streaming response chunks.
186    ///
187    /// Returns `Err(String)` if the request cannot be started (e.g. no model selected).
188    /// On success, the provider spawns a background task that streams `ChatMessageResponse`
189    /// chunks through the returned receiver. The caller drains the receiver until it
190    /// receives a chunk with `done: true`; dropping the receiver stops the stream.
191    ///
192    /// Token usage, when available, is included in the final `ChatMessageResponse`
193    /// chunk (in the `usage` field). No separate method is needed to retrieve it.
194    fn chat(
195        &mut self,
196        messages: Vec<Message>,
197        tools: Vec<ToolDefinition>,
198    ) -> impl Future<Output = Result<tokio::sync::mpsc::Receiver<ChatMessageResponse>, String>> + Send;
199
200    /// Set the request timeout in seconds. Only meaningful for providers that use timeouts.
201    fn set_timeout(&mut self, _timeout_secs: u64) {}
202
203    /// Set the maximum number of retries. Only meaningful for providers that use retries.
204    fn set_retries(&mut self, _max_retries: u32) {}
205}
206
207/// Concrete wrapper over all built-in provider backends.
208///
209/// The [`Provider`] trait is not dyn-compatible (its async methods use
210/// RPITIT return types), so dynamic dispatch over a runtime-selected
211/// backend goes through this enum instead of `Arc<Mutex<dyn Provider>>`.
212/// Commands and the agent loop share an `Arc<Mutex<AnyProvider>>`.
213pub enum AnyProvider {
214    OpenAiCompat(crate::provider::openai_compat_provider::OpenAiCompatProvider),
215    Sockudo(crate::provider::sockudo::SockudoProvider),
216    /// Test-only variant. Constructed via `MockProviderHandle` (feature
217    /// `test-util`); never built in production code paths.
218    #[cfg(feature = "test-util")]
219    Mock(crate::provider::mock::MockProviderHandle),
220}
221
222impl AnyProvider {
223    /// Build the provider for `kind` from resolved settings values.
224    ///
225    /// `url` and credentials come from the caller (CLI flags / settings);
226    /// `timeout_secs`/`max_retries` are the effective per-provider values
227    /// from `Settings::effective_timeout_secs()`/`effective_max_retries()`.
228    #[allow(clippy::too_many_arguments)]
229    pub fn build(
230        kind: crate::config::ProviderKind,
231        base_url: String,
232        api_key: Option<SecretString>,
233        timeout_secs: u64,
234        max_retries: u32,
235        sockudo: SockudoCredentials,
236    ) -> Result<Self, String> {
237        Ok(match kind {
238            // Ollama serves the OpenAI-compatible API at `/v1`, so it goes
239            // through the same provider as llama.cpp / vLLM / hosted gateways.
240            crate::config::ProviderKind::Ollama => AnyProvider::OpenAiCompat(
241                OpenAiCompatProvider::with_options(base_url, None, timeout_secs, max_retries),
242            ),
243            crate::config::ProviderKind::LlamaCpp => AnyProvider::OpenAiCompat(
244                OpenAiCompatProvider::with_options(base_url, None, timeout_secs, max_retries)
245                    .with_static_models(vec!["llama-cpp".to_string()]),
246            ),
247            crate::config::ProviderKind::Vllm => AnyProvider::OpenAiCompat(
248                OpenAiCompatProvider::with_options(base_url, None, timeout_secs, max_retries),
249            ),
250            crate::config::ProviderKind::OpenAiCompat => {
251                // Empty key = explicit no-auth (e.g. `--api-key ""`).
252                let key = api_key.filter(|k| !k.is_empty());
253                AnyProvider::OpenAiCompat(OpenAiCompatProvider::with_options(
254                    base_url,
255                    key,
256                    timeout_secs,
257                    max_retries,
258                ))
259            }
260            crate::config::ProviderKind::Sockudo => AnyProvider::Sockudo(SockudoProvider::new(
261                base_url,
262                sockudo.app_id,
263                sockudo.app_key,
264                sockudo.app_secret.unwrap_or_default(),
265            )),
266        })
267    }
268}
269
270/// Credentials needed to construct the Sockudo provider.
271#[derive(Debug, Clone, Default)]
272pub struct SockudoCredentials {
273    pub app_id: String,
274    pub app_key: String,
275    pub app_secret: Option<SecretString>,
276}
277
278impl From<&Settings> for SockudoCredentials {
279    fn from(settings: &Settings) -> Self {
280        SockudoCredentials {
281            app_id: settings.sockudo_app_id.clone().unwrap_or_default(),
282            app_key: settings.sockudo_app_key.clone().unwrap_or_default(),
283            app_secret: settings.sockudo_app_secret.clone(),
284        }
285    }
286}
287
288impl Provider for AnyProvider {
289    async fn health_check(&self) -> Result<(), String> {
290        match self {
291            AnyProvider::OpenAiCompat(p) => p.health_check().await,
292            AnyProvider::Sockudo(p) => p.health_check().await,
293            #[cfg(feature = "test-util")]
294            AnyProvider::Mock(p) => p.health_check().await,
295        }
296    }
297
298    async fn list_models(&self) -> Vec<String> {
299        match self {
300            AnyProvider::OpenAiCompat(p) => p.list_models().await,
301            AnyProvider::Sockudo(p) => p.list_models().await,
302            #[cfg(feature = "test-util")]
303            AnyProvider::Mock(p) => p.list_models().await,
304        }
305    }
306
307    fn select_model(&mut self, name: String) {
308        match self {
309            AnyProvider::OpenAiCompat(p) => p.select_model(name),
310            AnyProvider::Sockudo(p) => p.select_model(name),
311            #[cfg(feature = "test-util")]
312            AnyProvider::Mock(p) => p.select_model(name),
313        }
314    }
315
316    fn current_model(&self) -> Option<String> {
317        match self {
318            AnyProvider::OpenAiCompat(p) => p.current_model(),
319            AnyProvider::Sockudo(p) => p.current_model(),
320            #[cfg(feature = "test-util")]
321            AnyProvider::Mock(p) => p.current_model(),
322        }
323    }
324
325    async fn chat(
326        &mut self,
327        messages: Vec<Message>,
328        tools: Vec<ToolDefinition>,
329    ) -> Result<tokio::sync::mpsc::Receiver<ChatMessageResponse>, String> {
330        match self {
331            AnyProvider::OpenAiCompat(p) => p.chat(messages, tools).await,
332            AnyProvider::Sockudo(p) => p.chat(messages, tools).await,
333            #[cfg(feature = "test-util")]
334            AnyProvider::Mock(p) => p.chat(messages, tools).await,
335        }
336    }
337
338    fn set_timeout(&mut self, timeout_secs: u64) {
339        match self {
340            AnyProvider::OpenAiCompat(p) => p.set_timeout(timeout_secs),
341            AnyProvider::Sockudo(p) => p.set_timeout(timeout_secs),
342            #[cfg(feature = "test-util")]
343            AnyProvider::Mock(p) => p.set_timeout(timeout_secs),
344        }
345    }
346
347    fn set_retries(&mut self, max_retries: u32) {
348        match self {
349            AnyProvider::OpenAiCompat(p) => p.set_retries(max_retries),
350            AnyProvider::Sockudo(p) => p.set_retries(max_retries),
351            #[cfg(feature = "test-util")]
352            AnyProvider::Mock(p) => p.set_retries(max_retries),
353        }
354    }
355}
356
357#[cfg(test)]
358mod tests {
359    use super::*;
360    use crate::provider::mock::{MockProvider, MockProviderHandle};
361    use std::sync::{Arc, Mutex as StdMutex};
362
363    #[test]
364    fn build_returns_provider_for_each_kind() {
365        let creds = SockudoCredentials::default();
366        for kind in [
367            crate::config::ProviderKind::Ollama,
368            crate::config::ProviderKind::LlamaCpp,
369            crate::config::ProviderKind::Vllm,
370            crate::config::ProviderKind::OpenAiCompat,
371            crate::config::ProviderKind::Sockudo,
372        ] {
373            let provider = AnyProvider::build(
374                kind,
375                "http://localhost:1234".to_string(),
376                Some(SecretString::new("k")),
377                30,
378                0,
379                creds.clone(),
380            )
381            .unwrap_or_else(|e| panic!("build({kind}) failed: {e}"));
382            let _ = provider.current_model();
383        }
384    }
385
386    #[tokio::test]
387    async fn any_provider_delegates_to_mock() {
388        let mock = Arc::new(StdMutex::new(MockProvider::new()));
389        mock.lock().unwrap().enqueue_text("via any");
390        let mut provider = AnyProvider::Mock(MockProviderHandle::new(mock));
391
392        assert_eq!(provider.current_model().as_deref(), Some("mock-model"));
393        provider.select_model("m2".to_string());
394        assert_eq!(provider.current_model().as_deref(), Some("m2"));
395
396        let mut rx = provider
397            .chat(vec![Message::simple(Role::User, "hi")], vec![])
398            .await
399            .expect("chat via AnyProvider should work");
400        let chunk = rx.recv().await.expect("chunk");
401        assert!(chunk.done);
402        assert_eq!(chunk.message.content, "via any");
403    }
404
405    #[tokio::test]
406    async fn any_provider_health_check_delegates() {
407        let mut mock = MockProvider::new();
408        assert!(mock.health_check().await.is_ok());
409        mock.set_health_ok(false);
410        assert!(mock.health_check().await.is_err());
411    }
412}