Skip to main content

tinyharness_lib/provider/
mod.rs

1pub mod llama_cpp;
2pub mod ollama;
3pub mod openai_compat;
4pub mod vllm;
5
6use std::fmt::Display;
7use std::future::Future;
8use std::pin::Pin;
9
10use serde::{Deserialize, Serialize};
11
12use crate::config::OllamaThinkType;
13
14#[derive(Debug, Clone, Serialize, Deserialize)]
15pub struct ToolDefinition {
16    pub name: String,
17    pub description: String,
18    pub parameters: schemars::Schema,
19}
20
21#[derive(Debug, Clone, Serialize, Deserialize)]
22pub struct ToolCall {
23    pub function: ToolCallFunction,
24}
25
26#[derive(Debug, Clone, Serialize, Deserialize)]
27pub struct ToolCallFunction {
28    pub name: String,
29    pub arguments: serde_json::Value,
30    /// Gemini `thought_signature` required for multi-turn tool calling.
31    /// Gemini returns this on tool calls and requires it back in subsequent
32    /// turns. Ollama Cloud doesn't preserve it, so we capture and re-inject it.
33    #[serde(skip_serializing_if = "Option::is_none", default)]
34    pub thought_signature: Option<String>,
35}
36
37#[derive(Debug, Clone, Serialize, Deserialize)]
38pub struct ChatMessage {
39    pub content: String,
40    #[serde(default)]
41    pub tool_calls: Vec<ToolCall>,
42    /// Thinking/reasoning content from the model (Ollama's `thinking` field).
43    /// Only populated when the model supports reasoning (e.g. qwen2.5 variants).
44    #[serde(default)]
45    pub thinking: Option<String>,
46}
47
48#[derive(Debug, Clone, Deserialize)]
49pub struct ChatMessageResponse {
50    pub message: ChatMessage,
51    pub done: bool,
52    #[serde(default)]
53    pub is_error: bool,
54    #[serde(default)]
55    pub usage: Option<TokenUsage>,
56}
57
58/// Token usage information from the provider.
59#[derive(Debug, Clone, Default, Deserialize, Serialize)]
60pub struct TokenUsage {
61    pub prompt_tokens: u32,
62    pub completion_tokens: u32,
63    pub total_tokens: u32,
64}
65
66#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
67pub enum Role {
68    System,
69    User,
70    Assistant,
71    Tool,
72}
73
74impl Display for Role {
75    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
76        match self {
77            Role::System => f.write_str("System"),
78            Role::User => f.write_str("User"),
79            Role::Assistant => f.write_str("Assistant"),
80            Role::Tool => f.write_str("Tool"),
81        }
82    }
83}
84
85#[derive(Debug, Clone, Serialize, Deserialize)]
86pub struct Message {
87    pub role: Role,
88    pub content: String,
89    pub tool_calls: Vec<ToolCall>,
90}
91
92pub trait Provider: Send + Sync {
93    /// Check whether the backend is reachable and healthy.
94    fn health_check(&self) -> Pin<Box<dyn Future<Output = Result<(), String>> + Send>>;
95
96    fn list_models(&self) -> Pin<Box<dyn Future<Output = Vec<String>> + Send>>;
97
98    fn select_model(&mut self, name: String);
99
100    fn current_model(&self) -> Option<String>;
101
102    /// Send a chat request and return a receiver for streaming response chunks.
103    ///
104    /// Returns `Err(String)` if the request cannot be started (e.g. no model selected).
105    /// On success, the provider spawns a background task that streams `ChatMessageResponse`
106    /// chunks through the returned receiver. The caller drains the receiver until it
107    /// receives a chunk with `done: true`.
108    ///
109    /// Token usage, when available, is included in the final `ChatMessageResponse`
110    /// chunk (in the `usage` field). No separate method is needed to retrieve it.
111    fn chat(
112        &mut self,
113        messages: Vec<Message>,
114        tools: Vec<ToolDefinition>,
115    ) -> Pin<
116        Box<
117            dyn Future<Output = Result<tokio::sync::mpsc::Receiver<ChatMessageResponse>, String>>
118                + Send,
119        >,
120    >;
121
122    /// Set the request timeout in seconds. Only meaningful for providers that use timeouts.
123    fn set_timeout(&mut self, _timeout_secs: u64) {}
124
125    /// Set the maximum number of retries. Only meaningful for providers that use retries.
126    fn set_retries(&mut self, _max_retries: u32) {}
127
128    /// Set the think/reasoning level. Only meaningful for Ollama.
129    fn set_think_type(&mut self, _think_type: OllamaThinkType) {}
130}