steeldb/agent/provider.rs
1//! The provider abstraction: any chat model with tool-use, behind one async trait. Bedrock (Claude)
2//! and Paddock (local Qwen) both implement it; the agent loop is provider-agnostic.
3
4use crate::agent::types::{Msg, ToolSpec, Turn};
5use async_trait::async_trait;
6
7#[async_trait]
8pub trait LlmProvider: Send + Sync {
9 /// One turn: given the system prompt, conversation, and available tools, return the model's reply
10 /// (assistant text and/or tool calls).
11 async fn chat(&self, system: &str, msgs: &[Msg], tools: &[ToolSpec]) -> Result<Turn, String>;
12 fn name(&self) -> &str;
13
14 /// One turn whose output is **constrained to a JSON schema** by the decoder, rather than requested by a
15 /// tool definition and hoped for.
16 ///
17 /// This exists because tool calling is a capability many small models simply lack: asked to call a tool,
18 /// `qwen2.5:0.5b` returns `tool_calls: null` every time and answers in prose, so a caller that needs
19 /// structured output has nothing to work with. Constraining the sampler to a grammar needs no such
20 /// capability — the model can only emit tokens the schema permits.
21 ///
22 /// Returns `Ok(None)` when this provider cannot constrain decoding, so a caller can fall back rather than
23 /// treating it as an error. The default is `Ok(None)`, which keeps every existing provider unaffected.
24 async fn chat_json(
25 &self,
26 system: &str,
27 msgs: &[Msg],
28 schema: &serde_json::Value,
29 name: &str,
30 ) -> Result<Option<serde_json::Value>, String> {
31 let _ = (system, msgs, schema, name);
32 Ok(None)
33 }
34
35 /// Whether this provider produces trustworthy natural-language *synthesis* of tool results. Large
36 /// models (Bedrock, big OpenAI-compatible) do; small on-device tool-callers (Needle, the native
37 /// 1.7B) do NOT — for those the agent renders a deterministic template from the program result
38 /// instead of trusting the model to phrase the numbers. Default `true`.
39 fn synthesizes(&self) -> bool {
40 true
41 }
42}