haest_core/provider.rs
1//! The seam between the loop and a model API. An adapter turns a neutral
2//! request ([`ProviderRequest`]) into its wire format and yields neutral stream
3//! events ([`ProviderEvent`]), ending with exactly one
4//! [`Done`](ProviderEvent::Done) carrying the complete assistant message.
5//! Adapters never fail mid-stream: an API error ends with `Done` and
6//! [`StopReason::Error`](crate::StopReason::Error), so the loop has one code path for every failure.
7//!
8//! PLUG POINT: implement [`Provider`] for a new model API, or [`ModelResolver`]
9//! to decide which provider serves a `provider:model` spec. `haest-providers`
10//! has the adapters (Anthropic Messages, OpenAI Chat Completions for
11//! OpenRouter and any compatible server, OpenAI Responses) and a registry
12//! that resolves specs; `testing::ScriptedProvider` (the `testing` feature)
13//! answers from a script, for tests.
14//!
15//! Also here: the request's parts ([`ToolSpec`], [`Effort`],
16//! [`WebSearchOptions`]) and [`parse_model_spec`], which adapters share.
17//!
18//! # Writing a provider
19//!
20//! What every adapter in `haest-providers` uses, for one of your own:
21//!
22//! - [`mark_stream_failed`]: the `Done` of a call that failed or was aborted.
23//! - [`nearest_accepted_effort`]: the reasoning level the model's catalog row
24//! accepts, for an API that refuses any level it does not list.
25//! - [`book_priced_cost`]: a cost from the configured price, for an API that
26//! returns none.
27//! - [`data_url`](crate::media::data_url) and
28//! [`dropped_note`](crate::media::dropped_note): media the wire format
29//! carries as a `data:` URL, or cannot carry at all.
30
31use futures::{StreamExt, stream::BoxStream};
32use serde::{Deserialize, Serialize};
33use serde_json::Value;
34use tokio_util::sync::CancellationToken;
35
36pub use adapter::mark_stream_failed;
37pub use answer::{Deltas, answer_stream};
38pub use resolve::{ModelError, ModelResolver, ModelResolverRef, ResolvedModel, parse_model_spec};
39
40pub use crate::{effort::nearest_accepted_effort, pricing::book_priced_cost};
41
42use crate::messages::{AssistantMessage, Message};
43
44crate::word_enum! {
45 /// haest's reasoning levels, cheapest first. `Off` turns reasoning off
46 /// where the model allows it.
47 ///
48 /// A request carries one as asked; the loop fits it to what the model
49 /// accepts, per request. Written as its word (`"high"`) everywhere.
50 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
51 #[non_exhaustive]
52 pub enum Effort ("effort") {
53 /// No reasoning, where the model can stop.
54 Off = "off",
55 /// The least reasoning.
56 Low = "low",
57 /// More than `low`.
58 Medium = "medium",
59 /// More than `medium`.
60 High = "high",
61 /// More than `high`.
62 Xhigh = "xhigh",
63 /// The most the model does.
64 Max = "max",
65 }
66}
67
68/// A tool as the model sees it: what a provider puts in the request's tool list.
69#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
70pub struct ToolSpec {
71 /// The name the model calls it by.
72 pub name: String,
73 /// What it does and when to use it, for the model.
74 pub description: String,
75 /// JSON Schema of the input object.
76 pub parameters: Value,
77}
78
79crate::word_enum! {
80 /// A provider-side web search (OpenRouter's `openrouter:web_search` server tool).
81 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
82 #[non_exhaustive]
83 pub enum WebSearchEngine ("web search engine") {
84 /// The model's native search where it has one, else Exa.
85 Auto = "auto",
86 /// The model vendor's own search.
87 Native = "native",
88 /// Exa.
89 Exa = "exa",
90 /// Parallel.
91 Parallel = "parallel",
92 /// Firecrawl.
93 Firecrawl = "firecrawl",
94 /// Perplexity.
95 Perplexity = "perplexity",
96 }
97}
98
99impl WebSearchOptions {
100 /// `engine` with haest's result caps: 5 a search, 15 across one model call.
101 pub fn new(engine: WebSearchEngine) -> Self {
102 Self {
103 engine,
104 max_results: 5,
105 max_total_results: 15,
106 }
107 }
108}
109
110/// [`WebSearchEngine::Auto`] with haest's result caps, as
111/// [`new`](WebSearchOptions::new) sets them.
112impl Default for WebSearchOptions {
113 fn default() -> Self {
114 Self::new(WebSearchEngine::Auto)
115 }
116}
117
118/// A provider-side web search for every model call ([`WebSearchOptions::new`]
119/// sets haest's caps). Only providers that run one (OpenRouter) send it;
120/// the rest ignore it.
121///
122/// Plain data: set the fields a host cares about and take the rest from
123/// [`Default`], so a setting added later breaks no host.
124///
125/// ```
126/// use haest_core::provider::{WebSearchEngine, WebSearchOptions};
127///
128/// let exa = WebSearchOptions {
129/// engine: WebSearchEngine::Exa,
130/// max_total_results: 10,
131/// ..WebSearchOptions::default()
132/// };
133/// assert_eq!(exa.max_results, 5);
134/// ```
135#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
136#[serde(rename_all = "camelCase")]
137pub struct WebSearchOptions {
138 /// Which search engine runs it.
139 pub engine: WebSearchEngine,
140 /// Results per search.
141 pub max_results: u32,
142 /// Results across every search of one model call: the cost and context bound.
143 pub max_total_results: u32,
144}
145
146/// One model call, provider-neutral: everything an adapter turns into its wire
147/// request.
148///
149/// Built with [`ProviderRequest::new`] and a `with_*` per optional part, so a
150/// part added later breaks no caller; an adapter reads the fields.
151#[derive(Debug, Clone)]
152#[non_exhaustive]
153pub struct ProviderRequest {
154 /// The provider's own model id (the spec without `provider:`).
155 pub model: String,
156 /// The system prompt.
157 pub system: String,
158 /// The history. The loop sends it fitted to what the model accepts
159 /// (media it cannot take is withheld, with a note).
160 pub messages: Vec<Message>,
161 /// The tools the model may call; empty = none.
162 pub tools: Vec<ToolSpec>,
163 /// The turn's interrupt. When it fires, end the stream soon, with a
164 /// [`ProviderEvent::Done`] whose stop reason is [`StopReason::Aborted`](crate::StopReason::Aborted).
165 pub cancel: CancellationToken,
166 /// None = send nothing and let the model use its own default.
167 pub effort: Option<Effort>,
168 /// The output cap; None = the adapter's or the model's default.
169 pub max_tokens: Option<u32>,
170 /// Only providers that run one (OpenRouter) send it.
171 pub web_search: Option<WebSearchOptions>,
172}
173
174impl ProviderRequest {
175 /// A call to `model` with `messages`: no system prompt, no tools, a fresh
176 /// cancel token, and the model's own effort, output cap and search.
177 pub fn new(model: impl Into<String>, messages: Vec<Message>) -> Self {
178 Self {
179 model: model.into(),
180 system: String::new(),
181 messages,
182 tools: Vec::new(),
183 cancel: CancellationToken::new(),
184 effort: None,
185 max_tokens: None,
186 web_search: None,
187 }
188 }
189
190 /// With `system` as the system prompt.
191 #[must_use]
192 pub fn with_system(mut self, system: impl Into<String>) -> Self {
193 self.system = system.into();
194 self
195 }
196
197 /// With `tools` offered to the model.
198 #[must_use]
199 pub fn with_tools(mut self, tools: Vec<ToolSpec>) -> Self {
200 self.tools = tools;
201 self
202 }
203
204 /// Interrupted by `cancel`.
205 #[must_use]
206 pub fn with_cancel(mut self, cancel: CancellationToken) -> Self {
207 self.cancel = cancel;
208 self
209 }
210
211 /// At reasoning `effort`. Without it nothing is sent: the model's default.
212 #[must_use]
213 pub fn with_effort(mut self, effort: Effort) -> Self {
214 self.effort = Some(effort);
215 self
216 }
217
218 /// With at most `max_tokens` of output. Without it, the adapter's or the
219 /// model's default cap.
220 #[must_use]
221 pub fn with_max_tokens(mut self, max_tokens: u32) -> Self {
222 self.max_tokens = Some(max_tokens);
223 self
224 }
225
226 /// With a provider-side web search. Without it, none is sent.
227 #[must_use]
228 pub fn with_web_search(mut self, web_search: WebSearchOptions) -> Self {
229 self.web_search = Some(web_search);
230 self
231 }
232}
233
234/// One event of a model call's stream. The deltas are for showing progress;
235/// only [`ProviderEvent::Done`]'s message goes into history.
236#[derive(Debug, Clone, PartialEq)]
237#[non_exhaustive]
238pub enum ProviderEvent {
239 /// Answer text as it arrives.
240 TextDelta(String),
241 /// Reasoning text as it arrives, where the provider shows any.
242 ThinkingDelta(String),
243 /// A tool call began in the stream: the text before it was narration.
244 ToolCallStart {
245 /// The call's id, as in the final [`ToolCall`](crate::ToolCall).
246 id: String,
247 /// The tool's name.
248 name: String,
249 },
250 /// Always last, exactly once: the whole assistant message, its usage and
251 /// its stop reason, also when the call failed.
252 Done(AssistantMessage),
253}
254
255/// What [`Provider::stream`] returns: a boxed, `'static`, `Send` stream of
256/// [`ProviderEvent`]s.
257pub type ProviderStream = BoxStream<'static, ProviderEvent>;
258
259/// Why a model call has no answer when its stream ended without a
260/// [`ProviderEvent::Done`].
261pub(crate) const NO_RESPONSE: &str = "provider ended without a response";
262
263/// The answer `stream` ends with, every event before it handed to `delta`;
264/// None when the stream ended without a [`ProviderEvent::Done`] ([`NO_RESPONSE`]).
265pub(crate) async fn final_message(
266 mut stream: ProviderStream,
267 mut delta: impl FnMut(ProviderEvent),
268) -> Option<AssistantMessage> {
269 let mut done = None;
270 while let Some(ev) = stream.next().await {
271 match ev {
272 ProviderEvent::Done(m) => done = Some(m),
273 other => delta(other),
274 }
275 }
276 done
277}
278
279/// A model API: turns a [`ProviderRequest`] into a stream of
280/// [`ProviderEvent`]s.
281///
282/// The contract:
283/// - [`Provider::stream`] returns at once; the work runs as the stream is
284/// polled.
285/// - The stream ends with exactly one [`ProviderEvent::Done`], also on
286/// failure: an API or network error is a `Done` with
287/// [`StopReason::Error`](crate::StopReason::Error) and the reason in
288/// `error_message`, never a panic or a stream that stops short. A stream
289/// that ends without `Done` reaches history as an error message.
290/// - The loop reads the stream to its end and does not watch `req.cancel`
291/// itself: when it fires, the adapter ends the stream with a `Done` whose
292/// stop reason is [`StopReason::Aborted`](crate::StopReason::Aborted).
293///
294/// # Examples
295///
296/// A provider that answers every request with the last user message, built
297/// with [`answer_stream`], which ends the stream with the answer the async
298/// block returns:
299///
300/// ```
301/// use haest_core::{
302/// AssistantMessage, AssistantPart, Message, Provider, ProviderEvent, ProviderRequest,
303/// ProviderStream, answer_stream,
304/// };
305///
306/// struct Echo;
307///
308/// impl Provider for Echo {
309/// fn id(&self) -> &str {
310/// "echo"
311/// }
312///
313/// fn stream(&self, req: ProviderRequest) -> ProviderStream {
314/// let said = match req.messages.last() {
315/// Some(Message::User(u)) => u.content.clone(),
316/// _ => String::new(),
317/// };
318/// let mut answer = AssistantMessage::start(self.id(), &req.model);
319/// answer_stream(|deltas| async move {
320/// // A real provider sends each piece as its API streams it.
321/// deltas.text(said.as_str());
322/// answer.content.push(AssistantPart::Text { text: said });
323/// answer
324/// })
325/// }
326/// }
327///
328/// # #[tokio::main(flavor = "current_thread")]
329/// # async fn main() -> Result<(), haest_core::BoxError> {
330/// use futures::StreamExt;
331///
332/// let req = ProviderRequest::new("echo-1", vec![Message::user("hello")])
333/// .with_system("Repeat the user.");
334/// let events: Vec<ProviderEvent> = Echo.stream(req).collect().await;
335/// assert_eq!(events[0], ProviderEvent::TextDelta("hello".into()));
336/// let Some(ProviderEvent::Done(answer)) = events.last() else {
337/// return Err("Done comes last".into());
338/// };
339/// assert_eq!((answer.text(), answer.provider.as_str()), ("hello".into(), "echo"));
340/// # Ok(())
341/// # }
342/// ```
343pub trait Provider: Send + Sync {
344 /// Stable id stored on every assistant message ("anthropic", "openrouter", ...).
345 fn id(&self) -> &str;
346 /// Start the call. Must end with exactly one [`ProviderEvent::Done`], also on failure.
347 fn stream(&self, req: ProviderRequest) -> ProviderStream;
348}
349
350crate::plug_ref! {
351 /// A [`Provider`], shared: what a [`ResolvedModel`] holds. Built from the
352 /// provider itself, or from an `Arc` of one a resolver keeps.
353 pub struct ProviderRef(dyn Provider);
354 debug(provider, f) { f.debug_tuple("ProviderRef").field(&provider.id()).finish() }
355}
356
357mod adapter;
358mod answer;
359mod resolve;
360#[cfg(test)]
361mod tests;