Skip to main content

haest_core/
provider.rs

1//! The seam between the loop and a model API. An adapter turns a neutral
2//! request ([`ProviderRequest`]) into its wire format and yields neutral stream
3//! events ([`ProviderEvent`]), ending with exactly one
4//! [`Done`](ProviderEvent::Done) carrying the complete assistant message.
5//! Adapters never fail mid-stream: an API error ends with `Done` and
6//! [`StopReason::Error`](crate::StopReason::Error), so the loop has one code path for every failure.
7//!
8//! PLUG POINT: implement [`Provider`] for a new model API, or [`ModelResolver`]
9//! to decide which provider serves a `provider:model` spec. `haest-providers`
10//! has the adapters (Anthropic Messages, OpenAI Chat Completions for
11//! OpenRouter and any compatible server, OpenAI Responses) and a registry
12//! that resolves specs; `testing::ScriptedProvider` (the `testing` feature)
13//! answers from a script, for tests.
14//!
15//! Also here: the request's parts ([`ToolSpec`], [`Effort`],
16//! [`WebSearchOptions`]) and [`parse_model_spec`], which adapters share.
17//!
18//! # Writing a provider
19//!
20//! What every adapter in `haest-providers` uses, for one of your own:
21//!
22//! - [`mark_stream_failed`]: the `Done` of a call that failed or was aborted.
23//! - [`nearest_accepted_effort`]: the reasoning level the model's catalog row
24//!   accepts, for an API that refuses any level it does not list.
25//! - [`book_priced_cost`]: a cost from the configured price, for an API that
26//!   returns none.
27//! - [`data_url`](crate::media::data_url) and
28//!   [`dropped_note`](crate::media::dropped_note): media the wire format
29//!   carries as a `data:` URL, or cannot carry at all.
30
31use futures::{StreamExt, stream::BoxStream};
32use serde::{Deserialize, Serialize};
33use serde_json::Value;
34use tokio_util::sync::CancellationToken;
35
36pub use adapter::mark_stream_failed;
37pub use answer::{Deltas, answer_stream};
38pub use resolve::{ModelError, ModelResolver, ModelResolverRef, ResolvedModel, parse_model_spec};
39
40pub use crate::{effort::nearest_accepted_effort, pricing::book_priced_cost};
41
42use crate::messages::{AssistantMessage, Message};
43
44crate::word_enum! {
45    /// haest's reasoning levels, cheapest first. `Off` turns reasoning off
46    /// where the model allows it.
47    ///
48    /// A request carries one as asked; the loop fits it to what the model
49    /// accepts, per request. Written as its word (`"high"`) everywhere.
50    #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
51    #[non_exhaustive]
52    pub enum Effort ("effort") {
53        /// No reasoning, where the model can stop.
54        Off = "off",
55        /// The least reasoning.
56        Low = "low",
57        /// More than `low`.
58        Medium = "medium",
59        /// More than `medium`.
60        High = "high",
61        /// More than `high`.
62        Xhigh = "xhigh",
63        /// The most the model does.
64        Max = "max",
65    }
66}
67
68/// A tool as the model sees it: what a provider puts in the request's tool list.
69#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
70pub struct ToolSpec {
71    /// The name the model calls it by.
72    pub name: String,
73    /// What it does and when to use it, for the model.
74    pub description: String,
75    /// JSON Schema of the input object.
76    pub parameters: Value,
77}
78
79crate::word_enum! {
80    /// A provider-side web search (OpenRouter's `openrouter:web_search` server tool).
81    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
82    #[non_exhaustive]
83    pub enum WebSearchEngine ("web search engine") {
84        /// The model's native search where it has one, else Exa.
85        Auto = "auto",
86        /// The model vendor's own search.
87        Native = "native",
88        /// Exa.
89        Exa = "exa",
90        /// Parallel.
91        Parallel = "parallel",
92        /// Firecrawl.
93        Firecrawl = "firecrawl",
94        /// Perplexity.
95        Perplexity = "perplexity",
96    }
97}
98
99impl WebSearchOptions {
100    /// `engine` with haest's result caps: 5 a search, 15 across one model call.
101    pub fn new(engine: WebSearchEngine) -> Self {
102        Self {
103            engine,
104            max_results: 5,
105            max_total_results: 15,
106        }
107    }
108}
109
110/// [`WebSearchEngine::Auto`] with haest's result caps, as
111/// [`new`](WebSearchOptions::new) sets them.
112impl Default for WebSearchOptions {
113    fn default() -> Self {
114        Self::new(WebSearchEngine::Auto)
115    }
116}
117
118/// A provider-side web search for every model call ([`WebSearchOptions::new`]
119/// sets haest's caps). Only providers that run one (OpenRouter) send it;
120/// the rest ignore it.
121///
122/// Plain data: set the fields a host cares about and take the rest from
123/// [`Default`], so a setting added later breaks no host.
124///
125/// ```
126/// use haest_core::provider::{WebSearchEngine, WebSearchOptions};
127///
128/// let exa = WebSearchOptions {
129///     engine: WebSearchEngine::Exa,
130///     max_total_results: 10,
131///     ..WebSearchOptions::default()
132/// };
133/// assert_eq!(exa.max_results, 5);
134/// ```
135#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
136#[serde(rename_all = "camelCase")]
137pub struct WebSearchOptions {
138    /// Which search engine runs it.
139    pub engine: WebSearchEngine,
140    /// Results per search.
141    pub max_results: u32,
142    /// Results across every search of one model call: the cost and context bound.
143    pub max_total_results: u32,
144}
145
146/// One model call, provider-neutral: everything an adapter turns into its wire
147/// request.
148///
149/// Built with [`ProviderRequest::new`] and a `with_*` per optional part, so a
150/// part added later breaks no caller; an adapter reads the fields.
151#[derive(Debug, Clone)]
152#[non_exhaustive]
153pub struct ProviderRequest {
154    /// The provider's own model id (the spec without `provider:`).
155    pub model: String,
156    /// The system prompt.
157    pub system: String,
158    /// The history. The loop sends it fitted to what the model accepts
159    /// (media it cannot take is withheld, with a note).
160    pub messages: Vec<Message>,
161    /// The tools the model may call; empty = none.
162    pub tools: Vec<ToolSpec>,
163    /// The turn's interrupt. When it fires, end the stream soon, with a
164    /// [`ProviderEvent::Done`] whose stop reason is [`StopReason::Aborted`](crate::StopReason::Aborted).
165    pub cancel: CancellationToken,
166    /// None = send nothing and let the model use its own default.
167    pub effort: Option<Effort>,
168    /// The output cap; None = the adapter's or the model's default.
169    pub max_tokens: Option<u32>,
170    /// Only providers that run one (OpenRouter) send it.
171    pub web_search: Option<WebSearchOptions>,
172}
173
174impl ProviderRequest {
175    /// A call to `model` with `messages`: no system prompt, no tools, a fresh
176    /// cancel token, and the model's own effort, output cap and search.
177    pub fn new(model: impl Into<String>, messages: Vec<Message>) -> Self {
178        Self {
179            model: model.into(),
180            system: String::new(),
181            messages,
182            tools: Vec::new(),
183            cancel: CancellationToken::new(),
184            effort: None,
185            max_tokens: None,
186            web_search: None,
187        }
188    }
189
190    /// With `system` as the system prompt.
191    #[must_use]
192    pub fn with_system(mut self, system: impl Into<String>) -> Self {
193        self.system = system.into();
194        self
195    }
196
197    /// With `tools` offered to the model.
198    #[must_use]
199    pub fn with_tools(mut self, tools: Vec<ToolSpec>) -> Self {
200        self.tools = tools;
201        self
202    }
203
204    /// Interrupted by `cancel`.
205    #[must_use]
206    pub fn with_cancel(mut self, cancel: CancellationToken) -> Self {
207        self.cancel = cancel;
208        self
209    }
210
211    /// At reasoning `effort`. Without it nothing is sent: the model's default.
212    #[must_use]
213    pub fn with_effort(mut self, effort: Effort) -> Self {
214        self.effort = Some(effort);
215        self
216    }
217
218    /// With at most `max_tokens` of output. Without it, the adapter's or the
219    /// model's default cap.
220    #[must_use]
221    pub fn with_max_tokens(mut self, max_tokens: u32) -> Self {
222        self.max_tokens = Some(max_tokens);
223        self
224    }
225
226    /// With a provider-side web search. Without it, none is sent.
227    #[must_use]
228    pub fn with_web_search(mut self, web_search: WebSearchOptions) -> Self {
229        self.web_search = Some(web_search);
230        self
231    }
232}
233
234/// One event of a model call's stream. The deltas are for showing progress;
235/// only [`ProviderEvent::Done`]'s message goes into history.
236#[derive(Debug, Clone, PartialEq)]
237#[non_exhaustive]
238pub enum ProviderEvent {
239    /// Answer text as it arrives.
240    TextDelta(String),
241    /// Reasoning text as it arrives, where the provider shows any.
242    ThinkingDelta(String),
243    /// A tool call began in the stream: the text before it was narration.
244    ToolCallStart {
245        /// The call's id, as in the final [`ToolCall`](crate::ToolCall).
246        id: String,
247        /// The tool's name.
248        name: String,
249    },
250    /// Always last, exactly once: the whole assistant message, its usage and
251    /// its stop reason, also when the call failed.
252    Done(AssistantMessage),
253}
254
255/// What [`Provider::stream`] returns: a boxed, `'static`, `Send` stream of
256/// [`ProviderEvent`]s.
257pub type ProviderStream = BoxStream<'static, ProviderEvent>;
258
259/// Why a model call has no answer when its stream ended without a
260/// [`ProviderEvent::Done`].
261pub(crate) const NO_RESPONSE: &str = "provider ended without a response";
262
263/// The answer `stream` ends with, every event before it handed to `delta`;
264/// None when the stream ended without a [`ProviderEvent::Done`] ([`NO_RESPONSE`]).
265pub(crate) async fn final_message(
266    mut stream: ProviderStream,
267    mut delta: impl FnMut(ProviderEvent),
268) -> Option<AssistantMessage> {
269    let mut done = None;
270    while let Some(ev) = stream.next().await {
271        match ev {
272            ProviderEvent::Done(m) => done = Some(m),
273            other => delta(other),
274        }
275    }
276    done
277}
278
279/// A model API: turns a [`ProviderRequest`] into a stream of
280/// [`ProviderEvent`]s.
281///
282/// The contract:
283/// - [`Provider::stream`] returns at once; the work runs as the stream is
284///   polled.
285/// - The stream ends with exactly one [`ProviderEvent::Done`], also on
286///   failure: an API or network error is a `Done` with
287///   [`StopReason::Error`](crate::StopReason::Error) and the reason in
288///   `error_message`, never a panic or a stream that stops short. A stream
289///   that ends without `Done` reaches history as an error message.
290/// - The loop reads the stream to its end and does not watch `req.cancel`
291///   itself: when it fires, the adapter ends the stream with a `Done` whose
292///   stop reason is [`StopReason::Aborted`](crate::StopReason::Aborted).
293///
294/// # Examples
295///
296/// A provider that answers every request with the last user message, built
297/// with [`answer_stream`], which ends the stream with the answer the async
298/// block returns:
299///
300/// ```
301/// use haest_core::{
302///     AssistantMessage, AssistantPart, Message, Provider, ProviderEvent, ProviderRequest,
303///     ProviderStream, answer_stream,
304/// };
305///
306/// struct Echo;
307///
308/// impl Provider for Echo {
309///     fn id(&self) -> &str {
310///         "echo"
311///     }
312///
313///     fn stream(&self, req: ProviderRequest) -> ProviderStream {
314///         let said = match req.messages.last() {
315///             Some(Message::User(u)) => u.content.clone(),
316///             _ => String::new(),
317///         };
318///         let mut answer = AssistantMessage::start(self.id(), &req.model);
319///         answer_stream(|deltas| async move {
320///             // A real provider sends each piece as its API streams it.
321///             deltas.text(said.as_str());
322///             answer.content.push(AssistantPart::Text { text: said });
323///             answer
324///         })
325///     }
326/// }
327///
328/// # #[tokio::main(flavor = "current_thread")]
329/// # async fn main() -> Result<(), haest_core::BoxError> {
330/// use futures::StreamExt;
331///
332/// let req = ProviderRequest::new("echo-1", vec![Message::user("hello")])
333///     .with_system("Repeat the user.");
334/// let events: Vec<ProviderEvent> = Echo.stream(req).collect().await;
335/// assert_eq!(events[0], ProviderEvent::TextDelta("hello".into()));
336/// let Some(ProviderEvent::Done(answer)) = events.last() else {
337///     return Err("Done comes last".into());
338/// };
339/// assert_eq!((answer.text(), answer.provider.as_str()), ("hello".into(), "echo"));
340/// # Ok(())
341/// # }
342/// ```
343pub trait Provider: Send + Sync {
344    /// Stable id stored on every assistant message ("anthropic", "openrouter", ...).
345    fn id(&self) -> &str;
346    /// Start the call. Must end with exactly one [`ProviderEvent::Done`], also on failure.
347    fn stream(&self, req: ProviderRequest) -> ProviderStream;
348}
349
350crate::plug_ref! {
351    /// A [`Provider`], shared: what a [`ResolvedModel`] holds. Built from the
352    /// provider itself, or from an `Arc` of one a resolver keeps.
353    pub struct ProviderRef(dyn Provider);
354    debug(provider, f) { f.debug_tuple("ProviderRef").field(&provider.id()).finish() }
355}
356
357mod adapter;
358mod answer;
359mod resolve;
360#[cfg(test)]
361mod tests;