agent_framework_foundry_local/
lib.rs

1//! # agent-framework-foundry-local
2//!
3//! A [Microsoft Foundry Local](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-local/what-is-foundry-local)
4//! [`ChatClient`] for `agent-framework-rs`.
5//!
6//! Foundry Local runs models on-device and exposes them through an
7//! OpenAI-compatible REST endpoint (`POST {base_url}/chat/completions`,
8//! default base URL `http://localhost:5273/v1`). That compatibility layer
9//! speaks the exact same JSON shapes as OpenAI's Chat Completions API, so
10//! request/response conversion is reused from
11//! [`agent_framework_openai::convert`] rather than duplicated — mirroring how
12//! `agent-framework-ollama` reuses it for the same reason. Like Ollama, a
13//! stock local Foundry Local instance normally requires no API key, so
14//! [`FoundryLocalChatClient::new`] and [`FoundryLocalChatClient::from_env`]
15//! don't require one either.
16//!
17//! ```no_run
18//! use agent_framework_foundry_local::FoundryLocalChatClient;
19//! use agent_framework_core::prelude::*;
20//!
21//! # async fn demo() -> Result<()> {
22//! let client = FoundryLocalChatClient::new("phi-3.5-mini");
23//! let agent = Agent::builder(client)
24//!     .instructions("You are concise.")
25//!     .build();
26//! let reply = agent.run_once("Say hi").await?;
27//! println!("{}", reply.text());
28//! # Ok(())
29//! # }
30//! ```
31//!
32//! Pointing at a non-default port (Foundry Local can bind to a
33//! dynamically-chosen one):
34//!
35//! ```no_run
36//! use agent_framework_foundry_local::FoundryLocalChatClient;
37//!
38//! let client =
39//!     FoundryLocalChatClient::new("phi-3.5-mini").with_base_url("http://localhost:5273/v1");
40//! # let _ = client;
41//! ```
42//!
43//! The actual port Foundry Local's OpenAI-compatible endpoint listens on is
44//! discovered dynamically from the Foundry Local service in real
45//! deployments (it varies by install and can change across restarts); this
46//! crate does not perform that discovery itself. Point
47//! [`FoundryLocalChatClient`] at the right base URL with
48//! [`FoundryLocalChatClient::with_base_url`] or the `FOUNDRY_LOCAL_ENDPOINT_ENV`
49//! (`FOUNDRY_LOCAL_ENDPOINT`) environment variable, using whatever port the
50//! Foundry Local SDK/CLI reports (`5273` above is just the common default).
51
52pub mod convert;
53
54use std::collections::{HashMap, VecDeque};
55use std::sync::Arc;
56
57use agent_framework_core::client::{ChatClient, ChatStream};
58use agent_framework_core::error::{Error, Result};
59use agent_framework_core::streaming::Utf8StreamDecoder;
60use agent_framework_core::types::{ChatOptions, ChatResponse, ChatResponseUpdate, Message};
61use futures::StreamExt;
62use serde_json::Value;
63
64/// The default Foundry Local OpenAI-compatible base URL.
65const DEFAULT_BASE_URL: &str = "http://localhost:5273/v1";
66
67/// The environment variable read for a non-default Foundry Local endpoint.
68/// If set, [`FoundryLocalChatClient::from_env`] uses it as the base URL.
69const FOUNDRY_LOCAL_ENDPOINT_ENV: &str = "FOUNDRY_LOCAL_ENDPOINT";
70
71/// The environment variable read for an optional bearer token. Not required
72/// for a stock local Foundry Local instance; useful when one sits behind an
73/// authenticating proxy.
74const FOUNDRY_LOCAL_API_KEY_ENV: &str = "FOUNDRY_LOCAL_API_KEY";
75
76/// Parse a `Retry-After` header into a delay in seconds. Mirrors the
77/// OpenAI/Anthropic/Azure/Ollama clients (Foundry Local's compatibility layer
78/// doesn't generally emit this, but a proxy in front of it might).
79fn parse_retry_after(headers: &reqwest::header::HeaderMap) -> Option<f64> {
80    headers
81        .get(reqwest::header::RETRY_AFTER)
82        .and_then(|v| v.to_str().ok())
83        .and_then(|s| s.trim().parse::<f64>().ok())
84        .filter(|s| s.is_finite() && *s >= 0.0)
85}
86
87/// A Microsoft Foundry Local chat client (`POST {base_url}/chat/completions`).
88#[derive(Clone)]
89pub struct FoundryLocalChatClient {
90    inner: Arc<Inner>,
91}
92
93#[derive(Clone)]
94struct Inner {
95    http: reqwest::Client,
96    base_url: String,
97    model: String,
98    /// Optional bearer token. Unset by default — a stock Foundry Local
99    /// instance needs none — but some deployments sit behind a proxy that
100    /// requires one.
101    api_key: Option<String>,
102}
103
104impl std::fmt::Debug for FoundryLocalChatClient {
105    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
106        f.debug_struct("FoundryLocalChatClient")
107            .field("base_url", &self.inner.base_url)
108            .field("model", &self.inner.model)
109            .field("has_api_key", &self.inner.api_key.is_some())
110            .finish_non_exhaustive()
111    }
112}
113
114impl FoundryLocalChatClient {
115    /// Create a client for the given default model, targeting
116    /// `http://localhost:5273/v1`.
117    pub fn new(model: impl Into<String>) -> Self {
118        Self {
119            inner: Arc::new(Inner {
120                http: reqwest::Client::new(),
121                base_url: DEFAULT_BASE_URL.to_string(),
122                model: model.into(),
123                api_key: None,
124            }),
125        }
126    }
127
128    /// Build a client from the environment. Foundry Local has no required
129    /// credential env var (a stock instance is unauthenticated), so this
130    /// never fails on missing configuration; it only reads the optional
131    /// `FOUNDRY_LOCAL_ENDPOINT_ENV` (`FOUNDRY_LOCAL_ENDPOINT`) to override
132    /// the default base URL and `FOUNDRY_LOCAL_API_KEY_ENV`
133    /// (`FOUNDRY_LOCAL_API_KEY`) to set a bearer token, if present.
134    pub fn from_env(model: impl Into<String>) -> Result<Self> {
135        let mut client = Self::new(model);
136        if let Ok(endpoint) = std::env::var(FOUNDRY_LOCAL_ENDPOINT_ENV) {
137            if !endpoint.trim().is_empty() {
138                client = client.with_base_url(endpoint);
139            }
140        }
141        if let Ok(api_key) = std::env::var(FOUNDRY_LOCAL_API_KEY_ENV) {
142            if !api_key.trim().is_empty() {
143                client = client.with_api_key(api_key);
144            }
145        }
146        Ok(client)
147    }
148
149    /// Override the base URL (for a non-default port, or a remote Foundry
150    /// Local host). Must be the OpenAI-compatible root (i.e. include the
151    /// `/v1` suffix), matching [`FoundryLocalChatClient::new`]'s default.
152    pub fn with_base_url(mut self, base_url: impl Into<String>) -> Self {
153        Arc::make_mut(&mut self.inner).base_url = base_url.into();
154        self
155    }
156
157    /// Set a bearer token sent as `Authorization: Bearer <key>`. Not needed
158    /// for a stock local Foundry Local instance; useful when one sits behind
159    /// an authenticating proxy.
160    pub fn with_api_key(mut self, api_key: impl Into<String>) -> Self {
161        Arc::make_mut(&mut self.inner).api_key = Some(api_key.into());
162        self
163    }
164
165    /// The default model id.
166    pub fn model(&self) -> &str {
167        &self.inner.model
168    }
169
170    /// The configured base URL.
171    pub fn base_url(&self) -> &str {
172        &self.inner.base_url
173    }
174
175    fn build_body(&self, messages: &[Message], options: &ChatOptions, stream: bool) -> Value {
176        let model = options
177            .model
178            .clone()
179            .unwrap_or_else(|| self.inner.model.clone());
180        convert::build_request(messages, options, &model, stream)
181    }
182
183    async fn post(&self, body: &Value) -> Result<reqwest::Response> {
184        let url = format!(
185            "{}/chat/completions",
186            self.inner.base_url.trim_end_matches('/')
187        );
188        let mut req = self.inner.http.post(&url).json(body);
189        if let Some(key) = &self.inner.api_key {
190            req = req.bearer_auth(key);
191        }
192        let resp = req
193            .send()
194            .await
195            .map_err(|e| Error::service(format!("request failed: {e}")))?;
196        if !resp.status().is_success() {
197            let status = resp.status();
198            let retry_after = parse_retry_after(resp.headers());
199            let text = resp.text().await.unwrap_or_default();
200            // Foundry Local's OpenAI-compatibility layer is wire-compatible
201            // for errors too, so status/body classification is shared
202            // verbatim with `agent-framework-openai` rather than duplicated.
203            return Err(agent_framework_openai::classify_service_error(
204                status.as_u16(),
205                &text,
206                format!("Foundry Local API error {status}: {text}"),
207                retry_after,
208            ));
209        }
210        Ok(resp)
211    }
212}
213
214#[async_trait::async_trait]
215impl ChatClient for FoundryLocalChatClient {
216    async fn get_response(
217        &self,
218        messages: Vec<Message>,
219        options: ChatOptions,
220    ) -> Result<ChatResponse> {
221        let body = self.build_body(&messages, &options, false);
222        let resp = self.post(&body).await?;
223        let value: Value = resp
224            .json()
225            .await
226            .map_err(|e| Error::service(format!("invalid response json: {e}")))?;
227        Ok(agent_framework_openai::convert::parse_response(&value))
228    }
229
230    async fn get_streaming_response(
231        &self,
232        messages: Vec<Message>,
233        options: ChatOptions,
234    ) -> Result<ChatStream> {
235        let body = self.build_body(&messages, &options, true);
236        let resp = self.post(&body).await?;
237        Ok(parse_sse_stream(resp).boxed())
238    }
239
240    fn model(&self) -> Option<&str> {
241        Some(&self.inner.model)
242    }
243}
244
245type ByteStream =
246    std::pin::Pin<Box<dyn futures::Stream<Item = reqwest::Result<bytes::Bytes>> + Send>>;
247
248/// Turn an SSE HTTP response into a stream of [`ChatResponseUpdate`]s.
249fn parse_sse_stream(
250    resp: reqwest::Response,
251) -> impl futures::Stream<Item = Result<ChatResponseUpdate>> + Send {
252    let byte_stream: ByteStream = Box::pin(resp.bytes_stream());
253    futures::stream::unfold(
254        SseState {
255            byte_stream,
256            buffer: String::new(),
257            utf8: Utf8StreamDecoder::new(),
258            queued: VecDeque::new(),
259            tool_ids: HashMap::new(),
260            done: false,
261        },
262        |mut state| async move {
263            loop {
264                if let Some(update) = state.queued.pop_front() {
265                    return Some((Ok(update), state));
266                }
267                if state.done {
268                    return None;
269                }
270                match state.byte_stream.next().await {
271                    Some(Ok(bytes)) => {
272                        let decoded = state.utf8.push(&bytes);
273                        state.buffer.push_str(&decoded);
274                        while let Some(pos) = state.buffer.find('\n') {
275                            let line = state.buffer[..pos].trim().to_string();
276                            state.buffer.drain(..=pos);
277                            if let Some(data) = line.strip_prefix("data:") {
278                                let data = data.trim();
279                                if data == "[DONE]" {
280                                    return drain_or_end(state);
281                                }
282                                if data.is_empty() {
283                                    continue;
284                                }
285                                if let Ok(value) = serde_json::from_str::<Value>(data) {
286                                    if let Some(err) = value.get("error") {
287                                        let msg = err
288                                            .get("message")
289                                            .and_then(Value::as_str)
290                                            .unwrap_or("unknown stream error")
291                                            .to_string();
292                                        state.done = true;
293                                        return Some((Err(Error::service(msg)), state));
294                                    }
295                                    if let Some(update) =
296                                        convert::parse_delta(&value, &mut state.tool_ids)
297                                    {
298                                        state.queued.push_back(update);
299                                    }
300                                }
301                            }
302                        }
303                    }
304                    Some(Err(e)) => {
305                        state.done = true;
306                        return Some((Err(Error::service(format!("stream error: {e}"))), state));
307                    }
308                    None => return drain_or_end(state),
309                }
310            }
311        },
312    )
313}
314
315/// State carried across `unfold` iterations while parsing the SSE stream.
316struct SseState {
317    byte_stream: ByteStream,
318    buffer: String,
319    utf8: Utf8StreamDecoder,
320    queued: VecDeque<ChatResponseUpdate>,
321    tool_ids: HashMap<i64, String>,
322    done: bool,
323}
324
325fn drain_or_end(mut state: SseState) -> Option<(Result<ChatResponseUpdate>, SseState)> {
326    match state.queued.pop_front() {
327        Some(update) => {
328            state.done = true;
329            Some((Ok(update), state))
330        }
331        None => None,
332    }
333}
334
335#[cfg(test)]
336mod tests {
337    use super::*;
338
339    // region: base URL and constructor defaults
340
341    #[test]
342    fn default_base_url_is_localhost_v1() {
343        let client = FoundryLocalChatClient::new("phi-3.5-mini");
344        assert_eq!(client.base_url(), "http://localhost:5273/v1");
345        assert_eq!(client.model(), "phi-3.5-mini");
346    }
347
348    #[test]
349    fn with_base_url_overrides_default() {
350        let client = FoundryLocalChatClient::new("phi-3.5-mini")
351            .with_base_url("http://example.test:5273/v1");
352        assert_eq!(client.base_url(), "http://example.test:5273/v1");
353    }
354
355    #[test]
356    fn with_api_key_sets_bearer_token() {
357        let client = FoundryLocalChatClient::new("phi-3.5-mini").with_api_key("secret");
358        assert_eq!(client.inner.api_key.as_deref(), Some("secret"));
359    }
360
361    #[test]
362    fn no_api_key_by_default() {
363        let client = FoundryLocalChatClient::new("phi-3.5-mini");
364        assert!(client.inner.api_key.is_none());
365    }
366
367    // endregion
368
369    // region: from_env
370
371    /// Guards env mutation: tests within a crate run on multiple threads, and
372    /// env vars are process-global.
373    static ENV_MUTEX: std::sync::Mutex<()> = std::sync::Mutex::new(());
374
375    #[test]
376    fn from_env_never_fails_without_env_vars() {
377        let _guard = ENV_MUTEX.lock().unwrap();
378        // SAFETY: serialized by ENV_MUTEX against the other env-var test in
379        // this module; no other test in this crate touches these variables.
380        unsafe {
381            std::env::remove_var(FOUNDRY_LOCAL_ENDPOINT_ENV);
382            std::env::remove_var(FOUNDRY_LOCAL_API_KEY_ENV);
383        }
384        let client = FoundryLocalChatClient::from_env("phi-3.5-mini").unwrap();
385        assert_eq!(client.base_url(), DEFAULT_BASE_URL);
386        assert_eq!(client.model(), "phi-3.5-mini");
387        assert!(client.inner.api_key.is_none());
388    }
389
390    #[test]
391    fn from_env_reads_endpoint_and_api_key() {
392        let _guard = ENV_MUTEX.lock().unwrap();
393        // SAFETY: serialized by ENV_MUTEX; see above.
394        unsafe {
395            std::env::set_var(FOUNDRY_LOCAL_ENDPOINT_ENV, "http://192.168.1.50:5273/v1");
396            std::env::set_var(FOUNDRY_LOCAL_API_KEY_ENV, "secret");
397        }
398        let client = FoundryLocalChatClient::from_env("phi-3.5-mini").unwrap();
399        assert_eq!(client.base_url(), "http://192.168.1.50:5273/v1");
400        assert_eq!(client.inner.api_key.as_deref(), Some("secret"));
401        unsafe {
402            std::env::remove_var(FOUNDRY_LOCAL_ENDPOINT_ENV);
403            std::env::remove_var(FOUNDRY_LOCAL_API_KEY_ENV);
404        }
405    }
406
407    // endregion
408
409    // region: request building
410
411    #[test]
412    fn build_body_uses_client_default_model_when_options_model_unset() {
413        let client = FoundryLocalChatClient::new("phi-3.5-mini");
414        let body = client.build_body(&[Message::user("hi")], &ChatOptions::new(), false);
415        assert_eq!(body["model"], serde_json::json!("phi-3.5-mini"));
416        assert_eq!(
417            body["messages"],
418            serde_json::json!([{ "role": "user", "content": "hi" }])
419        );
420    }
421
422    #[test]
423    fn build_body_prefers_per_request_model() {
424        let client = FoundryLocalChatClient::new("phi-3.5-mini");
425        let options = ChatOptions {
426            model: Some("qwen2.5".to_string()),
427            ..ChatOptions::new()
428        };
429        let body = client.build_body(&[Message::user("hi")], &options, false);
430        assert_eq!(body["model"], serde_json::json!("qwen2.5"));
431    }
432
433    #[test]
434    fn build_body_sets_stream_flag() {
435        let client = FoundryLocalChatClient::new("phi-3.5-mini");
436        let body = client.build_body(&[Message::user("hi")], &ChatOptions::new(), true);
437        assert_eq!(body["stream"], serde_json::json!(true));
438    }
439
440    // endregion
441
442    // region: SSE stream parsing over a synthetic byte stream
443
444    fn sse_bytes(lines: &[String]) -> bytes::Bytes {
445        bytes::Bytes::from(lines.join("\n") + "\n\n")
446    }
447
448    async fn collect_via_state(text: bytes::Bytes) -> Vec<ChatResponseUpdate> {
449        let stream = futures::stream::once(async move { Ok::<_, reqwest::Error>(text) });
450        let byte_stream: ByteStream = Box::pin(stream);
451        let mut state = SseState {
452            byte_stream,
453            buffer: String::new(),
454            utf8: Utf8StreamDecoder::new(),
455            queued: VecDeque::new(),
456            tool_ids: HashMap::new(),
457            done: false,
458        };
459        let mut updates = Vec::new();
460        if let Some(Ok(bytes)) = state.byte_stream.next().await {
461            let decoded = state.utf8.push(&bytes);
462            state.buffer.push_str(&decoded);
463            while let Some(pos) = state.buffer.find('\n') {
464                let line = state.buffer[..pos].trim().to_string();
465                state.buffer.drain(..=pos);
466                let Some(data) = line.strip_prefix("data:") else {
467                    continue;
468                };
469                let data = data.trim();
470                if data.is_empty() || data == "[DONE]" {
471                    continue;
472                }
473                let value: Value = serde_json::from_str(data).unwrap();
474                if let Some(update) = convert::parse_delta(&value, &mut state.tool_ids) {
475                    updates.push(update);
476                }
477            }
478        }
479        updates
480    }
481
482    #[tokio::test]
483    async fn streaming_chunk_produces_text_update() {
484        let chunk = serde_json::json!({
485            "id": "chatcmpl-1",
486            "model": "phi-3.5-mini",
487            "choices": [{ "delta": { "role": "assistant", "content": "Hello" }, "finish_reason": null }],
488        });
489        let bytes = sse_bytes(&[format!("data: {chunk}")]);
490        let updates = collect_via_state(bytes).await;
491        assert_eq!(updates.len(), 1);
492        let resp = ChatResponse::from_updates(updates);
493        assert_eq!(resp.text(), "Hello");
494        assert_eq!(resp.response_id.as_deref(), Some("chatcmpl-1"));
495    }
496
497    #[tokio::test]
498    async fn streaming_tool_call_and_finish_reason_accumulate() {
499        let call_chunk = serde_json::json!({
500            "id": "chatcmpl-2",
501            "choices": [{
502                "delta": { "tool_calls": [{ "index": 0, "id": "call_1", "function": { "name": "get_weather", "arguments": "{}" } }] },
503                "finish_reason": null,
504            }],
505        });
506        let finish_chunk = serde_json::json!({
507            "id": "chatcmpl-2",
508            "choices": [{ "delta": {}, "finish_reason": "tool_calls" }],
509        });
510        let bytes = sse_bytes(&[
511            format!("data: {call_chunk}"),
512            format!("data: {finish_chunk}"),
513        ]);
514        let updates = collect_via_state(bytes).await;
515        assert_eq!(updates.len(), 2);
516        let resp = ChatResponse::from_updates(updates);
517        let calls = resp.function_calls();
518        assert_eq!(calls.len(), 1);
519        assert_eq!(calls[0].call_id, "call_1");
520        assert_eq!(calls[0].name, "get_weather");
521        assert_eq!(
522            resp.finish_reason,
523            Some(agent_framework_core::types::FinishReason::tool_calls())
524        );
525    }
526
527    #[tokio::test]
528    async fn streaming_done_sentinel_ends_stream_without_extra_update() {
529        let chunk = serde_json::json!({
530            "id": "chatcmpl-3",
531            "choices": [{ "delta": { "content": "hi" }, "finish_reason": null }],
532        });
533        let bytes = sse_bytes(&[format!("data: {chunk}"), "data: [DONE]".to_string()]);
534        let updates = collect_via_state(bytes).await;
535        // `collect_via_state` mirrors the raw line loop (no special-casing of
536        // [DONE] beyond `continue`), so this asserts the sentinel line never
537        // parses into a spurious update, matching the real stream (which
538        // instead terminates on it).
539        assert_eq!(updates.len(), 1);
540    }
541
542    // endregion
543}
agent_framework_foundry_local/lib.rs

agent_framework_foundry_local/
lib.rs
