1use std::collections::HashMap;
7use std::sync::Arc;
8use std::time::SystemTime;
9
10use async_trait::async_trait;
11use axum::http::HeaderMap;
12use serde::{Deserialize, Serialize};
13use serde_json::Value;
14
15#[derive(Debug, Clone, Serialize, Deserialize)]
21pub struct ChatMessage {
22 pub role: String,
24 pub content: Value,
26}
27
28impl ChatMessage {
29 #[must_use]
31 pub fn text(role: impl Into<String>, content: impl Into<String>) -> Self {
32 Self {
33 role: role.into(),
34 content: Value::String(content.into()),
35 }
36 }
37
38 #[must_use]
41 pub fn text_view(&self) -> String {
42 match &self.content {
43 Value::String(s) => s.clone(),
44 Value::Array(blocks) => blocks
45 .iter()
46 .filter_map(|b| b.get("text").and_then(Value::as_str))
47 .collect::<Vec<_>>()
48 .join("\n"),
49 _ => String::new(),
50 }
51 }
52}
53
54#[derive(Debug, Clone, Serialize, Deserialize)]
60pub struct ModelRequest {
61 pub model: String,
63 #[serde(default, skip_serializing_if = "Option::is_none")]
65 pub system: Option<String>,
66 pub messages: Vec<ChatMessage>,
68 pub max_tokens: u32,
70 #[serde(default)]
72 pub tools: Value,
73 #[serde(default, skip_serializing_if = "Value::is_null")]
79 pub raw: Value,
80 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
84 pub cache_prefix: bool,
85}
86
87#[derive(Debug, Clone, Serialize, Deserialize)]
89pub struct ModelResponse {
90 pub model: String,
92 pub text: String,
94 pub in_tokens: u64,
96 pub cache_write_tokens: u64,
99 pub cache_read_tokens: u64,
101 pub out_tokens: u64,
103 pub raw: Value,
105}
106
107#[derive(Debug, Clone, thiserror::Error)]
109pub enum ProviderError {
110 #[error("transport error: {0}")]
112 Transport(String),
113 #[error("http {status}: {body}")]
115 Http {
116 status: u16,
118 body: String,
120 },
121 #[error("decode error: {0}")]
123 Decode(String),
124}
125
126impl ProviderError {
127 #[must_use]
130 pub fn is_failover_eligible(&self) -> bool {
131 match self {
132 ProviderError::Transport(_) => true,
133 ProviderError::Http { status, .. } => *status >= 500,
134 ProviderError::Decode(_) => false,
135 }
136 }
137}
138
139#[derive(Clone, Default)]
143pub struct Auth {
144 pub anthropic_key: Option<String>,
146 pub openai_key: Option<String>,
148}
149
150impl std::fmt::Debug for Auth {
151 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
152 f.debug_struct("Auth")
153 .field("anthropic_key", &self.anthropic_key.as_ref().map(|_| "***"))
154 .field("openai_key", &self.openai_key.as_ref().map(|_| "***"))
155 .finish()
156 }
157}
158
159impl Auth {
160 #[must_use]
163 pub fn from_headers(headers: &HeaderMap) -> Self {
164 let anthropic_key = headers
165 .get("x-api-key")
166 .and_then(|v| v.to_str().ok())
167 .map(str::to_owned)
168 .or_else(|| std::env::var("ANTHROPIC_API_KEY").ok());
169 let openai_key = headers
170 .get(axum::http::header::AUTHORIZATION)
171 .and_then(|v| v.to_str().ok())
172 .and_then(|v| v.strip_prefix("Bearer "))
173 .map(str::to_owned)
174 .or_else(|| std::env::var("OPENAI_API_KEY").ok());
175 Self {
176 anthropic_key,
177 openai_key,
178 }
179 }
180}
181
182#[async_trait]
185pub trait Provider: Send + Sync + std::fmt::Debug {
186 async fn complete(
192 &self,
193 req: &ModelRequest,
194 auth: &Auth,
195 ) -> Result<ModelResponse, ProviderError>;
196
197 fn id(&self) -> &str;
199
200 fn carries_structured_verbatim(&self, _inbound: firstpass_core::Dialect) -> bool {
215 false
216 }
217}
218
219fn wire_model(model: &str) -> &str {
223 model.split_once('/').map_or(model, |(_, m)| m)
224}
225
226fn resolve_api_key(api_key_env: Option<&str>, byok_override: Option<&str>) -> String {
231 api_key_env
232 .and_then(|e| std::env::var(e).ok())
233 .or_else(|| byok_override.map(str::to_owned))
234 .unwrap_or_default()
235}
236
237#[must_use]
242pub fn anthropic_wire_body(req: &ModelRequest) -> Value {
243 if let Value::Object(raw) = &req.raw {
244 let mut body = raw.clone();
245 body.insert(
246 "model".to_owned(),
247 Value::String(wire_model(&req.model).to_owned()),
248 );
249 body.remove("stream");
250 normalize_reasoning(&mut body, Wire::Anthropic);
251 if req.cache_prefix {
252 add_cache_breakpoints(&mut body);
253 }
254 return Value::Object(body);
255 }
256 let messages: Vec<Value> = req
257 .messages
258 .iter()
259 .map(|m| serde_json::json!({ "role": m.role, "content": m.content }))
260 .collect();
261 let mut body = serde_json::json!({
262 "model": wire_model(&req.model),
263 "max_tokens": req.max_tokens,
264 "messages": messages,
265 });
266 if let Some(system) = req.system.as_deref() {
267 body["system"] = serde_json::json!(system);
268 }
269 if !req.tools.is_null() {
270 body["tools"] = req.tools.clone();
271 }
272 if req.cache_prefix
277 && let Some(map) = body.as_object_mut()
278 {
279 add_cache_breakpoints(map);
280 }
281 body
282}
283
284#[derive(Debug, Clone)]
290pub struct AnthropicProvider {
291 pub id: String,
293 pub base_url: String,
295 pub api_key_env: Option<String>,
298 pub http: reqwest::Client,
300}
301
302#[async_trait]
303impl Provider for AnthropicProvider {
304 fn id(&self) -> &str {
305 &self.id
306 }
307
308 fn carries_structured_verbatim(&self, inbound: firstpass_core::Dialect) -> bool {
309 inbound == firstpass_core::Dialect::Anthropic
310 }
311
312 async fn complete(
313 &self,
314 req: &ModelRequest,
315 auth: &Auth,
316 ) -> Result<ModelResponse, ProviderError> {
317 let key = resolve_api_key(self.api_key_env.as_deref(), auth.anthropic_key.as_deref());
318 let body = anthropic_wire_body(req);
319
320 let url = format!("{}/v1/messages", self.base_url.trim_end_matches('/'));
321 let resp = self
322 .http
323 .post(url)
324 .header("x-api-key", key)
325 .header("anthropic-version", "2023-06-01")
326 .json(&body)
327 .send()
328 .await
329 .map_err(|e| ProviderError::Transport(e.to_string()))?;
330
331 let status = resp.status();
332 let bytes = resp
333 .bytes()
334 .await
335 .map_err(|e| ProviderError::Transport(e.to_string()))?;
336
337 if !status.is_success() {
338 return Err(ProviderError::Http {
339 status: status.as_u16(),
340 body: String::from_utf8_lossy(&bytes).into_owned(),
341 });
342 }
343
344 let json: Value =
345 serde_json::from_slice(&bytes).map_err(|e| ProviderError::Decode(e.to_string()))?;
346 let u = anthropic_parse_response(&json)?;
347 let (text, in_tokens, out_tokens) = (u.text, u.in_tokens, u.out_tokens);
348 let (cache_write_tokens, cache_read_tokens) = (u.cache_write_tokens, u.cache_read_tokens);
349
350 Ok(ModelResponse {
351 cache_write_tokens,
352 cache_read_tokens,
353 model: req.model.clone(),
354 text,
355 in_tokens,
356 out_tokens,
357 raw: json,
358 })
359 }
360}
361
362fn anthropic_parse_response(json: &Value) -> Result<AnthropicUsage, ProviderError> {
366 let text = json
367 .get("content")
368 .and_then(Value::as_array)
369 .map(|blocks| {
370 blocks
371 .iter()
372 .filter_map(|b| b.get("text").and_then(Value::as_str))
373 .collect::<Vec<_>>()
374 .join("")
375 })
376 .ok_or_else(|| ProviderError::Decode("missing content[].text".to_owned()))?;
377
378 let in_tokens = json
379 .pointer("/usage/input_tokens")
380 .and_then(Value::as_u64)
381 .unwrap_or(0);
382 let out_tokens = json
383 .pointer("/usage/output_tokens")
384 .and_then(Value::as_u64)
385 .unwrap_or(0);
386 let cache_write_tokens = json
391 .pointer("/usage/cache_creation_input_tokens")
392 .and_then(Value::as_u64)
393 .unwrap_or(0);
394 let cache_read_tokens = json
395 .pointer("/usage/cache_read_input_tokens")
396 .and_then(Value::as_u64)
397 .unwrap_or(0);
398
399 Ok(AnthropicUsage {
400 text,
401 in_tokens,
402 cache_write_tokens,
403 cache_read_tokens,
404 out_tokens,
405 })
406}
407
408struct AnthropicUsage {
410 text: String,
411 in_tokens: u64,
412 cache_write_tokens: u64,
413 cache_read_tokens: u64,
414 out_tokens: u64,
415}
416
417fn anthropic_messages_body(req: &ModelRequest, anthropic_version: &str) -> Value {
422 if let Value::Object(raw) = &req.raw {
425 let mut body = raw.clone();
426 body.remove("model");
427 body.remove("stream");
428 body.insert(
429 "anthropic_version".to_owned(),
430 Value::String(anthropic_version.to_owned()),
431 );
432 return Value::Object(body);
433 }
434 let messages: Vec<Value> = req
435 .messages
436 .iter()
437 .map(|m| serde_json::json!({ "role": m.role, "content": m.content }))
438 .collect();
439 let mut body = serde_json::json!({
440 "anthropic_version": anthropic_version,
441 "max_tokens": req.max_tokens,
442 "messages": messages,
443 });
444 if let Some(system) = req.system.as_deref() {
445 body["system"] = serde_json::json!(system);
446 }
447 if !req.tools.is_null() {
448 body["tools"] = req.tools.clone();
449 }
450 body
451}
452
453#[must_use]
459pub fn openai_wire_body(req: &ModelRequest) -> Value {
460 if let Value::Object(raw) = &req.raw {
461 let mut body = raw.clone();
462 body.insert(
463 "model".to_owned(),
464 Value::String(wire_model(&req.model).to_owned()),
465 );
466 body.remove("stream");
467 normalize_reasoning(&mut body, Wire::OpenAi);
468 return Value::Object(body);
469 }
470 let mut messages: Vec<Value> = Vec::with_capacity(req.messages.len() + 1);
472 if let Some(system) = req.system.as_deref() {
473 messages.push(serde_json::json!({ "role": "system", "content": system }));
474 }
475 messages.extend(
476 req.messages
477 .iter()
478 .map(|m| serde_json::json!({ "role": m.role, "content": m.content })),
479 );
480 let mut body = serde_json::json!({
481 "model": wire_model(&req.model),
482 "max_tokens": req.max_tokens,
483 "messages": messages,
484 });
485 if !req.tools.is_null() {
486 body["tools"] = req.tools.clone();
487 }
488 body
489}
490
491#[derive(Debug, Clone, Copy, PartialEq, Eq)]
499pub enum Effort {
500 Low,
501 Medium,
502 High,
503}
504
505impl Effort {
506 const fn budget_tokens(self) -> u32 {
510 match self {
511 Self::Low => 1024, Self::Medium => 4096,
513 Self::High => 16384,
514 }
515 }
516
517 const fn from_budget(tokens: u64) -> Self {
520 if tokens < 2048 {
521 Self::Low
522 } else if tokens < 8192 {
523 Self::Medium
524 } else {
525 Self::High
526 }
527 }
528
529 fn as_str(self) -> &'static str {
530 match self {
531 Self::Low => "low",
532 Self::Medium => "medium",
533 Self::High => "high",
534 }
535 }
536}
537
538#[must_use]
541pub fn effort_of(raw: &Value) -> Option<Effort> {
542 read_effort(raw.as_object()?)
543}
544
545fn read_effort(body: &serde_json::Map<String, Value>) -> Option<Effort> {
547 if let Some(s) = body.get("reasoning_effort").and_then(Value::as_str) {
549 return match s {
550 "low" | "minimal" => Some(Effort::Low),
551 "medium" => Some(Effort::Medium),
552 "high" => Some(Effort::High),
553 _ => None,
554 };
555 }
556 if let Some(t) = body.get("thinking").and_then(Value::as_object)
558 && t.get("type").and_then(Value::as_str) != Some("disabled")
559 && let Some(b) = t.get("budget_tokens").and_then(Value::as_u64)
560 {
561 return Some(Effort::from_budget(b));
562 }
563 if let Some(b) = body
565 .get("thinkingConfig")
566 .and_then(Value::as_object)
567 .and_then(|c| c.get("thinkingBudget"))
568 .and_then(Value::as_u64)
569 {
570 return Some(Effort::from_budget(b));
571 }
572 None
573}
574
575#[derive(Debug, Clone, Copy, PartialEq, Eq)]
577pub enum Wire {
578 Anthropic,
579 OpenAi,
580 Gemini,
581}
582
583pub fn normalize_reasoning(body: &mut serde_json::Map<String, Value>, target: Wire) {
589 let native = match target {
596 Wire::OpenAi => "reasoning_effort",
597 Wire::Anthropic => "thinking",
598 Wire::Gemini => "thinkingConfig",
599 };
600 if body.contains_key(native) {
601 return;
602 }
603 let effort = read_effort(body);
604 body.remove("reasoning_effort");
606 body.remove("thinking");
607 body.remove("thinkingConfig");
608 let Some(effort) = effort else { return };
609
610 match target {
611 Wire::OpenAi => {
612 body.insert(
613 "reasoning_effort".to_owned(),
614 Value::String(effort.as_str().to_owned()),
615 );
616 }
617 Wire::Anthropic => {
618 let max = body.get("max_tokens").and_then(Value::as_u64).unwrap_or(0);
622 let budget = u64::from(effort.budget_tokens()).min(max.saturating_sub(1));
623 if budget >= 1024 {
624 body.insert(
625 "thinking".to_owned(),
626 serde_json::json!({ "type": "enabled", "budget_tokens": budget }),
627 );
628 body.remove("temperature");
631 body.remove("top_p");
632 }
633 }
634 Wire::Gemini => {
635 body.entry("generationConfig")
637 .or_insert_with(|| Value::Object(serde_json::Map::new()))
638 .as_object_mut()
639 .map(|c| {
640 c.insert(
641 "thinkingConfig".to_owned(),
642 serde_json::json!({ "thinkingBudget": effort.budget_tokens() }),
643 )
644 });
645 }
646 }
647}
648
649const MIN_CACHEABLE_PREFIX_CHARS: usize = 3000;
671
672pub fn add_cache_breakpoints(body: &mut serde_json::Map<String, Value>) {
673 if body_has_cache_control(body) {
674 return;
675 }
676 let prefix_chars = body.get("system").map_or(0, |v| v.to_string().len())
681 + body.get("tools").map_or(0, |v| v.to_string().len());
682 if prefix_chars < MIN_CACHEABLE_PREFIX_CHARS {
683 return;
684 }
685 let marker = serde_json::json!({ "type": "ephemeral" });
686
687 match body.get("system") {
689 Some(Value::String(text)) if !text.is_empty() => {
690 body.insert(
691 "system".to_owned(),
692 serde_json::json!([{
693 "type": "text", "text": text, "cache_control": marker,
694 }]),
695 );
696 }
697 Some(Value::Array(blocks)) if !blocks.is_empty() => {
698 let mut blocks = blocks.clone();
699 if let Some(last) = blocks.last_mut().and_then(Value::as_object_mut) {
700 last.insert("cache_control".to_owned(), marker.clone());
701 }
702 body.insert("system".to_owned(), Value::Array(blocks));
703 }
704 _ => {}
705 }
706
707 if let Some(Value::Array(tools)) = body.get("tools")
710 && !tools.is_empty()
711 {
712 let mut tools = tools.clone();
713 if let Some(last) = tools.last_mut().and_then(Value::as_object_mut) {
714 last.insert("cache_control".to_owned(), marker);
715 }
716 body.insert("tools".to_owned(), Value::Array(tools));
717 }
718}
719
720fn body_has_cache_control(body: &serde_json::Map<String, Value>) -> bool {
722 fn any_in(v: &Value) -> bool {
723 match v {
724 Value::Array(items) => items.iter().any(any_in),
725 Value::Object(o) => o.contains_key("cache_control") || o.values().any(any_in),
726 _ => false,
727 }
728 }
729 body.values().any(any_in)
730}
731
732#[derive(Debug, Clone)]
738pub struct OpenAiProvider {
739 pub id: String,
741 pub base_url: String,
743 pub api_key_env: Option<String>,
747 pub http: reqwest::Client,
749}
750
751#[async_trait]
752impl Provider for OpenAiProvider {
753 fn id(&self) -> &str {
754 &self.id
755 }
756
757 fn carries_structured_verbatim(&self, inbound: firstpass_core::Dialect) -> bool {
758 inbound == firstpass_core::Dialect::Openai
759 }
760
761 async fn complete(
762 &self,
763 req: &ModelRequest,
764 auth: &Auth,
765 ) -> Result<ModelResponse, ProviderError> {
766 let key = resolve_api_key(self.api_key_env.as_deref(), auth.openai_key.as_deref());
767 let body = openai_wire_body(req);
768
769 let url = format!(
770 "{}/v1/chat/completions",
771 self.base_url.trim_end_matches('/')
772 );
773 let resp = self
774 .http
775 .post(url)
776 .header(axum::http::header::AUTHORIZATION, format!("Bearer {key}"))
777 .json(&body)
778 .send()
779 .await
780 .map_err(|e| ProviderError::Transport(e.to_string()))?;
781
782 let status = resp.status();
783 let bytes = resp
784 .bytes()
785 .await
786 .map_err(|e| ProviderError::Transport(e.to_string()))?;
787
788 if !status.is_success() {
789 return Err(ProviderError::Http {
790 status: status.as_u16(),
791 body: String::from_utf8_lossy(&bytes).into_owned(),
792 });
793 }
794
795 let json: Value =
796 serde_json::from_slice(&bytes).map_err(|e| ProviderError::Decode(e.to_string()))?;
797
798 let text = json
800 .pointer("/choices/0/message/content")
801 .and_then(Value::as_str)
802 .unwrap_or("")
803 .to_owned();
804
805 let in_tokens = json
806 .pointer("/usage/prompt_tokens")
807 .and_then(Value::as_u64)
808 .unwrap_or(0);
809 let out_tokens = json
810 .pointer("/usage/completion_tokens")
811 .and_then(Value::as_u64)
812 .unwrap_or(0);
813
814 Ok(ModelResponse {
815 cache_write_tokens: 0,
817 cache_read_tokens: 0,
818 model: req.model.clone(),
819 text,
820 in_tokens,
821 out_tokens,
822 raw: json,
823 })
824 }
825}
826
827fn gemini_request_body(req: &ModelRequest) -> Value {
834 let contents: Vec<Value> = req
835 .messages
836 .iter()
837 .map(|m| {
838 let role = if m.role == "assistant" {
839 "model"
840 } else {
841 "user"
842 };
843 serde_json::json!({ "role": role, "parts": [{ "text": m.text_view() }] })
844 })
845 .collect();
846 let mut body = serde_json::json!({
847 "contents": contents,
848 "generationConfig": { "maxOutputTokens": req.max_tokens },
849 });
850 if let Some(system) = req.system.as_deref() {
851 body["system_instruction"] = serde_json::json!({ "parts": [{ "text": system }] });
852 }
853 if let (Some(effort), Some(map)) = (effort_of(&req.raw), body.as_object_mut()) {
856 let mut carry = serde_json::Map::new();
857 carry.insert(
858 "reasoning_effort".to_owned(),
859 Value::String(effort.as_str().to_owned()),
860 );
861 carry.insert(
862 "generationConfig".to_owned(),
863 map["generationConfig"].clone(),
864 );
865 normalize_reasoning(&mut carry, Wire::Gemini);
866 map.insert(
867 "generationConfig".to_owned(),
868 carry["generationConfig"].clone(),
869 );
870 }
871 body
872}
873
874fn gemini_parse_response(json: &Value) -> Result<(String, u64, u64), ProviderError> {
877 let parts = json
878 .pointer("/candidates/0/content/parts")
879 .and_then(Value::as_array)
880 .ok_or_else(|| ProviderError::Decode("missing candidates[0].content.parts".to_owned()))?;
881 let text = parts
882 .iter()
883 .filter_map(|p| p.get("text").and_then(Value::as_str))
884 .collect::<Vec<_>>()
885 .join("");
886 let in_tokens = json
887 .pointer("/usageMetadata/promptTokenCount")
888 .and_then(Value::as_u64)
889 .unwrap_or(0);
890 let out_tokens = json
891 .pointer("/usageMetadata/candidatesTokenCount")
892 .and_then(Value::as_u64)
893 .unwrap_or(0);
894 Ok((text, in_tokens, out_tokens))
895}
896
897#[derive(Debug, Clone)]
905pub struct GeminiProvider {
906 pub id: String,
908 pub base_url: String,
910 pub api_key_env: Option<String>,
912 pub http: reqwest::Client,
914}
915
916#[async_trait]
917impl Provider for GeminiProvider {
918 fn id(&self) -> &str {
919 &self.id
920 }
921
922 async fn complete(
923 &self,
924 req: &ModelRequest,
925 auth: &Auth,
926 ) -> Result<ModelResponse, ProviderError> {
927 let key = resolve_api_key(self.api_key_env.as_deref(), auth.openai_key.as_deref());
928 let body = gemini_request_body(req);
929 let url = format!(
930 "{}/v1beta/models/{}:generateContent",
931 self.base_url.trim_end_matches('/'),
932 wire_model(&req.model),
933 );
934 let resp = self
935 .http
936 .post(url)
937 .header("x-goog-api-key", key)
938 .json(&body)
939 .send()
940 .await
941 .map_err(|e| ProviderError::Transport(e.to_string()))?;
942
943 let status = resp.status();
944 let bytes = resp
945 .bytes()
946 .await
947 .map_err(|e| ProviderError::Transport(e.to_string()))?;
948 if !status.is_success() {
949 return Err(ProviderError::Http {
950 status: status.as_u16(),
951 body: String::from_utf8_lossy(&bytes).into_owned(),
952 });
953 }
954
955 let json: Value =
956 serde_json::from_slice(&bytes).map_err(|e| ProviderError::Decode(e.to_string()))?;
957 let (text, in_tokens, out_tokens) = gemini_parse_response(&json)?;
958 Ok(ModelResponse {
959 cache_write_tokens: 0,
961 cache_read_tokens: 0,
962 model: req.model.clone(),
963 text,
964 in_tokens,
965 out_tokens,
966 raw: json,
967 })
968 }
969}
970
971struct AwsEnvCredentials {
975 access_key_id: String,
976 secret_access_key: String,
977 session_token: Option<String>,
978}
979
980impl AwsEnvCredentials {
981 fn from_env() -> Result<Self, ProviderError> {
982 let access_key_id = std::env::var("AWS_ACCESS_KEY_ID")
983 .map_err(|_| ProviderError::Transport("AWS_ACCESS_KEY_ID is not set".to_owned()))?;
984 let secret_access_key = std::env::var("AWS_SECRET_ACCESS_KEY")
985 .map_err(|_| ProviderError::Transport("AWS_SECRET_ACCESS_KEY is not set".to_owned()))?;
986 let session_token = std::env::var("AWS_SESSION_TOKEN").ok();
987 Ok(Self {
988 access_key_id,
989 secret_access_key,
990 session_token,
991 })
992 }
993}
994
995fn bedrock_url(region: &str, model: &str) -> String {
997 format!("https://bedrock-runtime.{region}.amazonaws.com/model/{model}/invoke")
998}
999
1000fn sign_bedrock(
1011 url: &str,
1012 region: &str,
1013 body: &[u8],
1014 creds: &AwsEnvCredentials,
1015) -> Result<http::Request<Vec<u8>>, ProviderError> {
1016 let host = url
1017 .parse::<http::Uri>()
1018 .ok()
1019 .and_then(|u| u.host().map(str::to_owned))
1020 .ok_or_else(|| ProviderError::Transport(format!("invalid bedrock URL: {url}")))?;
1021
1022 let identity: aws_smithy_runtime_api::client::identity::Identity =
1023 aws_credential_types::Credentials::new(
1024 creds.access_key_id.clone(),
1025 creds.secret_access_key.clone(),
1026 creds.session_token.clone(),
1027 None,
1028 "firstpass",
1029 )
1030 .into();
1031
1032 let signing_params: aws_sigv4::http_request::SigningParams<'_> =
1033 aws_sigv4::sign::v4::SigningParams::builder()
1034 .identity(&identity)
1035 .region(region)
1036 .name("bedrock")
1037 .time(SystemTime::now())
1038 .settings(aws_sigv4::http_request::SigningSettings::default())
1039 .build()
1040 .map_err(|e| ProviderError::Transport(format!("sigv4 signing params: {e}")))?
1041 .into();
1042
1043 let headers = [
1044 ("host", host.as_str()),
1045 ("content-type", "application/json"),
1046 ];
1047 let signable = aws_sigv4::http_request::SignableRequest::new(
1048 "POST",
1049 url,
1050 headers.into_iter(),
1051 aws_sigv4::http_request::SignableBody::Bytes(body),
1052 )
1053 .map_err(|e| ProviderError::Transport(format!("sigv4 signable request: {e}")))?;
1054
1055 let (instructions, _signature) = aws_sigv4::http_request::sign(signable, &signing_params)
1056 .map_err(|e| ProviderError::Transport(format!("sigv4 sign: {e}")))?
1057 .into_parts();
1058
1059 let mut req = http::Request::builder()
1060 .method("POST")
1061 .uri(url)
1062 .header("host", host)
1063 .header("content-type", "application/json")
1064 .body(body.to_vec())
1065 .map_err(|e| ProviderError::Transport(format!("build bedrock request: {e}")))?;
1066 instructions.apply_to_request_http1x(&mut req);
1067 Ok(req)
1068}
1069
1070#[derive(Debug, Clone)]
1080pub struct BedrockProvider {
1081 pub id: String,
1083 pub region: Option<String>,
1085 pub http: reqwest::Client,
1087}
1088
1089#[async_trait]
1090impl Provider for BedrockProvider {
1091 fn id(&self) -> &str {
1092 &self.id
1093 }
1094
1095 fn carries_structured_verbatim(&self, inbound: firstpass_core::Dialect) -> bool {
1096 inbound == firstpass_core::Dialect::Anthropic
1097 }
1098
1099 async fn complete(
1100 &self,
1101 req: &ModelRequest,
1102 _auth: &Auth,
1103 ) -> Result<ModelResponse, ProviderError> {
1104 let region = self.region.as_deref().ok_or_else(|| {
1105 ProviderError::Transport("bedrock provider requires a region".to_owned())
1106 })?;
1107 let model = wire_model(&req.model);
1108 let url = bedrock_url(region, model);
1109 let body = anthropic_messages_body(req, "bedrock-2023-05-31");
1110 let body_bytes =
1111 serde_json::to_vec(&body).map_err(|e| ProviderError::Decode(e.to_string()))?;
1112
1113 let creds = AwsEnvCredentials::from_env()?;
1114 let signed = sign_bedrock(&url, region, &body_bytes, &creds)?;
1115 let http_req = reqwest::Request::try_from(signed)
1116 .map_err(|e| ProviderError::Transport(format!("build reqwest request: {e}")))?;
1117
1118 let resp = self
1119 .http
1120 .execute(http_req)
1121 .await
1122 .map_err(|e| ProviderError::Transport(e.to_string()))?;
1123
1124 let status = resp.status();
1125 let bytes = resp
1126 .bytes()
1127 .await
1128 .map_err(|e| ProviderError::Transport(e.to_string()))?;
1129 if !status.is_success() {
1130 return Err(ProviderError::Http {
1131 status: status.as_u16(),
1132 body: String::from_utf8_lossy(&bytes).into_owned(),
1133 });
1134 }
1135
1136 let json: Value =
1137 serde_json::from_slice(&bytes).map_err(|e| ProviderError::Decode(e.to_string()))?;
1138 let u = anthropic_parse_response(&json)?;
1139 let (text, in_tokens, out_tokens) = (u.text, u.in_tokens, u.out_tokens);
1140 let (cache_write_tokens, cache_read_tokens) = (u.cache_write_tokens, u.cache_read_tokens);
1141 Ok(ModelResponse {
1142 cache_write_tokens,
1143 cache_read_tokens,
1144 model: req.model.clone(),
1145 text,
1146 in_tokens,
1147 out_tokens,
1148 raw: json,
1149 })
1150 }
1151}
1152
1153fn vertex_url(region: &str, project: &str, model: &str) -> String {
1155 format!(
1156 "https://{region}-aiplatform.googleapis.com/v1/projects/{project}/locations/{region}/publishers/anthropic/models/{model}:rawPredict"
1157 )
1158}
1159
1160#[derive(Debug, Clone)]
1170pub struct VertexProvider {
1171 pub id: String,
1173 pub region: Option<String>,
1175 pub project: Option<String>,
1177 pub http: reqwest::Client,
1179}
1180
1181#[async_trait]
1182impl Provider for VertexProvider {
1183 fn id(&self) -> &str {
1184 &self.id
1185 }
1186
1187 fn carries_structured_verbatim(&self, inbound: firstpass_core::Dialect) -> bool {
1188 inbound == firstpass_core::Dialect::Anthropic
1189 }
1190
1191 async fn complete(
1192 &self,
1193 req: &ModelRequest,
1194 _auth: &Auth,
1195 ) -> Result<ModelResponse, ProviderError> {
1196 let region = self.region.as_deref().ok_or_else(|| {
1197 ProviderError::Transport("vertex provider requires a region".to_owned())
1198 })?;
1199 let project = self.project.as_deref().ok_or_else(|| {
1200 ProviderError::Transport("vertex provider requires a project".to_owned())
1201 })?;
1202 let model = wire_model(&req.model);
1203 let url = vertex_url(region, project, model);
1204 let body = anthropic_messages_body(req, "vertex-2023-10-16");
1205
1206 let provider = gcp_auth::provider()
1209 .await
1210 .map_err(|e| ProviderError::Transport(format!("gcp_auth provider: {e}")))?;
1211 let token = provider
1212 .token(&["https://www.googleapis.com/auth/cloud-platform"])
1213 .await
1214 .map_err(|e| ProviderError::Transport(format!("gcp_auth token: {e}")))?;
1215
1216 let resp = self
1217 .http
1218 .post(url)
1219 .header(
1220 axum::http::header::AUTHORIZATION,
1221 format!("Bearer {}", token.as_str()),
1222 )
1223 .json(&body)
1224 .send()
1225 .await
1226 .map_err(|e| ProviderError::Transport(e.to_string()))?;
1227
1228 let status = resp.status();
1229 let bytes = resp
1230 .bytes()
1231 .await
1232 .map_err(|e| ProviderError::Transport(e.to_string()))?;
1233 if !status.is_success() {
1234 return Err(ProviderError::Http {
1235 status: status.as_u16(),
1236 body: String::from_utf8_lossy(&bytes).into_owned(),
1237 });
1238 }
1239
1240 let json: Value =
1241 serde_json::from_slice(&bytes).map_err(|e| ProviderError::Decode(e.to_string()))?;
1242 let u = anthropic_parse_response(&json)?;
1243 let (text, in_tokens, out_tokens) = (u.text, u.in_tokens, u.out_tokens);
1244 let (cache_write_tokens, cache_read_tokens) = (u.cache_write_tokens, u.cache_read_tokens);
1245 Ok(ModelResponse {
1246 cache_write_tokens,
1247 cache_read_tokens,
1248 model: req.model.clone(),
1249 text,
1250 in_tokens,
1251 out_tokens,
1252 raw: json,
1253 })
1254 }
1255}
1256
1257#[derive(Clone)]
1259pub struct ProviderRegistry {
1260 providers: HashMap<String, Arc<dyn Provider>>,
1261}
1262
1263impl std::fmt::Debug for ProviderRegistry {
1264 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1265 f.debug_struct("ProviderRegistry")
1266 .field("providers", &self.providers.keys().collect::<Vec<_>>())
1267 .finish()
1268 }
1269}
1270
1271const BUILTIN_OPENAI_COMPATIBLE: &[(&str, &str, &str)] = &[
1289 ("groq", "https://api.groq.com/openai", "GROQ_API_KEY"),
1290 ("deepseek", "https://api.deepseek.com", "DEEPSEEK_API_KEY"),
1291 ("together", "https://api.together.xyz", "TOGETHER_API_KEY"),
1292 (
1293 "fireworks",
1294 "https://api.fireworks.ai/inference",
1295 "FIREWORKS_API_KEY",
1296 ),
1297 ("mistral", "https://api.mistral.ai", "MISTRAL_API_KEY"),
1298 (
1299 "openrouter",
1300 "https://openrouter.ai/api",
1301 "OPENROUTER_API_KEY",
1302 ),
1303 ("xai", "https://api.x.ai", "XAI_API_KEY"),
1304 ("cerebras", "https://api.cerebras.ai", "CEREBRAS_API_KEY"),
1305];
1306
1307#[must_use]
1312pub fn builtin_key_env(id: &str) -> Option<&'static str> {
1313 BUILTIN_OPENAI_COMPATIBLE
1314 .iter()
1315 .find(|(p, _, _)| *p == id)
1316 .map(|(_, _, k)| *k)
1317}
1318
1319impl ProviderRegistry {
1320 fn build_http_client() -> reqwest::Client {
1325 reqwest::Client::builder()
1326 .connect_timeout(std::time::Duration::from_secs(10))
1327 .timeout(std::time::Duration::from_secs(120))
1328 .build()
1329 .unwrap_or_else(|_| reqwest::Client::new())
1330 }
1331
1332 #[must_use]
1334 pub fn new(anthropic_base: impl Into<String>, openai_base: impl Into<String>) -> Self {
1335 Self::from_config(&[], anthropic_base, openai_base)
1336 }
1337
1338 #[must_use]
1344 pub fn from_config(
1345 defs: &[firstpass_core::ProviderDef],
1346 anthropic_base: impl Into<String>,
1347 openai_base: impl Into<String>,
1348 ) -> Self {
1349 let http = Self::build_http_client();
1350 let mut providers: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1351 providers.insert(
1352 "anthropic".to_owned(),
1353 Arc::new(AnthropicProvider {
1354 id: "anthropic".to_owned(),
1355 base_url: anthropic_base.into(),
1356 api_key_env: None,
1357 http: http.clone(),
1358 }),
1359 );
1360 providers.insert(
1361 "openai".to_owned(),
1362 Arc::new(OpenAiProvider {
1363 id: "openai".to_owned(),
1364 base_url: openai_base.into(),
1365 api_key_env: None,
1366 http: http.clone(),
1367 }),
1368 );
1369 for (id, base, key_env) in BUILTIN_OPENAI_COMPATIBLE {
1372 providers.insert(
1373 (*id).to_owned(),
1374 Arc::new(OpenAiProvider {
1375 id: (*id).to_owned(),
1376 base_url: (*base).to_owned(),
1377 api_key_env: Some((*key_env).to_owned()),
1378 http: http.clone(),
1379 }),
1380 );
1381 }
1382 for def in defs {
1383 let provider: Arc<dyn Provider> = match def.auth {
1387 firstpass_core::AuthScheme::AwsSigv4 => Arc::new(BedrockProvider {
1388 id: def.id.clone(),
1389 region: def.region.clone(),
1390 http: http.clone(),
1391 }),
1392 firstpass_core::AuthScheme::GcpOauth => Arc::new(VertexProvider {
1393 id: def.id.clone(),
1394 region: def.region.clone(),
1395 project: def.project.clone(),
1396 http: http.clone(),
1397 }),
1398 firstpass_core::AuthScheme::ApiKey => match def.dialect {
1399 firstpass_core::Dialect::Anthropic => Arc::new(AnthropicProvider {
1400 id: def.id.clone(),
1401 base_url: def.base_url.clone(),
1402 api_key_env: def.api_key_env.clone(),
1403 http: http.clone(),
1404 }),
1405 firstpass_core::Dialect::Openai => Arc::new(OpenAiProvider {
1406 id: def.id.clone(),
1407 base_url: def.base_url.clone(),
1408 api_key_env: def.api_key_env.clone(),
1409 http: http.clone(),
1410 }),
1411 firstpass_core::Dialect::Gemini => Arc::new(GeminiProvider {
1412 id: def.id.clone(),
1413 base_url: def.base_url.clone(),
1414 api_key_env: def.api_key_env.clone(),
1415 http: http.clone(),
1416 }),
1417 },
1418 };
1419 providers.insert(def.id.clone(), provider);
1420 }
1421 Self { providers }
1422 }
1423
1424 #[must_use]
1426 pub fn from_map(providers: HashMap<String, Arc<dyn Provider>>) -> Self {
1427 Self { providers }
1428 }
1429
1430 #[must_use]
1432 pub fn get(&self, provider_id: &str) -> Option<Arc<dyn Provider>> {
1433 self.providers.get(provider_id).cloned()
1434 }
1435}
1436
1437#[cfg(test)]
1439#[derive(Debug, Clone, Default)]
1440pub struct MockProvider {
1441 id: String,
1442 outcomes: HashMap<String, Result<ModelResponse, ProviderError>>,
1443 calls: Arc<std::sync::Mutex<Vec<String>>>,
1446 delay_ms: u64,
1449}
1450
1451#[cfg(test)]
1452impl MockProvider {
1453 #[must_use]
1455 pub fn new(
1456 id: impl Into<String>,
1457 outcomes: HashMap<String, Result<ModelResponse, ProviderError>>,
1458 ) -> Self {
1459 Self {
1460 id: id.into(),
1461 outcomes,
1462 calls: Arc::default(),
1463 delay_ms: 0,
1464 }
1465 }
1466
1467 #[must_use]
1469 pub fn with_delay(mut self, ms: u64) -> Self {
1470 self.delay_ms = ms;
1471 self
1472 }
1473
1474 #[must_use]
1477 pub fn call_log(&self) -> Arc<std::sync::Mutex<Vec<String>>> {
1478 Arc::clone(&self.calls)
1479 }
1480}
1481
1482#[cfg(test)]
1483#[async_trait]
1484impl Provider for MockProvider {
1485 fn id(&self) -> &str {
1486 &self.id
1487 }
1488
1489 fn carries_structured_verbatim(&self, _inbound: firstpass_core::Dialect) -> bool {
1492 true
1493 }
1494
1495 async fn complete(
1496 &self,
1497 req: &ModelRequest,
1498 _auth: &Auth,
1499 ) -> Result<ModelResponse, ProviderError> {
1500 self.calls.lock().unwrap().push(req.model.clone());
1501 if self.delay_ms > 0 {
1502 tokio::time::sleep(std::time::Duration::from_millis(self.delay_ms)).await;
1503 }
1504 self.outcomes.get(&req.model).cloned().unwrap_or_else(|| {
1505 Err(ProviderError::Decode(format!(
1506 "no mock outcome configured for {}",
1507 req.model
1508 )))
1509 })
1510 }
1511}
1512
1513#[cfg(test)]
1514mod tests {
1515 use super::*;
1516
1517 fn obj(v: Value) -> serde_json::Map<String, Value> {
1518 v.as_object().expect("test literal is an object").clone()
1519 }
1520
1521 #[test]
1522 fn openai_reasoning_effort_crosses_onto_an_anthropic_rung() {
1523 let mut body = obj(serde_json::json!({
1527 "model": "claude-sonnet-5", "max_tokens": 8192, "reasoning_effort": "high",
1528 }));
1529
1530 normalize_reasoning(&mut body, Wire::Anthropic);
1531
1532 assert!(
1533 !body.contains_key("reasoning_effort"),
1534 "the foreign spelling must not survive: {body:?}"
1535 );
1536 assert_eq!(body["thinking"]["type"], "enabled");
1537 assert_eq!(body["thinking"]["budget_tokens"], 8191);
1540 }
1541
1542 #[test]
1543 fn anthropic_thinking_budget_crosses_onto_an_openai_rung_as_a_word() {
1544 let mut body = obj(serde_json::json!({
1545 "model": "gpt-5.5",
1546 "thinking": { "type": "enabled", "budget_tokens": 16384 },
1547 }));
1548
1549 normalize_reasoning(&mut body, Wire::OpenAi);
1550
1551 assert!(!body.contains_key("thinking"));
1552 assert_eq!(body["reasoning_effort"], "high");
1553 }
1554
1555 #[test]
1556 fn thinking_is_dropped_rather_than_sent_invalid_when_max_tokens_cannot_fit_it() {
1557 let mut body = obj(serde_json::json!({
1561 "model": "claude-sonnet-5", "max_tokens": 500, "reasoning_effort": "high",
1562 }));
1563
1564 normalize_reasoning(&mut body, Wire::Anthropic);
1565
1566 assert!(
1567 !body.contains_key("thinking"),
1568 "must not emit a budget max_tokens cannot satisfy: {body:?}"
1569 );
1570 assert!(!body.contains_key("reasoning_effort"));
1571 }
1572
1573 #[test]
1574 fn enabling_thinking_clears_a_temperature_the_caller_chose_without_knowing() {
1575 let mut body = obj(serde_json::json!({
1578 "model": "claude-sonnet-5", "max_tokens": 8192,
1579 "reasoning_effort": "medium", "temperature": 0.2, "top_p": 0.9,
1580 }));
1581
1582 normalize_reasoning(&mut body, Wire::Anthropic);
1583
1584 assert_eq!(body["thinking"]["budget_tokens"], 4096);
1585 assert!(!body.contains_key("temperature"), "{body:?}");
1586 assert!(!body.contains_key("top_p"), "{body:?}");
1587 }
1588
1589 #[test]
1590 fn a_body_that_asked_for_no_reasoning_is_left_alone() {
1591 let before = serde_json::json!({
1592 "model": "claude-sonnet-5", "max_tokens": 1024, "temperature": 0.7,
1593 });
1594 let mut body = obj(before.clone());
1595
1596 normalize_reasoning(&mut body, Wire::Anthropic);
1597
1598 assert_eq!(
1599 Value::Object(body),
1600 before,
1601 "no reasoning request, no rewrite"
1602 );
1603 }
1604
1605 #[test]
1606 fn gemini_gets_a_nested_budget_not_a_top_level_one() {
1607 let mut body = obj(serde_json::json!({
1608 "contents": [], "generationConfig": { "maxOutputTokens": 4096 },
1609 "reasoning_effort": "low",
1610 }));
1611
1612 normalize_reasoning(&mut body, Wire::Gemini);
1613
1614 assert_eq!(
1615 body["generationConfig"]["thinkingConfig"]["thinkingBudget"],
1616 1024
1617 );
1618 assert_eq!(
1619 body["generationConfig"]["maxOutputTokens"], 4096,
1620 "must not clobber the rest of generationConfig"
1621 );
1622 assert!(
1623 !body.contains_key("thinkingConfig"),
1624 "top level is wrong for Gemini"
1625 );
1626 }
1627
1628 #[test]
1629 fn a_same_dialect_body_is_never_rewritten_even_when_the_caller_got_it_wrong() {
1630 let before = serde_json::json!({
1634 "max_tokens": 512,
1635 "thinking": { "type": "enabled", "budget_tokens": 1024 },
1636 });
1637 let mut body = obj(before.clone());
1638
1639 normalize_reasoning(&mut body, Wire::Anthropic);
1640
1641 assert_eq!(Value::Object(body), before);
1642 }
1643
1644 #[test]
1645 fn disabled_thinking_is_not_read_as_a_request_to_think() {
1646 let body = obj(serde_json::json!({
1647 "thinking": { "type": "disabled", "budget_tokens": 16384 },
1648 }));
1649 assert_eq!(read_effort(&body), None);
1650 }
1651
1652 #[test]
1653 fn a_cached_response_reports_its_cache_tokens_not_just_the_remainder() {
1654 let json = serde_json::json!({
1658 "content": [{ "type": "text", "text": "ok" }],
1659 "usage": {
1660 "input_tokens": 20,
1661 "cache_creation_input_tokens": 190_000,
1662 "cache_read_input_tokens": 1_234,
1663 "output_tokens": 400,
1664 },
1665 });
1666
1667 let u = anthropic_parse_response(&json).expect("parses");
1668
1669 assert_eq!(u.in_tokens, 20);
1670 assert_eq!(u.cache_write_tokens, 190_000);
1671 assert_eq!(u.cache_read_tokens, 1_234);
1672 assert_eq!(u.out_tokens, 400);
1673 }
1674
1675 #[test]
1676 fn an_uncached_response_reports_zero_cache_traffic() {
1677 let json = serde_json::json!({
1679 "content": [{ "type": "text", "text": "ok" }],
1680 "usage": { "input_tokens": 1000, "output_tokens": 200 },
1681 });
1682
1683 let u = anthropic_parse_response(&json).expect("parses");
1684
1685 assert_eq!(u.in_tokens, 1000);
1686 assert_eq!(u.cache_write_tokens, 0);
1687 assert_eq!(u.cache_read_tokens, 0);
1688 }
1689
1690 #[test]
1691 fn cache_breakpoints_land_on_the_stable_prefix_only() {
1692 let mut body = obj(serde_json::json!({
1695 "model": "claude-sonnet-5",
1696 "system": "you are a careful assistant. ".repeat(200),
1697 "tools": [{ "name": "a" }, { "name": "b" }],
1698 "messages": [{ "role": "user", "content": "hi" }],
1699 }));
1700
1701 add_cache_breakpoints(&mut body);
1702
1703 assert_eq!(body["system"][0]["type"], "text");
1705 assert_eq!(body["system"][0]["cache_control"]["type"], "ephemeral");
1706 assert!(
1707 body["system"][0]["text"]
1708 .as_str()
1709 .unwrap()
1710 .starts_with("you are a careful")
1711 );
1712 assert!(body["tools"][0].get("cache_control").is_none());
1715 assert_eq!(body["tools"][1]["cache_control"]["type"], "ephemeral");
1716 assert!(
1717 body["messages"][0].get("cache_control").is_none(),
1718 "the conversation is not the stable prefix"
1719 );
1720 }
1721
1722 #[test]
1723 fn a_prefix_too_small_to_qualify_is_not_marked() {
1724 let mut body = obj(serde_json::json!({
1727 "model": "claude-sonnet-5",
1728 "system": "be terse",
1729 "messages": [{ "role": "user", "content": "hi" }],
1730 }));
1731 let before = body.clone();
1732
1733 add_cache_breakpoints(&mut body);
1734
1735 assert_eq!(body, before, "a two-word system prompt cannot be cached");
1736 }
1737
1738 #[test]
1739 fn a_caller_that_placed_its_own_breakpoints_is_left_alone() {
1740 let before = serde_json::json!({
1743 "system": [{ "type": "text", "text": "s", "cache_control": { "type": "ephemeral" } }],
1744 "tools": [{ "name": "a" }],
1745 });
1746 let mut body = obj(before.clone());
1747
1748 add_cache_breakpoints(&mut body);
1749
1750 assert_eq!(Value::Object(body), before);
1751 }
1752
1753 #[test]
1754 fn nothing_is_marked_when_there_is_no_stable_prefix() {
1755 let mut body = obj(serde_json::json!({
1756 "model": "claude-sonnet-5",
1757 "messages": [{ "role": "user", "content": "one shot" }],
1758 }));
1759 let before = body.clone();
1760
1761 add_cache_breakpoints(&mut body);
1762
1763 assert_eq!(body, before, "no system, no tools — nothing to cache");
1764 }
1765
1766 #[test]
1767 fn breakpoints_are_off_unless_the_request_asks_for_them() {
1768 let raw = serde_json::json!({
1770 "model": "claude-sonnet-5", "max_tokens": 64,
1771 "system": "a long stable system prefix. ".repeat(200),
1772 "messages": [{ "role": "user", "content": "hi" }],
1773 });
1774 let req = ModelRequest {
1775 model: "anthropic/claude-sonnet-5".to_owned(),
1776 system: Some("a long stable system prefix. ".repeat(200)),
1777 messages: vec![ChatMessage::text("user", "hi")],
1778 max_tokens: 64,
1779 tools: Value::Null,
1780 raw: raw.clone(),
1781 cache_prefix: false,
1782 };
1783
1784 let body = anthropic_wire_body(&req);
1785 assert!(body["system"].is_string(), "unchanged when opted out");
1786
1787 let opted = ModelRequest {
1788 cache_prefix: true,
1789 ..req
1790 };
1791 let body = anthropic_wire_body(&opted);
1792 assert_eq!(body["system"][0]["cache_control"]["type"], "ephemeral");
1793 }
1794
1795 #[test]
1796 fn breakpoints_apply_on_the_normalized_path_too() {
1797 let req = ModelRequest {
1802 model: "anthropic/claude-sonnet-5".to_owned(),
1803 system: Some("stable prefix. ".repeat(300)),
1804 messages: vec![ChatMessage::text("user", "hi")],
1805 max_tokens: 64,
1806 tools: serde_json::json!([{ "name": "a" }]),
1807 raw: Value::Null, cache_prefix: true,
1809 };
1810
1811 let body = anthropic_wire_body(&req);
1812
1813 assert_eq!(body["system"][0]["cache_control"]["type"], "ephemeral");
1814 assert_eq!(body["tools"][0]["cache_control"]["type"], "ephemeral");
1815 }
1816
1817 #[test]
1818 fn common_openai_compatible_providers_work_without_a_config_block() {
1819 let reg = ProviderRegistry::from_config(
1822 &[],
1823 "https://api.anthropic.com",
1824 "https://api.openai.com",
1825 );
1826 for id in [
1827 "groq",
1828 "deepseek",
1829 "together",
1830 "fireworks",
1831 "mistral",
1832 "openrouter",
1833 "xai",
1834 "cerebras",
1835 ] {
1836 assert!(reg.get(id).is_some(), "{id} should be built in");
1837 }
1838 assert!(reg.get("anthropic").is_some());
1840 assert!(reg.get("openai").is_some());
1841 assert!(reg.get("not-a-provider").is_none());
1842 }
1843
1844 #[test]
1845 fn an_explicit_provider_block_overrides_the_builtin() {
1846 let defs = vec![firstpass_core::ProviderDef {
1849 id: "groq".to_owned(),
1850 base_url: "https://internal.example.com/groq".to_owned(),
1851 api_key_env: Some("MY_KEY".to_owned()),
1852 dialect: firstpass_core::Dialect::Openai,
1853 auth: firstpass_core::AuthScheme::ApiKey,
1854 region: None,
1855 project: None,
1856 }];
1857
1858 let reg = ProviderRegistry::from_config(
1859 &defs,
1860 "https://api.anthropic.com",
1861 "https://api.openai.com",
1862 );
1863
1864 let p = reg.get("groq").expect("groq present");
1865 assert!(
1866 format!("{p:?}").contains("internal.example.com"),
1867 "the explicit block must win: {p:?}"
1868 );
1869 }
1870
1871 #[test]
1872 fn wire_model_strips_the_provider_prefix() {
1873 assert_eq!(wire_model("anthropic/claude-haiku-4-5"), "claude-haiku-4-5");
1875 assert_eq!(wire_model("openai/gpt-5.5"), "gpt-5.5");
1876 assert_eq!(wire_model("claude-opus-4-8"), "claude-opus-4-8"); }
1878
1879 #[test]
1880 fn gemini_request_maps_roles_and_system_instruction() {
1881 let req = ModelRequest {
1882 model: "gemini/gemini-2.0-flash".to_owned(),
1883 system: Some("be terse".to_owned()),
1884 messages: vec![
1885 ChatMessage::text("user", "hi"),
1886 ChatMessage::text("assistant", "hello"),
1887 ],
1888 max_tokens: 256,
1889 tools: Value::Null,
1890 raw: Value::Null,
1891 cache_prefix: false,
1892 };
1893 let body = gemini_request_body(&req);
1894 assert_eq!(body["system_instruction"]["parts"][0]["text"], "be terse");
1896 assert_eq!(body["generationConfig"]["maxOutputTokens"], 256);
1897 assert_eq!(body["contents"][0]["role"], "user");
1899 assert_eq!(body["contents"][0]["parts"][0]["text"], "hi");
1900 assert_eq!(body["contents"][1]["role"], "model");
1901 assert_eq!(body["contents"][1]["parts"][0]["text"], "hello");
1902 }
1903
1904 #[test]
1905 fn gemini_response_parses_text_and_usage() {
1906 let json = serde_json::json!({
1907 "candidates": [{ "content": { "role": "model", "parts": [
1908 { "text": "the answer " }, { "text": "is 42" }
1909 ] } }],
1910 "usageMetadata": { "promptTokenCount": 11, "candidatesTokenCount": 4 }
1911 });
1912 let (text, in_tok, out_tok) = gemini_parse_response(&json).unwrap();
1913 assert_eq!(text, "the answer is 42");
1914 assert_eq!(in_tok, 11);
1915 assert_eq!(out_tok, 4);
1916 assert!(gemini_parse_response(&serde_json::json!({ "candidates": [] })).is_err());
1918 }
1919
1920 #[test]
1921 fn from_config_wires_the_gemini_dialect() {
1922 let defs = vec![firstpass_core::ProviderDef {
1923 id: "gemini".to_owned(),
1924 dialect: firstpass_core::Dialect::Gemini,
1925 base_url: "https://generativelanguage.googleapis.com".to_owned(),
1926 api_key_env: Some("GEMINI_API_KEY".to_owned()),
1927 auth: firstpass_core::AuthScheme::ApiKey,
1928 region: None,
1929 project: None,
1930 }];
1931 let reg = ProviderRegistry::from_config(
1932 &defs,
1933 "https://api.anthropic.com",
1934 "https://api.openai.com",
1935 );
1936 assert_eq!(reg.get("gemini").unwrap().id(), "gemini");
1937 }
1938
1939 #[test]
1940 fn from_config_registers_custom_providers_alongside_builtins() {
1941 let defs = vec![
1942 firstpass_core::ProviderDef {
1943 id: "groq".to_owned(),
1944 dialect: firstpass_core::Dialect::Openai,
1945 base_url: "https://api.groq.com/openai".to_owned(),
1946 api_key_env: Some("GROQ_API_KEY".to_owned()),
1947 auth: firstpass_core::AuthScheme::ApiKey,
1948 region: None,
1949 project: None,
1950 },
1951 firstpass_core::ProviderDef {
1953 id: "openai".to_owned(),
1954 dialect: firstpass_core::Dialect::Openai,
1955 base_url: "https://my-azure.openai.azure.com".to_owned(),
1956 api_key_env: Some("AZURE_OPENAI_KEY".to_owned()),
1957 auth: firstpass_core::AuthScheme::ApiKey,
1958 region: None,
1959 project: None,
1960 },
1961 ];
1962 let reg = ProviderRegistry::from_config(
1963 &defs,
1964 "https://api.anthropic.com",
1965 "https://api.openai.com",
1966 );
1967 assert_eq!(reg.get("anthropic").unwrap().id(), "anthropic");
1969 assert_eq!(reg.get("groq").unwrap().id(), "groq");
1970 assert!(reg.get("does-not-exist").is_none());
1972 }
1973
1974 #[test]
1975 fn anthropic_messages_body_omits_model_and_includes_system_only_when_set() {
1976 let req = ModelRequest {
1977 model: "bedrock/anthropic.claude-3-5-haiku".to_owned(),
1978 system: Some("be terse".to_owned()),
1979 messages: vec![
1980 ChatMessage::text("user", "hi"),
1981 ChatMessage {
1982 role: "assistant".to_owned(),
1983 content: serde_json::json!([{ "type": "text", "text": "hello" }]),
1984 },
1985 ],
1986 max_tokens: 128,
1987 tools: Value::Null,
1988 raw: Value::Null,
1989 cache_prefix: false,
1990 };
1991 let body = anthropic_messages_body(&req, "bedrock-2023-05-31");
1992 assert!(body.get("model").is_none());
1994 assert_eq!(body["anthropic_version"], "bedrock-2023-05-31");
1995 assert_eq!(body["max_tokens"], 128);
1996 assert_eq!(body["system"], "be terse");
1997 assert_eq!(body["messages"][0]["content"], serde_json::json!("hi"));
1998 assert_eq!(
2000 body["messages"][1]["content"],
2001 serde_json::json!([{ "type": "text", "text": "hello" }])
2002 );
2003
2004 let req_no_system = ModelRequest {
2006 system: None,
2007 ..req
2008 };
2009 let body2 = anthropic_messages_body(&req_no_system, "vertex-2023-10-16");
2010 assert!(body2.get("system").is_none());
2011 }
2012
2013 #[test]
2014 fn bedrock_url_construction_and_missing_region() {
2015 assert_eq!(
2016 bedrock_url("us-east-1", "anthropic.claude-3-5-haiku-20241022-v1:0"),
2017 "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke"
2018 );
2019 }
2020
2021 #[tokio::test]
2022 async fn bedrock_complete_errors_without_a_region() {
2023 let provider = BedrockProvider {
2024 id: "bedrock".to_owned(),
2025 region: None,
2026 http: reqwest::Client::new(),
2027 };
2028 let req = ModelRequest {
2029 model: "bedrock/anthropic.claude-3-5-haiku".to_owned(),
2030 system: None,
2031 messages: vec![],
2032 max_tokens: 16,
2033 tools: Value::Null,
2034 raw: Value::Null,
2035 cache_prefix: false,
2036 };
2037 let err = provider.complete(&req, &Auth::default()).await.unwrap_err();
2038 assert!(matches!(err, ProviderError::Transport(_)));
2039 }
2040
2041 #[test]
2042 fn bedrock_signing_produces_a_sigv4_authorization_header() {
2043 let creds = AwsEnvCredentials {
2046 access_key_id: "AKIDEXAMPLE".to_owned(),
2047 secret_access_key: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY".to_owned(),
2048 session_token: None,
2049 };
2050 let url = bedrock_url("us-east-1", "anthropic.claude-3-5-haiku");
2051 let body = br#"{"anthropic_version":"bedrock-2023-05-31"}"#;
2052 let signed = sign_bedrock(&url, "us-east-1", body, &creds).unwrap();
2053
2054 let auth_header = signed
2055 .headers()
2056 .get("authorization")
2057 .and_then(|v| v.to_str().ok())
2058 .expect("authorization header present");
2059 assert!(auth_header.starts_with("AWS4-HMAC-SHA256"));
2060
2061 let host_header = signed
2062 .headers()
2063 .get("host")
2064 .and_then(|v| v.to_str().ok())
2065 .expect("host header present");
2066 assert_eq!(host_header, "bedrock-runtime.us-east-1.amazonaws.com");
2067 }
2068
2069 #[test]
2070 fn vertex_url_construction_and_missing_project() {
2071 assert_eq!(
2072 vertex_url("us-central1", "my-project", "claude-3-5-sonnet"),
2073 "https://us-central1-aiplatform.googleapis.com/v1/projects/my-project/locations/us-central1/publishers/anthropic/models/claude-3-5-sonnet:rawPredict"
2074 );
2075 }
2076
2077 #[tokio::test]
2078 async fn vertex_complete_errors_without_a_project() {
2079 let provider = VertexProvider {
2080 id: "vertex".to_owned(),
2081 region: Some("us-central1".to_owned()),
2082 project: None,
2083 http: reqwest::Client::new(),
2084 };
2085 let req = ModelRequest {
2086 model: "vertex/claude-3-5-sonnet".to_owned(),
2087 system: None,
2088 messages: vec![],
2089 max_tokens: 16,
2090 tools: Value::Null,
2091 raw: Value::Null,
2092 cache_prefix: false,
2093 };
2094 let err = provider.complete(&req, &Auth::default()).await.unwrap_err();
2095 assert!(matches!(err, ProviderError::Transport(_)));
2096 }
2097
2098 #[test]
2099 fn from_config_wires_bedrock_and_vertex_auth_schemes() {
2100 let defs = vec![
2101 firstpass_core::ProviderDef {
2102 id: "bedrock".to_owned(),
2103 dialect: firstpass_core::Dialect::Anthropic,
2104 base_url: String::new(),
2105 api_key_env: None,
2106 auth: firstpass_core::AuthScheme::AwsSigv4,
2107 region: Some("us-east-1".to_owned()),
2108 project: None,
2109 },
2110 firstpass_core::ProviderDef {
2111 id: "vertex".to_owned(),
2112 dialect: firstpass_core::Dialect::Anthropic,
2113 base_url: String::new(),
2114 api_key_env: None,
2115 auth: firstpass_core::AuthScheme::GcpOauth,
2116 region: Some("us-central1".to_owned()),
2117 project: Some("my-project".to_owned()),
2118 },
2119 ];
2120 let reg = ProviderRegistry::from_config(
2121 &defs,
2122 "https://api.anthropic.com",
2123 "https://api.openai.com",
2124 );
2125 assert_eq!(reg.get("bedrock").unwrap().id(), "bedrock");
2126 assert_eq!(reg.get("vertex").unwrap().id(), "vertex");
2127 }
2128
2129 #[test]
2130 fn resolve_api_key_prefers_configured_env_then_byok() {
2131 let path = std::env::var("PATH").expect("PATH is set");
2133 assert_eq!(resolve_api_key(Some("PATH"), Some("byok")), path);
2135 assert_eq!(
2137 resolve_api_key(Some("FIRSTPASS_DEFINITELY_UNSET_KEY"), Some("byok")),
2138 "byok"
2139 );
2140 assert_eq!(resolve_api_key(None, Some("byok")), "byok");
2142 assert_eq!(resolve_api_key(None, None), "");
2144 }
2145
2146 #[test]
2147 fn anthropic_wire_forwards_tool_and_image_content_verbatim() {
2148 let messages = vec![
2152 ChatMessage::text("user", "hi"),
2153 ChatMessage {
2154 role: "assistant".to_owned(),
2155 content: serde_json::json!([
2156 { "type": "tool_use", "id": "t1", "name": "calc", "input": { "x": 1 } }
2157 ]),
2158 },
2159 ChatMessage {
2160 role: "user".to_owned(),
2161 content: serde_json::json!([
2162 { "type": "tool_result", "tool_use_id": "t1", "content": "2" },
2163 { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "AA==" } }
2164 ]),
2165 },
2166 ];
2167 let wire = anthropic_wire_body(&ModelRequest {
2168 model: "anthropic/claude-haiku-4-5".to_owned(),
2169 system: None,
2170 messages,
2171 max_tokens: 64,
2172 tools: Value::Null,
2173 raw: Value::Null,
2174 cache_prefix: false,
2175 });
2176 assert_eq!(wire["messages"][0]["content"], serde_json::json!("hi"));
2177 assert_eq!(
2178 wire["messages"][1]["content"],
2179 serde_json::json!([{ "type": "tool_use", "id": "t1", "name": "calc", "input": { "x": 1 } }])
2180 );
2181 assert_eq!(
2182 wire["messages"][2]["content"],
2183 serde_json::json!([
2184 { "type": "tool_result", "tool_use_id": "t1", "content": "2" },
2185 { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "AA==" } }
2186 ])
2187 );
2188 }
2189
2190 fn resp(model: &str, text: &str) -> ModelResponse {
2191 ModelResponse {
2192 model: model.to_owned(),
2193 text: text.to_owned(),
2194 in_tokens: 10,
2195 cache_write_tokens: 0,
2196 cache_read_tokens: 0,
2197 out_tokens: 5,
2198 raw: Value::Null,
2199 }
2200 }
2201
2202 #[test]
2203 fn transport_and_5xx_are_failover_eligible() {
2204 assert!(ProviderError::Transport("boom".into()).is_failover_eligible());
2205 assert!(
2206 ProviderError::Http {
2207 status: 503,
2208 body: String::new()
2209 }
2210 .is_failover_eligible()
2211 );
2212 }
2213
2214 #[test]
2215 fn client_errors_and_decode_failures_are_hard() {
2216 assert!(
2217 !ProviderError::Http {
2218 status: 400,
2219 body: String::new()
2220 }
2221 .is_failover_eligible()
2222 );
2223 assert!(!ProviderError::Decode("bad json".into()).is_failover_eligible());
2224 }
2225
2226 #[test]
2227 fn auth_debug_never_prints_key_material() {
2228 let auth = Auth {
2229 anthropic_key: Some("sk-ant-super-secret".to_owned()),
2230 openai_key: Some("sk-oai-super-secret".to_owned()),
2231 };
2232 let debug = format!("{auth:?}");
2233 assert!(!debug.contains("super-secret"));
2234 }
2235
2236 #[tokio::test]
2237 async fn mock_provider_returns_configured_outcome() {
2238 let mut outcomes = HashMap::new();
2239 outcomes.insert(
2240 "anthropic/claude-haiku-4-5".to_owned(),
2241 Ok(resp("anthropic/claude-haiku-4-5", "hello")),
2242 );
2243 let provider = MockProvider::new("anthropic", outcomes);
2244 let req = ModelRequest {
2245 model: "anthropic/claude-haiku-4-5".to_owned(),
2246 system: None,
2247 messages: vec![],
2248 max_tokens: 100,
2249 tools: Value::Null,
2250 raw: Value::Null,
2251 cache_prefix: false,
2252 };
2253 let out = provider.complete(&req, &Auth::default()).await.unwrap();
2254 assert_eq!(out.text, "hello");
2255 }
2256
2257 #[test]
2258 fn anthropic_wire_body_carries_raw_request_verbatim() {
2259 let raw = serde_json::json!({
2263 "model": "claude-opus-4-8",
2264 "stream": true,
2265 "max_tokens": 512,
2266 "temperature": 0.2,
2267 "tool_choice": { "type": "auto" },
2268 "tools": [{ "name": "get_weather", "input_schema": { "type": "object" } }],
2269 "thinking": { "type": "enabled", "budget_tokens": 1024 },
2270 "messages": [{ "role": "user", "content": "hi" }]
2271 });
2272 let req = ModelRequest {
2273 model: "anthropic/claude-haiku-4-5".to_owned(),
2274 system: None,
2275 messages: vec![ChatMessage::text("user", "hi")],
2276 max_tokens: 512,
2277 tools: raw["tools"].clone(),
2278 raw: raw.clone(),
2279 cache_prefix: false,
2280 };
2281 let wire = anthropic_wire_body(&req);
2282 assert_eq!(wire["model"], "claude-haiku-4-5", "rung model swapped in");
2283 assert!(wire.get("stream").is_none(), "stream stripped");
2284 for field in [
2285 "temperature",
2286 "tool_choice",
2287 "tools",
2288 "thinking",
2289 "max_tokens",
2290 "messages",
2291 ] {
2292 assert_eq!(
2293 wire[field], raw[field],
2294 "field {field} must survive verbatim"
2295 );
2296 }
2297 }
2298
2299 #[test]
2300 fn bedrock_vertex_body_carries_raw_minus_model_plus_version() {
2301 let raw = serde_json::json!({
2302 "model": "claude-haiku-4-5",
2303 "stream": true,
2304 "max_tokens": 64,
2305 "tools": [{ "name": "t" }],
2306 "messages": [{ "role": "user", "content": "hi" }]
2307 });
2308 let req = ModelRequest {
2309 model: "bedrock/anthropic.claude-haiku".to_owned(),
2310 system: None,
2311 messages: vec![ChatMessage::text("user", "hi")],
2312 max_tokens: 64,
2313 tools: raw["tools"].clone(),
2314 raw: raw.clone(),
2315 cache_prefix: false,
2316 };
2317 let body = anthropic_messages_body(&req, "bedrock-2023-05-31");
2318 assert!(body.get("model").is_none(), "model lives in the URL");
2319 assert!(body.get("stream").is_none());
2320 assert_eq!(body["anthropic_version"], "bedrock-2023-05-31");
2321 assert_eq!(body["tools"], raw["tools"]);
2322 assert_eq!(body["messages"], raw["messages"]);
2323 }
2324
2325 #[test]
2326 fn verbatim_carriers_are_dialect_aware() {
2327 use firstpass_core::Dialect;
2328 let reg = ProviderRegistry::new("http://localhost", "http://localhost");
2329 assert!(
2331 reg.get("anthropic")
2332 .unwrap()
2333 .carries_structured_verbatim(Dialect::Anthropic)
2334 );
2335 assert!(
2336 !reg.get("anthropic")
2337 .unwrap()
2338 .carries_structured_verbatim(Dialect::Openai)
2339 );
2340 assert!(
2342 reg.get("openai")
2343 .unwrap()
2344 .carries_structured_verbatim(Dialect::Openai)
2345 );
2346 assert!(
2347 !reg.get("openai")
2348 .unwrap()
2349 .carries_structured_verbatim(Dialect::Anthropic)
2350 );
2351 }
2352
2353 #[test]
2354 fn openai_wire_body_carries_raw_request_verbatim() {
2355 let raw = serde_json::json!({
2357 "model": "gpt-4o",
2358 "stream": true,
2359 "max_tokens": 256,
2360 "temperature": 0.7,
2361 "response_format": { "type": "json_object" },
2362 "tool_choice": "auto",
2363 "tools": [{ "type": "function", "function": { "name": "get_weather", "parameters": {} } }],
2364 "messages": [{ "role": "user", "content": "hi" }]
2365 });
2366 let req = ModelRequest {
2367 model: "openai/gpt-4.1-mini".to_owned(),
2368 system: None,
2369 messages: vec![ChatMessage::text("user", "hi")],
2370 max_tokens: 256,
2371 tools: Value::Null,
2372 raw: raw.clone(),
2373 cache_prefix: false,
2374 };
2375 let wire = openai_wire_body(&req);
2376 assert_eq!(wire["model"], "gpt-4.1-mini", "rung model swapped in");
2377 assert!(wire.get("stream").is_none(), "stream stripped");
2378 for field in [
2379 "temperature",
2380 "tool_choice",
2381 "tools",
2382 "response_format",
2383 "messages",
2384 "max_tokens",
2385 ] {
2386 assert_eq!(
2387 wire[field], raw[field],
2388 "field {field} must survive verbatim"
2389 );
2390 }
2391 }
2392
2393 #[test]
2394 fn openai_wire_body_falls_back_to_normalized_fields() {
2395 let req = ModelRequest {
2397 model: "openai/gpt-4.1-mini".to_owned(),
2398 system: Some("be terse".to_owned()),
2399 messages: vec![
2400 ChatMessage::text("user", "hello"),
2401 ChatMessage::text("assistant", "hi"),
2402 ],
2403 max_tokens: 128,
2404 tools: Value::Null,
2405 raw: Value::Null,
2406 cache_prefix: false,
2407 };
2408 let wire = openai_wire_body(&req);
2409 assert_eq!(wire["model"], "gpt-4.1-mini");
2410 assert_eq!(wire["messages"][0]["role"], "system");
2412 assert_eq!(wire["messages"][0]["content"], "be terse");
2413 assert_eq!(wire["messages"][1]["role"], "user");
2414 assert_eq!(wire["messages"][2]["role"], "assistant");
2415 }
2416}