1use std::collections::HashMap;
7use std::sync::Arc;
8use std::time::SystemTime;
9
10use async_trait::async_trait;
11use axum::http::HeaderMap;
12use serde::{Deserialize, Serialize};
13use serde_json::Value;
14
15#[derive(Debug, Clone, Serialize, Deserialize)]
21pub struct ChatMessage {
22 pub role: String,
24 pub content: Value,
26}
27
28impl ChatMessage {
29 #[must_use]
31 pub fn text(role: impl Into<String>, content: impl Into<String>) -> Self {
32 Self {
33 role: role.into(),
34 content: Value::String(content.into()),
35 }
36 }
37
38 #[must_use]
41 pub fn text_view(&self) -> String {
42 match &self.content {
43 Value::String(s) => s.clone(),
44 Value::Array(blocks) => blocks
45 .iter()
46 .filter_map(|b| b.get("text").and_then(Value::as_str))
47 .collect::<Vec<_>>()
48 .join("\n"),
49 _ => String::new(),
50 }
51 }
52}
53
54#[derive(Debug, Clone, Serialize, Deserialize)]
60pub struct ModelRequest {
61 pub model: String,
63 #[serde(default, skip_serializing_if = "Option::is_none")]
65 pub system: Option<String>,
66 pub messages: Vec<ChatMessage>,
68 pub max_tokens: u32,
70 #[serde(default)]
72 pub tools: Value,
73 #[serde(default, skip_serializing_if = "Value::is_null")]
79 pub raw: Value,
80 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
84 pub cache_prefix: bool,
85}
86
87#[derive(Debug, Clone, Serialize, Deserialize)]
89pub struct ModelResponse {
90 pub model: String,
92 pub text: String,
94 pub in_tokens: u64,
96 pub cache_write_tokens: u64,
99 pub cache_read_tokens: u64,
101 pub out_tokens: u64,
103 pub raw: Value,
105}
106
107impl ModelResponse {
108 #[must_use]
110 pub fn truncated(&self) -> bool {
111 hit_output_limit(&self.raw)
112 }
113}
114
115#[must_use]
119pub fn hit_output_limit(raw: &Value) -> bool {
120 let is = |ptr: &str, want: &str| raw.pointer(ptr).and_then(Value::as_str) == Some(want);
121 is("/stop_reason", "max_tokens")
122 || is("/choices/0/finish_reason", "length")
123 || is("/incomplete_details/reason", "max_output_tokens")
124 || is("/candidates/0/finishReason", "MAX_TOKENS")
125}
126
127#[derive(Debug, Clone, thiserror::Error)]
129pub enum ProviderError {
130 #[error("transport error: {0}")]
132 Transport(String),
133 #[error("http {status}: {body}")]
135 Http {
136 status: u16,
138 body: String,
140 },
141 #[error("decode error: {0}")]
143 Decode(String),
144}
145
146impl ProviderError {
147 #[must_use]
150 pub fn is_failover_eligible(&self) -> bool {
151 match self {
152 ProviderError::Transport(_) => true,
153 ProviderError::Http { status, .. } => *status >= 500,
154 ProviderError::Decode(_) => false,
155 }
156 }
157}
158
159#[derive(Clone, Default)]
163pub struct Auth {
164 pub anthropic_key: Option<String>,
166 pub openai_key: Option<String>,
168}
169
170impl std::fmt::Debug for Auth {
171 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
172 f.debug_struct("Auth")
173 .field("anthropic_key", &self.anthropic_key.as_ref().map(|_| "***"))
174 .field("openai_key", &self.openai_key.as_ref().map(|_| "***"))
175 .finish()
176 }
177}
178
179impl Auth {
180 #[must_use]
183 pub fn from_headers(headers: &HeaderMap) -> Self {
184 let anthropic_key = headers
185 .get("x-api-key")
186 .and_then(|v| v.to_str().ok())
187 .map(str::to_owned)
188 .or_else(|| std::env::var("ANTHROPIC_API_KEY").ok());
189 let openai_key = headers
190 .get(axum::http::header::AUTHORIZATION)
191 .and_then(|v| v.to_str().ok())
192 .and_then(|v| v.strip_prefix("Bearer "))
193 .map(str::to_owned)
194 .or_else(|| std::env::var("OPENAI_API_KEY").ok());
195 Self {
196 anthropic_key,
197 openai_key,
198 }
199 }
200}
201
202#[async_trait]
205pub trait Provider: Send + Sync + std::fmt::Debug {
206 async fn complete(
212 &self,
213 req: &ModelRequest,
214 auth: &Auth,
215 ) -> Result<ModelResponse, ProviderError>;
216
217 fn id(&self) -> &str;
219
220 fn carries_structured_verbatim(&self, _inbound: firstpass_core::Dialect) -> bool {
235 false
236 }
237}
238
239fn wire_model(model: &str) -> &str {
243 model.split_once('/').map_or(model, |(_, m)| m)
244}
245
246fn resolve_api_key(api_key_env: Option<&str>, byok_override: Option<&str>) -> String {
251 api_key_env
252 .and_then(|e| std::env::var(e).ok())
253 .or_else(|| byok_override.map(str::to_owned))
254 .unwrap_or_default()
255}
256
257#[must_use]
262pub fn anthropic_wire_body(req: &ModelRequest) -> Value {
263 if let Value::Object(raw) = &req.raw {
264 let mut body = raw.clone();
265 body.insert(
266 "model".to_owned(),
267 Value::String(wire_model(&req.model).to_owned()),
268 );
269 body.remove("stream");
270 normalize_reasoning(&mut body, Wire::Anthropic);
271 if req.cache_prefix {
272 add_cache_breakpoints(&mut body);
273 }
274 return Value::Object(body);
275 }
276 let messages: Vec<Value> = req
277 .messages
278 .iter()
279 .map(|m| serde_json::json!({ "role": m.role, "content": m.content }))
280 .collect();
281 let mut body = serde_json::json!({
282 "model": wire_model(&req.model),
283 "max_tokens": req.max_tokens,
284 "messages": messages,
285 });
286 if let Some(system) = req.system.as_deref() {
287 body["system"] = serde_json::json!(system);
288 }
289 if !req.tools.is_null() {
290 body["tools"] = req.tools.clone();
291 }
292 if req.cache_prefix
297 && let Some(map) = body.as_object_mut()
298 {
299 add_cache_breakpoints(map);
300 }
301 body
302}
303
304#[derive(Debug, Clone)]
310pub struct AnthropicProvider {
311 pub id: String,
313 pub base_url: String,
315 pub api_key_env: Option<String>,
318 pub http: reqwest::Client,
320}
321
322#[async_trait]
323impl Provider for AnthropicProvider {
324 fn id(&self) -> &str {
325 &self.id
326 }
327
328 fn carries_structured_verbatim(&self, inbound: firstpass_core::Dialect) -> bool {
329 inbound == firstpass_core::Dialect::Anthropic
330 }
331
332 async fn complete(
333 &self,
334 req: &ModelRequest,
335 auth: &Auth,
336 ) -> Result<ModelResponse, ProviderError> {
337 let key = resolve_api_key(self.api_key_env.as_deref(), auth.anthropic_key.as_deref());
338 let body = anthropic_wire_body(req);
339
340 let url = format!("{}/v1/messages", self.base_url.trim_end_matches('/'));
341 let resp = self
342 .http
343 .post(url)
344 .header("x-api-key", key)
345 .header("anthropic-version", "2023-06-01")
346 .json(&body)
347 .send()
348 .await
349 .map_err(|e| ProviderError::Transport(e.to_string()))?;
350
351 let status = resp.status();
352 let bytes = resp
353 .bytes()
354 .await
355 .map_err(|e| ProviderError::Transport(e.to_string()))?;
356
357 if !status.is_success() {
358 return Err(ProviderError::Http {
359 status: status.as_u16(),
360 body: String::from_utf8_lossy(&bytes).into_owned(),
361 });
362 }
363
364 let json: Value =
365 serde_json::from_slice(&bytes).map_err(|e| ProviderError::Decode(e.to_string()))?;
366 let u = anthropic_parse_response(&json)?;
367 let (text, in_tokens, out_tokens) = (u.text, u.in_tokens, u.out_tokens);
368 let (cache_write_tokens, cache_read_tokens) = (u.cache_write_tokens, u.cache_read_tokens);
369
370 Ok(ModelResponse {
371 cache_write_tokens,
372 cache_read_tokens,
373 model: req.model.clone(),
374 text,
375 in_tokens,
376 out_tokens,
377 raw: json,
378 })
379 }
380}
381
382fn anthropic_parse_response(json: &Value) -> Result<AnthropicUsage, ProviderError> {
386 let text = json
387 .get("content")
388 .and_then(Value::as_array)
389 .map(|blocks| {
390 blocks
391 .iter()
392 .filter_map(|b| b.get("text").and_then(Value::as_str))
393 .collect::<Vec<_>>()
394 .join("")
395 })
396 .ok_or_else(|| ProviderError::Decode("missing content[].text".to_owned()))?;
397
398 let in_tokens = json
399 .pointer("/usage/input_tokens")
400 .and_then(Value::as_u64)
401 .unwrap_or(0);
402 let out_tokens = json
403 .pointer("/usage/output_tokens")
404 .and_then(Value::as_u64)
405 .unwrap_or(0);
406 let cache_write_tokens = json
411 .pointer("/usage/cache_creation_input_tokens")
412 .and_then(Value::as_u64)
413 .unwrap_or(0);
414 let cache_read_tokens = json
415 .pointer("/usage/cache_read_input_tokens")
416 .and_then(Value::as_u64)
417 .unwrap_or(0);
418
419 Ok(AnthropicUsage {
420 text,
421 in_tokens,
422 cache_write_tokens,
423 cache_read_tokens,
424 out_tokens,
425 })
426}
427
428struct AnthropicUsage {
430 text: String,
431 in_tokens: u64,
432 cache_write_tokens: u64,
433 cache_read_tokens: u64,
434 out_tokens: u64,
435}
436
437fn anthropic_messages_body(req: &ModelRequest, anthropic_version: &str) -> Value {
442 if let Value::Object(raw) = &req.raw {
445 let mut body = raw.clone();
446 body.remove("model");
447 body.remove("stream");
448 body.insert(
449 "anthropic_version".to_owned(),
450 Value::String(anthropic_version.to_owned()),
451 );
452 return Value::Object(body);
453 }
454 let messages: Vec<Value> = req
455 .messages
456 .iter()
457 .map(|m| serde_json::json!({ "role": m.role, "content": m.content }))
458 .collect();
459 let mut body = serde_json::json!({
460 "anthropic_version": anthropic_version,
461 "max_tokens": req.max_tokens,
462 "messages": messages,
463 });
464 if let Some(system) = req.system.as_deref() {
465 body["system"] = serde_json::json!(system);
466 }
467 if !req.tools.is_null() {
468 body["tools"] = req.tools.clone();
469 }
470 body
471}
472
473#[must_use]
479pub fn openai_wire_body(req: &ModelRequest) -> Value {
480 if let Value::Object(raw) = &req.raw {
481 let mut body = raw.clone();
482 body.insert(
483 "model".to_owned(),
484 Value::String(wire_model(&req.model).to_owned()),
485 );
486 body.remove("stream");
487 normalize_reasoning(&mut body, Wire::OpenAi);
488 return Value::Object(body);
489 }
490 let mut messages: Vec<Value> = Vec::with_capacity(req.messages.len() + 1);
492 if let Some(system) = req.system.as_deref() {
493 messages.push(serde_json::json!({ "role": "system", "content": system }));
494 }
495 messages.extend(
496 req.messages
497 .iter()
498 .map(|m| serde_json::json!({ "role": m.role, "content": m.content })),
499 );
500 let mut body = serde_json::json!({
501 "model": wire_model(&req.model),
502 "max_tokens": req.max_tokens,
503 "messages": messages,
504 });
505 if !req.tools.is_null() {
506 body["tools"] = req.tools.clone();
507 }
508 body
509}
510
511#[derive(Debug, Clone, Copy, PartialEq, Eq)]
519pub enum Effort {
520 Low,
521 Medium,
522 High,
523}
524
525impl Effort {
526 const fn budget_tokens(self) -> u32 {
530 match self {
531 Self::Low => 1024, Self::Medium => 4096,
533 Self::High => 16384,
534 }
535 }
536
537 const fn from_budget(tokens: u64) -> Self {
540 if tokens < 2048 {
541 Self::Low
542 } else if tokens < 8192 {
543 Self::Medium
544 } else {
545 Self::High
546 }
547 }
548
549 fn as_str(self) -> &'static str {
550 match self {
551 Self::Low => "low",
552 Self::Medium => "medium",
553 Self::High => "high",
554 }
555 }
556}
557
558#[must_use]
561pub fn effort_of(raw: &Value) -> Option<Effort> {
562 read_effort(raw.as_object()?)
563}
564
565fn read_effort(body: &serde_json::Map<String, Value>) -> Option<Effort> {
567 if let Some(s) = body.get("reasoning_effort").and_then(Value::as_str) {
569 return match s {
570 "low" | "minimal" => Some(Effort::Low),
571 "medium" => Some(Effort::Medium),
572 "high" => Some(Effort::High),
573 _ => None,
574 };
575 }
576 if let Some(t) = body.get("thinking").and_then(Value::as_object)
578 && t.get("type").and_then(Value::as_str) != Some("disabled")
579 && let Some(b) = t.get("budget_tokens").and_then(Value::as_u64)
580 {
581 return Some(Effort::from_budget(b));
582 }
583 if let Some(b) = body
585 .get("thinkingConfig")
586 .and_then(Value::as_object)
587 .and_then(|c| c.get("thinkingBudget"))
588 .and_then(Value::as_u64)
589 {
590 return Some(Effort::from_budget(b));
591 }
592 None
593}
594
595#[derive(Debug, Clone, Copy, PartialEq, Eq)]
597pub enum Wire {
598 Anthropic,
599 OpenAi,
600 Gemini,
601}
602
603pub fn normalize_reasoning(body: &mut serde_json::Map<String, Value>, target: Wire) {
609 let native = match target {
616 Wire::OpenAi => "reasoning_effort",
617 Wire::Anthropic => "thinking",
618 Wire::Gemini => "thinkingConfig",
619 };
620 if body.contains_key(native) {
621 return;
622 }
623 let effort = read_effort(body);
624 body.remove("reasoning_effort");
626 body.remove("thinking");
627 body.remove("thinkingConfig");
628 let Some(effort) = effort else { return };
629
630 match target {
631 Wire::OpenAi => {
632 body.insert(
633 "reasoning_effort".to_owned(),
634 Value::String(effort.as_str().to_owned()),
635 );
636 }
637 Wire::Anthropic => {
638 let max = body.get("max_tokens").and_then(Value::as_u64).unwrap_or(0);
642 let budget = u64::from(effort.budget_tokens()).min(max.saturating_sub(1));
643 if budget >= 1024 {
644 body.insert(
645 "thinking".to_owned(),
646 serde_json::json!({ "type": "enabled", "budget_tokens": budget }),
647 );
648 body.remove("temperature");
651 body.remove("top_p");
652 }
653 }
654 Wire::Gemini => {
655 body.entry("generationConfig")
657 .or_insert_with(|| Value::Object(serde_json::Map::new()))
658 .as_object_mut()
659 .map(|c| {
660 c.insert(
661 "thinkingConfig".to_owned(),
662 serde_json::json!({ "thinkingBudget": effort.budget_tokens() }),
663 )
664 });
665 }
666 }
667}
668
669const MIN_CACHEABLE_PREFIX_CHARS: usize = 3000;
691
692pub fn add_cache_breakpoints(body: &mut serde_json::Map<String, Value>) {
693 if body_has_cache_control(body) {
694 return;
695 }
696 let prefix_chars = body.get("system").map_or(0, |v| v.to_string().len())
701 + body.get("tools").map_or(0, |v| v.to_string().len());
702 if prefix_chars < MIN_CACHEABLE_PREFIX_CHARS {
703 return;
704 }
705 let marker = serde_json::json!({ "type": "ephemeral" });
706
707 match body.get("system") {
709 Some(Value::String(text)) if !text.is_empty() => {
710 body.insert(
711 "system".to_owned(),
712 serde_json::json!([{
713 "type": "text", "text": text, "cache_control": marker,
714 }]),
715 );
716 }
717 Some(Value::Array(blocks)) if !blocks.is_empty() => {
718 let mut blocks = blocks.clone();
719 if let Some(last) = blocks.last_mut().and_then(Value::as_object_mut) {
720 last.insert("cache_control".to_owned(), marker.clone());
721 }
722 body.insert("system".to_owned(), Value::Array(blocks));
723 }
724 _ => {}
725 }
726
727 if let Some(Value::Array(tools)) = body.get("tools")
730 && !tools.is_empty()
731 {
732 let mut tools = tools.clone();
733 if let Some(last) = tools.last_mut().and_then(Value::as_object_mut) {
734 last.insert("cache_control".to_owned(), marker);
735 }
736 body.insert("tools".to_owned(), Value::Array(tools));
737 }
738}
739
740fn body_has_cache_control(body: &serde_json::Map<String, Value>) -> bool {
742 fn any_in(v: &Value) -> bool {
743 match v {
744 Value::Array(items) => items.iter().any(any_in),
745 Value::Object(o) => o.contains_key("cache_control") || o.values().any(any_in),
746 _ => false,
747 }
748 }
749 body.values().any(any_in)
750}
751
752#[derive(Debug, Clone)]
758pub struct OpenAiProvider {
759 pub id: String,
761 pub base_url: String,
763 pub api_key_env: Option<String>,
767 pub http: reqwest::Client,
769}
770
771#[async_trait]
772impl Provider for OpenAiProvider {
773 fn id(&self) -> &str {
774 &self.id
775 }
776
777 fn carries_structured_verbatim(&self, inbound: firstpass_core::Dialect) -> bool {
778 inbound == firstpass_core::Dialect::Openai
779 }
780
781 async fn complete(
782 &self,
783 req: &ModelRequest,
784 auth: &Auth,
785 ) -> Result<ModelResponse, ProviderError> {
786 let key = resolve_api_key(self.api_key_env.as_deref(), auth.openai_key.as_deref());
787 let body = openai_wire_body(req);
788
789 let url = format!(
790 "{}/v1/chat/completions",
791 self.base_url.trim_end_matches('/')
792 );
793 let resp = self
794 .http
795 .post(url)
796 .header(axum::http::header::AUTHORIZATION, format!("Bearer {key}"))
797 .json(&body)
798 .send()
799 .await
800 .map_err(|e| ProviderError::Transport(e.to_string()))?;
801
802 let status = resp.status();
803 let bytes = resp
804 .bytes()
805 .await
806 .map_err(|e| ProviderError::Transport(e.to_string()))?;
807
808 if !status.is_success() {
809 return Err(ProviderError::Http {
810 status: status.as_u16(),
811 body: String::from_utf8_lossy(&bytes).into_owned(),
812 });
813 }
814
815 let json: Value =
816 serde_json::from_slice(&bytes).map_err(|e| ProviderError::Decode(e.to_string()))?;
817
818 let text = json
820 .pointer("/choices/0/message/content")
821 .and_then(Value::as_str)
822 .unwrap_or("")
823 .to_owned();
824
825 let in_tokens = json
826 .pointer("/usage/prompt_tokens")
827 .and_then(Value::as_u64)
828 .unwrap_or(0);
829 let out_tokens = json
830 .pointer("/usage/completion_tokens")
831 .and_then(Value::as_u64)
832 .unwrap_or(0);
833
834 Ok(ModelResponse {
835 cache_write_tokens: 0,
837 cache_read_tokens: 0,
838 model: req.model.clone(),
839 text,
840 in_tokens,
841 out_tokens,
842 raw: json,
843 })
844 }
845}
846
847fn gemini_request_body(req: &ModelRequest) -> Value {
854 let contents: Vec<Value> = req
855 .messages
856 .iter()
857 .map(|m| {
858 let role = if m.role == "assistant" {
859 "model"
860 } else {
861 "user"
862 };
863 serde_json::json!({ "role": role, "parts": [{ "text": m.text_view() }] })
864 })
865 .collect();
866 let mut body = serde_json::json!({
867 "contents": contents,
868 "generationConfig": { "maxOutputTokens": req.max_tokens },
869 });
870 if let Some(system) = req.system.as_deref() {
871 body["system_instruction"] = serde_json::json!({ "parts": [{ "text": system }] });
872 }
873 if let (Some(effort), Some(map)) = (effort_of(&req.raw), body.as_object_mut()) {
876 let mut carry = serde_json::Map::new();
877 carry.insert(
878 "reasoning_effort".to_owned(),
879 Value::String(effort.as_str().to_owned()),
880 );
881 carry.insert(
882 "generationConfig".to_owned(),
883 map["generationConfig"].clone(),
884 );
885 normalize_reasoning(&mut carry, Wire::Gemini);
886 map.insert(
887 "generationConfig".to_owned(),
888 carry["generationConfig"].clone(),
889 );
890 }
891 body
892}
893
894fn gemini_parse_response(json: &Value) -> Result<(String, u64, u64), ProviderError> {
897 let parts = json
898 .pointer("/candidates/0/content/parts")
899 .and_then(Value::as_array)
900 .ok_or_else(|| ProviderError::Decode("missing candidates[0].content.parts".to_owned()))?;
901 let text = parts
902 .iter()
903 .filter_map(|p| p.get("text").and_then(Value::as_str))
904 .collect::<Vec<_>>()
905 .join("");
906 let in_tokens = json
907 .pointer("/usageMetadata/promptTokenCount")
908 .and_then(Value::as_u64)
909 .unwrap_or(0);
910 let out_tokens = json
911 .pointer("/usageMetadata/candidatesTokenCount")
912 .and_then(Value::as_u64)
913 .unwrap_or(0);
914 Ok((text, in_tokens, out_tokens))
915}
916
917#[derive(Debug, Clone)]
925pub struct GeminiProvider {
926 pub id: String,
928 pub base_url: String,
930 pub api_key_env: Option<String>,
932 pub http: reqwest::Client,
934}
935
936#[async_trait]
937impl Provider for GeminiProvider {
938 fn id(&self) -> &str {
939 &self.id
940 }
941
942 async fn complete(
943 &self,
944 req: &ModelRequest,
945 auth: &Auth,
946 ) -> Result<ModelResponse, ProviderError> {
947 let key = resolve_api_key(self.api_key_env.as_deref(), auth.openai_key.as_deref());
948 let body = gemini_request_body(req);
949 let url = format!(
950 "{}/v1beta/models/{}:generateContent",
951 self.base_url.trim_end_matches('/'),
952 wire_model(&req.model),
953 );
954 let resp = self
955 .http
956 .post(url)
957 .header("x-goog-api-key", key)
958 .json(&body)
959 .send()
960 .await
961 .map_err(|e| ProviderError::Transport(e.to_string()))?;
962
963 let status = resp.status();
964 let bytes = resp
965 .bytes()
966 .await
967 .map_err(|e| ProviderError::Transport(e.to_string()))?;
968 if !status.is_success() {
969 return Err(ProviderError::Http {
970 status: status.as_u16(),
971 body: String::from_utf8_lossy(&bytes).into_owned(),
972 });
973 }
974
975 let json: Value =
976 serde_json::from_slice(&bytes).map_err(|e| ProviderError::Decode(e.to_string()))?;
977 let (text, in_tokens, out_tokens) = gemini_parse_response(&json)?;
978 Ok(ModelResponse {
979 cache_write_tokens: 0,
981 cache_read_tokens: 0,
982 model: req.model.clone(),
983 text,
984 in_tokens,
985 out_tokens,
986 raw: json,
987 })
988 }
989}
990
991struct AwsEnvCredentials {
995 access_key_id: String,
996 secret_access_key: String,
997 session_token: Option<String>,
998}
999
1000impl AwsEnvCredentials {
1001 fn from_env() -> Result<Self, ProviderError> {
1002 let access_key_id = std::env::var("AWS_ACCESS_KEY_ID")
1003 .map_err(|_| ProviderError::Transport("AWS_ACCESS_KEY_ID is not set".to_owned()))?;
1004 let secret_access_key = std::env::var("AWS_SECRET_ACCESS_KEY")
1005 .map_err(|_| ProviderError::Transport("AWS_SECRET_ACCESS_KEY is not set".to_owned()))?;
1006 let session_token = std::env::var("AWS_SESSION_TOKEN").ok();
1007 Ok(Self {
1008 access_key_id,
1009 secret_access_key,
1010 session_token,
1011 })
1012 }
1013}
1014
1015fn bedrock_url(region: &str, model: &str) -> String {
1017 format!("https://bedrock-runtime.{region}.amazonaws.com/model/{model}/invoke")
1018}
1019
1020fn sign_bedrock(
1031 url: &str,
1032 region: &str,
1033 body: &[u8],
1034 creds: &AwsEnvCredentials,
1035) -> Result<http::Request<Vec<u8>>, ProviderError> {
1036 let host = url
1037 .parse::<http::Uri>()
1038 .ok()
1039 .and_then(|u| u.host().map(str::to_owned))
1040 .ok_or_else(|| ProviderError::Transport(format!("invalid bedrock URL: {url}")))?;
1041
1042 let identity: aws_smithy_runtime_api::client::identity::Identity =
1043 aws_credential_types::Credentials::new(
1044 creds.access_key_id.clone(),
1045 creds.secret_access_key.clone(),
1046 creds.session_token.clone(),
1047 None,
1048 "firstpass",
1049 )
1050 .into();
1051
1052 let signing_params: aws_sigv4::http_request::SigningParams<'_> =
1053 aws_sigv4::sign::v4::SigningParams::builder()
1054 .identity(&identity)
1055 .region(region)
1056 .name("bedrock")
1057 .time(SystemTime::now())
1058 .settings(aws_sigv4::http_request::SigningSettings::default())
1059 .build()
1060 .map_err(|e| ProviderError::Transport(format!("sigv4 signing params: {e}")))?
1061 .into();
1062
1063 let headers = [
1064 ("host", host.as_str()),
1065 ("content-type", "application/json"),
1066 ];
1067 let signable = aws_sigv4::http_request::SignableRequest::new(
1068 "POST",
1069 url,
1070 headers.into_iter(),
1071 aws_sigv4::http_request::SignableBody::Bytes(body),
1072 )
1073 .map_err(|e| ProviderError::Transport(format!("sigv4 signable request: {e}")))?;
1074
1075 let (instructions, _signature) = aws_sigv4::http_request::sign(signable, &signing_params)
1076 .map_err(|e| ProviderError::Transport(format!("sigv4 sign: {e}")))?
1077 .into_parts();
1078
1079 let mut req = http::Request::builder()
1080 .method("POST")
1081 .uri(url)
1082 .header("host", host)
1083 .header("content-type", "application/json")
1084 .body(body.to_vec())
1085 .map_err(|e| ProviderError::Transport(format!("build bedrock request: {e}")))?;
1086 instructions.apply_to_request_http1x(&mut req);
1087 Ok(req)
1088}
1089
1090#[derive(Debug, Clone)]
1100pub struct BedrockProvider {
1101 pub id: String,
1103 pub region: Option<String>,
1105 pub http: reqwest::Client,
1107}
1108
1109#[async_trait]
1110impl Provider for BedrockProvider {
1111 fn id(&self) -> &str {
1112 &self.id
1113 }
1114
1115 fn carries_structured_verbatim(&self, inbound: firstpass_core::Dialect) -> bool {
1116 inbound == firstpass_core::Dialect::Anthropic
1117 }
1118
1119 async fn complete(
1120 &self,
1121 req: &ModelRequest,
1122 _auth: &Auth,
1123 ) -> Result<ModelResponse, ProviderError> {
1124 let region = self.region.as_deref().ok_or_else(|| {
1125 ProviderError::Transport("bedrock provider requires a region".to_owned())
1126 })?;
1127 let model = wire_model(&req.model);
1128 let url = bedrock_url(region, model);
1129 let body = anthropic_messages_body(req, "bedrock-2023-05-31");
1130 let body_bytes =
1131 serde_json::to_vec(&body).map_err(|e| ProviderError::Decode(e.to_string()))?;
1132
1133 let creds = AwsEnvCredentials::from_env()?;
1134 let signed = sign_bedrock(&url, region, &body_bytes, &creds)?;
1135 let http_req = reqwest::Request::try_from(signed)
1136 .map_err(|e| ProviderError::Transport(format!("build reqwest request: {e}")))?;
1137
1138 let resp = self
1139 .http
1140 .execute(http_req)
1141 .await
1142 .map_err(|e| ProviderError::Transport(e.to_string()))?;
1143
1144 let status = resp.status();
1145 let bytes = resp
1146 .bytes()
1147 .await
1148 .map_err(|e| ProviderError::Transport(e.to_string()))?;
1149 if !status.is_success() {
1150 return Err(ProviderError::Http {
1151 status: status.as_u16(),
1152 body: String::from_utf8_lossy(&bytes).into_owned(),
1153 });
1154 }
1155
1156 let json: Value =
1157 serde_json::from_slice(&bytes).map_err(|e| ProviderError::Decode(e.to_string()))?;
1158 let u = anthropic_parse_response(&json)?;
1159 let (text, in_tokens, out_tokens) = (u.text, u.in_tokens, u.out_tokens);
1160 let (cache_write_tokens, cache_read_tokens) = (u.cache_write_tokens, u.cache_read_tokens);
1161 Ok(ModelResponse {
1162 cache_write_tokens,
1163 cache_read_tokens,
1164 model: req.model.clone(),
1165 text,
1166 in_tokens,
1167 out_tokens,
1168 raw: json,
1169 })
1170 }
1171}
1172
1173fn vertex_url(region: &str, project: &str, model: &str) -> String {
1175 format!(
1176 "https://{region}-aiplatform.googleapis.com/v1/projects/{project}/locations/{region}/publishers/anthropic/models/{model}:rawPredict"
1177 )
1178}
1179
1180#[derive(Debug, Clone)]
1190pub struct VertexProvider {
1191 pub id: String,
1193 pub region: Option<String>,
1195 pub project: Option<String>,
1197 pub http: reqwest::Client,
1199}
1200
1201#[async_trait]
1202impl Provider for VertexProvider {
1203 fn id(&self) -> &str {
1204 &self.id
1205 }
1206
1207 fn carries_structured_verbatim(&self, inbound: firstpass_core::Dialect) -> bool {
1208 inbound == firstpass_core::Dialect::Anthropic
1209 }
1210
1211 async fn complete(
1212 &self,
1213 req: &ModelRequest,
1214 _auth: &Auth,
1215 ) -> Result<ModelResponse, ProviderError> {
1216 let region = self.region.as_deref().ok_or_else(|| {
1217 ProviderError::Transport("vertex provider requires a region".to_owned())
1218 })?;
1219 let project = self.project.as_deref().ok_or_else(|| {
1220 ProviderError::Transport("vertex provider requires a project".to_owned())
1221 })?;
1222 let model = wire_model(&req.model);
1223 let url = vertex_url(region, project, model);
1224 let body = anthropic_messages_body(req, "vertex-2023-10-16");
1225
1226 let provider = gcp_auth::provider()
1229 .await
1230 .map_err(|e| ProviderError::Transport(format!("gcp_auth provider: {e}")))?;
1231 let token = provider
1232 .token(&["https://www.googleapis.com/auth/cloud-platform"])
1233 .await
1234 .map_err(|e| ProviderError::Transport(format!("gcp_auth token: {e}")))?;
1235
1236 let resp = self
1237 .http
1238 .post(url)
1239 .header(
1240 axum::http::header::AUTHORIZATION,
1241 format!("Bearer {}", token.as_str()),
1242 )
1243 .json(&body)
1244 .send()
1245 .await
1246 .map_err(|e| ProviderError::Transport(e.to_string()))?;
1247
1248 let status = resp.status();
1249 let bytes = resp
1250 .bytes()
1251 .await
1252 .map_err(|e| ProviderError::Transport(e.to_string()))?;
1253 if !status.is_success() {
1254 return Err(ProviderError::Http {
1255 status: status.as_u16(),
1256 body: String::from_utf8_lossy(&bytes).into_owned(),
1257 });
1258 }
1259
1260 let json: Value =
1261 serde_json::from_slice(&bytes).map_err(|e| ProviderError::Decode(e.to_string()))?;
1262 let u = anthropic_parse_response(&json)?;
1263 let (text, in_tokens, out_tokens) = (u.text, u.in_tokens, u.out_tokens);
1264 let (cache_write_tokens, cache_read_tokens) = (u.cache_write_tokens, u.cache_read_tokens);
1265 Ok(ModelResponse {
1266 cache_write_tokens,
1267 cache_read_tokens,
1268 model: req.model.clone(),
1269 text,
1270 in_tokens,
1271 out_tokens,
1272 raw: json,
1273 })
1274 }
1275}
1276
1277#[derive(Clone)]
1279pub struct ProviderRegistry {
1280 providers: HashMap<String, Arc<dyn Provider>>,
1281}
1282
1283impl std::fmt::Debug for ProviderRegistry {
1284 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1285 f.debug_struct("ProviderRegistry")
1286 .field("providers", &self.providers.keys().collect::<Vec<_>>())
1287 .finish()
1288 }
1289}
1290
1291const BUILTIN_OPENAI_COMPATIBLE: &[(&str, &str, &str)] = &[
1309 ("groq", "https://api.groq.com/openai", "GROQ_API_KEY"),
1310 ("deepseek", "https://api.deepseek.com", "DEEPSEEK_API_KEY"),
1311 ("together", "https://api.together.xyz", "TOGETHER_API_KEY"),
1312 (
1313 "fireworks",
1314 "https://api.fireworks.ai/inference",
1315 "FIREWORKS_API_KEY",
1316 ),
1317 ("mistral", "https://api.mistral.ai", "MISTRAL_API_KEY"),
1318 (
1319 "openrouter",
1320 "https://openrouter.ai/api",
1321 "OPENROUTER_API_KEY",
1322 ),
1323 ("xai", "https://api.x.ai", "XAI_API_KEY"),
1324 ("cerebras", "https://api.cerebras.ai", "CEREBRAS_API_KEY"),
1325];
1326
1327#[must_use]
1332pub fn builtin_key_env(id: &str) -> Option<&'static str> {
1333 BUILTIN_OPENAI_COMPATIBLE
1334 .iter()
1335 .find(|(p, _, _)| *p == id)
1336 .map(|(_, _, k)| *k)
1337}
1338
1339impl ProviderRegistry {
1340 fn build_http_client() -> reqwest::Client {
1348 reqwest::Client::builder()
1349 .connect_timeout(std::time::Duration::from_secs(10))
1350 .timeout(std::time::Duration::from_secs(600))
1351 .build()
1352 .unwrap_or_else(|_| reqwest::Client::new())
1353 }
1354
1355 #[must_use]
1357 pub fn new(anthropic_base: impl Into<String>, openai_base: impl Into<String>) -> Self {
1358 Self::from_config(&[], anthropic_base, openai_base)
1359 }
1360
1361 #[must_use]
1367 pub fn from_config(
1368 defs: &[firstpass_core::ProviderDef],
1369 anthropic_base: impl Into<String>,
1370 openai_base: impl Into<String>,
1371 ) -> Self {
1372 let http = Self::build_http_client();
1373 let mut providers: HashMap<String, Arc<dyn Provider>> = HashMap::new();
1374 providers.insert(
1375 "anthropic".to_owned(),
1376 Arc::new(AnthropicProvider {
1377 id: "anthropic".to_owned(),
1378 base_url: anthropic_base.into(),
1379 api_key_env: None,
1380 http: http.clone(),
1381 }),
1382 );
1383 providers.insert(
1384 "openai".to_owned(),
1385 Arc::new(OpenAiProvider {
1386 id: "openai".to_owned(),
1387 base_url: openai_base.into(),
1388 api_key_env: None,
1389 http: http.clone(),
1390 }),
1391 );
1392 for (id, base, key_env) in BUILTIN_OPENAI_COMPATIBLE {
1395 providers.insert(
1396 (*id).to_owned(),
1397 Arc::new(OpenAiProvider {
1398 id: (*id).to_owned(),
1399 base_url: (*base).to_owned(),
1400 api_key_env: Some((*key_env).to_owned()),
1401 http: http.clone(),
1402 }),
1403 );
1404 }
1405 for def in defs {
1406 let provider: Arc<dyn Provider> = match def.auth {
1410 firstpass_core::AuthScheme::AwsSigv4 => Arc::new(BedrockProvider {
1411 id: def.id.clone(),
1412 region: def.region.clone(),
1413 http: http.clone(),
1414 }),
1415 firstpass_core::AuthScheme::GcpOauth => Arc::new(VertexProvider {
1416 id: def.id.clone(),
1417 region: def.region.clone(),
1418 project: def.project.clone(),
1419 http: http.clone(),
1420 }),
1421 firstpass_core::AuthScheme::ApiKey => match def.dialect {
1422 firstpass_core::Dialect::Anthropic => Arc::new(AnthropicProvider {
1423 id: def.id.clone(),
1424 base_url: def.base_url.clone(),
1425 api_key_env: def.api_key_env.clone(),
1426 http: http.clone(),
1427 }),
1428 firstpass_core::Dialect::Openai => Arc::new(OpenAiProvider {
1429 id: def.id.clone(),
1430 base_url: def.base_url.clone(),
1431 api_key_env: def.api_key_env.clone(),
1432 http: http.clone(),
1433 }),
1434 firstpass_core::Dialect::Gemini => Arc::new(GeminiProvider {
1435 id: def.id.clone(),
1436 base_url: def.base_url.clone(),
1437 api_key_env: def.api_key_env.clone(),
1438 http: http.clone(),
1439 }),
1440 },
1441 };
1442 providers.insert(def.id.clone(), provider);
1443 }
1444 Self { providers }
1445 }
1446
1447 #[must_use]
1449 pub fn from_map(providers: HashMap<String, Arc<dyn Provider>>) -> Self {
1450 Self { providers }
1451 }
1452
1453 #[must_use]
1455 pub fn get(&self, provider_id: &str) -> Option<Arc<dyn Provider>> {
1456 self.providers.get(provider_id).cloned()
1457 }
1458}
1459
1460#[cfg(test)]
1462#[derive(Debug, Clone, Default)]
1463pub struct MockProvider {
1464 id: String,
1465 outcomes: HashMap<String, Result<ModelResponse, ProviderError>>,
1466 calls: Arc<std::sync::Mutex<Vec<String>>>,
1469 delay_ms: u64,
1472}
1473
1474#[cfg(test)]
1475impl MockProvider {
1476 #[must_use]
1478 pub fn new(
1479 id: impl Into<String>,
1480 outcomes: HashMap<String, Result<ModelResponse, ProviderError>>,
1481 ) -> Self {
1482 Self {
1483 id: id.into(),
1484 outcomes,
1485 calls: Arc::default(),
1486 delay_ms: 0,
1487 }
1488 }
1489
1490 #[must_use]
1492 pub fn with_delay(mut self, ms: u64) -> Self {
1493 self.delay_ms = ms;
1494 self
1495 }
1496
1497 #[must_use]
1500 pub fn call_log(&self) -> Arc<std::sync::Mutex<Vec<String>>> {
1501 Arc::clone(&self.calls)
1502 }
1503}
1504
1505#[cfg(test)]
1506#[async_trait]
1507impl Provider for MockProvider {
1508 fn id(&self) -> &str {
1509 &self.id
1510 }
1511
1512 fn carries_structured_verbatim(&self, _inbound: firstpass_core::Dialect) -> bool {
1515 true
1516 }
1517
1518 async fn complete(
1519 &self,
1520 req: &ModelRequest,
1521 _auth: &Auth,
1522 ) -> Result<ModelResponse, ProviderError> {
1523 self.calls.lock().unwrap().push(req.model.clone());
1524 if self.delay_ms > 0 {
1525 tokio::time::sleep(std::time::Duration::from_millis(self.delay_ms)).await;
1526 }
1527 self.outcomes.get(&req.model).cloned().unwrap_or_else(|| {
1528 Err(ProviderError::Decode(format!(
1529 "no mock outcome configured for {}",
1530 req.model
1531 )))
1532 })
1533 }
1534}
1535
1536#[cfg(test)]
1537mod tests {
1538 #[test]
1541 fn output_limit_is_detected_across_providers() {
1542 use serde_json::json;
1543 for raw in [
1544 json!({"stop_reason": "max_tokens"}),
1545 json!({"choices": [{"finish_reason": "length"}]}),
1546 json!({"status": "incomplete", "incomplete_details": {"reason": "max_output_tokens"}}),
1547 json!({"candidates": [{"finishReason": "MAX_TOKENS"}]}),
1548 ] {
1549 assert!(hit_output_limit(&raw), "{raw}");
1550 }
1551 for raw in [
1552 json!({"stop_reason": "end_turn"}),
1553 json!({"choices": [{"finish_reason": "stop"}]}),
1554 json!({"incomplete_details": {"reason": "content_filter"}}),
1555 json!({"candidates": [{"finishReason": "STOP"}]}),
1556 serde_json::Value::Null,
1557 ] {
1558 assert!(!hit_output_limit(&raw), "{raw}");
1559 }
1560 }
1561
1562 use super::*;
1563
1564 fn obj(v: Value) -> serde_json::Map<String, Value> {
1565 v.as_object().expect("test literal is an object").clone()
1566 }
1567
1568 #[test]
1569 fn openai_reasoning_effort_crosses_onto_an_anthropic_rung() {
1570 let mut body = obj(serde_json::json!({
1574 "model": "claude-sonnet-5", "max_tokens": 8192, "reasoning_effort": "high",
1575 }));
1576
1577 normalize_reasoning(&mut body, Wire::Anthropic);
1578
1579 assert!(
1580 !body.contains_key("reasoning_effort"),
1581 "the foreign spelling must not survive: {body:?}"
1582 );
1583 assert_eq!(body["thinking"]["type"], "enabled");
1584 assert_eq!(body["thinking"]["budget_tokens"], 8191);
1587 }
1588
1589 #[test]
1590 fn anthropic_thinking_budget_crosses_onto_an_openai_rung_as_a_word() {
1591 let mut body = obj(serde_json::json!({
1592 "model": "gpt-5.5",
1593 "thinking": { "type": "enabled", "budget_tokens": 16384 },
1594 }));
1595
1596 normalize_reasoning(&mut body, Wire::OpenAi);
1597
1598 assert!(!body.contains_key("thinking"));
1599 assert_eq!(body["reasoning_effort"], "high");
1600 }
1601
1602 #[test]
1603 fn thinking_is_dropped_rather_than_sent_invalid_when_max_tokens_cannot_fit_it() {
1604 let mut body = obj(serde_json::json!({
1608 "model": "claude-sonnet-5", "max_tokens": 500, "reasoning_effort": "high",
1609 }));
1610
1611 normalize_reasoning(&mut body, Wire::Anthropic);
1612
1613 assert!(
1614 !body.contains_key("thinking"),
1615 "must not emit a budget max_tokens cannot satisfy: {body:?}"
1616 );
1617 assert!(!body.contains_key("reasoning_effort"));
1618 }
1619
1620 #[test]
1621 fn enabling_thinking_clears_a_temperature_the_caller_chose_without_knowing() {
1622 let mut body = obj(serde_json::json!({
1625 "model": "claude-sonnet-5", "max_tokens": 8192,
1626 "reasoning_effort": "medium", "temperature": 0.2, "top_p": 0.9,
1627 }));
1628
1629 normalize_reasoning(&mut body, Wire::Anthropic);
1630
1631 assert_eq!(body["thinking"]["budget_tokens"], 4096);
1632 assert!(!body.contains_key("temperature"), "{body:?}");
1633 assert!(!body.contains_key("top_p"), "{body:?}");
1634 }
1635
1636 #[test]
1637 fn a_body_that_asked_for_no_reasoning_is_left_alone() {
1638 let before = serde_json::json!({
1639 "model": "claude-sonnet-5", "max_tokens": 1024, "temperature": 0.7,
1640 });
1641 let mut body = obj(before.clone());
1642
1643 normalize_reasoning(&mut body, Wire::Anthropic);
1644
1645 assert_eq!(
1646 Value::Object(body),
1647 before,
1648 "no reasoning request, no rewrite"
1649 );
1650 }
1651
1652 #[test]
1653 fn gemini_gets_a_nested_budget_not_a_top_level_one() {
1654 let mut body = obj(serde_json::json!({
1655 "contents": [], "generationConfig": { "maxOutputTokens": 4096 },
1656 "reasoning_effort": "low",
1657 }));
1658
1659 normalize_reasoning(&mut body, Wire::Gemini);
1660
1661 assert_eq!(
1662 body["generationConfig"]["thinkingConfig"]["thinkingBudget"],
1663 1024
1664 );
1665 assert_eq!(
1666 body["generationConfig"]["maxOutputTokens"], 4096,
1667 "must not clobber the rest of generationConfig"
1668 );
1669 assert!(
1670 !body.contains_key("thinkingConfig"),
1671 "top level is wrong for Gemini"
1672 );
1673 }
1674
1675 #[test]
1676 fn a_same_dialect_body_is_never_rewritten_even_when_the_caller_got_it_wrong() {
1677 let before = serde_json::json!({
1681 "max_tokens": 512,
1682 "thinking": { "type": "enabled", "budget_tokens": 1024 },
1683 });
1684 let mut body = obj(before.clone());
1685
1686 normalize_reasoning(&mut body, Wire::Anthropic);
1687
1688 assert_eq!(Value::Object(body), before);
1689 }
1690
1691 #[test]
1692 fn disabled_thinking_is_not_read_as_a_request_to_think() {
1693 let body = obj(serde_json::json!({
1694 "thinking": { "type": "disabled", "budget_tokens": 16384 },
1695 }));
1696 assert_eq!(read_effort(&body), None);
1697 }
1698
1699 #[test]
1700 fn a_cached_response_reports_its_cache_tokens_not_just_the_remainder() {
1701 let json = serde_json::json!({
1705 "content": [{ "type": "text", "text": "ok" }],
1706 "usage": {
1707 "input_tokens": 20,
1708 "cache_creation_input_tokens": 190_000,
1709 "cache_read_input_tokens": 1_234,
1710 "output_tokens": 400,
1711 },
1712 });
1713
1714 let u = anthropic_parse_response(&json).expect("parses");
1715
1716 assert_eq!(u.in_tokens, 20);
1717 assert_eq!(u.cache_write_tokens, 190_000);
1718 assert_eq!(u.cache_read_tokens, 1_234);
1719 assert_eq!(u.out_tokens, 400);
1720 }
1721
1722 #[test]
1723 fn an_uncached_response_reports_zero_cache_traffic() {
1724 let json = serde_json::json!({
1726 "content": [{ "type": "text", "text": "ok" }],
1727 "usage": { "input_tokens": 1000, "output_tokens": 200 },
1728 });
1729
1730 let u = anthropic_parse_response(&json).expect("parses");
1731
1732 assert_eq!(u.in_tokens, 1000);
1733 assert_eq!(u.cache_write_tokens, 0);
1734 assert_eq!(u.cache_read_tokens, 0);
1735 }
1736
1737 #[test]
1738 fn cache_breakpoints_land_on_the_stable_prefix_only() {
1739 let mut body = obj(serde_json::json!({
1742 "model": "claude-sonnet-5",
1743 "system": "you are a careful assistant. ".repeat(200),
1744 "tools": [{ "name": "a" }, { "name": "b" }],
1745 "messages": [{ "role": "user", "content": "hi" }],
1746 }));
1747
1748 add_cache_breakpoints(&mut body);
1749
1750 assert_eq!(body["system"][0]["type"], "text");
1752 assert_eq!(body["system"][0]["cache_control"]["type"], "ephemeral");
1753 assert!(
1754 body["system"][0]["text"]
1755 .as_str()
1756 .unwrap()
1757 .starts_with("you are a careful")
1758 );
1759 assert!(body["tools"][0].get("cache_control").is_none());
1762 assert_eq!(body["tools"][1]["cache_control"]["type"], "ephemeral");
1763 assert!(
1764 body["messages"][0].get("cache_control").is_none(),
1765 "the conversation is not the stable prefix"
1766 );
1767 }
1768
1769 #[test]
1770 fn a_prefix_too_small_to_qualify_is_not_marked() {
1771 let mut body = obj(serde_json::json!({
1774 "model": "claude-sonnet-5",
1775 "system": "be terse",
1776 "messages": [{ "role": "user", "content": "hi" }],
1777 }));
1778 let before = body.clone();
1779
1780 add_cache_breakpoints(&mut body);
1781
1782 assert_eq!(body, before, "a two-word system prompt cannot be cached");
1783 }
1784
1785 #[test]
1786 fn a_caller_that_placed_its_own_breakpoints_is_left_alone() {
1787 let before = serde_json::json!({
1790 "system": [{ "type": "text", "text": "s", "cache_control": { "type": "ephemeral" } }],
1791 "tools": [{ "name": "a" }],
1792 });
1793 let mut body = obj(before.clone());
1794
1795 add_cache_breakpoints(&mut body);
1796
1797 assert_eq!(Value::Object(body), before);
1798 }
1799
1800 #[test]
1801 fn nothing_is_marked_when_there_is_no_stable_prefix() {
1802 let mut body = obj(serde_json::json!({
1803 "model": "claude-sonnet-5",
1804 "messages": [{ "role": "user", "content": "one shot" }],
1805 }));
1806 let before = body.clone();
1807
1808 add_cache_breakpoints(&mut body);
1809
1810 assert_eq!(body, before, "no system, no tools — nothing to cache");
1811 }
1812
1813 #[test]
1814 fn breakpoints_are_off_unless_the_request_asks_for_them() {
1815 let raw = serde_json::json!({
1817 "model": "claude-sonnet-5", "max_tokens": 64,
1818 "system": "a long stable system prefix. ".repeat(200),
1819 "messages": [{ "role": "user", "content": "hi" }],
1820 });
1821 let req = ModelRequest {
1822 model: "anthropic/claude-sonnet-5".to_owned(),
1823 system: Some("a long stable system prefix. ".repeat(200)),
1824 messages: vec![ChatMessage::text("user", "hi")],
1825 max_tokens: 64,
1826 tools: Value::Null,
1827 raw: raw.clone(),
1828 cache_prefix: false,
1829 };
1830
1831 let body = anthropic_wire_body(&req);
1832 assert!(body["system"].is_string(), "unchanged when opted out");
1833
1834 let opted = ModelRequest {
1835 cache_prefix: true,
1836 ..req
1837 };
1838 let body = anthropic_wire_body(&opted);
1839 assert_eq!(body["system"][0]["cache_control"]["type"], "ephemeral");
1840 }
1841
1842 #[test]
1843 fn breakpoints_apply_on_the_normalized_path_too() {
1844 let req = ModelRequest {
1849 model: "anthropic/claude-sonnet-5".to_owned(),
1850 system: Some("stable prefix. ".repeat(300)),
1851 messages: vec![ChatMessage::text("user", "hi")],
1852 max_tokens: 64,
1853 tools: serde_json::json!([{ "name": "a" }]),
1854 raw: Value::Null, cache_prefix: true,
1856 };
1857
1858 let body = anthropic_wire_body(&req);
1859
1860 assert_eq!(body["system"][0]["cache_control"]["type"], "ephemeral");
1861 assert_eq!(body["tools"][0]["cache_control"]["type"], "ephemeral");
1862 }
1863
1864 #[test]
1865 fn common_openai_compatible_providers_work_without_a_config_block() {
1866 let reg = ProviderRegistry::from_config(
1869 &[],
1870 "https://api.anthropic.com",
1871 "https://api.openai.com",
1872 );
1873 for id in [
1874 "groq",
1875 "deepseek",
1876 "together",
1877 "fireworks",
1878 "mistral",
1879 "openrouter",
1880 "xai",
1881 "cerebras",
1882 ] {
1883 assert!(reg.get(id).is_some(), "{id} should be built in");
1884 }
1885 assert!(reg.get("anthropic").is_some());
1887 assert!(reg.get("openai").is_some());
1888 assert!(reg.get("not-a-provider").is_none());
1889 }
1890
1891 #[test]
1892 fn an_explicit_provider_block_overrides_the_builtin() {
1893 let defs = vec![firstpass_core::ProviderDef {
1896 id: "groq".to_owned(),
1897 base_url: "https://internal.example.com/groq".to_owned(),
1898 api_key_env: Some("MY_KEY".to_owned()),
1899 dialect: firstpass_core::Dialect::Openai,
1900 auth: firstpass_core::AuthScheme::ApiKey,
1901 region: None,
1902 project: None,
1903 }];
1904
1905 let reg = ProviderRegistry::from_config(
1906 &defs,
1907 "https://api.anthropic.com",
1908 "https://api.openai.com",
1909 );
1910
1911 let p = reg.get("groq").expect("groq present");
1912 assert!(
1913 format!("{p:?}").contains("internal.example.com"),
1914 "the explicit block must win: {p:?}"
1915 );
1916 }
1917
1918 #[test]
1919 fn wire_model_strips_the_provider_prefix() {
1920 assert_eq!(wire_model("anthropic/claude-haiku-4-5"), "claude-haiku-4-5");
1922 assert_eq!(wire_model("openai/gpt-5.5"), "gpt-5.5");
1923 assert_eq!(wire_model("claude-opus-4-8"), "claude-opus-4-8"); }
1925
1926 #[test]
1927 fn gemini_request_maps_roles_and_system_instruction() {
1928 let req = ModelRequest {
1929 model: "gemini/gemini-2.0-flash".to_owned(),
1930 system: Some("be terse".to_owned()),
1931 messages: vec![
1932 ChatMessage::text("user", "hi"),
1933 ChatMessage::text("assistant", "hello"),
1934 ],
1935 max_tokens: 256,
1936 tools: Value::Null,
1937 raw: Value::Null,
1938 cache_prefix: false,
1939 };
1940 let body = gemini_request_body(&req);
1941 assert_eq!(body["system_instruction"]["parts"][0]["text"], "be terse");
1943 assert_eq!(body["generationConfig"]["maxOutputTokens"], 256);
1944 assert_eq!(body["contents"][0]["role"], "user");
1946 assert_eq!(body["contents"][0]["parts"][0]["text"], "hi");
1947 assert_eq!(body["contents"][1]["role"], "model");
1948 assert_eq!(body["contents"][1]["parts"][0]["text"], "hello");
1949 }
1950
1951 #[test]
1952 fn gemini_response_parses_text_and_usage() {
1953 let json = serde_json::json!({
1954 "candidates": [{ "content": { "role": "model", "parts": [
1955 { "text": "the answer " }, { "text": "is 42" }
1956 ] } }],
1957 "usageMetadata": { "promptTokenCount": 11, "candidatesTokenCount": 4 }
1958 });
1959 let (text, in_tok, out_tok) = gemini_parse_response(&json).unwrap();
1960 assert_eq!(text, "the answer is 42");
1961 assert_eq!(in_tok, 11);
1962 assert_eq!(out_tok, 4);
1963 assert!(gemini_parse_response(&serde_json::json!({ "candidates": [] })).is_err());
1965 }
1966
1967 #[test]
1968 fn from_config_wires_the_gemini_dialect() {
1969 let defs = vec![firstpass_core::ProviderDef {
1970 id: "gemini".to_owned(),
1971 dialect: firstpass_core::Dialect::Gemini,
1972 base_url: "https://generativelanguage.googleapis.com".to_owned(),
1973 api_key_env: Some("GEMINI_API_KEY".to_owned()),
1974 auth: firstpass_core::AuthScheme::ApiKey,
1975 region: None,
1976 project: None,
1977 }];
1978 let reg = ProviderRegistry::from_config(
1979 &defs,
1980 "https://api.anthropic.com",
1981 "https://api.openai.com",
1982 );
1983 assert_eq!(reg.get("gemini").unwrap().id(), "gemini");
1984 }
1985
1986 #[test]
1987 fn from_config_registers_custom_providers_alongside_builtins() {
1988 let defs = vec![
1989 firstpass_core::ProviderDef {
1990 id: "groq".to_owned(),
1991 dialect: firstpass_core::Dialect::Openai,
1992 base_url: "https://api.groq.com/openai".to_owned(),
1993 api_key_env: Some("GROQ_API_KEY".to_owned()),
1994 auth: firstpass_core::AuthScheme::ApiKey,
1995 region: None,
1996 project: None,
1997 },
1998 firstpass_core::ProviderDef {
2000 id: "openai".to_owned(),
2001 dialect: firstpass_core::Dialect::Openai,
2002 base_url: "https://my-azure.openai.azure.com".to_owned(),
2003 api_key_env: Some("AZURE_OPENAI_KEY".to_owned()),
2004 auth: firstpass_core::AuthScheme::ApiKey,
2005 region: None,
2006 project: None,
2007 },
2008 ];
2009 let reg = ProviderRegistry::from_config(
2010 &defs,
2011 "https://api.anthropic.com",
2012 "https://api.openai.com",
2013 );
2014 assert_eq!(reg.get("anthropic").unwrap().id(), "anthropic");
2016 assert_eq!(reg.get("groq").unwrap().id(), "groq");
2017 assert!(reg.get("does-not-exist").is_none());
2019 }
2020
2021 #[test]
2022 fn anthropic_messages_body_omits_model_and_includes_system_only_when_set() {
2023 let req = ModelRequest {
2024 model: "bedrock/anthropic.claude-3-5-haiku".to_owned(),
2025 system: Some("be terse".to_owned()),
2026 messages: vec![
2027 ChatMessage::text("user", "hi"),
2028 ChatMessage {
2029 role: "assistant".to_owned(),
2030 content: serde_json::json!([{ "type": "text", "text": "hello" }]),
2031 },
2032 ],
2033 max_tokens: 128,
2034 tools: Value::Null,
2035 raw: Value::Null,
2036 cache_prefix: false,
2037 };
2038 let body = anthropic_messages_body(&req, "bedrock-2023-05-31");
2039 assert!(body.get("model").is_none());
2041 assert_eq!(body["anthropic_version"], "bedrock-2023-05-31");
2042 assert_eq!(body["max_tokens"], 128);
2043 assert_eq!(body["system"], "be terse");
2044 assert_eq!(body["messages"][0]["content"], serde_json::json!("hi"));
2045 assert_eq!(
2047 body["messages"][1]["content"],
2048 serde_json::json!([{ "type": "text", "text": "hello" }])
2049 );
2050
2051 let req_no_system = ModelRequest {
2053 system: None,
2054 ..req
2055 };
2056 let body2 = anthropic_messages_body(&req_no_system, "vertex-2023-10-16");
2057 assert!(body2.get("system").is_none());
2058 }
2059
2060 #[test]
2061 fn bedrock_url_construction_and_missing_region() {
2062 assert_eq!(
2063 bedrock_url("us-east-1", "anthropic.claude-3-5-haiku-20241022-v1:0"),
2064 "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke"
2065 );
2066 }
2067
2068 #[tokio::test]
2069 async fn bedrock_complete_errors_without_a_region() {
2070 let provider = BedrockProvider {
2071 id: "bedrock".to_owned(),
2072 region: None,
2073 http: reqwest::Client::new(),
2074 };
2075 let req = ModelRequest {
2076 model: "bedrock/anthropic.claude-3-5-haiku".to_owned(),
2077 system: None,
2078 messages: vec![],
2079 max_tokens: 16,
2080 tools: Value::Null,
2081 raw: Value::Null,
2082 cache_prefix: false,
2083 };
2084 let err = provider.complete(&req, &Auth::default()).await.unwrap_err();
2085 assert!(matches!(err, ProviderError::Transport(_)));
2086 }
2087
2088 #[test]
2089 fn bedrock_signing_produces_a_sigv4_authorization_header() {
2090 let creds = AwsEnvCredentials {
2093 access_key_id: "AKIDEXAMPLE".to_owned(),
2094 secret_access_key: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY".to_owned(),
2095 session_token: None,
2096 };
2097 let url = bedrock_url("us-east-1", "anthropic.claude-3-5-haiku");
2098 let body = br#"{"anthropic_version":"bedrock-2023-05-31"}"#;
2099 let signed = sign_bedrock(&url, "us-east-1", body, &creds).unwrap();
2100
2101 let auth_header = signed
2102 .headers()
2103 .get("authorization")
2104 .and_then(|v| v.to_str().ok())
2105 .expect("authorization header present");
2106 assert!(auth_header.starts_with("AWS4-HMAC-SHA256"));
2107
2108 let host_header = signed
2109 .headers()
2110 .get("host")
2111 .and_then(|v| v.to_str().ok())
2112 .expect("host header present");
2113 assert_eq!(host_header, "bedrock-runtime.us-east-1.amazonaws.com");
2114 }
2115
2116 #[test]
2117 fn vertex_url_construction_and_missing_project() {
2118 assert_eq!(
2119 vertex_url("us-central1", "my-project", "claude-3-5-sonnet"),
2120 "https://us-central1-aiplatform.googleapis.com/v1/projects/my-project/locations/us-central1/publishers/anthropic/models/claude-3-5-sonnet:rawPredict"
2121 );
2122 }
2123
2124 #[tokio::test]
2125 async fn vertex_complete_errors_without_a_project() {
2126 let provider = VertexProvider {
2127 id: "vertex".to_owned(),
2128 region: Some("us-central1".to_owned()),
2129 project: None,
2130 http: reqwest::Client::new(),
2131 };
2132 let req = ModelRequest {
2133 model: "vertex/claude-3-5-sonnet".to_owned(),
2134 system: None,
2135 messages: vec![],
2136 max_tokens: 16,
2137 tools: Value::Null,
2138 raw: Value::Null,
2139 cache_prefix: false,
2140 };
2141 let err = provider.complete(&req, &Auth::default()).await.unwrap_err();
2142 assert!(matches!(err, ProviderError::Transport(_)));
2143 }
2144
2145 #[test]
2146 fn from_config_wires_bedrock_and_vertex_auth_schemes() {
2147 let defs = vec![
2148 firstpass_core::ProviderDef {
2149 id: "bedrock".to_owned(),
2150 dialect: firstpass_core::Dialect::Anthropic,
2151 base_url: String::new(),
2152 api_key_env: None,
2153 auth: firstpass_core::AuthScheme::AwsSigv4,
2154 region: Some("us-east-1".to_owned()),
2155 project: None,
2156 },
2157 firstpass_core::ProviderDef {
2158 id: "vertex".to_owned(),
2159 dialect: firstpass_core::Dialect::Anthropic,
2160 base_url: String::new(),
2161 api_key_env: None,
2162 auth: firstpass_core::AuthScheme::GcpOauth,
2163 region: Some("us-central1".to_owned()),
2164 project: Some("my-project".to_owned()),
2165 },
2166 ];
2167 let reg = ProviderRegistry::from_config(
2168 &defs,
2169 "https://api.anthropic.com",
2170 "https://api.openai.com",
2171 );
2172 assert_eq!(reg.get("bedrock").unwrap().id(), "bedrock");
2173 assert_eq!(reg.get("vertex").unwrap().id(), "vertex");
2174 }
2175
2176 #[test]
2177 fn resolve_api_key_prefers_configured_env_then_byok() {
2178 let path = std::env::var("PATH").expect("PATH is set");
2180 assert_eq!(resolve_api_key(Some("PATH"), Some("byok")), path);
2182 assert_eq!(
2184 resolve_api_key(Some("FIRSTPASS_DEFINITELY_UNSET_KEY"), Some("byok")),
2185 "byok"
2186 );
2187 assert_eq!(resolve_api_key(None, Some("byok")), "byok");
2189 assert_eq!(resolve_api_key(None, None), "");
2191 }
2192
2193 #[test]
2194 fn anthropic_wire_forwards_tool_and_image_content_verbatim() {
2195 let messages = vec![
2199 ChatMessage::text("user", "hi"),
2200 ChatMessage {
2201 role: "assistant".to_owned(),
2202 content: serde_json::json!([
2203 { "type": "tool_use", "id": "t1", "name": "calc", "input": { "x": 1 } }
2204 ]),
2205 },
2206 ChatMessage {
2207 role: "user".to_owned(),
2208 content: serde_json::json!([
2209 { "type": "tool_result", "tool_use_id": "t1", "content": "2" },
2210 { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "AA==" } }
2211 ]),
2212 },
2213 ];
2214 let wire = anthropic_wire_body(&ModelRequest {
2215 model: "anthropic/claude-haiku-4-5".to_owned(),
2216 system: None,
2217 messages,
2218 max_tokens: 64,
2219 tools: Value::Null,
2220 raw: Value::Null,
2221 cache_prefix: false,
2222 });
2223 assert_eq!(wire["messages"][0]["content"], serde_json::json!("hi"));
2224 assert_eq!(
2225 wire["messages"][1]["content"],
2226 serde_json::json!([{ "type": "tool_use", "id": "t1", "name": "calc", "input": { "x": 1 } }])
2227 );
2228 assert_eq!(
2229 wire["messages"][2]["content"],
2230 serde_json::json!([
2231 { "type": "tool_result", "tool_use_id": "t1", "content": "2" },
2232 { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "AA==" } }
2233 ])
2234 );
2235 }
2236
2237 fn resp(model: &str, text: &str) -> ModelResponse {
2238 ModelResponse {
2239 model: model.to_owned(),
2240 text: text.to_owned(),
2241 in_tokens: 10,
2242 cache_write_tokens: 0,
2243 cache_read_tokens: 0,
2244 out_tokens: 5,
2245 raw: Value::Null,
2246 }
2247 }
2248
2249 #[test]
2250 fn transport_and_5xx_are_failover_eligible() {
2251 assert!(ProviderError::Transport("boom".into()).is_failover_eligible());
2252 assert!(
2253 ProviderError::Http {
2254 status: 503,
2255 body: String::new()
2256 }
2257 .is_failover_eligible()
2258 );
2259 }
2260
2261 #[test]
2262 fn client_errors_and_decode_failures_are_hard() {
2263 assert!(
2264 !ProviderError::Http {
2265 status: 400,
2266 body: String::new()
2267 }
2268 .is_failover_eligible()
2269 );
2270 assert!(!ProviderError::Decode("bad json".into()).is_failover_eligible());
2271 }
2272
2273 #[test]
2274 fn auth_debug_never_prints_key_material() {
2275 let auth = Auth {
2276 anthropic_key: Some("sk-ant-super-secret".to_owned()),
2277 openai_key: Some("sk-oai-super-secret".to_owned()),
2278 };
2279 let debug = format!("{auth:?}");
2280 assert!(!debug.contains("super-secret"));
2281 }
2282
2283 #[tokio::test]
2284 async fn mock_provider_returns_configured_outcome() {
2285 let mut outcomes = HashMap::new();
2286 outcomes.insert(
2287 "anthropic/claude-haiku-4-5".to_owned(),
2288 Ok(resp("anthropic/claude-haiku-4-5", "hello")),
2289 );
2290 let provider = MockProvider::new("anthropic", outcomes);
2291 let req = ModelRequest {
2292 model: "anthropic/claude-haiku-4-5".to_owned(),
2293 system: None,
2294 messages: vec![],
2295 max_tokens: 100,
2296 tools: Value::Null,
2297 raw: Value::Null,
2298 cache_prefix: false,
2299 };
2300 let out = provider.complete(&req, &Auth::default()).await.unwrap();
2301 assert_eq!(out.text, "hello");
2302 }
2303
2304 #[test]
2305 fn anthropic_wire_body_carries_raw_request_verbatim() {
2306 let raw = serde_json::json!({
2310 "model": "claude-opus-4-8",
2311 "stream": true,
2312 "max_tokens": 512,
2313 "temperature": 0.2,
2314 "tool_choice": { "type": "auto" },
2315 "tools": [{ "name": "get_weather", "input_schema": { "type": "object" } }],
2316 "thinking": { "type": "enabled", "budget_tokens": 1024 },
2317 "messages": [{ "role": "user", "content": "hi" }]
2318 });
2319 let req = ModelRequest {
2320 model: "anthropic/claude-haiku-4-5".to_owned(),
2321 system: None,
2322 messages: vec![ChatMessage::text("user", "hi")],
2323 max_tokens: 512,
2324 tools: raw["tools"].clone(),
2325 raw: raw.clone(),
2326 cache_prefix: false,
2327 };
2328 let wire = anthropic_wire_body(&req);
2329 assert_eq!(wire["model"], "claude-haiku-4-5", "rung model swapped in");
2330 assert!(wire.get("stream").is_none(), "stream stripped");
2331 for field in [
2332 "temperature",
2333 "tool_choice",
2334 "tools",
2335 "thinking",
2336 "max_tokens",
2337 "messages",
2338 ] {
2339 assert_eq!(
2340 wire[field], raw[field],
2341 "field {field} must survive verbatim"
2342 );
2343 }
2344 }
2345
2346 #[test]
2347 fn bedrock_vertex_body_carries_raw_minus_model_plus_version() {
2348 let raw = serde_json::json!({
2349 "model": "claude-haiku-4-5",
2350 "stream": true,
2351 "max_tokens": 64,
2352 "tools": [{ "name": "t" }],
2353 "messages": [{ "role": "user", "content": "hi" }]
2354 });
2355 let req = ModelRequest {
2356 model: "bedrock/anthropic.claude-haiku".to_owned(),
2357 system: None,
2358 messages: vec![ChatMessage::text("user", "hi")],
2359 max_tokens: 64,
2360 tools: raw["tools"].clone(),
2361 raw: raw.clone(),
2362 cache_prefix: false,
2363 };
2364 let body = anthropic_messages_body(&req, "bedrock-2023-05-31");
2365 assert!(body.get("model").is_none(), "model lives in the URL");
2366 assert!(body.get("stream").is_none());
2367 assert_eq!(body["anthropic_version"], "bedrock-2023-05-31");
2368 assert_eq!(body["tools"], raw["tools"]);
2369 assert_eq!(body["messages"], raw["messages"]);
2370 }
2371
2372 #[test]
2373 fn verbatim_carriers_are_dialect_aware() {
2374 use firstpass_core::Dialect;
2375 let reg = ProviderRegistry::new("http://localhost", "http://localhost");
2376 assert!(
2378 reg.get("anthropic")
2379 .unwrap()
2380 .carries_structured_verbatim(Dialect::Anthropic)
2381 );
2382 assert!(
2383 !reg.get("anthropic")
2384 .unwrap()
2385 .carries_structured_verbatim(Dialect::Openai)
2386 );
2387 assert!(
2389 reg.get("openai")
2390 .unwrap()
2391 .carries_structured_verbatim(Dialect::Openai)
2392 );
2393 assert!(
2394 !reg.get("openai")
2395 .unwrap()
2396 .carries_structured_verbatim(Dialect::Anthropic)
2397 );
2398 }
2399
2400 #[test]
2401 fn openai_wire_body_carries_raw_request_verbatim() {
2402 let raw = serde_json::json!({
2404 "model": "gpt-4o",
2405 "stream": true,
2406 "max_tokens": 256,
2407 "temperature": 0.7,
2408 "response_format": { "type": "json_object" },
2409 "tool_choice": "auto",
2410 "tools": [{ "type": "function", "function": { "name": "get_weather", "parameters": {} } }],
2411 "messages": [{ "role": "user", "content": "hi" }]
2412 });
2413 let req = ModelRequest {
2414 model: "openai/gpt-4.1-mini".to_owned(),
2415 system: None,
2416 messages: vec![ChatMessage::text("user", "hi")],
2417 max_tokens: 256,
2418 tools: Value::Null,
2419 raw: raw.clone(),
2420 cache_prefix: false,
2421 };
2422 let wire = openai_wire_body(&req);
2423 assert_eq!(wire["model"], "gpt-4.1-mini", "rung model swapped in");
2424 assert!(wire.get("stream").is_none(), "stream stripped");
2425 for field in [
2426 "temperature",
2427 "tool_choice",
2428 "tools",
2429 "response_format",
2430 "messages",
2431 "max_tokens",
2432 ] {
2433 assert_eq!(
2434 wire[field], raw[field],
2435 "field {field} must survive verbatim"
2436 );
2437 }
2438 }
2439
2440 #[test]
2441 fn openai_wire_body_falls_back_to_normalized_fields() {
2442 let req = ModelRequest {
2444 model: "openai/gpt-4.1-mini".to_owned(),
2445 system: Some("be terse".to_owned()),
2446 messages: vec![
2447 ChatMessage::text("user", "hello"),
2448 ChatMessage::text("assistant", "hi"),
2449 ],
2450 max_tokens: 128,
2451 tools: Value::Null,
2452 raw: Value::Null,
2453 cache_prefix: false,
2454 };
2455 let wire = openai_wire_body(&req);
2456 assert_eq!(wire["model"], "gpt-4.1-mini");
2457 assert_eq!(wire["messages"][0]["role"], "system");
2459 assert_eq!(wire["messages"][0]["content"], "be terse");
2460 assert_eq!(wire["messages"][1]["role"], "user");
2461 assert_eq!(wire["messages"][2]["role"], "assistant");
2462 }
2463}