1use async_trait::async_trait;
9use serde::{Deserialize, Serialize};
10use serde_json::Value;
11use std::collections::HashMap;
12use std::path::Path;
13
14use crate::error::Error;
15use crate::error::Result;
16use crate::message::Message;
17
18use tokio::sync::mpsc;
19
20#[cfg(feature = "anthropic")]
21pub mod anthropic;
22pub mod mock;
23pub mod openai;
24
25#[cfg(feature = "anthropic")]
26pub use anthropic::AnthropicProvider;
27#[cfg(any(test, feature = "test-utils"))]
28pub use mock::MockProvider;
29pub use openai::OpenAiProvider;
30
31pub type StreamSender = mpsc::UnboundedSender<String>;
34
35#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
37pub struct TokenUsage {
38 pub prompt_tokens: u32,
39 pub completion_tokens: u32,
40 pub total_tokens: u32,
41 pub cache_hit_tokens: u32,
42 pub cache_miss_tokens: u32,
43}
44
45impl TokenUsage {
46 pub fn accumulate(self, other: TokenUsage) -> TokenUsage {
48 TokenUsage {
49 prompt_tokens: self.prompt_tokens.saturating_add(other.prompt_tokens),
50 completion_tokens: self
51 .completion_tokens
52 .saturating_add(other.completion_tokens),
53 total_tokens: self.total_tokens.saturating_add(other.total_tokens),
54 cache_hit_tokens: self.cache_hit_tokens.saturating_add(other.cache_hit_tokens),
55 cache_miss_tokens: self
56 .cache_miss_tokens
57 .saturating_add(other.cache_miss_tokens),
58 }
59 }
60}
61
62#[derive(Debug, Clone, Copy, PartialEq)]
64pub struct ModelPricing {
65 pub input_per_million: f64,
66 pub output_per_million: f64,
67 pub cache_hit_input_per_million: f64,
70}
71
72impl ModelPricing {
73 pub fn cost_usd(&self, usage: TokenUsage) -> f64 {
75 let in_cost = if usage.cache_hit_tokens > 0 {
76 let cache_hit =
78 usage.cache_hit_tokens as f64 * self.cache_hit_input_per_million / 1_000_000.0;
79 let cache_miss = usage.cache_miss_tokens as f64 * self.input_per_million / 1_000_000.0;
83 cache_hit + cache_miss
84 } else {
85 (usage.prompt_tokens as f64) * self.input_per_million / 1_000_000.0
86 };
87 let out_cost = (usage.completion_tokens as f64) * self.output_per_million / 1_000_000.0;
88 in_cost + out_cost
89 }
90}
91
92pub fn pricing_for(model: &str) -> Option<ModelPricing> {
94 match model {
95 "MiniMax-M2" => Some(ModelPricing {
96 input_per_million: 0.30,
97 output_per_million: 1.20,
98 cache_hit_input_per_million: 0.30,
100 }),
101 "deepseek-chat" | "deepseek-v4-flash" => Some(ModelPricing {
102 input_per_million: 0.27,
103 output_per_million: 1.10,
104 cache_hit_input_per_million: 0.027,
106 }),
107 "deepseek-v4-pro" => Some(ModelPricing {
110 input_per_million: 1.89,
111 output_per_million: 7.70,
112 cache_hit_input_per_million: 0.189,
114 }),
115 "glm-4-flash" => Some(ModelPricing {
116 input_per_million: 0.10,
117 output_per_million: 0.10,
118 cache_hit_input_per_million: 0.10,
120 }),
121 "glm-5.1" => Some(ModelPricing {
125 input_per_million: 0.50,
126 output_per_million: 2.00,
127 cache_hit_input_per_million: 0.50,
129 }),
130 _ => {
131 None
135 }
136 }
137}
138
139pub fn load_pricing_from_yaml(path: &Path) -> Result<HashMap<String, ModelPricing>> {
143 use std::fs;
144 use std::io::{self, BufRead};
145
146 let file = fs::File::open(path).map_err(Error::Io)?;
147 let reader = io::BufReader::new(file);
148
149 let mut models: HashMap<String, ModelPricing> = HashMap::new();
150 let mut current_model: Option<String> = None;
151 let mut current_pricing: Option<ModelPricingBuilder> = None;
152 let mut in_models_section = false;
153
154 for line in reader.lines() {
160 let line = line.map_err(Error::Io)?;
161
162 if line.trim().is_empty() || line.trim().starts_with('#') {
164 continue;
165 }
166
167 let leading_spaces = line.len() - line.trim_start().len();
169 let trimmed = line.trim();
170
171 if trimmed == "models:" {
173 in_models_section = true;
174 continue;
175 }
176
177 if in_models_section {
179 if leading_spaces == 2 && trimmed.ends_with(':') {
182 if let (Some(name), Some(builder)) = (current_model.take(), current_pricing.take())
184 {
185 if let Some(pricing) = builder.build() {
186 models.insert(name, pricing);
187 }
188 }
189 let model_name = trimmed.trim_end_matches(':').to_string();
191 current_model = Some(model_name);
192 current_pricing = Some(ModelPricingBuilder::default());
193 continue;
194 }
195
196 if leading_spaces >= 4 && current_model.is_some() {
199 if let Some(ref mut builder) = current_pricing {
200 if let Some((key, value)) = trimmed.split_once(':') {
201 let key = key.trim();
202 let value = value.split('#').next().unwrap_or(value).trim();
204 if let Err(e) = builder.parse_field(key, value) {
205 return Err(Error::Config {
206 message: format!("error parsing {}: {}", path.display(), e),
207 });
208 }
209 }
210 }
211 }
212 }
213 }
214
215 if let (Some(name), Some(builder)) = (current_model, current_pricing) {
217 if let Some(pricing) = builder.build() {
218 models.insert(name, pricing);
219 }
220 }
221
222 Ok(models)
223}
224
225#[derive(Default)]
227struct ModelPricingBuilder {
228 input_per_million: Option<f64>,
229 output_per_million: Option<f64>,
230 cache_hit_input_per_million: Option<f64>,
231}
232
233impl ModelPricingBuilder {
234 fn parse_field(&mut self, key: &str, value: &str) -> Result<(), String> {
235 let value: f64 = value
236 .parse()
237 .map_err(|_| format!("invalid float: {}", value))?;
238 match key {
239 "input_per_million" => self.input_per_million = Some(value),
240 "output_per_million" => self.output_per_million = Some(value),
241 "cache_hit_input_per_million" => self.cache_hit_input_per_million = Some(value),
242 _ => return Err(format!("unknown field: {}", key)),
243 }
244 Ok(())
245 }
246
247 fn build(self) -> Option<ModelPricing> {
248 Some(ModelPricing {
249 input_per_million: self.input_per_million?,
250 output_per_million: self.output_per_million?,
251 cache_hit_input_per_million: self
252 .cache_hit_input_per_million
253 .unwrap_or(self.input_per_million.unwrap_or(0.0)),
254 })
255 }
256}
257
258#[derive(Debug, Clone, Serialize, Deserialize)]
260pub struct ToolSpec {
261 pub name: String,
262 pub description: String,
263 pub parameters: Value,
265}
266
267#[derive(Debug, Clone, Serialize, Deserialize)]
269pub struct ToolCall {
270 pub id: String,
271 pub name: String,
272 pub arguments: Value,
274}
275
276pub struct StructuredRequest {
278 pub messages: Vec<Message>,
279 pub schema: Value,
281 pub schema_name: String,
283}
284
285#[derive(Debug, Clone, Default)]
287pub struct Completion {
288 pub content: String,
289 pub tool_calls: Vec<ToolCall>,
290 pub finish_reason: Option<String>,
291 pub usage: Option<TokenUsage>,
292 pub reasoning_content: Option<String>,
295}
296
297#[async_trait]
298pub trait LlmProvider: Send + Sync {
299 async fn complete(&self, messages: &[Message], tools: &[ToolSpec]) -> Result<Completion>;
300
301 async fn complete_structured(&self, _req: StructuredRequest) -> Result<Value> {
305 Err(Error::Config {
306 message: "provider does not support structured output".into(),
307 })
308 }
309
310 async fn stream(
319 &self,
320 messages: &[Message],
321 tools: &[ToolSpec],
322 stream_tx: Option<StreamSender>,
323 ) -> Result<Completion> {
324 let completion = self.complete(messages, tools).await?;
325 if let Some(tx) = stream_tx {
326 if !completion.content.is_empty() {
327 let _ = tx.send(completion.content.clone());
328 }
329 }
330 Ok(completion)
331 }
332}
333
334#[cfg(test)]
335mod tests {
336 use super::*;
337
338 #[tokio::test]
339 async fn mock_structured_returns_default_error() {
340 let provider = MockProvider::new(vec![]).with_structured_responses(vec![]);
343 let req = StructuredRequest {
344 messages: vec![Message::user("hi".to_string())],
345 schema: serde_json::json!({"type": "object", "properties": {"answer": {"type": "string"}}}),
346 schema_name: "test_schema".to_string(),
347 };
348 let result = provider.complete_structured(req).await;
349 assert!(result.is_err());
350 let err = result.unwrap_err();
351 let msg = err.to_string();
352 assert!(
353 msg.contains("no structured responses configured"),
354 "error should mention no structured responses: {msg}"
355 );
356 }
357
358 #[test]
359 fn token_usage_default_is_all_zeros() {
360 let u = TokenUsage::default();
361 assert_eq!(u.prompt_tokens, 0);
362 assert_eq!(u.completion_tokens, 0);
363 assert_eq!(u.total_tokens, 0);
364 }
365
366 #[test]
367 fn token_usage_accumulate_is_saturating() {
368 let u1 = TokenUsage {
369 prompt_tokens: 10,
370 completion_tokens: 5,
371 total_tokens: 15,
372 cache_hit_tokens: 0,
373 cache_miss_tokens: 0,
374 };
375 let u2 = TokenUsage {
376 prompt_tokens: 20,
377 completion_tokens: 30,
378 total_tokens: 50,
379 cache_hit_tokens: 0,
380 cache_miss_tokens: 0,
381 };
382 let acc = u1.accumulate(u2);
383 assert_eq!(acc.prompt_tokens, 30);
384 assert_eq!(acc.completion_tokens, 35);
385 assert_eq!(acc.total_tokens, 65);
386 }
387
388 #[test]
389 fn token_usage_accumulate_is_commutative() {
390 let u1 = TokenUsage {
391 prompt_tokens: 10,
392 completion_tokens: 5,
393 total_tokens: 15,
394 cache_hit_tokens: 0,
395 cache_miss_tokens: 0,
396 };
397 let u2 = TokenUsage {
398 prompt_tokens: 20,
399 completion_tokens: 30,
400 total_tokens: 50,
401 cache_hit_tokens: 0,
402 cache_miss_tokens: 0,
403 };
404 assert_eq!(u1.accumulate(u2), u2.accumulate(u1));
405 }
406
407 #[test]
408 fn token_usage_accumulate_saturates() {
409 let u1 = TokenUsage {
410 prompt_tokens: u32::MAX,
411 completion_tokens: 1,
412 total_tokens: u32::MAX,
413 cache_hit_tokens: 0,
414 cache_miss_tokens: 0,
415 };
416 let u2 = TokenUsage {
417 prompt_tokens: 1,
418 completion_tokens: u32::MAX,
419 total_tokens: u32::MAX,
420 cache_hit_tokens: 0,
421 cache_miss_tokens: 0,
422 };
423 let acc = u1.accumulate(u2);
424 assert_eq!(acc.prompt_tokens, u32::MAX);
425 assert_eq!(acc.completion_tokens, u32::MAX);
426 assert_eq!(acc.total_tokens, u32::MAX);
427 }
428
429 #[test]
430 fn cost_usd_handles_zero_usage() {
431 let pricing = ModelPricing {
432 input_per_million: 1.0,
433 output_per_million: 2.0,
434 cache_hit_input_per_million: 0.1, };
436 let usage = TokenUsage::default();
437 let cost = pricing.cost_usd(usage);
438 assert!((cost - 0.0).abs() < 1e-9);
439 }
440
441 #[test]
442 fn cost_usd_computes_simple_case() {
443 let pricing = ModelPricing {
444 input_per_million: 1.0,
445 output_per_million: 1.0,
446 cache_hit_input_per_million: 0.1,
447 };
448 let usage = TokenUsage {
450 prompt_tokens: 1_000_000,
451 completion_tokens: 0,
452 total_tokens: 1_000_000,
453 cache_hit_tokens: 0,
454 cache_miss_tokens: 0,
455 };
456 let cost = pricing.cost_usd(usage);
457 assert!((cost - 1.0).abs() < 1e-9);
458 }
459
460 #[test]
461 fn cost_usd_mixes_input_and_output() {
462 let pricing = ModelPricing {
463 input_per_million: 1.0,
464 output_per_million: 2.0,
465 cache_hit_input_per_million: 0.1,
466 };
467 let usage = TokenUsage {
469 prompt_tokens: 500_000,
470 completion_tokens: 250_000,
471 total_tokens: 750_000,
472 cache_hit_tokens: 0,
473 cache_miss_tokens: 0,
474 };
475 let cost = pricing.cost_usd(usage);
476 assert!((cost - 1.0).abs() < 1e-9);
478 }
479
480 #[test]
481 fn pricing_for_known_models() {
482 let p1 = pricing_for("MiniMax-M2");
483 assert!(p1.is_some());
484 assert!((p1.unwrap().input_per_million - 0.30).abs() < 1e-9);
485
486 let p2 = pricing_for("deepseek-chat");
487 assert!(p2.is_some());
488 assert!((p2.unwrap().input_per_million - 0.27).abs() < 1e-9);
489 }
490
491 #[test]
492 fn pricing_for_unknown_returns_none() {
493 let p = pricing_for("unknown-model-xyz");
494 assert!(p.is_none());
495 }
496
497 #[test]
498 fn token_usage_accumulate_cache_fields() {
499 let u1 = TokenUsage {
500 prompt_tokens: 100,
501 completion_tokens: 50,
502 total_tokens: 150,
503 cache_hit_tokens: 60,
504 cache_miss_tokens: 40,
505 };
506 let u2 = TokenUsage {
507 prompt_tokens: 200,
508 completion_tokens: 100,
509 total_tokens: 300,
510 cache_hit_tokens: 120,
511 cache_miss_tokens: 80,
512 };
513 let acc = u1.accumulate(u2);
514 assert_eq!(acc.cache_hit_tokens, 180);
515 assert_eq!(acc.cache_miss_tokens, 120);
516 assert_eq!(acc.prompt_tokens, 300);
517 assert_eq!(acc.completion_tokens, 150);
518 assert_eq!(acc.total_tokens, 450);
519 }
520
521 #[test]
523 fn cost_usd_with_no_cache_hit_matches_old_behavior() {
524 let pricing = ModelPricing {
525 input_per_million: 1.0,
526 output_per_million: 2.0,
527 cache_hit_input_per_million: 0.1, };
529 let usage = TokenUsage {
531 prompt_tokens: 1_000_000,
532 completion_tokens: 500_000,
533 total_tokens: 1_500_000,
534 cache_hit_tokens: 0,
535 cache_miss_tokens: 1_000_000,
536 };
537 let cost = pricing.cost_usd(usage);
539 assert!((cost - 2.0).abs() < 1e-9);
540 }
541
542 #[test]
544 fn cost_usd_with_cache_hit_applies_discount() {
545 let pricing = ModelPricing {
547 input_per_million: 0.27,
548 output_per_million: 1.10,
549 cache_hit_input_per_million: 0.027,
550 };
551 let usage = TokenUsage {
553 prompt_tokens: 1_000,
554 completion_tokens: 500,
555 total_tokens: 1_500,
556 cache_hit_tokens: 900,
557 cache_miss_tokens: 100,
558 };
559 let cost = pricing.cost_usd(usage);
560 let expected =
565 900.0 * 0.027 / 1_000_000.0 + 100.0 * 0.27 / 1_000_000.0 + 500.0 * 1.10 / 1_000_000.0;
566 assert!((cost - expected).abs() < 1e-9);
567 }
568
569 #[test]
571 fn pricing_for_deepseek_has_cache_discount() {
572 let pricing = pricing_for("deepseek-chat").expect("deepseek-chat should be known");
573 assert!((pricing.input_per_million - 0.27).abs() < 1e-9);
575 assert!((pricing.cache_hit_input_per_million - 0.027).abs() < 1e-9);
576 }
577
578 #[test]
580 fn pricing_for_unknown_model_returns_none() {
581 let p = pricing_for("unknown-model-xyz");
582 assert!(p.is_none());
583 }
584
585 #[test]
587 fn token_usage_accumulate_preserves_cache_tokens() {
588 let u1 = TokenUsage {
589 prompt_tokens: 1000,
590 completion_tokens: 100,
591 total_tokens: 1100,
592 cache_hit_tokens: 900,
593 cache_miss_tokens: 100,
594 };
595 let u2 = TokenUsage {
596 prompt_tokens: 2000,
597 completion_tokens: 200,
598 total_tokens: 2200,
599 cache_hit_tokens: 1800,
600 cache_miss_tokens: 200,
601 };
602 let acc = u1.accumulate(u2);
603 assert_eq!(acc.cache_hit_tokens, 2700);
604 assert_eq!(acc.cache_miss_tokens, 300);
605 assert_eq!(acc.prompt_tokens, 3000);
606 }
607
608 #[test]
610 fn load_pricing_from_yaml_parses_file() {
611 let temp_dir = std::env::temp_dir();
612 let yaml_path = temp_dir.join("test_pricing.yaml");
613 std::fs::write(
614 &yaml_path,
615 r#"
616models:
617 test-model:
618 input_per_million: 1.0
619 output_per_million: 2.0
620 cache_hit_input_per_million: 0.1
621"#,
622 )
623 .unwrap();
624
625 let result = load_pricing_from_yaml(&yaml_path);
626 assert!(result.is_ok(), "load should succeed: {:?}", result.err());
627 let pricing = result.unwrap();
628 assert!(
629 pricing.contains_key("test-model"),
630 "should contain test-model: {:?}",
631 pricing.keys().collect::<Vec<_>>()
632 );
633 let p = pricing.get("test-model").unwrap();
634 assert!((p.input_per_million - 1.0).abs() < 1e-9);
635 assert!((p.output_per_million - 2.0).abs() < 1e-9);
636 assert!((p.cache_hit_input_per_million - 0.1).abs() < 1e-9);
637
638 std::fs::remove_file(&yaml_path).ok();
639 }
640
641 #[test]
643 fn load_pricing_from_yaml_overrides_hardcoded() {
644 let temp_dir = std::env::temp_dir();
645 let yaml_path = temp_dir.join("test_pricing_override.yaml");
646 std::fs::write(
648 &yaml_path,
649 r#"
650models:
651 deepseek-chat:
652 input_per_million: 99.0
653 output_per_million: 99.0
654 cache_hit_input_per_million: 9.9
655"#,
656 )
657 .unwrap();
658
659 let result = load_pricing_from_yaml(&yaml_path);
660 assert!(result.is_ok());
661 let pricing = result.unwrap();
662 let p = pricing
663 .get("deepseek-chat")
664 .expect("should have deepseek-chat");
665 assert!((p.input_per_million - 99.0).abs() < 1e-9);
667
668 assert!(pricing.contains_key("deepseek-chat"));
670 assert!(!pricing.contains_key("MiniMax-M2"));
672
673 std::fs::remove_file(&yaml_path).ok();
674 }
675
676 #[test]
678 fn load_pricing_from_yaml_missing_model_falls_back() {
679 let temp_dir = std::env::temp_dir();
684 let yaml_path = temp_dir.join("test_pricing_partial.yaml");
685 std::fs::write(
687 &yaml_path,
688 r#"
689models:
690 test-only:
691 input_per_million: 1.0
692 output_per_million: 1.0
693 cache_hit_input_per_million: 0.1
694"#,
695 )
696 .unwrap();
697
698 let result = load_pricing_from_yaml(&yaml_path);
699 assert!(result.is_ok());
700 let external = result.unwrap();
701
702 assert!(
704 external.contains_key("test-only"),
705 "should have test-only: {:?}",
706 external.keys().collect::<Vec<_>>()
707 );
708 let fallback = pricing_for("MiniMax-M2");
710 assert!(fallback.is_some());
711 assert!((fallback.unwrap().input_per_million - 0.30).abs() < 1e-9);
712
713 std::fs::remove_file(&yaml_path).ok();
714 }
715
716 #[test]
718 fn load_pricing_from_yaml_malformed_error() {
719 let temp_dir = std::env::temp_dir();
720 let yaml_path = temp_dir.join("test_pricing_malformed.yaml");
721 std::fs::write(
723 &yaml_path,
724 r#"
725models:
726 bad-model:
727 input_per_million: not_a_number
728"#,
729 )
730 .unwrap();
731
732 let result = load_pricing_from_yaml(&yaml_path);
733 assert!(result.is_err(), "should fail with bad data");
734 let err = result.unwrap_err();
735 let err_str = err.to_string();
737 assert!(
738 err_str.contains("error parsing") || err_str.contains("invalid float"),
739 "error should mention parsing issue: {}",
740 err_str
741 );
742
743 std::fs::remove_file(&yaml_path).ok();
744 }
745}