qai-sdk 0.1.26

Universal Rust SDK for AI Providers
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111

//! # DeepSeek Multi-round Chat Example
//!
//! Demonstrates multi-round conversational interaction and the DeepSeek KV context cache.

use qai_sdk::*;
use std::io::{self, Write};

#[tokio::main]
async fn main() -> Result<()> {
    dotenvy::dotenv().ok();
    // tracing_subscriber::fmt::init(); // Optional, commented to avoid noisy logs by default

    let api_key = std::env::var("DEEPSEEK_API_KEY").unwrap_or_else(|_| {
        println!("Warning: DEEPSEEK_API_KEY not set.");
        String::new()
    });
    
    if api_key.is_empty() {
        println!("Please set DEEPSEEK_API_KEY to run this example.");
        return Ok(());
    }

    let model = DeepSeekModel::new(api_key);

    println!("============================================================");
    println!(" DeepSeek Multi-round Chat (with Context Caching)");
    println!(" Type 'quit' or 'exit' to stop.");
    println!("============================================================\n");

    // The context vector to hold our multi-round conversation
    let mut messages = vec![
        Message {
            role: Role::System,
            content: vec![Content::Text {
                text: "You are a helpful and concise assistant. Always respond in english.".to_string(),
            }],
        }
    ];

    loop {
        print!("You: ");
        io::stdout().flush()?;

        let mut input = String::new();
        io::stdin().read_line(&mut input)?;
        let input = input.trim();

        if input.eq_ignore_ascii_case("quit") || input.eq_ignore_ascii_case("exit") {
            break;
        }
        if input.is_empty() {
            continue;
        }

        // Add user message to the context
        messages.push(Message {
            role: Role::User,
            content: vec![Content::Text {
                text: input.to_string(),
            }],
        });

        let prompt = Prompt {
            messages: messages.clone(),
        };

        // Generate the response statelessly
        let response = match model
            .generate(
                prompt,
                GenerateOptions {
                    model_id: "deepseek-chat".to_string(),
                    ..Default::default()
                },
            )
            .await {
                Ok(r) => r,
                Err(e) => {
                    println!("\nError generating response: {}", e);
                    // Remove the failed user message from history
                    messages.pop();
                    continue;
                }
            };

        println!("\nDeepSeek: {}", response.text.trim());
        
        // Print cache metrics
        println!("\n--- Usage Stats ---");
        println!("Prompt tokens: {}", response.usage.prompt_tokens);
        println!("Completion tokens: {}", response.usage.completion_tokens);
        
        if let Some(hit) = response.usage.cache_hit_tokens {
            println!("KV Cache Hit tokens: {}", hit);
        }
        if let Some(miss) = response.usage.cache_miss_tokens {
            println!("KV Cache Miss tokens: {}", miss);
        }
        println!("-------------------\n");

        // Add assistant's response to the context
        messages.push(Message {
            role: Role::Assistant,
            content: vec![Content::Text {
                text: response.text,
            }],
        });
    }

    Ok(())
}