inklings 0.1.0

A unified Rust API for various Large Language Model (LLM) providers
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123

use async_trait::async_trait;
use serde_json::json;
use crate::types::{Error, Message, Role};
use super::Provider;
use futures::{Stream, StreamExt};
use eventsource_stream::Eventsource;
use std::pin::Pin;

pub struct AnthropicProvider {
    api_key: String,
    client: reqwest::Client,
    model: String,
}

impl AnthropicProvider {
    pub fn new(api_key: String, model: Option<String>) -> Self {
        Self {
            api_key,
            client: reqwest::Client::new(),
            model: model.unwrap_or_else(|| "claude-3-5-haiku-20241022".to_string()),
        }
    }

}

#[async_trait]
impl Provider for AnthropicProvider {
    async fn complete(&self, prompt: &str) -> Result<String, Error> {
        let messages = vec![Message {
            role: Role::User,
            content: prompt.to_string(),
        }];
        self.chat(messages).await
    }

    async fn chat(&self, messages: Vec<Message>) -> Result<String, Error> {
        let response = self
            .client
            .post("https://api.anthropic.com/v1/messages")
            .header("x-api-key", &self.api_key)
            .header("anthropic-version", "2023-06-01")
            .json(&json!({
                "model": self.model,
                "messages": messages.iter().map(|m| json!({
                    "role": match m.role {
                        Role::System => "system",
                        Role::User => "user",
                        Role::Assistant => "assistant"
                    },
                    "content": m.content
                })).collect::<Vec<_>>(),
                "max_tokens": 1000,
            }))
            .send()
            .await?;

        if !response.status().is_success() {
            return Err(Error::ProviderError(format!(
                "Anthropic API error: {}",
                response.text().await?
            )));
        }

        let response: serde_json::Value = response.json().await?;
        Ok(response["content"][0]["text"]
            .as_str()
            .ok_or_else(|| Error::ProviderError("Invalid response format".to_string()))?
            .to_string())
    }

    async fn stream_chat(&self, messages: Vec<Message>) -> Result<Pin<Box<dyn Stream<Item = Result<String, Error>> + Send>>, Error> {
        let response = self
            .client
            .post("https://api.anthropic.com/v1/messages")
            .header("x-api-key", &self.api_key)
            .header("anthropic-version", "2023-06-01")
            .json(&json!({
                "model": self.model,
                "messages": messages.iter().map(|m| json!({
                    "role": match m.role {
                        Role::System => "system",
                        Role::User => "user",
                        Role::Assistant => "assistant"
                    },
                    "content": m.content
                })).collect::<Vec<_>>(),
                "max_tokens": 1000,
                "stream": true,
            }))
            .send()
            .await?;

        if !response.status().is_success() {
            return Err(Error::ProviderError(format!(
                "Anthropic API error: {}",
                response.text().await?
            )));
        }

        Ok(Box::pin(response
            .bytes_stream()
            .eventsource()
            .map(|event| {
                event
                    .map_err(|e| Error::ProviderError(e.to_string()))
                    .and_then(|event| {
                        let data = event.data;
                        if data == "[DONE]" {
                            return Ok("".to_string());
                        }
                        let json: serde_json::Value = serde_json::from_str(&data)
                            .map_err(|e| Error::ProviderError(e.to_string()))?;
                        
                        // Handle the delta format from Anthropic's streaming response
                        Ok(json["delta"]["text"]
                            .as_str()
                            .unwrap_or("")
                            .to_string())
                    })
            })
            .filter(|result| futures::future::ready(!matches!(result, Ok(s) if s.is_empty())))))
    }
}