tkach 0.3.0

A provider-independent agent runtime for Rust with built-in tools
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108

use async_trait::async_trait;
use serde_json::Value;

use crate::error::ProviderError;
use crate::message::{CacheControl, Content, Message, StopReason, Usage};
use crate::stream::ProviderEventStream;

/// Definition of a tool that gets sent to the LLM.
///
/// `cache_control` terminates a cached prefix segment at this tool
/// definition. The Anthropic API caches tool definitions in the order
/// they appear in the request — placing a breakpoint on the **last**
/// tool caches the entire toolset. Non-Anthropic providers ignore
/// this field.
#[derive(Debug, Clone)]
pub struct ToolDefinition {
    pub name: String,
    pub description: String,
    pub input_schema: Value,
    pub cache_control: Option<CacheControl>,
}

/// One block of the system prompt.
///
/// `Request.system` is a list because Anthropic's API accepts multiple
/// system blocks, each individually markable with [`CacheControl`] —
/// useful when part of the system prompt is stable (e.g. base
/// instructions) and part rotates per-call (e.g. user-specific context).
///
/// Non-Anthropic providers concatenate all blocks with `\n\n` into a
/// single system string and drop `cache_control`.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct SystemBlock {
    pub text: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub cache_control: Option<CacheControl>,
}

impl SystemBlock {
    /// Plain system block, no cache breakpoint.
    pub fn text(s: impl Into<String>) -> Self {
        Self {
            text: s.into(),
            cache_control: None,
        }
    }

    /// System block marked as a cache breakpoint with the default 5m TTL.
    pub fn cached(s: impl Into<String>) -> Self {
        Self {
            text: s.into(),
            cache_control: Some(CacheControl::ephemeral()),
        }
    }

    /// System block marked as a cache breakpoint with 1-hour TTL.
    pub fn cached_1h(s: impl Into<String>) -> Self {
        Self {
            text: s.into(),
            cache_control: Some(CacheControl::ephemeral_1h()),
        }
    }
}

/// Request to the LLM provider.
#[derive(Debug, Clone)]
pub struct Request {
    pub model: String,
    pub system: Option<Vec<SystemBlock>>,
    pub messages: Vec<Message>,
    pub tools: Vec<ToolDefinition>,
    pub max_tokens: u32,
    pub temperature: Option<f32>,
}

/// Response from the LLM provider.
#[derive(Debug, Clone)]
pub struct Response {
    pub content: Vec<Content>,
    pub stop_reason: StopReason,
    pub usage: Usage,
}

/// Trait for LLM providers (Anthropic, OpenAI, etc.).
///
/// Implement this trait to add support for a new LLM provider.
///
/// Two API surfaces, two transports:
///
/// - [`complete`](LlmProvider::complete) — single-shot HTTP, fully
///   buffered response. Lowest overhead when the caller wants the
///   final answer in one go.
/// - [`stream`](LlmProvider::stream) — Server-Sent Events HTTP, emits
///   incremental [`StreamEvent`](crate::stream::StreamEvent)s as the
///   model produces them.
///
/// Implementations are **independent code paths**: streaming is not
/// derived from `complete()` and vice-versa. Errors that happen
/// before the stream begins (auth, malformed request, connection
/// refused) surface from the `stream(...)` async fn itself; errors
/// that surface mid-stream (parse failures, mid-body HTTP errors)
/// arrive as `Err` items inside the stream.
#[async_trait]
pub trait LlmProvider: Send + Sync {
    async fn complete(&self, request: Request) -> Result<Response, ProviderError>;

    async fn stream(&self, request: Request) -> Result<ProviderEventStream, ProviderError>;
}