atomr-infer-runtime-openai 0.8.0

OpenAI Chat Completions provider for atomr-infer — implements ModelRunner against api.openai.com, with cost table, error classification, and SSE streaming.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145

//! OpenAI Chat Completions wire types — request envelope + SSE chunk
//! shape. Kept minimal: only what's needed to round-trip the prompts /
//! deltas / usage info that the actor system needs.

use serde::{Deserialize, Serialize};

use atomr_infer_core::batch::{ExecuteBatch, Message, MessageContent, Role};

#[derive(Debug, Serialize)]
pub struct ChatRequest<'a> {
    pub model: &'a str,
    pub messages: Vec<ChatMessage>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub temperature: Option<f32>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub top_p: Option<f32>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub max_tokens: Option<u32>,
    #[serde(skip_serializing_if = "<[String]>::is_empty")]
    pub stop: Vec<String>,
    pub stream: bool,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub seed: Option<u64>,
}

#[derive(Debug, Serialize, Deserialize)]
pub struct ChatMessage {
    pub role: String,
    pub content: serde_json::Value,
}

impl ChatRequest<'_> {
    pub fn from_batch<'b>(b: &'b ExecuteBatch) -> ChatRequest<'b> {
        ChatRequest {
            model: &b.model,
            messages: b.messages.iter().map(serialize_message).collect(),
            temperature: b.sampling.temperature,
            top_p: b.sampling.top_p,
            max_tokens: b.sampling.max_tokens,
            stop: b.sampling.stop.clone(),
            stream: b.stream,
            seed: b.sampling.seed,
        }
    }
}

fn serialize_message(m: &Message) -> ChatMessage {
    let role = match m.role {
        Role::System => "system",
        Role::User => "user",
        Role::Assistant => "assistant",
        Role::Tool => "tool",
        // `Role` is `#[non_exhaustive]`; default unknown roles to
        // "user" so the request still goes through.
        _ => "user",
    }
    .to_string();
    let content = match &m.content {
        MessageContent::Text(t) => serde_json::Value::String(t.clone()),
        MessageContent::Parts(parts) => {
            serde_json::to_value(parts).unwrap_or(serde_json::Value::String(String::new()))
        }
        _ => serde_json::Value::String(String::new()),
    };
    ChatMessage { role, content }
}

// ---- streaming chunk ------------------------------------------------------

#[derive(Debug, Deserialize)]
pub struct ChatChunk {
    #[serde(default)]
    pub id: String,
    #[serde(default)]
    pub choices: Vec<ChoiceDelta>,
    #[serde(default)]
    pub usage: Option<UsageDelta>,
}

#[derive(Debug, Deserialize)]
pub struct ChoiceDelta {
    #[serde(default)]
    pub delta: ContentDelta,
    #[serde(default)]
    pub finish_reason: Option<String>,
}

#[derive(Debug, Default, Deserialize)]
pub struct ContentDelta {
    #[serde(default)]
    pub content: Option<String>,
    #[serde(default)]
    pub tool_calls: Option<serde_json::Value>,
}

#[derive(Debug, Deserialize)]
pub struct UsageDelta {
    #[serde(default)]
    pub prompt_tokens: u32,
    #[serde(default)]
    pub completion_tokens: u32,
    #[serde(default)]
    pub prompt_tokens_details: Option<PromptTokensDetails>,
    #[serde(default)]
    pub completion_tokens_details: Option<CompletionTokensDetails>,
}

#[derive(Debug, Deserialize, Default)]
pub struct PromptTokensDetails {
    #[serde(default)]
    pub cached_tokens: u32,
}

#[derive(Debug, Deserialize, Default)]
pub struct CompletionTokensDetails {
    #[serde(default)]
    pub reasoning_tokens: u32,
}

// ---- non-streaming response ----------------------------------------------

#[derive(Debug, Deserialize)]
pub struct ChatResponse {
    pub id: String,
    pub choices: Vec<ChatChoice>,
    #[serde(default)]
    pub usage: Option<UsageDelta>,
}

#[derive(Debug, Deserialize)]
pub struct ChatChoice {
    #[serde(default)]
    pub message: ChatMessage,
    #[serde(default)]
    pub finish_reason: Option<String>,
}

impl Default for ChatMessage {
    fn default() -> Self {
        Self {
            role: "assistant".into(),
            content: serde_json::Value::Null,
        }
    }
}