use std::{
env,
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
time::Duration,
};
use nachalnik::{
BoxError, Content, DeltaSink, Message, ModelInfo, ModelRequest, ModelResponse, Provider,
StopReason, ToolCall, ToolCallId, Usage, async_trait,
};
use parking_lot::Mutex;
use serde_json::{Value, json};
pub(crate) const RETRIES: usize = 4;
const HEARTBEAT: Duration = Duration::from_millis(120);
pub struct OpenAiCompatible {
client: reqwest::Client,
base_url: Mutex<String>,
api_key: String,
model: Mutex<String>,
context_limit: Mutex<Option<usize>>,
attempts: AtomicUsize,
notice: Mutex<Option<String>>,
}
#[derive(Default)]
struct PartialCall {
id: String,
name: String,
args: String,
extra: Value,
}
pub(crate) fn install_crypto() {
let _ = rustls::crypto::ring::default_provider().install_default();
}
impl OpenAiCompatible {
pub fn new(
model: impl Into<String>,
base_url: impl Into<String>,
api_key: impl Into<String>,
) -> Self {
install_crypto();
Self {
client: reqwest::Client::new(),
base_url: Mutex::new(base_url.into()),
api_key: api_key.into(),
model: Mutex::new(model.into()),
context_limit: Mutex::new(configured_limit()),
attempts: AtomicUsize::new(0),
notice: Mutex::new(None),
}
}
pub fn endpoint(&self) -> String {
self.base_url.lock().clone()
}
pub fn host(&self) -> String {
Endpoint::host(self)
}
pub async fn set_endpoint(&self, url: impl Into<String>, model: Option<String>) {
*self.base_url.lock() = url.into();
if let Some(model) = model {
*self.model.lock() = model;
}
*self.context_limit.lock() = configured_limit();
self.probe().await;
self.say_if_the_model_is_not_there().await;
}
pub async fn models(&self) -> Vec<String> {
let base = self.endpoint();
let listed = self.listed_names(&format!("{base}/models"), true).await;
if !listed.is_empty() {
return listed;
}
match base.strip_suffix("/openai") {
Some(native) => self.listed_names(&format!("{native}/models"), false).await,
None => listed,
}
}
async fn say_if_the_model_is_not_there(&self) {
let model = self.model.lock().clone();
let listed = self.models().await;
if listed.is_empty() || listed.iter().any(|name| same_model(name, &model)) {
return;
}
let some: Vec<&str> = listed.iter().take(3).map(String::as_str).collect();
*self.notice.lock() = Some(format!(
"{model} is not one of the {} models at this address ({}{}); /model to pick one",
listed.len(),
some.join(", "),
match listed.len() > some.len() {
true => ", …",
false => "",
}
));
}
async fn listed_names(&self, url: &str, bearer: bool) -> Vec<String> {
let request = match bearer {
true => self.client.get(url).bearer_auth(&self.api_key),
false => self.client.get(format!("{url}?key={}", self.api_key)),
};
let Ok(response) = request.send().await else {
return Vec::new();
};
let Ok(body) = response.json::<Value>().await else {
return Vec::new();
};
body["data"]
.as_array()
.or_else(|| body["models"].as_array())
.map(|entries| {
entries
.iter()
.filter_map(|entry| entry["id"].as_str().or_else(|| entry["name"].as_str()))
.map(|name| name.strip_prefix("models/").unwrap_or(name).to_owned())
.collect()
})
.unwrap_or_default()
}
pub async fn set_model(&self, model: impl Into<String>) {
*self.model.lock() = model.into();
*self.context_limit.lock() = configured_limit();
self.probe().await;
self.say_if_the_model_is_not_there().await;
}
pub fn take_notice(&self) -> Option<String> {
self.notice.lock().take()
}
pub async fn probe(&self) {
if self.context_limit.lock().is_some() {
return;
}
let base = self.endpoint();
let mut limit = self.listed_limit(&format!("{base}/models"), true).await;
if limit.is_none()
&& let Some(native) = base.strip_suffix("/openai")
{
limit = self.listed_limit(&format!("{native}/models"), false).await;
}
if limit.is_none()
&& let Some(root) = base.strip_suffix("/v1")
{
limit = self.loaded_limit(root).await;
}
*self.context_limit.lock() = limit;
}
async fn loaded_limit(&self, root: &str) -> Option<usize> {
if let Some(limit) = self.running_limit(root).await {
return Some(limit);
}
self.client
.post(format!("{root}/api/generate"))
.json(&json!({ "model": *self.model.lock(), "prompt": "" }))
.send()
.await
.ok()?;
self.running_limit(root).await
}
async fn running_limit(&self, root: &str) -> Option<usize> {
let model = self.model.lock().clone();
let body = self
.client
.get(format!("{root}/api/ps"))
.send()
.await
.ok()?
.json::<Value>()
.await
.ok()?;
body["models"]
.as_array()?
.iter()
.find(|entry| same_model(entry["name"].as_str().unwrap_or_default(), &model))
.and_then(|entry| entry["context_length"].as_u64())
.map(|limit| limit as usize)
}
async fn listed_limit(&self, url: &str, bearer: bool) -> Option<usize> {
let model = self.model.lock().clone();
let request = match bearer {
true => self.client.get(url).bearer_auth(&self.api_key),
false => self.client.get(format!("{url}?key={}", self.api_key)),
};
let body = request.send().await.ok()?.json::<Value>().await.ok()?;
let entries = body["data"]
.as_array()
.or_else(|| body["models"].as_array())?;
let entry = entries.iter().find(|entry| {
let listed = entry["id"]
.as_str()
.or_else(|| entry["name"].as_str())
.unwrap_or_default();
same_model(listed, &model)
})?;
entry["context_length"]
.as_u64()
.or_else(|| entry["top_provider"]["context_length"].as_u64())
.or_else(|| entry["inputTokenLimit"].as_u64())
.map(|limit| limit as usize)
}
}
#[async_trait]
impl Provider for OpenAiCompatible {
fn info(&self) -> ModelInfo {
ModelInfo {
context_limit: *self.context_limit.lock(),
tool_calling: true,
reasoning: true,
..ModelInfo::new("openai-compatible", self.model.lock().clone())
}
}
fn render(&self, request: &ModelRequest) -> Option<Value> {
let mut body = json!({
"model": *self.model.lock(),
"messages": request.messages.iter().map(to_wire).collect::<Vec<_>>(),
"stream": true,
"stream_options": { "include_usage": true },
});
if !request.tools.is_empty() {
body["tools"] = Value::Array(
request
.tools
.iter()
.map(|spec| {
json!({
"type": "function",
"function": {
"name": spec.id,
"description": spec.description,
"parameters": spec.schema,
},
})
})
.collect(),
);
}
for (key, value) in &request.params {
body[key] = value.clone();
}
Some(body)
}
async fn respond(
&self,
request: ModelRequest,
deltas: DeltaSink,
) -> Result<ModelResponse, BoxError> {
let body = self.render(&request).expect("this provider always renders");
let mut response = loop {
let response = self
.client
.post(format!("{}/chat/completions", self.endpoint()))
.bearer_auth(&self.api_key)
.json(&body)
.send()
.await?;
let status = response.status();
if status.is_success() {
break response;
}
let transient = status.as_u16() == 429 || status.is_server_error();
let attempt = self.attempts.fetch_add(1, Ordering::SeqCst) + 1;
if !transient || attempt >= RETRIES {
let body = response.text().await.unwrap_or_default();
return Err(format!("{status}: {body}").into());
}
let wait = Duration::from_secs(1 << attempt);
*self.notice.lock() = Some(format!(
"{} answered {}; trying again in {}s",
self.model.lock(),
status.as_u16(),
wait.as_secs()
));
tokio::time::sleep(wait).await;
};
let mut buffer = String::new();
let mut text = String::new();
let mut reasoning = String::new();
let mut calls: Vec<PartialCall> = Vec::new();
let mut finish = None;
let mut usage = None;
let mut chunks = Vec::new();
loop {
let bytes = match tokio::time::timeout(HEARTBEAT, response.chunk()).await {
Ok(Ok(Some(bytes))) => bytes,
Ok(Ok(None)) => break,
Ok(Err(e)) => return Err(e.into()),
Err(_) => {
if deltas.is_interrupted() {
finish = Some("interrupted".to_owned());
break;
}
continue;
}
};
buffer.push_str(&String::from_utf8_lossy(&bytes));
while let Some(end) = buffer.find('\n') {
if deltas.is_interrupted() {
finish = Some("interrupted".to_owned());
break;
}
let line = buffer[..end].trim().to_owned();
buffer.drain(..=end);
let Some(data) = line.strip_prefix("data:") else {
continue;
};
let data = data.trim();
if data == "[DONE]" {
continue;
}
let Ok(chunk) = serde_json::from_str::<Value>(data) else {
continue;
};
if let Some(error) = chunk.get("error").filter(|e| !e.is_null()) {
return Err(format!("{error}").into());
}
if let Some(reported) = chunk.get("usage").filter(|u| !u.is_null()) {
usage = Some(Usage {
input_tokens: reported["prompt_tokens"].as_u64(),
output_tokens: reported["completion_tokens"].as_u64(),
reasoning_tokens: reported["completion_tokens_details"]["reasoning_tokens"]
.as_u64(),
cached_input_tokens: reported["prompt_tokens_details"]["cached_tokens"]
.as_u64(),
});
}
let choice = &chunk["choices"][0];
if let Some(reason) = choice["finish_reason"].as_str() {
finish = Some(reason.to_owned());
}
let delta = &choice["delta"];
if let Some(fragment) = delta["content"].as_str().filter(|f| !f.is_empty()) {
deltas.text(fragment);
text.push_str(fragment);
}
if let Some(fragment) = delta["reasoning"].as_str().filter(|f| !f.is_empty()) {
deltas.reasoning(fragment);
reasoning.push_str(fragment);
}
for requested in delta["tool_calls"].as_array().into_iter().flatten() {
let at = match requested["index"].as_u64() {
Some(index) => index as usize,
None => match requested["id"].as_str().filter(|id| !id.is_empty()) {
Some(id) => calls
.iter()
.position(|call| call.id == id)
.unwrap_or(calls.len()),
None => calls.len().saturating_sub(1),
},
};
while calls.len() <= at {
calls.push(PartialCall::default());
}
let call = &mut calls[at];
if let Some(id) = requested["id"].as_str() {
call.id = id.to_owned();
}
if let Some(name) = requested["function"]["name"].as_str() {
call.name.push_str(name);
}
if !requested["extra_content"].is_null() {
call.extra = requested["extra_content"].clone();
}
if let Some(fragment) = requested["function"]["arguments"].as_str() {
call.args.push_str(fragment);
deltas.tool_args(ToolCallId(call.id.clone()), fragment);
}
}
chunks.push(chunk);
}
if finish.as_deref() == Some("interrupted") {
break;
}
}
if chunks.is_empty() {
if finish.as_deref() == Some("interrupted") || deltas.is_interrupted() {
return Ok(ModelResponse {
content: None,
reasoning: None,
tool_calls: Vec::new(),
stop: StopReason::Other("interrupted".to_owned()),
usage: None,
raw: None,
});
}
let payload: Value = serde_json::from_str(&buffer).unwrap_or(Value::Null);
return match payload.get("error").filter(|e| !e.is_null()) {
Some(error) => Err(format!("{error}").into()),
None => Err(format!("the stream carried no data: {buffer}").into()),
};
}
Ok(ModelResponse {
content: (!text.is_empty()).then_some(Content::text(text)),
reasoning: (!reasoning.is_empty()).then_some(Content::text(reasoning)),
tool_calls: calls
.into_iter()
.map(|call| {
let args: Value = serde_json::from_str(&call.args)
.unwrap_or_else(|_| json!({ "_unparsed": call.args }));
ToolCall::new(call.id, call.name, args).with_extra(call.extra)
})
.collect(),
stop: match finish.as_deref() {
Some("stop") => StopReason::EndTurn,
Some("interrupted") => StopReason::Other("interrupted".to_owned()),
Some("tool_calls") | Some("function_call") => StopReason::ToolUse,
Some("length") => StopReason::Length,
Some("content_filter") => StopReason::Refusal,
Some(other) => StopReason::Other(other.to_owned()),
None => StopReason::Other("unreported".to_owned()),
},
usage,
raw: Some(json!({ "stream": chunks })),
})
}
}
#[async_trait]
pub trait Endpoint: Provider {
fn endpoint(&self) -> String;
fn model(&self) -> String;
fn host(&self) -> String {
let endpoint = self.endpoint();
let after_scheme = endpoint
.split_once("://")
.map_or(endpoint.as_str(), |(_, rest)| rest);
after_scheme
.split('/')
.next()
.filter(|host| !host.is_empty())
.unwrap_or(&endpoint)
.to_owned()
}
async fn models(&self) -> Vec<String>;
async fn set_model(&self, model: String);
async fn set_endpoint(&self, url: String, model: Option<String>);
fn take_notice(&self) -> Option<String>;
}
#[async_trait]
impl Endpoint for OpenAiCompatible {
fn endpoint(&self) -> String {
self.endpoint()
}
fn model(&self) -> String {
self.model.lock().clone()
}
async fn models(&self) -> Vec<String> {
self.models().await
}
async fn set_model(&self, model: String) {
self.set_model(model).await;
}
async fn set_endpoint(&self, url: String, model: Option<String>) {
self.set_endpoint(url, model).await;
}
fn take_notice(&self) -> Option<String> {
self.take_notice()
}
}
fn to_wire(message: &Message) -> Value {
let mut wire = json!({ "role": message.role.as_str() });
if let Some(content) = &message.content {
wire["content"] = json!(content.to_text());
}
let calls: Vec<_> = message.calls().collect();
if !calls.is_empty() {
wire["tool_calls"] = Value::Array(
calls
.iter()
.map(|call| {
let mut wire = json!({
"id": call.id.0,
"type": "function",
"function": { "name": call.tool, "arguments": call.args.to_string() },
});
if !call.extra.is_null() {
wire["extra_content"] = (*call.extra).clone();
}
wire
})
.collect(),
);
}
if let Some(id) = &message.tool_call_id {
wire["tool_call_id"] = json!(id.0);
}
if let Some(name) = &message.name {
wire["name"] = json!(name);
}
wire
}
pub fn same_model(listed: &str, model: &str) -> bool {
listed == model
|| listed.strip_prefix("models/") == Some(model)
|| listed.strip_suffix(":latest") == Some(model)
}
pub(crate) fn configured_limit() -> Option<usize> {
env::var("KAMCHATKA_CONTEXT_LIMIT")
.ok()
.and_then(|limit| limit.parse().ok())
}
pub fn api_key() -> Result<String, BoxError> {
env::var("KAMCHATKA_API_KEY")
.or_else(|_| env::var("OPENROUTER_API_KEY"))
.or_else(|_| env::var("OPENAI_API_KEY"))
.map_err(|_| "set KAMCHATKA_API_KEY (or OPENROUTER_API_KEY / OPENAI_API_KEY)".into())
}
pub fn base_url() -> String {
env::var("KAMCHATKA_BASE_URL").unwrap_or_else(|_| "https://openrouter.ai/api/v1".to_owned())
}
pub async fn connect(model: impl Into<String>) -> Result<Arc<OpenAiCompatible>, BoxError> {
let provider = Arc::new(OpenAiCompatible::new(model, base_url(), api_key()?));
provider.probe().await;
Ok(provider)
}