use std::convert::Infallible;
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use axum::Json;
use axum::extract::State;
use axum::response::sse::{Event, KeepAlive, Sse};
use axum::response::{IntoResponse, Response};
use futures::{Stream, StreamExt};
use kjarni::chat::Message;
use kjarni::generation::GenerationOverrides;
use serde::Deserialize;
use serde_json::{Value, json};
use tokio::sync::mpsc;
use super::AppState;
use super::error::{ApiError, ApiJson};
use super::models::{LoadedDecoder, resolve};
use super::stop::{StopFilter, Trim};
#[derive(Debug, Deserialize)]
#[serde(untagged)]
pub enum Stop {
One(String),
Many(Vec<String>),
}
impl Stop {
fn into_vec(self) -> Vec<String> {
match self {
Stop::One(s) => vec![s],
Stop::Many(v) => v,
}
}
}
#[derive(Debug, Default, Deserialize)]
pub struct StreamOptions {
#[serde(default)]
pub include_usage: bool,
}
#[derive(Debug, Default, Deserialize)]
pub struct Sampling {
pub temperature: Option<f32>,
pub top_p: Option<f32>,
pub top_k: Option<usize>,
pub min_p: Option<f32>,
pub repetition_penalty: Option<f32>,
pub max_tokens: Option<usize>,
pub max_completion_tokens: Option<usize>,
pub n: Option<usize>,
pub logprobs: Option<Value>,
pub top_logprobs: Option<Value>,
}
impl Sampling {
pub fn overrides(&self) -> Result<GenerationOverrides, ApiError> {
if self.n.is_some_and(|n| n != 1) {
return Err(ApiError::unsupported("n", "only n = 1 is supported"));
}
if self
.logprobs
.as_ref()
.is_some_and(|v| !v.is_null() && v != &json!(false))
|| self.top_logprobs.as_ref().is_some_and(|v| !v.is_null())
{
return Err(ApiError::unsupported(
"logprobs",
"log probabilities are not supported yet",
));
}
let mut overrides = GenerationOverrides::default();
match self.temperature {
Some(t) if !(0.0..=2.0).contains(&t) => {
return Err(ApiError::invalid(
"temperature",
"temperature must be between 0 and 2",
));
}
Some(0.0) => overrides.do_sample = Some(false),
Some(t) => {
overrides.do_sample = Some(true);
overrides.temperature = Some(t);
}
None => {}
}
if let Some(p) = self.top_p {
if !(p > 0.0 && p <= 1.0) {
return Err(ApiError::invalid(
"top_p",
"top_p must be above 0 and at most 1",
));
}
overrides.top_p = Some(p);
}
overrides.top_k = self.top_k;
overrides.min_p = self.min_p;
overrides.repetition_penalty = self.repetition_penalty;
match self.max_completion_tokens.or(self.max_tokens) {
Some(0) => {
return Err(ApiError::invalid(
"max_tokens",
"max_tokens must be at least 1",
));
}
max => overrides.max_new_tokens = max,
}
Ok(overrides)
}
}
#[derive(Debug, Deserialize)]
pub struct ChatRequest {
pub model: Option<String>,
pub messages: Vec<InMessage>,
#[serde(flatten)]
pub sampling: Sampling,
#[serde(default)]
pub stream: bool,
pub stream_options: Option<StreamOptions>,
pub stop: Option<Stop>,
pub tools: Option<Vec<Value>>,
pub tool_choice: Option<Value>,
pub response_format: Option<Value>,
}
#[derive(Debug, Deserialize)]
pub struct InMessage {
pub role: String,
#[serde(default)]
pub content: Option<Content>,
}
#[derive(Debug, Deserialize)]
#[serde(untagged)]
pub enum Content {
Text(String),
Parts(Vec<Part>),
}
#[derive(Debug, Deserialize)]
pub struct Part {
#[serde(rename = "type")]
pub kind: String,
pub text: Option<String>,
}
pub fn messages(input: Vec<InMessage>) -> Result<Vec<Message>, ApiError> {
if input.is_empty() {
return Err(ApiError::invalid("messages", "messages must not be empty"));
}
input
.into_iter()
.map(|m| {
let content = match m.content {
None => String::new(),
Some(Content::Text(text)) => text,
Some(Content::Parts(parts)) => {
let mut text = String::new();
for part in parts {
match (part.kind.as_str(), part.text) {
("text", Some(t)) => text.push_str(&t),
(kind, _) => {
return Err(ApiError::unsupported(
"messages",
format!("message content of type '{kind}' is not supported; only text is"),
));
}
}
}
text
}
};
match m.role.as_str() {
"system" | "developer" => Ok(Message::system(content)),
"user" => Ok(Message::user(content)),
"assistant" => Ok(Message::assistant(content)),
"tool" | "function" => Err(ApiError::unsupported(
"messages",
"tool calling is not supported yet",
)),
other => Err(ApiError::invalid(
"messages",
format!("unknown message role '{other}'"),
)),
}
})
.collect()
}
#[derive(Debug, Deserialize)]
pub struct CompletionRequest {
pub model: Option<String>,
pub prompt: Prompt,
#[serde(flatten)]
pub sampling: Sampling,
#[serde(default)]
pub stream: bool,
pub stream_options: Option<StreamOptions>,
pub stop: Option<Stop>,
#[serde(default)]
pub echo: bool,
pub suffix: Option<String>,
}
#[derive(Debug, Deserialize)]
#[serde(untagged)]
pub enum Prompt {
One(String),
Many(Vec<String>),
Tokens(serde::de::IgnoredAny),
}
#[derive(Debug, PartialEq)]
pub enum Piece {
Text(String),
Done {
finish_reason: &'static str,
completion_tokens: usize,
},
Failed(String),
}
pub struct Run {
pub id: String,
pub created: u64,
pub model: String,
pub prompt_tokens: usize,
pub pieces: mpsc::Receiver<Piece>,
}
pub fn new_id(prefix: &str) -> String {
use std::sync::atomic::{AtomicU64, Ordering};
static COUNTER: AtomicU64 = AtomicU64::new(0);
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_nanos() as u64)
.unwrap_or(0);
let n = COUNTER.fetch_add(1, Ordering::Relaxed);
format!("{prefix}-{nanos:x}{n:04x}")
}
pub fn unix_now() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0)
}
pub async fn start(
decoder: Arc<LoadedDecoder>,
model: String,
prompt: String,
overrides: GenerationOverrides,
stops: Vec<String>,
trim: bool,
) -> Result<Run, ApiError> {
let generator = decoder.generator();
let prompt_tokens = generator
.count_tokens(&prompt)
.map_err(ApiError::internal)?;
let context = generator.context_size();
if prompt_tokens >= context {
return Err(ApiError {
code: Some("context_length_exceeded"),
param: Some("messages"),
..ApiError::bad_request(format!(
"the prompt is {prompt_tokens} tokens and '{model}' takes at most {context}"
))
});
}
let max_new_tokens = generator.max_new_tokens(&overrides);
let mut stops = stops;
if decoder.chat().is_some() {
stops.extend(generator.decoder().stop_sequences());
}
let opening = decoder
.chat()
.map(|chat| chat.reply_prefix(&prompt).to_string())
.unwrap_or_default();
let (tx, rx) = mpsc::channel(64);
tokio::spawn(async move {
let Ok(_turn) = decoder.turn.clone().acquire_owned().await else {
return;
};
let generator = decoder.generator();
let mut tokens = match generator.stream_with_config(&prompt, overrides).await {
Ok(tokens) => tokens,
Err(e) => {
let _ = tx.send(Piece::Failed(e.to_string())).await;
return;
}
};
let mut filter = StopFilter::new(stops);
let mut trimmer = trim.then(Trim::default);
if !opening.is_empty() && !send_text(&tx, opening, &mut trimmer).await {
return;
}
let mut count = 0;
while let Some(token) = tokens.next().await {
let token = match token {
Ok(token) => token,
Err(e) => {
let _ = tx.send(Piece::Failed(e.to_string())).await;
return;
}
};
count += 1;
if token.is_special {
continue;
}
let (text, stopped) = filter.push(&token.text);
if !send_text(&tx, text, &mut trimmer).await {
return;
}
if stopped {
let _ = tx
.send(Piece::Done {
finish_reason: "stop",
completion_tokens: count,
})
.await;
return;
}
}
if !send_text(&tx, filter.finish(), &mut trimmer).await {
return;
}
let finish_reason = if max_new_tokens.is_some_and(|max| count >= max) {
"length"
} else {
"stop"
};
let _ = tx
.send(Piece::Done {
finish_reason,
completion_tokens: count,
})
.await;
});
Ok(Run {
id: String::new(),
created: unix_now(),
model,
prompt_tokens,
pieces: rx,
})
}
async fn send_text(tx: &mpsc::Sender<Piece>, text: String, trimmer: &mut Option<Trim>) -> bool {
let text = match trimmer {
Some(trimmer) => trimmer.push(&text),
None => text,
};
text.is_empty() || tx.send(Piece::Text(text)).await.is_ok()
}
pub async fn collect(run: &mut Run) -> Result<(String, &'static str, usize), ApiError> {
let mut text = String::new();
while let Some(piece) = run.pieces.recv().await {
match piece {
Piece::Text(t) => text.push_str(&t),
Piece::Done {
finish_reason,
completion_tokens,
} => {
return Ok((text, finish_reason, completion_tokens));
}
Piece::Failed(message) => return Err(ApiError::internal(message)),
}
}
Err(ApiError::internal("generation ended without finishing"))
}
fn usage(prompt_tokens: usize, completion_tokens: usize) -> Value {
json!({
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": prompt_tokens + completion_tokens,
})
}
fn events<F>(
mut run: Run,
include_usage: bool,
to_chunk: F,
) -> impl Stream<Item = Result<Event, Infallible>>
where
F: Fn(&Run, Option<&str>, Option<&str>) -> Value + Send + 'static,
{
async_stream_events(move |tx| async move {
while let Some(piece) = run.pieces.recv().await {
match piece {
Piece::Text(text) => {
if tx.send(to_chunk(&run, Some(&text), None)).await.is_err() {
return;
}
}
Piece::Done {
finish_reason,
completion_tokens,
} => {
let _ = tx.send(to_chunk(&run, None, Some(finish_reason))).await;
if include_usage {
let mut chunk = to_chunk(&run, None, None);
chunk["choices"] = json!([]);
chunk["usage"] = usage(run.prompt_tokens, completion_tokens);
let _ = tx.send(chunk).await;
}
break;
}
Piece::Failed(message) => {
let _ = tx.send(ApiError::internal(message).body()).await;
break;
}
}
}
})
}
fn async_stream_events<P, Fut>(produce: P) -> impl Stream<Item = Result<Event, Infallible>>
where
P: FnOnce(mpsc::Sender<Value>) -> Fut,
Fut: std::future::Future<Output = ()> + Send + 'static,
{
let (tx, rx) = mpsc::channel::<Value>(64);
tokio::spawn(produce(tx));
let json_events = futures::stream::unfold(rx, |mut rx| async move {
rx.recv()
.await
.map(|value| (Ok(Event::default().data(value.to_string())), rx))
});
json_events.chain(futures::stream::once(async {
Ok(Event::default().data("[DONE]"))
}))
}
pub async fn chat_completions(
State(app): State<Arc<AppState>>,
ApiJson(request): ApiJson<ChatRequest>,
) -> Result<Response, ApiError> {
if request.tools.as_ref().is_some_and(|t| !t.is_empty())
&& request.tool_choice.as_ref().is_none_or(|c| c != "none")
{
return Err(ApiError::unsupported(
"tools",
"tool calling is not supported yet",
));
}
if let Some(format) = &request.response_format
&& format.get("type").and_then(Value::as_str) != Some("text")
{
return Err(ApiError::unsupported(
"response_format",
"only response_format {\"type\": \"text\"} is supported; JSON output can't be guaranteed yet",
));
}
let overrides = request.sampling.overrides()?;
let messages = messages(request.messages)?;
let model = resolve(request.model.as_deref(), &app.models.defaults.chat)?;
let decoder = app.models.decoder(&model).await?;
let Some(chat) = decoder.chat() else {
return Err(ApiError::invalid(
"model",
format!("'{model}' has no chat template; use /v1/completions with it instead"),
));
};
let prompt = chat
.render_messages(&messages)
.map_err(|e| ApiError::invalid("messages", e.to_string()))?;
let stops = request.stop.map(Stop::into_vec).unwrap_or_default();
let mut run = start(decoder.clone(), model, prompt, overrides, stops, true).await?;
run.id = new_id("chatcmpl");
if request.stream {
let include_usage = request.stream_options.unwrap_or_default().include_usage;
let mut first = true;
let chunk = move |run: &Run, text: Option<&str>, finish: Option<&str>| {
let mut delta = json!({});
if first {
delta["role"] = json!("assistant");
first = false;
}
if let Some(text) = text {
delta["content"] = json!(text);
}
json!({
"id": run.id,
"object": "chat.completion.chunk",
"created": run.created,
"model": run.model,
"choices": [{
"index": 0,
"delta": delta,
"logprobs": null,
"finish_reason": finish,
}],
})
};
let chunk = std::sync::Mutex::new(chunk);
let stream = events(run, include_usage, move |run, text, finish| {
(chunk.lock().expect("chunk builder"))(run, text, finish)
});
return Ok(Sse::new(stream)
.keep_alive(KeepAlive::default())
.into_response());
}
let (text, finish_reason, completion_tokens) = collect(&mut run).await?;
Ok(Json(json!({
"id": run.id,
"object": "chat.completion",
"created": run.created,
"model": run.model,
"choices": [{
"index": 0,
"message": { "role": "assistant", "content": text, "refusal": null },
"logprobs": null,
"finish_reason": finish_reason,
}],
"usage": usage(run.prompt_tokens, completion_tokens),
}))
.into_response())
}
pub async fn completions(
State(app): State<Arc<AppState>>,
ApiJson(request): ApiJson<CompletionRequest>,
) -> Result<Response, ApiError> {
if request.echo {
return Err(ApiError::unsupported("echo", "echo is not supported"));
}
if request.suffix.is_some() {
return Err(ApiError::unsupported("suffix", "suffix is not supported"));
}
let prompt = match request.prompt {
Prompt::One(p) => p,
Prompt::Many(mut many) if many.len() == 1 => many.remove(0),
Prompt::Many(_) => {
return Err(ApiError::unsupported(
"prompt",
"send one prompt per request",
));
}
Prompt::Tokens(_) => {
return Err(ApiError::unsupported(
"prompt",
"token-id prompts are not supported; send text",
));
}
};
let overrides = request.sampling.overrides()?;
let model = resolve(request.model.as_deref(), &app.models.defaults.chat)?;
let decoder = app.models.decoder(&model).await?;
let stops = request.stop.map(Stop::into_vec).unwrap_or_default();
let mut run = start(decoder, model, prompt, overrides, stops, false).await?;
run.id = new_id("cmpl");
let chunk = |run: &Run, text: Option<&str>, finish: Option<&str>| {
json!({
"id": run.id,
"object": "text_completion",
"created": run.created,
"model": run.model,
"choices": [{
"index": 0,
"text": text.unwrap_or(""),
"logprobs": null,
"finish_reason": finish,
}],
})
};
if request.stream {
let include_usage = request.stream_options.unwrap_or_default().include_usage;
let stream = events(run, include_usage, chunk);
return Ok(Sse::new(stream)
.keep_alive(KeepAlive::default())
.into_response());
}
let (text, finish_reason, completion_tokens) = collect(&mut run).await?;
let mut body = chunk(&run, Some(&text), Some(finish_reason));
body["usage"] = usage(run.prompt_tokens, completion_tokens);
Ok(Json(body).into_response())
}