use std::io::{Read, Write};
use std::net::{Ipv4Addr, SocketAddr};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
use std::thread::JoinHandle;
use std::time::Duration;
use thiserror::Error;
use tiny_http::{Header, Method, Request, Response, Server};
use crate::engine::{Engine, NullSink, Plan, StreamError, TokenSink, hush_stream_aborts, unix_now};
use crate::error::ServerError;
use crate::request::ChatCompletionRequest;
use crate::response::{
COMPLETION_ID_PREFIX, ChatCompletion, ChunkBuilder, FinishReason, Health, ModelList,
ResponseMessage,
};
use crate::sse::{sse_data, sse_done, sse_event};
use crate::toolcall::{CallStream, Step, wire_call};
use crate::wire::{KEEPALIVE, StreamState};
pub const PATH_CHAT_COMPLETIONS: &str = "/v1/chat/completions";
pub const PATH_MODELS: &str = "/v1/models";
pub const PATH_HEALTH: &str = "/health";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ServeConfig {
pub port: u16,
pub keepalive: Duration,
pub max_body_bytes: usize,
}
impl Default for ServeConfig {
fn default() -> Self {
ServeConfig {
port: 8080,
keepalive: Duration::from_secs(10),
max_body_bytes: max_body_bytes_for_context(DEFAULT_CONTEXT),
}
}
}
pub const DEFAULT_CONTEXT: usize = 4096;
pub const BODY_BYTES_PER_CONTEXT_TOKEN: usize = 256;
pub const MIN_BODY_BYTES: usize = 64 << 10;
#[must_use]
pub fn max_body_bytes_for_context(context: usize) -> usize {
context
.saturating_mul(BODY_BYTES_PER_CONTEXT_TOKEN)
.max(MIN_BODY_BYTES)
}
#[derive(Debug, Error)]
pub enum ServeError {
#[error("binding 127.0.0.1:{port}: {message}")]
Bind {
port: u16,
message: String,
},
#[error("accepting connections: {0}")]
Accept(#[from] std::io::Error),
}
pub fn serve(engine: &mut dyn Engine, config: ServeConfig) -> Result<(), ServeError> {
serve_with(engine, config, |_| {})
}
pub fn serve_with(
engine: &mut dyn Engine,
config: ServeConfig,
ready: impl FnOnce(SocketAddr),
) -> Result<(), ServeError> {
hush_stream_aborts();
let bind = SocketAddr::from((Ipv4Addr::LOCALHOST, config.port));
let server = Server::http(bind).map_err(|e| ServeError::Bind {
port: config.port,
message: e.to_string(),
})?;
let address = match server.server_addr().to_ip() {
Some(address) => address,
None => bind,
};
tracing::info!(%address, model = engine.model_id(), "listening");
ready(address);
loop {
let request = server.recv()?;
handle(engine, request, &config);
}
}
fn handle(engine: &mut dyn Engine, request: Request, config: &ServeConfig) {
let path = request
.url()
.split('?')
.next()
.unwrap_or_default()
.to_owned();
let method = request.method().clone();
tracing::debug!(%method, %path, "request");
match (&method, path.as_str()) {
(Method::Post, PATH_CHAT_COMPLETIONS) => chat_completions(engine, request, config),
(Method::Get, PATH_MODELS) => {
let body = ModelList::single(engine.model_id(), engine.created());
respond_json(request, 200, &body);
}
(Method::Get, PATH_HEALTH) => respond_json(request, 200, &Health::ok()),
_ => respond_error(
request,
&ServerError::NotFound {
method: method.as_str().to_owned(),
path,
},
),
}
}
fn chat_completions(engine: &mut dyn Engine, mut request: Request, config: &ServeConfig) {
let body = match read_body(&mut request, config.max_body_bytes) {
Ok(body) => body,
Err(e) => return respond_error(request, &e),
};
let parsed = match ChatCompletionRequest::from_json(&body) {
Ok(parsed) => parsed,
Err(e) => return respond_error(request, &e),
};
tracing::debug!(
stop = parsed.stop_sequences().len(),
tools = parsed.tools().len(),
tool_choice = ?parsed.tool_choice,
stream = parsed.stream(),
"chat request"
);
let plan = match engine.prepare(&parsed) {
Ok(plan) => plan,
Err(e) => return respond_error(request, &e),
};
let id = completion_id();
let created = unix_now();
if parsed.stream() {
stream_into(
engine,
&parsed,
plan,
request.into_writer(),
&id,
created,
config.keepalive,
);
} else {
let mut sink = NullSink;
match engine.run(plan, &mut sink) {
Ok(completion) => {
let extracted = if parsed.tools().is_empty() {
crate::toolcall::Extracted::none(&completion.text)
} else {
crate::toolcall::extract(&completion.text)
};
let calls = crate::toolcall::wire_calls(&id, &extracted.calls);
let finish_reason = if calls.is_empty() {
completion.finish_reason
} else {
FinishReason::ToolCalls
};
let body = ChatCompletion::from_message(
id,
created,
&parsed.model,
ResponseMessage::calling(extracted.content, calls),
finish_reason,
completion.usage,
);
respond_json(request, 200, &body);
}
Err(e) => respond_error(request, &e),
}
}
}
pub fn stream_into<W: Write + Send + 'static>(
engine: &mut dyn Engine,
request: &ChatCompletionRequest,
plan: Plan,
writer: W,
id: &str,
created: u64,
keepalive: Duration,
) {
let state = Arc::new(Mutex::new(StreamState::new(writer)));
if let Err(e) = lock(&state).head() {
tracing::debug!(?e, "client left before the response head");
return;
}
let builder = ChunkBuilder::new(id, created, &request.model);
if emit(&state, &sse_or_empty(&builder.role())).is_err() {
return;
}
let keeper = KeepAlive::spawn(Arc::clone(&state), keepalive);
let mut sink = SseSink {
state: Arc::clone(&state),
builder: builder.clone(),
keepalive,
calls: CallStream::new(!request.tools().is_empty()),
};
let include_usage = request.include_usage();
let outcome = engine.run(plan, &mut sink);
drop(keeper);
match outcome {
Ok(completion) if completion.aborted => {
tracing::debug!("client left mid-reply; the stream is abandoned unterminated");
}
Ok(completion) => {
if let Some(unterminated) = sink.calls.flush() {
let _ = emit(&state, &sse_or_empty(&builder.content(unterminated)));
}
let finish_reason = if sink.calls.emitted() > 0 {
FinishReason::ToolCalls
} else {
completion.finish_reason
};
let _ = emit(&state, &sse_or_empty(&builder.finish(finish_reason)));
if include_usage {
let _ = emit(&state, &sse_or_empty(&builder.usage(completion.usage)));
}
let _ = emit(&state, &sse_done());
let _ = lock(&state).finish();
}
Err(e) => {
tracing::error!(error = %e, "generation failed mid-stream");
let _ = emit(&state, &sse_data(&e.body_json()));
let _ = emit(&state, &sse_done());
let _ = lock(&state).finish();
}
}
}
struct SseSink<W: Write> {
state: Arc<Mutex<StreamState<W>>>,
builder: ChunkBuilder,
keepalive: Duration,
calls: CallStream,
}
impl<W: Write> TokenSink for SseSink<W> {
fn prefill(&mut self, done: usize, total: usize) -> Result<(), StreamError> {
let mut state = lock(&self.state);
if let Some(failed) = state.failure() {
return Err(failed);
}
if state.idle_for() >= self.keepalive {
state.emit(KEEPALIVE)?;
tracing::trace!(done, total, "keep-alive on prefill progress");
}
Ok(())
}
fn token(&mut self, id: u32, text: &str) -> Result<(), StreamError> {
let event = match self.calls.push(id, text) {
Step::Nothing => return Ok(()),
Step::Content(text) if text.is_empty() => return Ok(()),
Step::Content(text) => sse_or_empty(&self.builder.content(text)),
Step::Call { index, call } => {
tracing::debug!(index, name = %call.name, "recovered a streamed tool call");
let wire = wire_call(self.builder.id(), index, &call);
sse_or_empty(&self.builder.tool_call(index, wire))
}
};
lock(&self.state).emit(&event)
}
}
fn sse_or_empty(chunk: &crate::response::ChatCompletionChunk) -> String {
match sse_event(chunk) {
Ok(event) => event,
Err(e) => {
tracing::error!(error = %e, "serializing a chunk failed; dropping it");
String::new()
}
}
}
struct KeepAlive {
stop: Arc<AtomicBool>,
handle: Option<JoinHandle<()>>,
}
impl KeepAlive {
fn spawn<W: Write + Send + 'static>(
state: Arc<Mutex<StreamState<W>>>,
interval: Duration,
) -> Self {
let stop = Arc::new(AtomicBool::new(false));
let flag = Arc::clone(&stop);
let tick = (interval / 4).clamp(Duration::from_millis(5), Duration::from_secs(1));
let handle = std::thread::Builder::new()
.name("rvmp-keepalive".to_owned())
.spawn(move || {
while !flag.load(Ordering::Relaxed) {
std::thread::sleep(tick);
if flag.load(Ordering::Relaxed) {
return;
}
let mut state = lock(&state);
if state.failure().is_some() {
return;
}
if state.idle_for() >= interval && state.emit(KEEPALIVE).is_err() {
return;
}
}
});
match handle {
Ok(handle) => KeepAlive {
stop,
handle: Some(handle),
},
Err(e) => {
tracing::warn!(error = %e, "no keep-alive thread; long prefills may time out");
KeepAlive { stop, handle: None }
}
}
}
}
impl Drop for KeepAlive {
fn drop(&mut self) {
self.stop.store(true, Ordering::Relaxed);
if let Some(handle) = self.handle.take() {
let _ = handle.join();
}
}
}
fn lock<W: Write>(state: &Mutex<StreamState<W>>) -> MutexGuard<'_, StreamState<W>> {
state.lock().unwrap_or_else(PoisonError::into_inner)
}
fn emit<W: Write>(state: &Mutex<StreamState<W>>, payload: &str) -> Result<(), StreamError> {
lock(state).emit(payload)
}
fn read_body(request: &mut Request, limit: usize) -> Result<String, ServerError> {
if request.body_length().is_some_and(|len| len > limit) {
return Err(ServerError::BodyTooLarge { limit });
}
let mut body = Vec::new();
let capped = u64::try_from(limit).unwrap_or(u64::MAX).saturating_add(1);
request
.as_reader()
.take(capped)
.read_to_end(&mut body)
.map_err(|e| ServerError::MalformedJson(format!("reading the request body: {e}")))?;
if body.len() > limit {
return Err(ServerError::BodyTooLarge { limit });
}
String::from_utf8(body)
.map_err(|_| ServerError::MalformedJson("request body is not valid UTF-8".to_owned()))
}
fn completion_id() -> String {
static COUNTER: AtomicU64 = AtomicU64::new(0);
let n = COUNTER.fetch_add(1, Ordering::Relaxed);
format!("{COMPLETION_ID_PREFIX}{:x}{:04x}", unix_now(), n & 0xffff)
}
fn header(name: &str, value: &str) -> Option<Header> {
match Header::from_bytes(name.as_bytes(), value.as_bytes()) {
Ok(header) => Some(header),
Err(()) => {
tracing::error!(name, value, "non-ASCII header; dropped");
None
}
}
}
fn respond_json<T: serde::Serialize>(request: Request, status: u16, body: &T) {
match serde_json::to_string(body) {
Ok(json) => send(request, status, json, &[]),
Err(e) => {
let error = ServerError::Serialize(e);
respond_error(request, &error);
}
}
}
fn respond_error(request: Request, error: &ServerError) {
tracing::debug!(status = error.status(), error = %error, "refusing a request");
send(request, error.status(), error.body_json(), error.headers());
}
fn send(request: Request, status: u16, body: String, extra: &[(&str, &str)]) {
let mut response = Response::from_string(body).with_status_code(status);
if let Some(header) = header("Content-Type", "application/json") {
response = response.with_header(header);
}
for (name, value) in extra {
if let Some(header) = header(name, value) {
response = response.with_header(header);
}
}
if let Err(e) = request.respond(response) {
tracing::debug!(error = %e, "failed to send a response");
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::engine::Completion;
use crate::response::{FinishReason, Usage};
use crate::wire::{SSE_HEAD, TERMINATOR};
use std::sync::atomic::AtomicUsize;
use std::time::Instant;
struct StubEngine {
prefill_steps: usize,
prefill_delay: Duration,
tokens: Vec<(u32, String)>,
token_delay: Duration,
finish_reason: FinishReason,
produced: Arc<AtomicUsize>,
}
const TEXT_ID: u32 = 7;
impl StubEngine {
fn new(tokens: &[&str]) -> Self {
StubEngine::with_ids(&tokens.iter().map(|t| (TEXT_ID, *t)).collect::<Vec<_>>())
}
fn with_ids(tokens: &[(u32, &str)]) -> Self {
StubEngine {
prefill_steps: 0,
prefill_delay: Duration::ZERO,
tokens: tokens
.iter()
.map(|(id, text)| (*id, (*text).to_owned()))
.collect(),
token_delay: Duration::ZERO,
finish_reason: FinishReason::Stop,
produced: Arc::new(AtomicUsize::new(0)),
}
}
}
impl Engine for StubEngine {
fn model_id(&self) -> &str {
"stub-model"
}
fn created(&self) -> u64 {
1_700_000_000
}
fn prepare(&mut self, request: &ChatCompletionRequest) -> Result<Plan, ServerError> {
request.validate()?;
Ok(Plan {
prompt_tokens: 11,
max_new: self.tokens.len(),
cached_prefix: 0,
prompt_ids: vec![1, 2, 3],
params: ramvamp_core::generate::GenerateParams {
max_new: self.tokens.len(),
temperature: 0.7,
top_k: Some(20),
top_p: 0.8,
seed: 42,
greedy: false,
},
})
}
fn run(&mut self, plan: Plan, sink: &mut dyn TokenSink) -> Result<Completion, ServerError> {
let mut text = String::new();
for step in 1..=self.prefill_steps {
std::thread::sleep(self.prefill_delay);
if sink.prefill(step, self.prefill_steps).is_err() {
return Ok(aborted(text, plan.prompt_tokens, 0));
}
}
for (index, (id, token)) in self.tokens.clone().iter().enumerate() {
std::thread::sleep(self.token_delay);
self.produced.store(index + 1, Ordering::Relaxed);
text.push_str(token);
if sink.token(*id, token).is_err() {
return Ok(aborted(text, plan.prompt_tokens, index + 1));
}
}
let generated = self.tokens.len();
Ok(Completion {
text,
usage: Usage::new(plan.prompt_tokens as u32, generated as u32),
finish_reason: self.finish_reason,
aborted: false,
})
}
}
fn aborted(text: String, prompt: usize, generated: usize) -> Completion {
Completion {
text,
usage: Usage::new(prompt as u32, generated as u32),
finish_reason: FinishReason::Stop,
aborted: true,
}
}
type Flushed = Arc<Mutex<Vec<(Duration, Vec<u8>)>>>;
#[derive(Clone)]
struct Recorder {
events: Flushed,
pending: Arc<Mutex<Vec<u8>>>,
flushes: Arc<AtomicUsize>,
start: Instant,
die_after_flushes: Option<usize>,
}
impl Recorder {
fn new() -> Self {
Recorder {
events: Arc::new(Mutex::new(Vec::new())),
pending: Arc::new(Mutex::new(Vec::new())),
flushes: Arc::new(AtomicUsize::new(0)),
start: Instant::now(),
die_after_flushes: None,
}
}
fn dying_after(mut self, flushes: usize) -> Self {
self.die_after_flushes = Some(flushes);
self
}
fn events(&self) -> Vec<(Duration, String)> {
self.events
.lock()
.unwrap_or_else(PoisonError::into_inner)
.iter()
.map(|(at, bytes)| (*at, String::from_utf8_lossy(bytes).into_owned()))
.collect()
}
fn body(&self) -> String {
self.events().into_iter().map(|(_, text)| text).collect()
}
}
impl Write for Recorder {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.pending
.lock()
.unwrap_or_else(PoisonError::into_inner)
.extend_from_slice(buf);
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
let bytes =
std::mem::take(&mut *self.pending.lock().unwrap_or_else(PoisonError::into_inner));
let n = self.flushes.fetch_add(1, Ordering::Relaxed) + 1;
if self.die_after_flushes.is_some_and(|limit| n > limit) {
return Err(std::io::Error::from(std::io::ErrorKind::BrokenPipe));
}
self.events
.lock()
.unwrap_or_else(PoisonError::into_inner)
.push((self.start.elapsed(), bytes));
Ok(())
}
}
fn request(stream: bool) -> ChatCompletionRequest {
let body = format!(
r#"{{"model":"stub-model","messages":[{{"role":"user","content":"hi"}}],"stream":{stream}}}"#
);
ChatCompletionRequest::from_json(&body).expect("valid body")
}
fn request_with_tools() -> ChatCompletionRequest {
ChatCompletionRequest::from_json(
r#"{"model":"stub-model","messages":[{"role":"user","content":"list the rust files"}],
"stream":true,
"tools":[{"type":"function","function":{"name":"glob","parameters":{}}}]}"#,
)
.expect("valid body")
}
fn run_stream(engine: &mut StubEngine, recorder: Recorder, keepalive: Duration) {
run_stream_for(engine, &request(true), recorder, keepalive);
}
fn run_stream_for(
engine: &mut StubEngine,
request: &ChatCompletionRequest,
recorder: Recorder,
keepalive: Duration,
) {
let plan = engine.prepare(request).expect("a plan");
stream_into(
engine,
request,
plan,
recorder,
"chatcmpl-test",
1_700_000_000,
keepalive,
);
}
fn chunks(body: &str) -> Vec<serde_json::Value> {
body.split("data: ")
.skip(1)
.map(|event| event.split("\n\n").next().unwrap_or_default())
.filter(|payload| *payload != crate::sse::DONE_SENTINEL)
.map(|payload| {
serde_json::from_str(payload)
.unwrap_or_else(|e| panic!("a chunk payload: {e}\n{payload}"))
})
.collect()
}
fn streamed_calls(body: &str) -> Vec<serde_json::Value> {
chunks(body)
.into_iter()
.filter_map(|chunk| {
chunk["choices"][0]["delta"]["tool_calls"]
.as_array()
.cloned()
})
.flatten()
.collect()
}
fn streamed_text(body: &str) -> String {
chunks(body)
.iter()
.filter_map(|chunk| chunk["choices"][0]["delta"]["content"].as_str())
.collect()
}
fn streamed_finish(body: &str) -> String {
chunks(body)
.iter()
.find_map(|chunk| {
chunk["choices"][0]["finish_reason"]
.as_str()
.map(str::to_owned)
})
.expect("a finish chunk")
}
fn glob_call_tokens() -> Vec<(u32, &'static str)> {
vec![
(crate::toolcall::OPEN_TAG_ID, "<tool_call>"),
(TEXT_ID, "\n{\"name\": \"glob\", "),
(TEXT_ID, "\"arguments\": {\"pattern\": \"*/.rs\"}}\n"),
(crate::toolcall::CLOSE_TAG_ID, "</tool_call>"),
]
}
#[test]
fn events_reach_the_client_at_the_sources_pace() {
let mut engine = StubEngine::new(&["a", "b", "c", "d", "e"]);
engine.token_delay = Duration::from_millis(60);
let recorder = Recorder::new();
run_stream(&mut engine, recorder.clone(), Duration::from_secs(60));
let content: Vec<Duration> = recorder
.events()
.into_iter()
.filter(|(_, text)| text.contains(r#""content":"#) && !text.contains(r#""role""#))
.map(|(at, _)| at)
.collect();
assert_eq!(content.len(), 5, "{:?}", recorder.events());
for pair in content.windows(2) {
assert!(
pair[1] - pair[0] >= Duration::from_millis(40),
"events arrived in a burst: {content:?}"
);
}
assert!(
content[4] - content[0] >= Duration::from_millis(200),
"{content:?}"
);
}
#[test]
fn the_head_is_written_before_the_engine_runs() {
let mut engine = StubEngine::new(&["x"]);
engine.prefill_steps = 1;
engine.prefill_delay = Duration::from_millis(250);
let recorder = Recorder::new();
run_stream(&mut engine, recorder.clone(), Duration::from_secs(60));
let events = recorder.events();
assert_eq!(events[0].1, SSE_HEAD);
assert!(
events[0].0 < Duration::from_millis(100),
"the head waited for the model: {:?}",
events[0].0
);
assert!(events[1].1.contains(r#""role":"assistant""#));
assert!(events[1].0 < Duration::from_millis(100));
}
#[test]
fn a_slow_prefill_emits_keep_alive_comments() {
let mut engine = StubEngine::new(&["done"]);
engine.prefill_steps = 1;
engine.prefill_delay = Duration::from_millis(400);
let recorder = Recorder::new();
run_stream(&mut engine, recorder.clone(), Duration::from_millis(50));
let comments: Vec<Duration> = recorder
.events()
.into_iter()
.filter(|(_, text)| text.contains(KEEPALIVE))
.map(|(at, _)| at)
.collect();
assert!(
comments.len() >= 3,
"expected keep-alives during a 400ms prefill, got {comments:?}"
);
assert!(recorder.body().contains(r#""content":"done""#));
}
#[test]
fn a_dropped_client_surfaces_as_a_write_error_and_aborts_generation() {
let mut engine = StubEngine::new(&["a", "b", "c", "d", "e", "f"]);
let produced = Arc::clone(&engine.produced);
let recorder = Recorder::new().dying_after(3);
run_stream(&mut engine, recorder.clone(), Duration::from_secs(60));
let got = produced.load(Ordering::Relaxed);
assert!(
(1..=2).contains(&got),
"generation should have stopped at the hangup, produced {got}"
);
assert!(!recorder.body().contains("[DONE]"), "{}", recorder.body());
}
#[test]
fn the_sentinel_precedes_the_zero_length_chunk() {
let mut engine = StubEngine::new(&["he", "llo"]);
let recorder = Recorder::new();
run_stream(&mut engine, recorder.clone(), Duration::from_secs(60));
let body = recorder.body();
let rest = body.strip_prefix(SSE_HEAD).expect("head first");
assert!(rest.ends_with(TERMINATOR), "{rest:?}");
let done = rest.find("data: [DONE]").expect("sentinel");
let end = rest.rfind(TERMINATOR).expect("terminator");
assert!(done < end, "the sentinel must precede the terminator");
}
#[test]
fn the_chunk_framing_decodes_back_to_the_reply() {
let mut engine = StubEngine::new(&["Hé", "llo", "→!"]);
let recorder = Recorder::new();
run_stream(&mut engine, recorder.clone(), Duration::from_secs(60));
let body = recorder.body();
let mut rest = body.strip_prefix(SSE_HEAD).expect("head first");
let mut payloads = Vec::new();
loop {
let (head, tail) = rest.split_once("\r\n").expect("a chunk header");
let len = usize::from_str_radix(head, 16).expect("hex length");
if len == 0 {
assert_eq!(tail, "\r\n", "the terminator ends the body");
break;
}
assert!(tail.len() >= len + 2, "chunk shorter than its header says");
payloads.push(tail[..len].to_owned());
assert_eq!(&tail[len..len + 2], "\r\n", "chunk not CRLF-terminated");
rest = &tail[len + 2..];
}
let text: String = payloads
.iter()
.filter_map(|event| event.strip_prefix("data: "))
.filter_map(|json| serde_json::from_str::<serde_json::Value>(json.trim_end()).ok())
.filter_map(|chunk| {
chunk["choices"][0]["delta"]["content"]
.as_str()
.map(str::to_owned)
})
.collect();
assert_eq!(text, "Héllo→!");
}
#[test]
fn usage_is_emitted_only_when_asked_for_and_before_the_sentinel() {
let mut engine = StubEngine::new(&["x"]);
let recorder = Recorder::new();
run_stream(&mut engine, recorder.clone(), Duration::from_secs(60));
assert!(!recorder.body().contains(r#""usage""#));
let asked = ChatCompletionRequest::from_json(
r#"{"model":"stub-model","messages":[{"role":"user","content":"hi"}],
"stream":true,"stream_options":{"include_usage":true}}"#,
)
.expect("valid body");
let plan = engine.prepare(&asked).expect("a plan");
let recorder = Recorder::new();
stream_into(
&mut engine,
&asked,
plan,
recorder.clone(),
"chatcmpl-test",
1_700_000_000,
Duration::from_secs(60),
);
let body = recorder.body();
let usage = body.find(r#""usage""#).expect("a usage chunk");
let finish = body.find(r#""finish_reason":"stop""#).expect("a finish");
let done = body.find("[DONE]").expect("the sentinel");
assert!(finish < usage && usage < done, "{body}");
assert!(body.contains(r#""prompt_tokens":11"#), "{body}");
}
#[test]
fn a_mid_stream_failure_becomes_an_error_event() {
struct Failing;
impl Engine for Failing {
fn model_id(&self) -> &str {
"stub-model"
}
fn created(&self) -> u64 {
0
}
fn prepare(&mut self, _: &ChatCompletionRequest) -> Result<Plan, ServerError> {
unreachable!("the test builds the plan itself")
}
fn run(&mut self, _: Plan, _: &mut dyn TokenSink) -> Result<Completion, ServerError> {
Err(ServerError::Generate(
ramvamp_core::generate::GenerateError::EmptyPrompt,
))
}
}
let mut stub = StubEngine::new(&[]);
let request = request(true);
let plan = stub.prepare(&request).expect("a plan");
let recorder = Recorder::new();
stream_into(
&mut Failing,
&request,
plan,
recorder.clone(),
"chatcmpl-test",
0,
Duration::from_secs(60),
);
let body = recorder.body();
assert!(body.starts_with(SSE_HEAD));
assert!(body.contains(r#""code":"generate_error""#), "{body}");
assert!(body.contains("[DONE]"));
assert!(body.ends_with(TERMINATOR));
}
fn spawn_server() -> SocketAddr {
spawn_server_with(&["ok"], FinishReason::Stop)
}
fn spawn_server_with(tokens: &[&str], finish_reason: FinishReason) -> SocketAddr {
let tokens: Vec<String> = tokens.iter().map(|t| (*t).to_owned()).collect();
let (tx, rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
let borrowed: Vec<&str> = tokens.iter().map(String::as_str).collect();
let mut engine = StubEngine::new(&borrowed);
engine.finish_reason = finish_reason;
let _ = serve_with(
&mut engine,
ServeConfig {
port: 0,
..ServeConfig::default()
},
|address| {
let _ = tx.send(address);
},
);
});
rx.recv_timeout(Duration::from_secs(10))
.expect("the server binds")
}
fn roundtrip(address: SocketAddr, raw: &str) -> String {
let mut stream = std::net::TcpStream::connect(address).expect("connect");
stream
.set_read_timeout(Some(Duration::from_secs(10)))
.expect("timeout");
stream.write_all(raw.as_bytes()).expect("write");
stream.flush().expect("flush");
let mut out = Vec::new();
let mut buffer = [0u8; 4096];
loop {
match stream.read(&mut buffer) {
Ok(0) | Err(_) => break,
Ok(n) => out.extend_from_slice(&buffer[..n]),
}
}
String::from_utf8_lossy(&out).into_owned()
}
fn get(address: SocketAddr, path: &str) -> String {
roundtrip(
address,
&format!("GET {path} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"),
)
}
fn post(address: SocketAddr, path: &str, body: &str) -> String {
roundtrip(
address,
&format!(
"POST {path} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\
Content-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}",
body.len()
),
)
}
#[test]
fn the_buffered_routes_answer_over_a_socket() {
let address = spawn_server();
assert!(address.ip().is_loopback(), "{address}");
let health = get(address, PATH_HEALTH);
assert!(health.starts_with("HTTP/1.1 200"), "{health}");
assert!(health.contains(r#"{"status":"ok"}"#), "{health}");
let models = get(address, PATH_MODELS);
assert!(models.starts_with("HTTP/1.1 200"), "{models}");
assert!(models.contains(r#""id":"stub-model""#), "{models}");
assert!(models.contains(r#""object":"list""#), "{models}");
let versioned = get(address, "/v1/models?api-version=2024-02-01");
assert!(versioned.contains(r#""id":"stub-model""#), "{versioned}");
let missing = get(address, "/v1/embeddings");
assert!(missing.starts_with("HTTP/1.1 404"), "{missing}");
assert!(
missing.contains(r#""type":"invalid_request_error""#),
"{missing}"
);
assert!(missing.contains(r#""code":"not_found""#), "{missing}");
let wrong = get(address, PATH_CHAT_COMPLETIONS);
assert!(wrong.starts_with("HTTP/1.1 404"), "{wrong}");
assert!(wrong.contains(r#""error""#), "{wrong}");
}
#[test]
fn a_non_streaming_completion_comes_back_as_one_json_body() {
let address = spawn_server();
let response = post(
address,
PATH_CHAT_COMPLETIONS,
r#"{"model":"stub-model","messages":[{"role":"user","content":"hi"}]}"#,
);
assert!(response.starts_with("HTTP/1.1 200"), "{response}");
assert!(
response.contains(r#""object":"chat.completion""#),
"{response}"
);
assert!(response.contains(r#""content":"ok""#), "{response}");
assert!(response.contains(r#""finish_reason":"stop""#), "{response}");
}
const TOOL_CALL_REPLY: &str = "Let me look.\n<tool_call>\n{\"name\": \"read\", \"arguments\": {\"path\": \"a.txt\"}}\n</tool_call>";
fn body_json(response: &str) -> serde_json::Value {
let body = response
.split_once("\r\n\r\n")
.map_or(response, |(_, body)| body);
serde_json::from_str(body).unwrap_or_else(|e| panic!("a JSON body: {e}\n{response}"))
}
#[test]
fn a_tool_call_reply_comes_back_as_structured_calls() {
let address = spawn_server_with(&[TOOL_CALL_REPLY], FinishReason::Stop);
let response = post(
address,
PATH_CHAT_COMPLETIONS,
r#"{"model":"stub-model","messages":[{"role":"user","content":"read a.txt"}],
"tools":[{"type":"function","function":{"name":"read","parameters":{}}}]}"#,
);
assert!(response.starts_with("HTTP/1.1 200"), "{response}");
let json = body_json(&response);
let choice = &json["choices"][0];
assert_eq!(choice["finish_reason"], "tool_calls");
let calls = choice["message"]["tool_calls"]
.as_array()
.unwrap_or_else(|| panic!("a tool_calls array: {json}"));
assert_eq!(calls.len(), 1);
assert_eq!(calls[0]["type"], "function");
assert_eq!(calls[0]["function"]["name"], "read");
assert_eq!(calls[0]["function"]["arguments"], r#"{"path": "a.txt"}"#);
let call_id = calls[0]["id"].as_str().expect("an id");
assert!(call_id.starts_with("call_"), "{call_id}");
assert!(call_id.ends_with("_0"), "{call_id}");
assert!(
call_id.contains(
json["id"]
.as_str()
.and_then(|id| id.strip_prefix("chatcmpl-"))
.expect("a completion id")
),
"the call id is derived from the completion id: {call_id}"
);
assert_eq!(choice["message"]["content"], "Let me look.");
assert!(
!response.contains("<tool_call>"),
"the raw block is still in the body: {response}"
);
}
#[test]
fn a_block_in_a_reply_to_a_request_without_tools_stays_text() {
let address = spawn_server_with(&[TOOL_CALL_REPLY], FinishReason::Stop);
let response = post(
address,
PATH_CHAT_COMPLETIONS,
r#"{"model":"stub-model","messages":[{"role":"user","content":"show me a tool call"}]}"#,
);
assert!(response.starts_with("HTTP/1.1 200"), "{response}");
let json = body_json(&response);
let choice = &json["choices"][0];
assert_eq!(choice["finish_reason"], "stop");
assert!(
choice["message"]["tool_calls"].is_null(),
"a request without tools got tool_calls back: {json}"
);
assert_eq!(
choice["message"]["content"].as_str().expect("content"),
TOOL_CALL_REPLY
);
}
#[test]
fn a_plain_reply_carries_no_tool_calls_key() {
let address = spawn_server();
let response = post(
address,
PATH_CHAT_COMPLETIONS,
r#"{"model":"stub-model","messages":[{"role":"user","content":"hi"}]}"#,
);
let json = body_json(&response);
assert_eq!(json["choices"][0]["message"]["content"], "ok");
assert_eq!(json["choices"][0]["finish_reason"], "stop");
assert!(
json["choices"][0]["message"].get("tool_calls").is_none(),
"{json}"
);
}
#[test]
fn a_block_truncated_by_the_token_budget_keeps_the_length_reason() {
let address = spawn_server_with(
&["<tool_call>\n{\"name\": \"read\", \"argum"],
FinishReason::Length,
);
let response = post(
address,
PATH_CHAT_COMPLETIONS,
r#"{"model":"stub-model","messages":[{"role":"user","content":"hi"}],"max_tokens":8}"#,
);
let json = body_json(&response);
assert_eq!(json["choices"][0]["finish_reason"], "length");
assert!(
json["choices"][0]["message"].get("tool_calls").is_none(),
"{json}"
);
assert_eq!(
json["choices"][0]["message"]["content"], "<tool_call>\n{\"name\": \"read\", \"argum",
"half a block is text, and text is not dropped"
);
}
#[test]
fn a_streamed_call_becomes_a_tool_calls_delta_and_no_raw_markup() {
let mut engine = StubEngine::with_ids(&glob_call_tokens());
let recorder = Recorder::new();
run_stream_for(
&mut engine,
&request_with_tools(),
recorder.clone(),
Duration::from_secs(60),
);
let body = recorder.body();
assert!(
!body.contains("tool_call>"),
"the raw markup is still on the wire: {body}"
);
let calls = streamed_calls(&body);
assert_eq!(calls.len(), 1, "{body}");
assert_eq!(calls[0]["index"], 0);
assert_eq!(calls[0]["type"], "function");
assert_eq!(calls[0]["function"]["name"], "glob");
assert_eq!(
calls[0]["function"]["arguments"], r#"{"pattern": "*/.rs"}"#,
"the model's argument bytes reach the client unaltered"
);
assert_eq!(streamed_finish(&body), "tool_calls");
assert_eq!(streamed_text(&body), "", "the block left no prose behind");
assert!(body.contains("[DONE]"), "{body}");
assert!(body.ends_with(TERMINATOR), "{body}");
}
#[test]
fn prose_before_between_and_after_streamed_calls_is_still_content() {
let mut tokens = vec![(TEXT_ID, "Let me "), (TEXT_ID, "look.\n")];
tokens.extend(glob_call_tokens());
tokens.push((TEXT_ID, "\nand the tests\n"));
tokens.extend(glob_call_tokens());
tokens.push((TEXT_ID, "\nThen I will compare."));
let mut engine = StubEngine::with_ids(&tokens);
let recorder = Recorder::new();
run_stream_for(
&mut engine,
&request_with_tools(),
recorder.clone(),
Duration::from_secs(60),
);
let body = recorder.body();
assert_eq!(
streamed_text(&body),
"Let me look.\n\nand the tests\n\nThen I will compare."
);
assert!(!body.contains("tool_call>"), "{body}");
}
#[test]
fn two_streamed_calls_get_indices_zero_and_one() {
let mut tokens = glob_call_tokens();
tokens.extend(glob_call_tokens());
let mut engine = StubEngine::with_ids(&tokens);
let recorder = Recorder::new();
run_stream_for(
&mut engine,
&request_with_tools(),
recorder.clone(),
Duration::from_secs(60),
);
let body = recorder.body();
let calls = streamed_calls(&body);
assert_eq!(calls.len(), 2, "{body}");
assert_eq!(calls[0]["index"], 0);
assert_eq!(calls[1]["index"], 1);
assert_ne!(calls[0]["id"], calls[1]["id"]);
assert_eq!(streamed_finish(&body), "tool_calls");
for call in &calls {
assert_eq!(call["function"]["arguments"], r#"{"pattern": "*/.rs"}"#);
}
}
#[test]
fn an_unterminated_streamed_block_flushes_as_content_and_keeps_length() {
let mut engine = StubEngine::with_ids(&[
(TEXT_ID, "Reading it.\n"),
(crate::toolcall::OPEN_TAG_ID, "<tool_call>"),
(TEXT_ID, "\n{\"name\": \"glob\", \"argum"),
]);
engine.finish_reason = FinishReason::Length;
let recorder = Recorder::new();
run_stream_for(
&mut engine,
&request_with_tools(),
recorder.clone(),
Duration::from_secs(60),
);
let body = recorder.body();
assert!(streamed_calls(&body).is_empty(), "{body}");
assert_eq!(streamed_finish(&body), "length");
assert_eq!(
streamed_text(&body),
"Reading it.\n<tool_call>\n{\"name\": \"glob\", \"argum",
"half a block is text, and text is not dropped"
);
}
#[test]
fn a_streamed_block_without_tools_stays_ordinary_text() {
let mut engine = StubEngine::with_ids(&glob_call_tokens());
let recorder = Recorder::new();
run_stream(&mut engine, recorder.clone(), Duration::from_secs(60));
let body = recorder.body();
assert!(
streamed_calls(&body).is_empty(),
"a request without tools got tool_calls back: {body}"
);
assert_eq!(streamed_finish(&body), "stop");
assert_eq!(
streamed_text(&body),
"<tool_call>\n{\"name\": \"glob\", \"arguments\": {\"pattern\": \"*/.rs\"}}\n</tool_call>",
"byte for byte, markers and all"
);
}
#[test]
fn malformed_json_in_a_streamed_block_still_yields_a_call() {
let mut engine = StubEngine::with_ids(&[
(crate::toolcall::OPEN_TAG_ID, "<tool_call>"),
(
TEXT_ID,
"\n{\"name\": \"glob\", \"arguments\": {\"pattern\": \"*\",}}\n",
),
(crate::toolcall::CLOSE_TAG_ID, "</tool_call>"),
]);
let recorder = Recorder::new();
run_stream_for(
&mut engine,
&request_with_tools(),
recorder.clone(),
Duration::from_secs(60),
);
let body = recorder.body();
let calls = streamed_calls(&body);
assert_eq!(calls.len(), 1, "{body}");
assert_eq!(calls[0]["function"]["name"], "glob");
assert_eq!(calls[0]["function"]["arguments"], r#"{"pattern": "*",}"#);
assert_eq!(streamed_finish(&body), "tool_calls");
}
#[test]
fn streamed_call_ids_match_the_buffered_paths_for_the_same_completion() {
const ID: &str = "chatcmpl-18f3a0001";
let mut engine = StubEngine::with_ids(&glob_call_tokens());
let recorder = Recorder::new();
let request = request_with_tools();
let plan = engine.prepare(&request).expect("a plan");
stream_into(
&mut engine,
&request,
plan,
recorder.clone(),
ID,
1_700_000_000,
Duration::from_secs(60),
);
let body = recorder.body();
let streamed = streamed_calls(&body);
let reply: String = glob_call_tokens().iter().map(|(_, text)| *text).collect();
let buffered = crate::toolcall::wire_calls(ID, &crate::toolcall::extract(&reply).calls);
assert_eq!(buffered.len(), 1);
assert_eq!(streamed[0]["id"], buffered[0].id);
assert_eq!(streamed[0]["id"], "call_18f3a0001_0");
}
#[test]
fn a_tool_call_delta_is_framed_like_every_other_event() {
let mut engine = StubEngine::with_ids(&glob_call_tokens());
let recorder = Recorder::new();
run_stream_for(
&mut engine,
&request_with_tools(),
recorder.clone(),
Duration::from_secs(60),
);
let body = recorder.body();
let mut rest = body.strip_prefix(SSE_HEAD).expect("head first");
let mut events = 0usize;
loop {
let (head, tail) = rest.split_once("\r\n").expect("a chunk header");
let len = usize::from_str_radix(head, 16).expect("hex length");
if len == 0 {
assert_eq!(tail, "\r\n", "the terminator ends the body");
break;
}
assert_eq!(&tail[len..len + 2], "\r\n", "chunk not CRLF-terminated");
let payload = &tail[..len];
assert_eq!(payload.matches("\n\n").count(), 1, "{payload:?}");
events += 1;
rest = &tail[len + 2..];
}
assert_eq!(events, 4, "{body}");
}
#[test]
fn the_array_form_content_reaches_the_engine() {
let address = spawn_server();
let response = post(
address,
PATH_CHAT_COMPLETIONS,
r#"{"model":"stub-model","messages":[
{"role":"user","content":[{"type":"text","text":"hi"}]}]}"#,
);
assert!(response.starts_with("HTTP/1.1 200"), "{response}");
assert!(response.contains(r#""content":"ok""#), "{response}");
}
#[test]
fn a_malformed_body_is_a_400_with_the_openai_shape() {
let address = spawn_server();
let response = post(address, PATH_CHAT_COMPLETIONS, "{not json");
assert!(response.starts_with("HTTP/1.1 400"), "{response}");
assert!(response.contains(r#""code":"invalid_json""#), "{response}");
assert!(!response.contains(r#""message":null"#), "{response}");
}
#[test]
fn an_oversized_body_is_refused_by_its_declared_length() {
const BODY: &str = "0123456789abcdef0123456789abcdef";
let mut request = tiny_http::TestRequest::new()
.with_method(Method::Post)
.with_path(PATH_CHAT_COMPLETIONS)
.with_body(BODY)
.into();
let error = read_body(&mut request, 16).expect_err("over the cap");
assert!(matches!(error, ServerError::BodyTooLarge { limit: 16 }));
assert_eq!(error.status(), 413);
}
#[test]
fn a_body_at_the_cap_is_accepted() {
const BODY: &str = "0123456789abcdef";
let mut request = tiny_http::TestRequest::new()
.with_method(Method::Post)
.with_path(PATH_CHAT_COMPLETIONS)
.with_body(BODY)
.into();
assert_eq!(
read_body(&mut request, BODY.len()).expect("at the cap"),
BODY
);
}
#[test]
fn completion_ids_do_not_repeat() {
let ids: std::collections::HashSet<String> = (0..64).map(|_| completion_id()).collect();
assert_eq!(ids.len(), 64);
assert!(ids.iter().all(|id| id.starts_with("chatcmpl-")));
}
}