Skip to main content

ramvamp_server/
http.rs

1//! The HTTP transport: one thread, one socket, one request at a time.
2//!
3//! # Loopback only, and not configurable
4//!
5//! The listener binds `127.0.0.1` and there is no flag to change it. That is a
6//! security decision, not an oversight: `tiny_http`'s `read_next_line` has
7//! neither a header-size cap nor a read timeout, so a single connection
8//! sending an endless header line grows a `Vec` until the process dies — and
9//! this process is meant to run inside a `memory.max=3G` cgroup next to a 30B
10//! model, where "grows a `Vec`" means the OOM killer takes the model with it.
11//! Loopback is the mitigation. Anyone who wants this reachable from elsewhere
12//! should put a real reverse proxy in front of it, which is also where request
13//! limits, TLS and authentication belong.
14//!
15//! # Concurrency is one, structurally
16//!
17//! There is no semaphore and no worker pool. One thread calls `Server::recv`
18//! in a loop and handles each request inline, which *is* the concurrency
19//! limit: a request that arrives mid-generation waits in the accept queue
20//! until the current one finishes. That is the desired behaviour, and it is
21//! why [`ServerError::AtCapacity`] is never returned here.
22//!
23//! Rejecting on arrival would be much worse than it looks. The OpenAI SDKs
24//! default to `max_retries = 2` with 0.5 s and 1 s backoff, so a client burns
25//! all three attempts in about 1.5 s — against a request that occupies the
26//! model for minutes. An immediate 503 therefore guarantees a hard failure,
27//! where queueing guarantees an answer. `AtCapacity` stays in the error enum
28//! for a future build that adds a *bounded* queue and can overflow it.
29
30use std::io::{Read, Write};
31use std::net::{Ipv4Addr, SocketAddr};
32use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
33use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
34use std::thread::JoinHandle;
35use std::time::Duration;
36
37use thiserror::Error;
38use tiny_http::{Header, Method, Request, Response, Server};
39
40use crate::engine::{Engine, NullSink, Plan, StreamError, TokenSink, hush_stream_aborts, unix_now};
41use crate::error::ServerError;
42use crate::request::ChatCompletionRequest;
43use crate::response::{
44    COMPLETION_ID_PREFIX, ChatCompletion, ChunkBuilder, FinishReason, Health, ModelList,
45    ResponseMessage,
46};
47use crate::sse::{sse_data, sse_done, sse_event};
48use crate::toolcall::{CallStream, Step, wire_call};
49use crate::wire::{KEEPALIVE, StreamState};
50
51/// `POST` this to generate.
52pub const PATH_CHAT_COMPLETIONS: &str = "/v1/chat/completions";
53/// `GET` this for the one-model listing.
54pub const PATH_MODELS: &str = "/v1/models";
55/// `GET` this for a liveness probe.
56pub const PATH_HEALTH: &str = "/health";
57
58/// How the server was started.
59#[derive(Debug, Clone, Copy, PartialEq, Eq)]
60pub struct ServeConfig {
61    /// TCP port on `127.0.0.1`. `0` binds an ephemeral port.
62    pub port: u16,
63    /// How long a stream may go silent before a keep-alive comment is sent.
64    ///
65    /// Every client that matters gives up after 300 s of no bytes — undici's
66    /// `bodyTimeout`, the `openai` provider's `headerTimeout`, and
67    /// `CLAUDE_CODE_BYTE_STREAM_IDLE_TIMEOUT_MS` all land on the same number —
68    /// and a 4K prompt takes about six minutes to prefill. Ten seconds is
69    /// thirty times under the wall and costs fourteen bytes a tick.
70    pub keepalive: Duration,
71    /// Largest request body this server will buffer, in bytes.
72    pub max_body_bytes: usize,
73}
74
75impl Default for ServeConfig {
76    fn default() -> Self {
77        ServeConfig {
78            port: 8080,
79            keepalive: Duration::from_secs(10),
80            // Derived from the default 4K context, which is what it has always
81            // worked out to: 1 MiB. `ramvamp serve` overrides this from the
82            // window it actually resolved `--context` to.
83            max_body_bytes: max_body_bytes_for_context(DEFAULT_CONTEXT),
84        }
85    }
86}
87
88/// The context window [`ServeConfig::default`] sizes itself against.
89///
90/// The CLI owns the real default and passes the resolved window in; this is
91/// only what a library caller who sets nothing gets, and it is the same 4096.
92pub const DEFAULT_CONTEXT: usize = 4096;
93
94/// Request-body bytes allowed per token of context.
95///
96/// A prompt runs about 4 bytes of text per token, so 256 is sixty-four times
97/// the text a completely full window can hold — headroom for JSON escaping,
98/// for the `tools` array, and for the whitespace a pretty-printing client
99/// sends. It is also small against a 3 GB budget at any window that budget can
100/// host: a 32K context allows 8 MiB.
101pub const BODY_BYTES_PER_CONTEXT_TOKEN: usize = 256;
102
103/// Floor under [`max_body_bytes_for_context`].
104///
105/// The JSON envelope — role names, tool-call ids, the `model` string, the
106/// framing — does not shrink with the context, so a deliberately tiny window
107/// must not produce a cap that a well-formed request cannot fit inside.
108pub const MIN_BODY_BYTES: usize = 64 << 10;
109
110/// The request-body cap for a server configured with a `context`-token window.
111///
112/// This used to be a flat 1 MiB, justified by "a 4K context's whole prompt is
113/// on the order of 16 KB of text". That reasoning does not survive `--context`
114/// being a dial: an agent client that sends tool schemas against a 16K window
115/// writes a request that approaches the old constant, and would have been
116/// answered with a 413 for asking for exactly what the server was configured
117/// to allow. Deriving it means the cap moves with the window instead.
118///
119/// Saturating, and floored at [`MIN_BODY_BYTES`], so no window produces a cap
120/// that wraps or one too small to hold an envelope.
121#[must_use]
122pub fn max_body_bytes_for_context(context: usize) -> usize {
123    context
124        .saturating_mul(BODY_BYTES_PER_CONTEXT_TOKEN)
125        .max(MIN_BODY_BYTES)
126}
127
128/// Why the server could not run. Per-request failures are [`ServerError`]s and
129/// are answered, not returned.
130#[derive(Debug, Error)]
131pub enum ServeError {
132    /// The port could not be bound.
133    #[error("binding 127.0.0.1:{port}: {message}")]
134    Bind {
135        /// The port that was asked for.
136        port: u16,
137        /// What the listener reported.
138        message: String,
139    },
140    /// The accept loop broke.
141    #[error("accepting connections: {0}")]
142    Accept(#[from] std::io::Error),
143}
144
145/// Serve until the process ends.
146///
147/// # Errors
148///
149/// [`ServeError::Bind`] if the port is taken, [`ServeError::Accept`] if the
150/// listener dies. Everything a client can cause is answered as HTTP.
151pub fn serve(engine: &mut dyn Engine, config: ServeConfig) -> Result<(), ServeError> {
152    serve_with(engine, config, |_| {})
153}
154
155/// [`serve`], reporting the bound address before the first accept.
156///
157/// The hook exists so a test can bind port 0 and find out where it landed;
158/// nothing in production passes anything but a no-op.
159///
160/// # Errors
161///
162/// Exactly [`serve`]'s.
163pub fn serve_with(
164    engine: &mut dyn Engine,
165    config: ServeConfig,
166    ready: impl FnOnce(SocketAddr),
167) -> Result<(), ServeError> {
168    hush_stream_aborts();
169    // Loopback, always. See the module docs: there is no flag for this and
170    // there must not be one until the header reader has a cap.
171    let bind = SocketAddr::from((Ipv4Addr::LOCALHOST, config.port));
172    let server = Server::http(bind).map_err(|e| ServeError::Bind {
173        port: config.port,
174        message: e.to_string(),
175    })?;
176    let address = match server.server_addr().to_ip() {
177        Some(address) => address,
178        None => bind,
179    };
180    tracing::info!(%address, model = engine.model_id(), "listening");
181    ready(address);
182
183    loop {
184        let request = server.recv()?;
185        handle(engine, request, &config);
186    }
187}
188
189/// Route one request. Never panics on anything a client can send.
190fn handle(engine: &mut dyn Engine, request: Request, config: &ServeConfig) {
191    // The query string is not part of the route: a client appending
192    // `?api-version=...` (Azure-flavoured SDKs do) must still be served.
193    let path = request
194        .url()
195        .split('?')
196        .next()
197        .unwrap_or_default()
198        .to_owned();
199    let method = request.method().clone();
200    tracing::debug!(%method, %path, "request");
201
202    match (&method, path.as_str()) {
203        (Method::Post, PATH_CHAT_COMPLETIONS) => chat_completions(engine, request, config),
204        (Method::Get, PATH_MODELS) => {
205            let body = ModelList::single(engine.model_id(), engine.created());
206            respond_json(request, 200, &body);
207        }
208        (Method::Get, PATH_HEALTH) => respond_json(request, 200, &Health::ok()),
209        _ => respond_error(
210            request,
211            &ServerError::NotFound {
212                method: method.as_str().to_owned(),
213                path,
214            },
215        ),
216    }
217}
218
219/// `POST /v1/chat/completions`, streaming or not.
220fn chat_completions(engine: &mut dyn Engine, mut request: Request, config: &ServeConfig) {
221    let body = match read_body(&mut request, config.max_body_bytes) {
222        Ok(body) => body,
223        Err(e) => return respond_error(request, &e),
224    };
225    let parsed = match ChatCompletionRequest::from_json(&body) {
226        Ok(parsed) => parsed,
227        Err(e) => return respond_error(request, &e),
228    };
229    // Which optional capabilities this client actually asked for, logged
230    // before anything is refused. Kept because the refusals above it are only
231    // defensible against evidence: `stop` and a forced `tool_choice` are 400s
232    // here, and whether that makes the server unusable depends entirely on
233    // what the clients pointed at it send on every request.
234    tracing::debug!(
235        stop = parsed.stop_sequences().len(),
236        tools = parsed.tools().len(),
237        tool_choice = ?parsed.tool_choice,
238        stream = parsed.stream(),
239        "chat request"
240    );
241    // Everything that can be refused is refused here, before a status is
242    // committed: once a stream's head is out, 200 is the answer whatever
243    // happens next.
244    let plan = match engine.prepare(&parsed) {
245        Ok(plan) => plan,
246        Err(e) => return respond_error(request, &e),
247    };
248
249    let id = completion_id();
250    let created = unix_now();
251    if parsed.stream() {
252        stream_into(
253            engine,
254            &parsed,
255            plan,
256            request.into_writer(),
257            &id,
258            created,
259            config.keepalive,
260        );
261    } else {
262        // No head to write early and nothing to keep alive: a non-streaming
263        // client asked for silence until the reply is complete, and at this
264        // runtime's speeds that is very likely to hit its own timeout. That
265        // is the client's choice to make, and `stream: true` is the fix.
266        let mut sink = NullSink;
267        match engine.run(plan, &mut sink) {
268            Ok(completion) => {
269                // The whole reply is in hand, so the `<tool_call>` blocks in it
270                // can be turned into calls before anything is sent. The
271                // streaming path reaches the same outcome by a different route
272                // — [`CallStream`], driven by the marker token ids — because it
273                // has to decide per token, without the text that follows.
274                //
275                // Gated on the request having declared tools. Without that, a
276                // conversation *about* tool calls ("show me what a tool call
277                // looks like") gets its example swallowed: the prose vanishes
278                // into a structured call the client never enabled, and the
279                // reply comes back `finish_reason: "tool_calls"` for a request
280                // that offered none. The model only emits these blocks because
281                // the tools branch of the template told it to, so when no tools
282                // were sent the markers are ordinary text and stay that way.
283                let extracted = if parsed.tools().is_empty() {
284                    crate::toolcall::Extracted::none(&completion.text)
285                } else {
286                    crate::toolcall::extract(&completion.text)
287                };
288                let calls = crate::toolcall::wire_calls(&id, &extracted.calls);
289                // A recovered call overrides the stop reason: the model
290                // stopped because it wanted a tool, whatever token ended it.
291                // Nothing recovered leaves the reason alone, so a block cut off
292                // by the token budget still reports `length`.
293                let finish_reason = if calls.is_empty() {
294                    completion.finish_reason
295                } else {
296                    FinishReason::ToolCalls
297                };
298                let body = ChatCompletion::from_message(
299                    id,
300                    created,
301                    &parsed.model,
302                    ResponseMessage::calling(extracted.content, calls),
303                    finish_reason,
304                    completion.usage,
305                );
306                respond_json(request, 200, &body);
307            }
308            Err(e) => respond_error(request, &e),
309        }
310    }
311}
312
313/// Run `plan` and write an SSE stream into `writer`.
314///
315/// Split out from the `tiny_http` plumbing on purpose: `writer` is the only
316/// thing this needs from the connection, so a test can hand it a recorder that
317/// timestamps every flush and assert what a buffered implementation could
318/// never pass — that the events arrive *when* the engine produced them.
319pub fn stream_into<W: Write + Send + 'static>(
320    engine: &mut dyn Engine,
321    request: &ChatCompletionRequest,
322    plan: Plan,
323    writer: W,
324    id: &str,
325    created: u64,
326    keepalive: Duration,
327) {
328    let state = Arc::new(Mutex::new(StreamState::new(writer)));
329
330    // The head goes out before a single expert is read. A 4K prompt is about
331    // six minutes of prefill and the clients give up after five, so the order
332    // of these two statements is the difference between a working server and
333    // one that is dead on arrival.
334    if let Err(e) = lock(&state).head() {
335        tracing::debug!(?e, "client left before the response head");
336        return;
337    }
338
339    let builder = ChunkBuilder::new(id, created, &request.model);
340    // The role chunk rides out with the head, so a client sees a well-formed
341    // stream begin before the model has done any work at all.
342    if emit(&state, &sse_or_empty(&builder.role())).is_err() {
343        return;
344    }
345
346    let keeper = KeepAlive::spawn(Arc::clone(&state), keepalive);
347    let mut sink = SseSink {
348        state: Arc::clone(&state),
349        builder: builder.clone(),
350        keepalive,
351        // The same gate the buffered path applies: without declared tools the
352        // markers are ordinary text a user asked about, and swallowing them
353        // deletes their answer.
354        calls: CallStream::new(!request.tools().is_empty()),
355    };
356    let include_usage = request.include_usage();
357    let outcome = engine.run(plan, &mut sink);
358    drop(keeper);
359
360    match outcome {
361        Ok(completion) if completion.aborted => {
362            tracing::debug!("client left mid-reply; the stream is abandoned unterminated");
363        }
364        Ok(completion) => {
365            // A block the token budget cut in half is not a call, but it is
366            // text the model wrote, and it has been held back until now.
367            if let Some(unterminated) = sink.calls.flush() {
368                let _ = emit(&state, &sse_or_empty(&builder.content(unterminated)));
369            }
370            // A recovered call overrides the stop reason: the model stopped
371            // because it wanted a tool, whatever token ended it. Nothing
372            // recovered leaves the reason alone, so a block cut off by the
373            // budget still reports `length` — the same rule as the buffered
374            // path, reached from a counter instead of from a list.
375            let finish_reason = if sink.calls.emitted() > 0 {
376                FinishReason::ToolCalls
377            } else {
378                completion.finish_reason
379            };
380            let _ = emit(&state, &sse_or_empty(&builder.finish(finish_reason)));
381            // Usage goes immediately *before* the sentinel: a client stops
382            // reading at `[DONE]`, so usage after it is usage nobody gets.
383            if include_usage {
384                let _ = emit(&state, &sse_or_empty(&builder.usage(completion.usage)));
385            }
386            let _ = emit(&state, &sse_done());
387            let _ = lock(&state).finish();
388        }
389        Err(e) => {
390            // The status is already spent — the head said 200 minutes ago —
391            // so the only honest place left to report this is inside the
392            // stream, in the same body shape a 4xx would have had.
393            tracing::error!(error = %e, "generation failed mid-stream");
394            let _ = emit(&state, &sse_data(&e.body_json()));
395            let _ = emit(&state, &sse_done());
396            let _ = lock(&state).finish();
397        }
398    }
399}
400
401/// A [`TokenSink`] that frames each token as an SSE chunk on the socket.
402struct SseSink<W: Write> {
403    state: Arc<Mutex<StreamState<W>>>,
404    builder: ChunkBuilder,
405    keepalive: Duration,
406    /// Recovers `<tool_call>` blocks as they arrive, so an agent client gets
407    /// structure rather than the markup it would have to parse itself.
408    calls: CallStream,
409}
410
411impl<W: Write> TokenSink for SseSink<W> {
412    fn prefill(&mut self, done: usize, total: usize) -> Result<(), StreamError> {
413        let mut state = lock(&self.state);
414        if let Some(failed) = state.failure() {
415            return Err(failed);
416        }
417        // Progress has no OpenAI chunk shape, so nothing is reported as
418        // progress. What it is good for is a second, cheaper source of
419        // keep-alives: for a 4K prompt these fire roughly every 45 s, which
420        // is under the 300 s wall on its own but far too sparse to be the
421        // only source — hence [`KeepAlive`] as well, and hence the shared
422        // idle clock, so the two do not double up.
423        if state.idle_for() >= self.keepalive {
424            state.emit(KEEPALIVE)?;
425            tracing::trace!(done, total, "keep-alive on prefill progress");
426        }
427        Ok(())
428    }
429
430    fn token(&mut self, id: u32, text: &str) -> Result<(), StreamError> {
431        // The id, not the text, decides: `<tool_call>` and `</tool_call>` are
432        // single added tokens, so the block boundaries are an integer compare
433        // and cannot be split across two deltas.
434        let event = match self.calls.push(id, text) {
435            // Inside a block. Nothing goes out until it closes, which is what
436            // keeps the raw markup off the wire.
437            Step::Nothing => return Ok(()),
438            // A token whose bytes are still an incomplete UTF-8 character
439            // decodes to nothing; sending an empty delta would be a chunk that
440            // says nothing, and the next token carries the character anyway.
441            Step::Content(text) if text.is_empty() => return Ok(()),
442            Step::Content(text) => sse_or_empty(&self.builder.content(text)),
443            Step::Call { index, call } => {
444                tracing::debug!(index, name = %call.name, "recovered a streamed tool call");
445                let wire = wire_call(self.builder.id(), index, &call);
446                sse_or_empty(&self.builder.tool_call(index, wire))
447            }
448        };
449        lock(&self.state).emit(&event)
450    }
451}
452
453/// Serialize one chunk, or the empty string.
454///
455/// [`sse_event`] can only fail if `serde_json` cannot serialize a struct of
456/// `String`s and numbers, which it can. The failure branch is logged rather
457/// than unwrapped so that stays true by construction, and an empty payload is
458/// dropped by the framer rather than sent as a zero-length chunk.
459fn sse_or_empty(chunk: &crate::response::ChatCompletionChunk) -> String {
460    match sse_event(chunk) {
461        Ok(event) => event,
462        Err(e) => {
463            tracing::error!(error = %e, "serializing a chunk failed; dropping it");
464            String::new()
465        }
466    }
467}
468
469/// A thread that writes an SSE comment whenever the stream has gone quiet.
470///
471/// It exists because prefill progress is not a clock. `GenerateProgress`
472/// fires once per prefill chunk — about eight events for a 4K prompt — so the
473/// gap between two of them is tens of seconds and depends on the chunk width,
474/// the cache state and the drive. A timer does not.
475///
476/// It shares the writer through the same mutex the generation thread uses, so
477/// a comment can never interleave with half a chunk, and it reads the shared
478/// idle clock so it stays quiet while content is flowing.
479struct KeepAlive {
480    stop: Arc<AtomicBool>,
481    handle: Option<JoinHandle<()>>,
482}
483
484impl KeepAlive {
485    fn spawn<W: Write + Send + 'static>(
486        state: Arc<Mutex<StreamState<W>>>,
487        interval: Duration,
488    ) -> Self {
489        let stop = Arc::new(AtomicBool::new(false));
490        let flag = Arc::clone(&stop);
491        // Quarter of the interval, so a comment is at worst 25% late, and
492        // bounded either way so a tiny interval does not spin and a huge one
493        // does not delay the join.
494        let tick = (interval / 4).clamp(Duration::from_millis(5), Duration::from_secs(1));
495        let handle = std::thread::Builder::new()
496            .name("rvmp-keepalive".to_owned())
497            .spawn(move || {
498                while !flag.load(Ordering::Relaxed) {
499                    std::thread::sleep(tick);
500                    if flag.load(Ordering::Relaxed) {
501                        return;
502                    }
503                    let mut state = lock(&state);
504                    if state.failure().is_some() {
505                        return;
506                    }
507                    if state.idle_for() >= interval && state.emit(KEEPALIVE).is_err() {
508                        // The generation loop will see the same sticky
509                        // failure at its next token and unwind there, where
510                        // leaving is safe.
511                        return;
512                    }
513                }
514            });
515        match handle {
516            Ok(handle) => KeepAlive {
517                stop,
518                handle: Some(handle),
519            },
520            Err(e) => {
521                // A stream without keep-alives still works for short prompts,
522                // so this is degraded rather than fatal.
523                tracing::warn!(error = %e, "no keep-alive thread; long prefills may time out");
524                KeepAlive { stop, handle: None }
525            }
526        }
527    }
528}
529
530impl Drop for KeepAlive {
531    fn drop(&mut self) {
532        self.stop.store(true, Ordering::Relaxed);
533        if let Some(handle) = self.handle.take() {
534            // Joined, not detached: the thread holds a writer clone, and the
535            // terminating chunk must not race a comment.
536            let _ = handle.join();
537        }
538    }
539}
540
541/// Lock the shared stream state, ignoring poisoning.
542///
543/// A poisoned mutex here means a panic while a write was in progress. The
544/// state behind it is a writer and two flags — there is no invariant a panic
545/// could have broken halfway — and refusing to write would turn one failed
546/// request into a hung one.
547fn lock<W: Write>(state: &Mutex<StreamState<W>>) -> MutexGuard<'_, StreamState<W>> {
548    state.lock().unwrap_or_else(PoisonError::into_inner)
549}
550
551fn emit<W: Write>(state: &Mutex<StreamState<W>>, payload: &str) -> Result<(), StreamError> {
552    lock(state).emit(payload)
553}
554
555/// Read a request body, refusing anything over `limit`.
556///
557/// Checked twice: the declared `Content-Length` first, so an oversized body is
558/// refused without reading it, and the actual byte count second, because
559/// `Content-Length` is a claim by the client and a chunked request does not
560/// make one at all.
561fn read_body(request: &mut Request, limit: usize) -> Result<String, ServerError> {
562    if request.body_length().is_some_and(|len| len > limit) {
563        return Err(ServerError::BodyTooLarge { limit });
564    }
565    let mut body = Vec::new();
566    // `limit + 1`, so "exactly at the limit" and "over it" are
567    // distinguishable without reading the whole thing.
568    let capped = u64::try_from(limit).unwrap_or(u64::MAX).saturating_add(1);
569    request
570        .as_reader()
571        .take(capped)
572        .read_to_end(&mut body)
573        .map_err(|e| ServerError::MalformedJson(format!("reading the request body: {e}")))?;
574    if body.len() > limit {
575        return Err(ServerError::BodyTooLarge { limit });
576    }
577    String::from_utf8(body)
578        .map_err(|_| ServerError::MalformedJson("request body is not valid UTF-8".to_owned()))
579}
580
581/// A completion id: `chatcmpl-` plus something that does not repeat.
582///
583/// Not random, because a random source is a dependency and this is an
584/// identifier, not a secret. The clock plus a counter cannot collide within a
585/// process and cannot collide across two processes unless they start in the
586/// same nanosecond.
587fn completion_id() -> String {
588    static COUNTER: AtomicU64 = AtomicU64::new(0);
589    let n = COUNTER.fetch_add(1, Ordering::Relaxed);
590    format!("{COMPLETION_ID_PREFIX}{:x}{:04x}", unix_now(), n & 0xffff)
591}
592
593/// Build a header from ASCII.
594///
595/// `Header::from_bytes` fails only on non-ASCII, and every call site here
596/// passes a literal, so the failure branch is unreachable — reported rather
597/// than unwrapped so it stays that way by construction.
598fn header(name: &str, value: &str) -> Option<Header> {
599    match Header::from_bytes(name.as_bytes(), value.as_bytes()) {
600        Ok(header) => Some(header),
601        Err(()) => {
602            tracing::error!(name, value, "non-ASCII header; dropped");
603            None
604        }
605    }
606}
607
608/// Answer with a buffered JSON body.
609///
610/// The buffered path is correct for everything that is not a stream: it is one
611/// short body, written once, and `tiny_http`'s own encoder handles the
612/// framing.
613fn respond_json<T: serde::Serialize>(request: Request, status: u16, body: &T) {
614    match serde_json::to_string(body) {
615        Ok(json) => send(request, status, json, &[]),
616        Err(e) => {
617            let error = ServerError::Serialize(e);
618            respond_error(request, &error);
619        }
620    }
621}
622
623/// Answer with an error body, its status and its headers.
624fn respond_error(request: Request, error: &ServerError) {
625    tracing::debug!(status = error.status(), error = %error, "refusing a request");
626    send(request, error.status(), error.body_json(), error.headers());
627}
628
629fn send(request: Request, status: u16, body: String, extra: &[(&str, &str)]) {
630    let mut response = Response::from_string(body).with_status_code(status);
631    if let Some(header) = header("Content-Type", "application/json") {
632        response = response.with_header(header);
633    }
634    for (name, value) in extra {
635        if let Some(header) = header(name, value) {
636            response = response.with_header(header);
637        }
638    }
639    if let Err(e) = request.respond(response) {
640        // `respond` maps the client-gone kinds to `Ok`, so anything reported
641        // here is a real write failure.
642        tracing::debug!(error = %e, "failed to send a response");
643    }
644}
645
646#[cfg(test)]
647mod tests {
648    use super::*;
649    use crate::engine::Completion;
650    use crate::response::{FinishReason, Usage};
651    use crate::wire::{SSE_HEAD, TERMINATOR};
652    use std::sync::atomic::AtomicUsize;
653    use std::time::Instant;
654
655    /// A token source with no model in it: known tokens, at known intervals.
656    ///
657    /// This is what makes the streaming contract testable at all. A buffered
658    /// stream and a live one produce byte-identical transcripts and differ
659    /// only in *when* the bytes appear, so the assertion has to be about time,
660    /// and an assertion about time needs a source whose timing is known.
661    struct StubEngine {
662        prefill_steps: usize,
663        prefill_delay: Duration,
664        /// Each token as (id, text), because the streaming tool-call lane keys
665        /// off the id and a source that could only produce text could not
666        /// exercise it.
667        tokens: Vec<(u32, String)>,
668        token_delay: Duration,
669        /// Why the stub says it stopped. The tool-call lane overrides this, so
670        /// what it overrides has to be settable.
671        finish_reason: FinishReason,
672        /// How many tokens the engine got as far as producing. The evidence
673        /// that a hung-up client actually stops generation.
674        produced: Arc<AtomicUsize>,
675    }
676
677    /// The id of an ordinary token: anything that is not a marker.
678    const TEXT_ID: u32 = 7;
679
680    impl StubEngine {
681        fn new(tokens: &[&str]) -> Self {
682            StubEngine::with_ids(&tokens.iter().map(|t| (TEXT_ID, *t)).collect::<Vec<_>>())
683        }
684
685        fn with_ids(tokens: &[(u32, &str)]) -> Self {
686            StubEngine {
687                prefill_steps: 0,
688                prefill_delay: Duration::ZERO,
689                tokens: tokens
690                    .iter()
691                    .map(|(id, text)| (*id, (*text).to_owned()))
692                    .collect(),
693                token_delay: Duration::ZERO,
694                finish_reason: FinishReason::Stop,
695                produced: Arc::new(AtomicUsize::new(0)),
696            }
697        }
698    }
699
700    impl Engine for StubEngine {
701        fn model_id(&self) -> &str {
702            "stub-model"
703        }
704
705        fn created(&self) -> u64 {
706            1_700_000_000
707        }
708
709        fn prepare(&mut self, request: &ChatCompletionRequest) -> Result<Plan, ServerError> {
710            request.validate()?;
711            Ok(Plan {
712                prompt_tokens: 11,
713                max_new: self.tokens.len(),
714                cached_prefix: 0,
715                prompt_ids: vec![1, 2, 3],
716                params: ramvamp_core::generate::GenerateParams {
717                    max_new: self.tokens.len(),
718                    temperature: 0.7,
719                    top_k: Some(20),
720                    top_p: 0.8,
721                    seed: 42,
722                    greedy: false,
723                },
724            })
725        }
726
727        fn run(&mut self, plan: Plan, sink: &mut dyn TokenSink) -> Result<Completion, ServerError> {
728            let mut text = String::new();
729            for step in 1..=self.prefill_steps {
730                std::thread::sleep(self.prefill_delay);
731                if sink.prefill(step, self.prefill_steps).is_err() {
732                    return Ok(aborted(text, plan.prompt_tokens, 0));
733                }
734            }
735            for (index, (id, token)) in self.tokens.clone().iter().enumerate() {
736                std::thread::sleep(self.token_delay);
737                self.produced.store(index + 1, Ordering::Relaxed);
738                text.push_str(token);
739                if sink.token(*id, token).is_err() {
740                    return Ok(aborted(text, plan.prompt_tokens, index + 1));
741                }
742            }
743            let generated = self.tokens.len();
744            Ok(Completion {
745                text,
746                usage: Usage::new(plan.prompt_tokens as u32, generated as u32),
747                finish_reason: self.finish_reason,
748                aborted: false,
749            })
750        }
751    }
752
753    fn aborted(text: String, prompt: usize, generated: usize) -> Completion {
754        Completion {
755            text,
756            usage: Usage::new(prompt as u32, generated as u32),
757            finish_reason: FinishReason::Stop,
758            aborted: true,
759        }
760    }
761
762    /// One flushed group: when it went out, and what it was.
763    type Flushed = Arc<Mutex<Vec<(Duration, Vec<u8>)>>>;
764
765    /// A writer that timestamps every flush and can be told to die.
766    #[derive(Clone)]
767    struct Recorder {
768        events: Flushed,
769        pending: Arc<Mutex<Vec<u8>>>,
770        flushes: Arc<AtomicUsize>,
771        start: Instant,
772        die_after_flushes: Option<usize>,
773    }
774
775    impl Recorder {
776        fn new() -> Self {
777            Recorder {
778                events: Arc::new(Mutex::new(Vec::new())),
779                pending: Arc::new(Mutex::new(Vec::new())),
780                flushes: Arc::new(AtomicUsize::new(0)),
781                start: Instant::now(),
782                die_after_flushes: None,
783            }
784        }
785
786        fn dying_after(mut self, flushes: usize) -> Self {
787            self.die_after_flushes = Some(flushes);
788            self
789        }
790
791        /// Every flushed group, as (elapsed, text).
792        fn events(&self) -> Vec<(Duration, String)> {
793            self.events
794                .lock()
795                .unwrap_or_else(PoisonError::into_inner)
796                .iter()
797                .map(|(at, bytes)| (*at, String::from_utf8_lossy(bytes).into_owned()))
798                .collect()
799        }
800
801        fn body(&self) -> String {
802            self.events().into_iter().map(|(_, text)| text).collect()
803        }
804    }
805
806    impl Write for Recorder {
807        fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
808            self.pending
809                .lock()
810                .unwrap_or_else(PoisonError::into_inner)
811                .extend_from_slice(buf);
812            Ok(buf.len())
813        }
814
815        fn flush(&mut self) -> std::io::Result<()> {
816            let bytes =
817                std::mem::take(&mut *self.pending.lock().unwrap_or_else(PoisonError::into_inner));
818            let n = self.flushes.fetch_add(1, Ordering::Relaxed) + 1;
819            if self.die_after_flushes.is_some_and(|limit| n > limit) {
820                return Err(std::io::Error::from(std::io::ErrorKind::BrokenPipe));
821            }
822            self.events
823                .lock()
824                .unwrap_or_else(PoisonError::into_inner)
825                .push((self.start.elapsed(), bytes));
826            Ok(())
827        }
828    }
829
830    fn request(stream: bool) -> ChatCompletionRequest {
831        let body = format!(
832            r#"{{"model":"stub-model","messages":[{{"role":"user","content":"hi"}}],"stream":{stream}}}"#
833        );
834        ChatCompletionRequest::from_json(&body).expect("valid body")
835    }
836
837    /// The same body with a `glob` tool declared, which is the gate the
838    /// streaming tool-call lane is behind.
839    fn request_with_tools() -> ChatCompletionRequest {
840        ChatCompletionRequest::from_json(
841            r#"{"model":"stub-model","messages":[{"role":"user","content":"list the rust files"}],
842                "stream":true,
843                "tools":[{"type":"function","function":{"name":"glob","parameters":{}}}]}"#,
844        )
845        .expect("valid body")
846    }
847
848    fn run_stream(engine: &mut StubEngine, recorder: Recorder, keepalive: Duration) {
849        run_stream_for(engine, &request(true), recorder, keepalive);
850    }
851
852    fn run_stream_for(
853        engine: &mut StubEngine,
854        request: &ChatCompletionRequest,
855        recorder: Recorder,
856        keepalive: Duration,
857    ) {
858        let plan = engine.prepare(request).expect("a plan");
859        stream_into(
860            engine,
861            request,
862            plan,
863            recorder,
864            "chatcmpl-test",
865            1_700_000_000,
866            keepalive,
867        );
868    }
869
870    /// The `data:` payloads of a recorded stream, parsed, with the sentinel
871    /// dropped. What a client actually gets.
872    fn chunks(body: &str) -> Vec<serde_json::Value> {
873        body.split("data: ")
874            .skip(1)
875            .map(|event| event.split("\n\n").next().unwrap_or_default())
876            .filter(|payload| *payload != crate::sse::DONE_SENTINEL)
877            .map(|payload| {
878                serde_json::from_str(payload)
879                    .unwrap_or_else(|e| panic!("a chunk payload: {e}\n{payload}"))
880            })
881            .collect()
882    }
883
884    /// The `tool_calls` entries of a recorded stream, flattened, in order.
885    fn streamed_calls(body: &str) -> Vec<serde_json::Value> {
886        chunks(body)
887            .into_iter()
888            .filter_map(|chunk| {
889                chunk["choices"][0]["delta"]["tool_calls"]
890                    .as_array()
891                    .cloned()
892            })
893            .flatten()
894            .collect()
895    }
896
897    /// The content deltas of a recorded stream, concatenated.
898    fn streamed_text(body: &str) -> String {
899        chunks(body)
900            .iter()
901            .filter_map(|chunk| chunk["choices"][0]["delta"]["content"].as_str())
902            .collect()
903    }
904
905    /// The reason the stream ended, from its finish chunk.
906    fn streamed_finish(body: &str) -> String {
907        chunks(body)
908            .iter()
909            .find_map(|chunk| {
910                chunk["choices"][0]["finish_reason"]
911                    .as_str()
912                    .map(str::to_owned)
913            })
914            .expect("a finish chunk")
915    }
916
917    /// The block a model writes when it wants `glob`, as the tokens it writes
918    /// it in: the markers are single added tokens, the body is not.
919    fn glob_call_tokens() -> Vec<(u32, &'static str)> {
920        vec![
921            (crate::toolcall::OPEN_TAG_ID, "<tool_call>"),
922            (TEXT_ID, "\n{\"name\": \"glob\", "),
923            (TEXT_ID, "\"arguments\": {\"pattern\": \"*/.rs\"}}\n"),
924            (crate::toolcall::CLOSE_TAG_ID, "</tool_call>"),
925        ]
926    }
927
928    /// The assertion a buffered implementation cannot pass: the content
929    /// events are separated on the wire by the interval the source produced
930    /// them at, rather than arriving together at the end.
931    #[test]
932    fn events_reach_the_client_at_the_sources_pace() {
933        let mut engine = StubEngine::new(&["a", "b", "c", "d", "e"]);
934        engine.token_delay = Duration::from_millis(60);
935        let recorder = Recorder::new();
936        run_stream(&mut engine, recorder.clone(), Duration::from_secs(60));
937
938        let content: Vec<Duration> = recorder
939            .events()
940            .into_iter()
941            .filter(|(_, text)| text.contains(r#""content":"#) && !text.contains(r#""role""#))
942            .map(|(at, _)| at)
943            .collect();
944        assert_eq!(content.len(), 5, "{:?}", recorder.events());
945        for pair in content.windows(2) {
946            assert!(
947                pair[1] - pair[0] >= Duration::from_millis(40),
948                "events arrived in a burst: {content:?}"
949            );
950        }
951        assert!(
952            content[4] - content[0] >= Duration::from_millis(200),
953            "{content:?}"
954        );
955    }
956
957    /// The head is committed before the engine does any work, which is what
958    /// keeps a six-minute prefill under a five-minute client timeout.
959    #[test]
960    fn the_head_is_written_before_the_engine_runs() {
961        let mut engine = StubEngine::new(&["x"]);
962        engine.prefill_steps = 1;
963        engine.prefill_delay = Duration::from_millis(250);
964        let recorder = Recorder::new();
965        run_stream(&mut engine, recorder.clone(), Duration::from_secs(60));
966
967        let events = recorder.events();
968        assert_eq!(events[0].1, SSE_HEAD);
969        assert!(
970            events[0].0 < Duration::from_millis(100),
971            "the head waited for the model: {:?}",
972            events[0].0
973        );
974        // And the role chunk rides out with it.
975        assert!(events[1].1.contains(r#""role":"assistant""#));
976        assert!(events[1].0 < Duration::from_millis(100));
977    }
978
979    /// A prompt that takes minutes to prefill produces no content events, so
980    /// something else has to keep the byte stream alive.
981    #[test]
982    fn a_slow_prefill_emits_keep_alive_comments() {
983        let mut engine = StubEngine::new(&["done"]);
984        engine.prefill_steps = 1;
985        engine.prefill_delay = Duration::from_millis(400);
986        let recorder = Recorder::new();
987        run_stream(&mut engine, recorder.clone(), Duration::from_millis(50));
988
989        let comments: Vec<Duration> = recorder
990            .events()
991            .into_iter()
992            .filter(|(_, text)| text.contains(KEEPALIVE))
993            .map(|(at, _)| at)
994            .collect();
995        assert!(
996            comments.len() >= 3,
997            "expected keep-alives during a 400ms prefill, got {comments:?}"
998        );
999        // They are comments, so a client discards them: the reply is intact.
1000        assert!(recorder.body().contains(r#""content":"done""#));
1001    }
1002
1003    /// A client that hangs up surfaces as a write error, and the engine stops
1004    /// generating instead of running for minutes with nobody listening.
1005    #[test]
1006    fn a_dropped_client_surfaces_as_a_write_error_and_aborts_generation() {
1007        let mut engine = StubEngine::new(&["a", "b", "c", "d", "e", "f"]);
1008        let produced = Arc::clone(&engine.produced);
1009        // head, role, then one content event; the next flush is a broken pipe.
1010        let recorder = Recorder::new().dying_after(3);
1011        run_stream(&mut engine, recorder.clone(), Duration::from_secs(60));
1012
1013        let got = produced.load(Ordering::Relaxed);
1014        assert!(
1015            (1..=2).contains(&got),
1016            "generation should have stopped at the hangup, produced {got}"
1017        );
1018        // Nothing is appended after the client is gone: no sentinel, no
1019        // terminator, because there is nowhere to put them.
1020        assert!(!recorder.body().contains("[DONE]"), "{}", recorder.body());
1021    }
1022
1023    /// The whole wire sequence: head, framing, order, sentinel, terminator.
1024    #[test]
1025    fn the_sentinel_precedes_the_zero_length_chunk() {
1026        let mut engine = StubEngine::new(&["he", "llo"]);
1027        let recorder = Recorder::new();
1028        run_stream(&mut engine, recorder.clone(), Duration::from_secs(60));
1029
1030        let body = recorder.body();
1031        let rest = body.strip_prefix(SSE_HEAD).expect("head first");
1032        assert!(rest.ends_with(TERMINATOR), "{rest:?}");
1033        let done = rest.find("data: [DONE]").expect("sentinel");
1034        let end = rest.rfind(TERMINATOR).expect("terminator");
1035        assert!(done < end, "the sentinel must precede the terminator");
1036    }
1037
1038    /// Every chunk's declared length is the byte length of its payload, and
1039    /// the reassembled deltas are the reply. This is the framing check a
1040    /// client actually performs.
1041    #[test]
1042    fn the_chunk_framing_decodes_back_to_the_reply() {
1043        let mut engine = StubEngine::new(&["Hé", "llo", "→!"]);
1044        let recorder = Recorder::new();
1045        run_stream(&mut engine, recorder.clone(), Duration::from_secs(60));
1046
1047        let body = recorder.body();
1048        let mut rest = body.strip_prefix(SSE_HEAD).expect("head first");
1049        let mut payloads = Vec::new();
1050        loop {
1051            let (head, tail) = rest.split_once("\r\n").expect("a chunk header");
1052            let len = usize::from_str_radix(head, 16).expect("hex length");
1053            if len == 0 {
1054                assert_eq!(tail, "\r\n", "the terminator ends the body");
1055                break;
1056            }
1057            assert!(tail.len() >= len + 2, "chunk shorter than its header says");
1058            payloads.push(tail[..len].to_owned());
1059            assert_eq!(&tail[len..len + 2], "\r\n", "chunk not CRLF-terminated");
1060            rest = &tail[len + 2..];
1061        }
1062
1063        let text: String = payloads
1064            .iter()
1065            .filter_map(|event| event.strip_prefix("data: "))
1066            .filter_map(|json| serde_json::from_str::<serde_json::Value>(json.trim_end()).ok())
1067            .filter_map(|chunk| {
1068                chunk["choices"][0]["delta"]["content"]
1069                    .as_str()
1070                    .map(str::to_owned)
1071            })
1072            .collect();
1073        assert_eq!(text, "Héllo→!");
1074    }
1075
1076    /// `include_usage` puts the usage chunk between the finish chunk and the
1077    /// sentinel, and leaves it out entirely when it was not asked for.
1078    #[test]
1079    fn usage_is_emitted_only_when_asked_for_and_before_the_sentinel() {
1080        let mut engine = StubEngine::new(&["x"]);
1081        let recorder = Recorder::new();
1082        run_stream(&mut engine, recorder.clone(), Duration::from_secs(60));
1083        assert!(!recorder.body().contains(r#""usage""#));
1084
1085        let asked = ChatCompletionRequest::from_json(
1086            r#"{"model":"stub-model","messages":[{"role":"user","content":"hi"}],
1087                "stream":true,"stream_options":{"include_usage":true}}"#,
1088        )
1089        .expect("valid body");
1090        let plan = engine.prepare(&asked).expect("a plan");
1091        let recorder = Recorder::new();
1092        stream_into(
1093            &mut engine,
1094            &asked,
1095            plan,
1096            recorder.clone(),
1097            "chatcmpl-test",
1098            1_700_000_000,
1099            Duration::from_secs(60),
1100        );
1101        let body = recorder.body();
1102        let usage = body.find(r#""usage""#).expect("a usage chunk");
1103        let finish = body.find(r#""finish_reason":"stop""#).expect("a finish");
1104        let done = body.find("[DONE]").expect("the sentinel");
1105        assert!(finish < usage && usage < done, "{body}");
1106        assert!(body.contains(r#""prompt_tokens":11"#), "{body}");
1107    }
1108
1109    /// A runtime failure after the head is out cannot be a status code, so it
1110    /// is an error event inside the stream followed by a clean terminator.
1111    #[test]
1112    fn a_mid_stream_failure_becomes_an_error_event() {
1113        struct Failing;
1114        impl Engine for Failing {
1115            fn model_id(&self) -> &str {
1116                "stub-model"
1117            }
1118            fn created(&self) -> u64 {
1119                0
1120            }
1121            fn prepare(&mut self, _: &ChatCompletionRequest) -> Result<Plan, ServerError> {
1122                unreachable!("the test builds the plan itself")
1123            }
1124            fn run(&mut self, _: Plan, _: &mut dyn TokenSink) -> Result<Completion, ServerError> {
1125                Err(ServerError::Generate(
1126                    ramvamp_core::generate::GenerateError::EmptyPrompt,
1127                ))
1128            }
1129        }
1130        let mut stub = StubEngine::new(&[]);
1131        let request = request(true);
1132        let plan = stub.prepare(&request).expect("a plan");
1133        let recorder = Recorder::new();
1134        stream_into(
1135            &mut Failing,
1136            &request,
1137            plan,
1138            recorder.clone(),
1139            "chatcmpl-test",
1140            0,
1141            Duration::from_secs(60),
1142        );
1143        let body = recorder.body();
1144        assert!(body.starts_with(SSE_HEAD));
1145        assert!(body.contains(r#""code":"generate_error""#), "{body}");
1146        assert!(body.contains("[DONE]"));
1147        assert!(body.ends_with(TERMINATOR));
1148    }
1149
1150    // ---- the buffered endpoints, over a real socket ----
1151
1152    fn spawn_server() -> SocketAddr {
1153        spawn_server_with(&["ok"], FinishReason::Stop)
1154    }
1155
1156    fn spawn_server_with(tokens: &[&str], finish_reason: FinishReason) -> SocketAddr {
1157        let tokens: Vec<String> = tokens.iter().map(|t| (*t).to_owned()).collect();
1158        let (tx, rx) = std::sync::mpsc::channel();
1159        std::thread::spawn(move || {
1160            let borrowed: Vec<&str> = tokens.iter().map(String::as_str).collect();
1161            let mut engine = StubEngine::new(&borrowed);
1162            engine.finish_reason = finish_reason;
1163            let _ = serve_with(
1164                &mut engine,
1165                ServeConfig {
1166                    port: 0,
1167                    ..ServeConfig::default()
1168                },
1169                |address| {
1170                    let _ = tx.send(address);
1171                },
1172            );
1173        });
1174        rx.recv_timeout(Duration::from_secs(10))
1175            .expect("the server binds")
1176    }
1177
1178    fn roundtrip(address: SocketAddr, raw: &str) -> String {
1179        let mut stream = std::net::TcpStream::connect(address).expect("connect");
1180        stream
1181            .set_read_timeout(Some(Duration::from_secs(10)))
1182            .expect("timeout");
1183        stream.write_all(raw.as_bytes()).expect("write");
1184        stream.flush().expect("flush");
1185        let mut out = Vec::new();
1186        let mut buffer = [0u8; 4096];
1187        loop {
1188            match stream.read(&mut buffer) {
1189                Ok(0) | Err(_) => break,
1190                Ok(n) => out.extend_from_slice(&buffer[..n]),
1191            }
1192        }
1193        String::from_utf8_lossy(&out).into_owned()
1194    }
1195
1196    fn get(address: SocketAddr, path: &str) -> String {
1197        roundtrip(
1198            address,
1199            &format!("GET {path} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"),
1200        )
1201    }
1202
1203    fn post(address: SocketAddr, path: &str, body: &str) -> String {
1204        roundtrip(
1205            address,
1206            &format!(
1207                "POST {path} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\
1208                 Content-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}",
1209                body.len()
1210            ),
1211        )
1212    }
1213
1214    #[test]
1215    fn the_buffered_routes_answer_over_a_socket() {
1216        let address = spawn_server();
1217        // Loopback only, and the bound address proves it.
1218        assert!(address.ip().is_loopback(), "{address}");
1219
1220        let health = get(address, PATH_HEALTH);
1221        assert!(health.starts_with("HTTP/1.1 200"), "{health}");
1222        assert!(health.contains(r#"{"status":"ok"}"#), "{health}");
1223
1224        let models = get(address, PATH_MODELS);
1225        assert!(models.starts_with("HTTP/1.1 200"), "{models}");
1226        assert!(models.contains(r#""id":"stub-model""#), "{models}");
1227        assert!(models.contains(r#""object":"list""#), "{models}");
1228
1229        // A query string is not part of the route.
1230        let versioned = get(address, "/v1/models?api-version=2024-02-01");
1231        assert!(versioned.contains(r#""id":"stub-model""#), "{versioned}");
1232
1233        let missing = get(address, "/v1/embeddings");
1234        assert!(missing.starts_with("HTTP/1.1 404"), "{missing}");
1235        assert!(
1236            missing.contains(r#""type":"invalid_request_error""#),
1237            "{missing}"
1238        );
1239        assert!(missing.contains(r#""code":"not_found""#), "{missing}");
1240
1241        // Right path, wrong method: still a 404 with a parseable body, never
1242        // an empty one.
1243        let wrong = get(address, PATH_CHAT_COMPLETIONS);
1244        assert!(wrong.starts_with("HTTP/1.1 404"), "{wrong}");
1245        assert!(wrong.contains(r#""error""#), "{wrong}");
1246    }
1247
1248    #[test]
1249    fn a_non_streaming_completion_comes_back_as_one_json_body() {
1250        let address = spawn_server();
1251        let response = post(
1252            address,
1253            PATH_CHAT_COMPLETIONS,
1254            r#"{"model":"stub-model","messages":[{"role":"user","content":"hi"}]}"#,
1255        );
1256        assert!(response.starts_with("HTTP/1.1 200"), "{response}");
1257        assert!(
1258            response.contains(r#""object":"chat.completion""#),
1259            "{response}"
1260        );
1261        assert!(response.contains(r#""content":"ok""#), "{response}");
1262        assert!(response.contains(r#""finish_reason":"stop""#), "{response}");
1263    }
1264
1265    /// What the model actually writes when it wants a tool: prose, then the
1266    /// block the template asked for.
1267    const TOOL_CALL_REPLY: &str = "Let me look.\n<tool_call>\n{\"name\": \"read\", \"arguments\": {\"path\": \"a.txt\"}}\n</tool_call>";
1268
1269    /// The JSON body of an HTTP response, headers stripped.
1270    fn body_json(response: &str) -> serde_json::Value {
1271        let body = response
1272            .split_once("\r\n\r\n")
1273            .map_or(response, |(_, body)| body);
1274        serde_json::from_str(body).unwrap_or_else(|e| panic!("a JSON body: {e}\n{response}"))
1275    }
1276
1277    /// The end of this wave: a client can act on the reply. The text the model
1278    /// wrote is gone from `content` and present as structure, with an id the
1279    /// client can put on its `tool` message.
1280    #[test]
1281    fn a_tool_call_reply_comes_back_as_structured_calls() {
1282        let address = spawn_server_with(&[TOOL_CALL_REPLY], FinishReason::Stop);
1283        let response = post(
1284            address,
1285            PATH_CHAT_COMPLETIONS,
1286            r#"{"model":"stub-model","messages":[{"role":"user","content":"read a.txt"}],
1287                "tools":[{"type":"function","function":{"name":"read","parameters":{}}}]}"#,
1288        );
1289        assert!(response.starts_with("HTTP/1.1 200"), "{response}");
1290        let json = body_json(&response);
1291        let choice = &json["choices"][0];
1292
1293        // The stop reason the engine reported is overridden: the model stopped
1294        // because it wants a tool run.
1295        assert_eq!(choice["finish_reason"], "tool_calls");
1296
1297        let calls = choice["message"]["tool_calls"]
1298            .as_array()
1299            .unwrap_or_else(|| panic!("a tool_calls array: {json}"));
1300        assert_eq!(calls.len(), 1);
1301        assert_eq!(calls[0]["type"], "function");
1302        assert_eq!(calls[0]["function"]["name"], "read");
1303        // `arguments` is a JSON *string*, byte for byte what the model wrote.
1304        assert_eq!(calls[0]["function"]["arguments"], r#"{"path": "a.txt"}"#);
1305        let call_id = calls[0]["id"].as_str().expect("an id");
1306        assert!(call_id.starts_with("call_"), "{call_id}");
1307        assert!(call_id.ends_with("_0"), "{call_id}");
1308        assert!(
1309            call_id.contains(
1310                json["id"]
1311                    .as_str()
1312                    .and_then(|id| id.strip_prefix("chatcmpl-"))
1313                    .expect("a completion id")
1314            ),
1315            "the call id is derived from the completion id: {call_id}"
1316        );
1317
1318        // And the prose survives without the markup.
1319        assert_eq!(choice["message"]["content"], "Let me look.");
1320        assert!(
1321            !response.contains("<tool_call>"),
1322            "the raw block is still in the body: {response}"
1323        );
1324    }
1325
1326    /// The same reply, from a request that declared no tools, stays text.
1327    ///
1328    /// A conversation *about* tool calling is the case this protects: asked
1329    /// what a call looks like, the model writes one, and extracting it would
1330    /// delete the answer from `content` and hand back a call the caller never
1331    /// enabled, on a request that offered no tools to call.
1332    #[test]
1333    fn a_block_in_a_reply_to_a_request_without_tools_stays_text() {
1334        let address = spawn_server_with(&[TOOL_CALL_REPLY], FinishReason::Stop);
1335        let response = post(
1336            address,
1337            PATH_CHAT_COMPLETIONS,
1338            r#"{"model":"stub-model","messages":[{"role":"user","content":"show me a tool call"}]}"#,
1339        );
1340        assert!(response.starts_with("HTTP/1.1 200"), "{response}");
1341        let json = body_json(&response);
1342        let choice = &json["choices"][0];
1343
1344        assert_eq!(choice["finish_reason"], "stop");
1345        assert!(
1346            choice["message"]["tool_calls"].is_null(),
1347            "a request without tools got tool_calls back: {json}"
1348        );
1349        // Byte for byte, markers and all.
1350        assert_eq!(
1351            choice["message"]["content"].as_str().expect("content"),
1352            TOOL_CALL_REPLY
1353        );
1354    }
1355
1356    /// A reply with no block is untouched, and the key stays absent rather
1357    /// than becoming an empty array a client would read as "tools wanted".
1358    #[test]
1359    fn a_plain_reply_carries_no_tool_calls_key() {
1360        let address = spawn_server();
1361        let response = post(
1362            address,
1363            PATH_CHAT_COMPLETIONS,
1364            r#"{"model":"stub-model","messages":[{"role":"user","content":"hi"}]}"#,
1365        );
1366        let json = body_json(&response);
1367        assert_eq!(json["choices"][0]["message"]["content"], "ok");
1368        assert_eq!(json["choices"][0]["finish_reason"], "stop");
1369        assert!(
1370            json["choices"][0]["message"].get("tool_calls").is_none(),
1371            "{json}"
1372        );
1373    }
1374
1375    /// A block cut off by the token budget yields no call, so the reason stays
1376    /// `length` — the client's cue to raise `max_tokens`, not to run something
1377    /// that was never fully asked for.
1378    #[test]
1379    fn a_block_truncated_by_the_token_budget_keeps_the_length_reason() {
1380        let address = spawn_server_with(
1381            &["<tool_call>\n{\"name\": \"read\", \"argum"],
1382            FinishReason::Length,
1383        );
1384        let response = post(
1385            address,
1386            PATH_CHAT_COMPLETIONS,
1387            r#"{"model":"stub-model","messages":[{"role":"user","content":"hi"}],"max_tokens":8}"#,
1388        );
1389        let json = body_json(&response);
1390        assert_eq!(json["choices"][0]["finish_reason"], "length");
1391        assert!(
1392            json["choices"][0]["message"].get("tool_calls").is_none(),
1393            "{json}"
1394        );
1395        assert_eq!(
1396            json["choices"][0]["message"]["content"], "<tool_call>\n{\"name\": \"read\", \"argum",
1397            "half a block is text, and text is not dropped"
1398        );
1399    }
1400
1401    // ---- streaming tool calls ----
1402
1403    /// The bug this lane exists to fix, in one assertion: OpenCode streams by
1404    /// default, and every agentic turn used to render the raw block as
1405    /// assistant prose. Now it is structure, and the markup is gone.
1406    #[test]
1407    fn a_streamed_call_becomes_a_tool_calls_delta_and_no_raw_markup() {
1408        let mut engine = StubEngine::with_ids(&glob_call_tokens());
1409        let recorder = Recorder::new();
1410        run_stream_for(
1411            &mut engine,
1412            &request_with_tools(),
1413            recorder.clone(),
1414            Duration::from_secs(60),
1415        );
1416
1417        let body = recorder.body();
1418        assert!(
1419            !body.contains("tool_call>"),
1420            "the raw markup is still on the wire: {body}"
1421        );
1422
1423        let calls = streamed_calls(&body);
1424        assert_eq!(calls.len(), 1, "{body}");
1425        assert_eq!(calls[0]["index"], 0);
1426        assert_eq!(calls[0]["type"], "function");
1427        assert_eq!(calls[0]["function"]["name"], "glob");
1428        assert_eq!(
1429            calls[0]["function"]["arguments"], r#"{"pattern": "*/.rs"}"#,
1430            "the model's argument bytes reach the client unaltered"
1431        );
1432        // The stop reason is overridden: the model stopped to call a tool.
1433        assert_eq!(streamed_finish(&body), "tool_calls");
1434        assert_eq!(streamed_text(&body), "", "the block left no prose behind");
1435        assert!(body.contains("[DONE]"), "{body}");
1436        assert!(body.ends_with(TERMINATOR), "{body}");
1437    }
1438
1439    /// Prose is still prose. Only the blocks are withheld, and only for as long
1440    /// as they are open.
1441    #[test]
1442    fn prose_before_between_and_after_streamed_calls_is_still_content() {
1443        let mut tokens = vec![(TEXT_ID, "Let me "), (TEXT_ID, "look.\n")];
1444        tokens.extend(glob_call_tokens());
1445        tokens.push((TEXT_ID, "\nand the tests\n"));
1446        tokens.extend(glob_call_tokens());
1447        tokens.push((TEXT_ID, "\nThen I will compare."));
1448
1449        let mut engine = StubEngine::with_ids(&tokens);
1450        let recorder = Recorder::new();
1451        run_stream_for(
1452            &mut engine,
1453            &request_with_tools(),
1454            recorder.clone(),
1455            Duration::from_secs(60),
1456        );
1457
1458        let body = recorder.body();
1459        assert_eq!(
1460            streamed_text(&body),
1461            "Let me look.\n\nand the tests\n\nThen I will compare."
1462        );
1463        assert!(!body.contains("tool_call>"), "{body}");
1464    }
1465
1466    /// Two calls in one turn are told apart by `index` and by nothing else.
1467    #[test]
1468    fn two_streamed_calls_get_indices_zero_and_one() {
1469        let mut tokens = glob_call_tokens();
1470        tokens.extend(glob_call_tokens());
1471        let mut engine = StubEngine::with_ids(&tokens);
1472        let recorder = Recorder::new();
1473        run_stream_for(
1474            &mut engine,
1475            &request_with_tools(),
1476            recorder.clone(),
1477            Duration::from_secs(60),
1478        );
1479
1480        let body = recorder.body();
1481        let calls = streamed_calls(&body);
1482        assert_eq!(calls.len(), 2, "{body}");
1483        assert_eq!(calls[0]["index"], 0);
1484        assert_eq!(calls[1]["index"], 1);
1485        assert_ne!(calls[0]["id"], calls[1]["id"]);
1486        assert_eq!(streamed_finish(&body), "tool_calls");
1487        // Each call is complete in one delta, so a client never has to
1488        // reassemble a half-written `arguments` string.
1489        for call in &calls {
1490            assert_eq!(call["function"]["arguments"], r#"{"pattern": "*/.rs"}"#);
1491        }
1492    }
1493
1494    /// A block the token budget cut in half is not a call. It is text, it is
1495    /// flushed, and the reason the generation actually ended stands.
1496    #[test]
1497    fn an_unterminated_streamed_block_flushes_as_content_and_keeps_length() {
1498        let mut engine = StubEngine::with_ids(&[
1499            (TEXT_ID, "Reading it.\n"),
1500            (crate::toolcall::OPEN_TAG_ID, "<tool_call>"),
1501            (TEXT_ID, "\n{\"name\": \"glob\", \"argum"),
1502        ]);
1503        engine.finish_reason = FinishReason::Length;
1504        let recorder = Recorder::new();
1505        run_stream_for(
1506            &mut engine,
1507            &request_with_tools(),
1508            recorder.clone(),
1509            Duration::from_secs(60),
1510        );
1511
1512        let body = recorder.body();
1513        assert!(streamed_calls(&body).is_empty(), "{body}");
1514        assert_eq!(streamed_finish(&body), "length");
1515        assert_eq!(
1516            streamed_text(&body),
1517            "Reading it.\n<tool_call>\n{\"name\": \"glob\", \"argum",
1518            "half a block is text, and text is not dropped"
1519        );
1520    }
1521
1522    /// A request that declared no tools gets the block back as text. The model
1523    /// only writes these markers because the template's tools branch asked for
1524    /// them, so without tools they are an answer, not a call.
1525    #[test]
1526    fn a_streamed_block_without_tools_stays_ordinary_text() {
1527        let mut engine = StubEngine::with_ids(&glob_call_tokens());
1528        let recorder = Recorder::new();
1529        run_stream(&mut engine, recorder.clone(), Duration::from_secs(60));
1530
1531        let body = recorder.body();
1532        assert!(
1533            streamed_calls(&body).is_empty(),
1534            "a request without tools got tool_calls back: {body}"
1535        );
1536        assert_eq!(streamed_finish(&body), "stop");
1537        assert_eq!(
1538            streamed_text(&body),
1539            "<tool_call>\n{\"name\": \"glob\", \"arguments\": {\"pattern\": \"*/.rs\"}}\n</tool_call>",
1540            "byte for byte, markers and all"
1541        );
1542    }
1543
1544    /// Models get this JSON wrong routinely, and a dropped turn would be the
1545    /// cost. The call goes out with whatever the `arguments` span held.
1546    #[test]
1547    fn malformed_json_in_a_streamed_block_still_yields_a_call() {
1548        let mut engine = StubEngine::with_ids(&[
1549            (crate::toolcall::OPEN_TAG_ID, "<tool_call>"),
1550            (
1551                TEXT_ID,
1552                "\n{\"name\": \"glob\", \"arguments\": {\"pattern\": \"*\",}}\n",
1553            ),
1554            (crate::toolcall::CLOSE_TAG_ID, "</tool_call>"),
1555        ]);
1556        let recorder = Recorder::new();
1557        run_stream_for(
1558            &mut engine,
1559            &request_with_tools(),
1560            recorder.clone(),
1561            Duration::from_secs(60),
1562        );
1563
1564        let body = recorder.body();
1565        let calls = streamed_calls(&body);
1566        assert_eq!(calls.len(), 1, "{body}");
1567        assert_eq!(calls[0]["function"]["name"], "glob");
1568        assert_eq!(calls[0]["function"]["arguments"], r#"{"pattern": "*",}"#);
1569        assert_eq!(streamed_finish(&body), "tool_calls");
1570    }
1571
1572    /// The two paths name the same call identically, so a client that retries a
1573    /// turn without `stream` gets ids it can still match against its table.
1574    #[test]
1575    fn streamed_call_ids_match_the_buffered_paths_for_the_same_completion() {
1576        const ID: &str = "chatcmpl-18f3a0001";
1577        let mut engine = StubEngine::with_ids(&glob_call_tokens());
1578        let recorder = Recorder::new();
1579        let request = request_with_tools();
1580        let plan = engine.prepare(&request).expect("a plan");
1581        stream_into(
1582            &mut engine,
1583            &request,
1584            plan,
1585            recorder.clone(),
1586            ID,
1587            1_700_000_000,
1588            Duration::from_secs(60),
1589        );
1590
1591        let body = recorder.body();
1592        let streamed = streamed_calls(&body);
1593        // What the buffered path would have produced from the same reply.
1594        let reply: String = glob_call_tokens().iter().map(|(_, text)| *text).collect();
1595        let buffered = crate::toolcall::wire_calls(ID, &crate::toolcall::extract(&reply).calls);
1596        assert_eq!(buffered.len(), 1);
1597        assert_eq!(streamed[0]["id"], buffered[0].id);
1598        assert_eq!(streamed[0]["id"], "call_18f3a0001_0");
1599    }
1600
1601    /// The framing rules do not change because a delta carries a call: one
1602    /// event per chunk, hex length of the payload only.
1603    #[test]
1604    fn a_tool_call_delta_is_framed_like_every_other_event() {
1605        let mut engine = StubEngine::with_ids(&glob_call_tokens());
1606        let recorder = Recorder::new();
1607        run_stream_for(
1608            &mut engine,
1609            &request_with_tools(),
1610            recorder.clone(),
1611            Duration::from_secs(60),
1612        );
1613
1614        let body = recorder.body();
1615        let mut rest = body.strip_prefix(SSE_HEAD).expect("head first");
1616        let mut events = 0usize;
1617        loop {
1618            let (head, tail) = rest.split_once("\r\n").expect("a chunk header");
1619            let len = usize::from_str_radix(head, 16).expect("hex length");
1620            if len == 0 {
1621                assert_eq!(tail, "\r\n", "the terminator ends the body");
1622                break;
1623            }
1624            assert_eq!(&tail[len..len + 2], "\r\n", "chunk not CRLF-terminated");
1625            let payload = &tail[..len];
1626            assert_eq!(payload.matches("\n\n").count(), 1, "{payload:?}");
1627            events += 1;
1628            rest = &tail[len + 2..];
1629        }
1630        // role, the call, finish, [DONE].
1631        assert_eq!(events, 4, "{body}");
1632    }
1633
1634    /// The array-form `content` every SDK sends, end to end. A server that
1635    /// models `content` as a `String` answers a question it never received.
1636    #[test]
1637    fn the_array_form_content_reaches_the_engine() {
1638        let address = spawn_server();
1639        let response = post(
1640            address,
1641            PATH_CHAT_COMPLETIONS,
1642            r#"{"model":"stub-model","messages":[
1643                {"role":"user","content":[{"type":"text","text":"hi"}]}]}"#,
1644        );
1645        assert!(response.starts_with("HTTP/1.1 200"), "{response}");
1646        assert!(response.contains(r#""content":"ok""#), "{response}");
1647    }
1648
1649    #[test]
1650    fn a_malformed_body_is_a_400_with_the_openai_shape() {
1651        let address = spawn_server();
1652        let response = post(address, PATH_CHAT_COMPLETIONS, "{not json");
1653        assert!(response.starts_with("HTTP/1.1 400"), "{response}");
1654        assert!(response.contains(r#""code":"invalid_json""#), "{response}");
1655        assert!(!response.contains(r#""message":null"#), "{response}");
1656    }
1657
1658    /// Every byte off the socket is untrusted, and the body is read into
1659    /// memory to be parsed. Inside a 3 GB cgroup an uncapped read is an OOM.
1660    #[test]
1661    fn an_oversized_body_is_refused_by_its_declared_length() {
1662        const BODY: &str = "0123456789abcdef0123456789abcdef";
1663        let mut request = tiny_http::TestRequest::new()
1664            .with_method(Method::Post)
1665            .with_path(PATH_CHAT_COMPLETIONS)
1666            .with_body(BODY)
1667            .into();
1668        let error = read_body(&mut request, 16).expect_err("over the cap");
1669        assert!(matches!(error, ServerError::BodyTooLarge { limit: 16 }));
1670        assert_eq!(error.status(), 413);
1671    }
1672
1673    #[test]
1674    fn a_body_at_the_cap_is_accepted() {
1675        const BODY: &str = "0123456789abcdef";
1676        let mut request = tiny_http::TestRequest::new()
1677            .with_method(Method::Post)
1678            .with_path(PATH_CHAT_COMPLETIONS)
1679            .with_body(BODY)
1680            .into();
1681        assert_eq!(
1682            read_body(&mut request, BODY.len()).expect("at the cap"),
1683            BODY
1684        );
1685    }
1686
1687    #[test]
1688    fn completion_ids_do_not_repeat() {
1689        let ids: std::collections::HashSet<String> = (0..64).map(|_| completion_id()).collect();
1690        assert_eq!(ids.len(), 64);
1691        assert!(ids.iter().all(|id| id.starts_with("chatcmpl-")));
1692    }
1693}