ramvamp_server/lib.rs
1//! OpenAI-compatible HTTP server for the ramvamp runtime.
2//!
3//! The crate is layered so that the wire contract can be tested without a 30B
4//! model on disk, and so that the *timing* of a stream can be tested without a
5//! socket. Every rule below is asserted by a unit test that runs in
6//! milliseconds, and a streaming sequence is pinned byte for byte.
7//!
8//! # The pure layer: shapes and rules
9//!
10//! * [`request`] — the `POST /v1/chat/completions` body, including the
11//! array-form `content` that clients send routinely and a naive server
12//! silently discards.
13//! * [`response`] — `chat.completion`, `chat.completion.chunk`, the
14//! `/v1/models` listing and the `/health` body.
15//! * [`sse`] — SSE payload construction: `data: {json}\n\n` and the bare
16//! `[DONE]` sentinel.
17//! * [`prompt`] — mapping messages onto the vendored ChatML renderer, through
18//! the *sanitizing* encoder because the input is untrusted.
19//! * [`toolcall`] — the return leg: recovering `tool_calls` from the
20//! `<tool_call>` text the model writes, preserving the argument bytes so the
21//! client's echo of them re-renders into the block the model emitted. Twice
22//! over, because a stream cannot wait for the whole reply: [`extract`] for
23//! the buffered path and [`CallStream`] for the streaming one, the latter
24//! keyed off the marker *token ids* so a boundary cannot fall between two
25//! deltas.
26//! * [`error`] — one typed error per refusal, each knowing its status, body and
27//! headers.
28//!
29//! # The engine layer
30//!
31//! * [`engine`] — the [`Engine`] trait, which is the seam a stub token source
32//! substitutes into, plus [`ModelEngine`]: model, forward state and
33//! tokenizer on one thread, with the KV cache reused across requests by
34//! longest-common-prefix match. The trap in that match — and why the match
35//! is against generated ids and never against a re-render of the transcript
36//! — is documented there, because getting it wrong is silent.
37//!
38//! # The transport layer
39//!
40//! * [`wire`] — HTTP/1.1 chunked framing written by hand, because
41//! `tiny_http::Response` buffers a stream to completion and hides client
42//! disconnects.
43//! * [`http`] — routing, body limits and the accept loop. Binds loopback only.
44//!
45//! # What it is sized for
46//!
47//! One session, on loopback, in front of a runtime that decodes at about 2
48//! tok/s. That shapes decisions that would look wrong on a hosted API:
49//! `n > 1` is refused instead of approximated, a busy server answers 503 rather
50//! than 429, and `Retry-After` is a small fiction rather than an honest wait.
51//! Each of those has its reasoning recorded next to the code, because each of
52//! them looks like a bug until you know why.
53//!
54//! The same rule governs the parameters this build cannot implement rather
55//! than cannot afford: a non-empty `stop` and a `tool_choice` that forces a
56//! particular call are 400s, because nothing downstream applies either one and
57//! a 200 that quietly ignored them would be indistinguishable from a correct
58//! answer. `tools` themselves *are* served — see [`prompt`].
59
60pub mod engine;
61pub mod error;
62pub mod http;
63pub mod prompt;
64pub mod request;
65pub mod response;
66pub mod sse;
67pub mod toolcall;
68pub mod wire;
69
70pub use engine::{
71 Completion, Engine, EngineConfig, ModelEngine, NullSink, Plan, StreamError, TokenSink,
72 common_prefix, hush_stream_aborts,
73};
74pub use error::{ErrorDetail, ErrorResponse, RETRY_HEADERS, ServerError};
75pub use http::{
76 BODY_BYTES_PER_CONTEXT_TOKEN, MIN_BODY_BYTES, ServeConfig, ServeError,
77 max_body_bytes_for_context, serve, serve_with,
78};
79pub use prompt::{Prompt, check_context};
80pub use request::{
81 ChatCompletionRequest, Content, ContentPart, Message, MessageRole, StreamOptions,
82 StringOrArray, ToolCall,
83};
84pub use response::{
85 ChatCompletion, ChatCompletionChunk, ChunkBuilder, Delta, FinishReason, Health, ModelList,
86 ResponseMessage, ToolCallDelta, Usage,
87};
88pub use sse::{DONE_SENTINEL, sse_data, sse_done, sse_event};
89pub use toolcall::{
90 CallStream, Extracted, ParsedCall, Step, call_id, extract, wire_call, wire_calls,
91};
92
93#[cfg(test)]
94mod tests {
95 use super::*;
96
97 /// One request through every pure stage: parse, validate, map, respond.
98 /// The tokenizer step is the only thing missing, and it is the only thing
99 /// that needs an install on disk.
100 #[test]
101 fn a_request_walks_the_whole_pure_path() -> Result<(), ServerError> {
102 let request = ChatCompletionRequest::from_json(
103 r#"{"model":"qwen3-30b-a3b",
104 "messages":[
105 {"role":"system","content":"be terse"},
106 {"role":"user","content":[{"type":"text","text":"2+2?"}]}
107 ],
108 "stream":true,
109 "stream_options":{"include_usage":true},
110 "max_completion_tokens":8}"#,
111 )?;
112 request.validate()?;
113 assert!(request.stream());
114 assert!(request.include_usage());
115 assert_eq!(request.max_new_tokens(), Some(8));
116
117 let mapped = prompt::chat_messages(&request.messages)?;
118 assert_eq!(mapped.len(), 2);
119 assert_eq!(mapped[1].content, "2+2?");
120
121 check_context(64, 8, 4096)?;
122
123 let builder = ChunkBuilder::new("chatcmpl-1", 0, &request.model);
124 let stream = format!(
125 "{}{}{}{}{}",
126 sse_event(&builder.role())?,
127 sse_event(&builder.content("4"))?,
128 sse_event(&builder.finish(FinishReason::Stop))?,
129 sse_event(&builder.usage(Usage::new(20, 1)))?,
130 sse_done(),
131 );
132 assert!(stream.ends_with("data: [DONE]\n\n"));
133 assert_eq!(stream.matches("\n\n").count(), 5);
134 Ok(())
135 }
136}