Skip to main content

ramvamp_server/
lib.rs

1//! OpenAI-compatible HTTP server for the ramvamp runtime.
2//!
3//! The crate is layered so that the wire contract can be tested without a 30B
4//! model on disk, and so that the *timing* of a stream can be tested without a
5//! socket. Every rule below is asserted by a unit test that runs in
6//! milliseconds, and a streaming sequence is pinned byte for byte.
7//!
8//! # The pure layer: shapes and rules
9//!
10//! * [`request`] — the `POST /v1/chat/completions` body, including the
11//!   array-form `content` that clients send routinely and a naive server
12//!   silently discards.
13//! * [`response`] — `chat.completion`, `chat.completion.chunk`, the
14//!   `/v1/models` listing and the `/health` body.
15//! * [`sse`] — SSE payload construction: `data: {json}\n\n` and the bare
16//!   `[DONE]` sentinel.
17//! * [`prompt`] — mapping messages onto the vendored ChatML renderer, through
18//!   the *sanitizing* encoder because the input is untrusted.
19//! * [`toolcall`] — the return leg: recovering `tool_calls` from the
20//!   `<tool_call>` text the model writes, preserving the argument bytes so the
21//!   client's echo of them re-renders into the block the model emitted. Twice
22//!   over, because a stream cannot wait for the whole reply: [`extract`] for
23//!   the buffered path and [`CallStream`] for the streaming one, the latter
24//!   keyed off the marker *token ids* so a boundary cannot fall between two
25//!   deltas.
26//! * [`error`] — one typed error per refusal, each knowing its status, body and
27//!   headers.
28//!
29//! # The engine layer
30//!
31//! * [`engine`] — the [`Engine`] trait, which is the seam a stub token source
32//!   substitutes into, plus [`ModelEngine`]: model, forward state and
33//!   tokenizer on one thread, with the KV cache reused across requests by
34//!   longest-common-prefix match. The trap in that match — and why the match
35//!   is against generated ids and never against a re-render of the transcript
36//!   — is documented there, because getting it wrong is silent.
37//!
38//! # The transport layer
39//!
40//! * [`wire`] — HTTP/1.1 chunked framing written by hand, because
41//!   `tiny_http::Response` buffers a stream to completion and hides client
42//!   disconnects.
43//! * [`http`] — routing, body limits and the accept loop. Binds loopback only.
44//!
45//! # What it is sized for
46//!
47//! One session, on loopback, in front of a runtime that decodes at about 2
48//! tok/s. That shapes decisions that would look wrong on a hosted API:
49//! `n > 1` is refused instead of approximated, a busy server answers 503 rather
50//! than 429, and `Retry-After` is a small fiction rather than an honest wait.
51//! Each of those has its reasoning recorded next to the code, because each of
52//! them looks like a bug until you know why.
53//!
54//! The same rule governs the parameters this build cannot implement rather
55//! than cannot afford: a non-empty `stop` and a `tool_choice` that forces a
56//! particular call are 400s, because nothing downstream applies either one and
57//! a 200 that quietly ignored them would be indistinguishable from a correct
58//! answer. `tools` themselves *are* served — see [`prompt`].
59
60pub mod engine;
61pub mod error;
62pub mod http;
63pub mod prompt;
64pub mod request;
65pub mod response;
66pub mod sse;
67pub mod toolcall;
68pub mod wire;
69
70pub use engine::{
71    Completion, Engine, EngineConfig, ModelEngine, NullSink, Plan, StreamError, TokenSink,
72    common_prefix, hush_stream_aborts,
73};
74pub use error::{ErrorDetail, ErrorResponse, RETRY_HEADERS, ServerError};
75pub use http::{
76    BODY_BYTES_PER_CONTEXT_TOKEN, MIN_BODY_BYTES, ServeConfig, ServeError,
77    max_body_bytes_for_context, serve, serve_with,
78};
79pub use prompt::{Prompt, check_context};
80pub use request::{
81    ChatCompletionRequest, Content, ContentPart, Message, MessageRole, StreamOptions,
82    StringOrArray, ToolCall,
83};
84pub use response::{
85    ChatCompletion, ChatCompletionChunk, ChunkBuilder, Delta, FinishReason, Health, ModelList,
86    ResponseMessage, ToolCallDelta, Usage,
87};
88pub use sse::{DONE_SENTINEL, sse_data, sse_done, sse_event};
89pub use toolcall::{
90    CallStream, Extracted, ParsedCall, Step, call_id, extract, wire_call, wire_calls,
91};
92
93#[cfg(test)]
94mod tests {
95    use super::*;
96
97    /// One request through every pure stage: parse, validate, map, respond.
98    /// The tokenizer step is the only thing missing, and it is the only thing
99    /// that needs an install on disk.
100    #[test]
101    fn a_request_walks_the_whole_pure_path() -> Result<(), ServerError> {
102        let request = ChatCompletionRequest::from_json(
103            r#"{"model":"qwen3-30b-a3b",
104                "messages":[
105                    {"role":"system","content":"be terse"},
106                    {"role":"user","content":[{"type":"text","text":"2+2?"}]}
107                ],
108                "stream":true,
109                "stream_options":{"include_usage":true},
110                "max_completion_tokens":8}"#,
111        )?;
112        request.validate()?;
113        assert!(request.stream());
114        assert!(request.include_usage());
115        assert_eq!(request.max_new_tokens(), Some(8));
116
117        let mapped = prompt::chat_messages(&request.messages)?;
118        assert_eq!(mapped.len(), 2);
119        assert_eq!(mapped[1].content, "2+2?");
120
121        check_context(64, 8, 4096)?;
122
123        let builder = ChunkBuilder::new("chatcmpl-1", 0, &request.model);
124        let stream = format!(
125            "{}{}{}{}{}",
126            sse_event(&builder.role())?,
127            sse_event(&builder.content("4"))?,
128            sse_event(&builder.finish(FinishReason::Stop))?,
129            sse_event(&builder.usage(Usage::new(20, 1)))?,
130            sse_done(),
131        );
132        assert!(stream.ends_with("data: [DONE]\n\n"));
133        assert_eq!(stream.matches("\n\n").count(), 5);
134        Ok(())
135    }
136}