1use std::io::{Read, Write};
31use std::net::{Ipv4Addr, SocketAddr};
32use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
33use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
34use std::thread::JoinHandle;
35use std::time::Duration;
36
37use thiserror::Error;
38use tiny_http::{Header, Method, Request, Response, Server};
39
40use crate::engine::{Engine, NullSink, Plan, StreamError, TokenSink, hush_stream_aborts, unix_now};
41use crate::error::ServerError;
42use crate::request::ChatCompletionRequest;
43use crate::response::{
44 COMPLETION_ID_PREFIX, ChatCompletion, ChunkBuilder, FinishReason, Health, ModelList,
45 ResponseMessage,
46};
47use crate::sse::{sse_data, sse_done, sse_event};
48use crate::toolcall::{CallStream, Step, wire_call};
49use crate::wire::{KEEPALIVE, StreamState};
50
51pub const PATH_CHAT_COMPLETIONS: &str = "/v1/chat/completions";
53pub const PATH_MODELS: &str = "/v1/models";
55pub const PATH_HEALTH: &str = "/health";
57
58#[derive(Debug, Clone, Copy, PartialEq, Eq)]
60pub struct ServeConfig {
61 pub port: u16,
63 pub keepalive: Duration,
71 pub max_body_bytes: usize,
73}
74
75impl Default for ServeConfig {
76 fn default() -> Self {
77 ServeConfig {
78 port: 8080,
79 keepalive: Duration::from_secs(10),
80 max_body_bytes: max_body_bytes_for_context(DEFAULT_CONTEXT),
84 }
85 }
86}
87
88pub const DEFAULT_CONTEXT: usize = 4096;
93
94pub const BODY_BYTES_PER_CONTEXT_TOKEN: usize = 256;
102
103pub const MIN_BODY_BYTES: usize = 64 << 10;
109
110#[must_use]
122pub fn max_body_bytes_for_context(context: usize) -> usize {
123 context
124 .saturating_mul(BODY_BYTES_PER_CONTEXT_TOKEN)
125 .max(MIN_BODY_BYTES)
126}
127
128#[derive(Debug, Error)]
131pub enum ServeError {
132 #[error("binding 127.0.0.1:{port}: {message}")]
134 Bind {
135 port: u16,
137 message: String,
139 },
140 #[error("accepting connections: {0}")]
142 Accept(#[from] std::io::Error),
143}
144
145pub fn serve(engine: &mut dyn Engine, config: ServeConfig) -> Result<(), ServeError> {
152 serve_with(engine, config, |_| {})
153}
154
155pub fn serve_with(
164 engine: &mut dyn Engine,
165 config: ServeConfig,
166 ready: impl FnOnce(SocketAddr),
167) -> Result<(), ServeError> {
168 hush_stream_aborts();
169 let bind = SocketAddr::from((Ipv4Addr::LOCALHOST, config.port));
172 let server = Server::http(bind).map_err(|e| ServeError::Bind {
173 port: config.port,
174 message: e.to_string(),
175 })?;
176 let address = match server.server_addr().to_ip() {
177 Some(address) => address,
178 None => bind,
179 };
180 tracing::info!(%address, model = engine.model_id(), "listening");
181 ready(address);
182
183 loop {
184 let request = server.recv()?;
185 handle(engine, request, &config);
186 }
187}
188
189fn handle(engine: &mut dyn Engine, request: Request, config: &ServeConfig) {
191 let path = request
194 .url()
195 .split('?')
196 .next()
197 .unwrap_or_default()
198 .to_owned();
199 let method = request.method().clone();
200 tracing::debug!(%method, %path, "request");
201
202 match (&method, path.as_str()) {
203 (Method::Post, PATH_CHAT_COMPLETIONS) => chat_completions(engine, request, config),
204 (Method::Get, PATH_MODELS) => {
205 let body = ModelList::single(engine.model_id(), engine.created());
206 respond_json(request, 200, &body);
207 }
208 (Method::Get, PATH_HEALTH) => respond_json(request, 200, &Health::ok()),
209 _ => respond_error(
210 request,
211 &ServerError::NotFound {
212 method: method.as_str().to_owned(),
213 path,
214 },
215 ),
216 }
217}
218
219fn chat_completions(engine: &mut dyn Engine, mut request: Request, config: &ServeConfig) {
221 let body = match read_body(&mut request, config.max_body_bytes) {
222 Ok(body) => body,
223 Err(e) => return respond_error(request, &e),
224 };
225 let parsed = match ChatCompletionRequest::from_json(&body) {
226 Ok(parsed) => parsed,
227 Err(e) => return respond_error(request, &e),
228 };
229 tracing::debug!(
235 stop = parsed.stop_sequences().len(),
236 tools = parsed.tools().len(),
237 tool_choice = ?parsed.tool_choice,
238 stream = parsed.stream(),
239 "chat request"
240 );
241 let plan = match engine.prepare(&parsed) {
245 Ok(plan) => plan,
246 Err(e) => return respond_error(request, &e),
247 };
248
249 let id = completion_id();
250 let created = unix_now();
251 if parsed.stream() {
252 stream_into(
253 engine,
254 &parsed,
255 plan,
256 request.into_writer(),
257 &id,
258 created,
259 config.keepalive,
260 );
261 } else {
262 let mut sink = NullSink;
267 match engine.run(plan, &mut sink) {
268 Ok(completion) => {
269 let extracted = if parsed.tools().is_empty() {
284 crate::toolcall::Extracted::none(&completion.text)
285 } else {
286 crate::toolcall::extract(&completion.text)
287 };
288 let calls = crate::toolcall::wire_calls(&id, &extracted.calls);
289 let finish_reason = if calls.is_empty() {
294 completion.finish_reason
295 } else {
296 FinishReason::ToolCalls
297 };
298 let body = ChatCompletion::from_message(
299 id,
300 created,
301 &parsed.model,
302 ResponseMessage::calling(extracted.content, calls),
303 finish_reason,
304 completion.usage,
305 );
306 respond_json(request, 200, &body);
307 }
308 Err(e) => respond_error(request, &e),
309 }
310 }
311}
312
313pub fn stream_into<W: Write + Send + 'static>(
320 engine: &mut dyn Engine,
321 request: &ChatCompletionRequest,
322 plan: Plan,
323 writer: W,
324 id: &str,
325 created: u64,
326 keepalive: Duration,
327) {
328 let state = Arc::new(Mutex::new(StreamState::new(writer)));
329
330 if let Err(e) = lock(&state).head() {
335 tracing::debug!(?e, "client left before the response head");
336 return;
337 }
338
339 let builder = ChunkBuilder::new(id, created, &request.model);
340 if emit(&state, &sse_or_empty(&builder.role())).is_err() {
343 return;
344 }
345
346 let keeper = KeepAlive::spawn(Arc::clone(&state), keepalive);
347 let mut sink = SseSink {
348 state: Arc::clone(&state),
349 builder: builder.clone(),
350 keepalive,
351 calls: CallStream::new(!request.tools().is_empty()),
355 };
356 let include_usage = request.include_usage();
357 let outcome = engine.run(plan, &mut sink);
358 drop(keeper);
359
360 match outcome {
361 Ok(completion) if completion.aborted => {
362 tracing::debug!("client left mid-reply; the stream is abandoned unterminated");
363 }
364 Ok(completion) => {
365 if let Some(unterminated) = sink.calls.flush() {
368 let _ = emit(&state, &sse_or_empty(&builder.content(unterminated)));
369 }
370 let finish_reason = if sink.calls.emitted() > 0 {
376 FinishReason::ToolCalls
377 } else {
378 completion.finish_reason
379 };
380 let _ = emit(&state, &sse_or_empty(&builder.finish(finish_reason)));
381 if include_usage {
384 let _ = emit(&state, &sse_or_empty(&builder.usage(completion.usage)));
385 }
386 let _ = emit(&state, &sse_done());
387 let _ = lock(&state).finish();
388 }
389 Err(e) => {
390 tracing::error!(error = %e, "generation failed mid-stream");
394 let _ = emit(&state, &sse_data(&e.body_json()));
395 let _ = emit(&state, &sse_done());
396 let _ = lock(&state).finish();
397 }
398 }
399}
400
401struct SseSink<W: Write> {
403 state: Arc<Mutex<StreamState<W>>>,
404 builder: ChunkBuilder,
405 keepalive: Duration,
406 calls: CallStream,
409}
410
411impl<W: Write> TokenSink for SseSink<W> {
412 fn prefill(&mut self, done: usize, total: usize) -> Result<(), StreamError> {
413 let mut state = lock(&self.state);
414 if let Some(failed) = state.failure() {
415 return Err(failed);
416 }
417 if state.idle_for() >= self.keepalive {
424 state.emit(KEEPALIVE)?;
425 tracing::trace!(done, total, "keep-alive on prefill progress");
426 }
427 Ok(())
428 }
429
430 fn token(&mut self, id: u32, text: &str) -> Result<(), StreamError> {
431 let event = match self.calls.push(id, text) {
435 Step::Nothing => return Ok(()),
438 Step::Content(text) if text.is_empty() => return Ok(()),
442 Step::Content(text) => sse_or_empty(&self.builder.content(text)),
443 Step::Call { index, call } => {
444 tracing::debug!(index, name = %call.name, "recovered a streamed tool call");
445 let wire = wire_call(self.builder.id(), index, &call);
446 sse_or_empty(&self.builder.tool_call(index, wire))
447 }
448 };
449 lock(&self.state).emit(&event)
450 }
451}
452
453fn sse_or_empty(chunk: &crate::response::ChatCompletionChunk) -> String {
460 match sse_event(chunk) {
461 Ok(event) => event,
462 Err(e) => {
463 tracing::error!(error = %e, "serializing a chunk failed; dropping it");
464 String::new()
465 }
466 }
467}
468
469struct KeepAlive {
480 stop: Arc<AtomicBool>,
481 handle: Option<JoinHandle<()>>,
482}
483
484impl KeepAlive {
485 fn spawn<W: Write + Send + 'static>(
486 state: Arc<Mutex<StreamState<W>>>,
487 interval: Duration,
488 ) -> Self {
489 let stop = Arc::new(AtomicBool::new(false));
490 let flag = Arc::clone(&stop);
491 let tick = (interval / 4).clamp(Duration::from_millis(5), Duration::from_secs(1));
495 let handle = std::thread::Builder::new()
496 .name("rvmp-keepalive".to_owned())
497 .spawn(move || {
498 while !flag.load(Ordering::Relaxed) {
499 std::thread::sleep(tick);
500 if flag.load(Ordering::Relaxed) {
501 return;
502 }
503 let mut state = lock(&state);
504 if state.failure().is_some() {
505 return;
506 }
507 if state.idle_for() >= interval && state.emit(KEEPALIVE).is_err() {
508 return;
512 }
513 }
514 });
515 match handle {
516 Ok(handle) => KeepAlive {
517 stop,
518 handle: Some(handle),
519 },
520 Err(e) => {
521 tracing::warn!(error = %e, "no keep-alive thread; long prefills may time out");
524 KeepAlive { stop, handle: None }
525 }
526 }
527 }
528}
529
530impl Drop for KeepAlive {
531 fn drop(&mut self) {
532 self.stop.store(true, Ordering::Relaxed);
533 if let Some(handle) = self.handle.take() {
534 let _ = handle.join();
537 }
538 }
539}
540
541fn lock<W: Write>(state: &Mutex<StreamState<W>>) -> MutexGuard<'_, StreamState<W>> {
548 state.lock().unwrap_or_else(PoisonError::into_inner)
549}
550
551fn emit<W: Write>(state: &Mutex<StreamState<W>>, payload: &str) -> Result<(), StreamError> {
552 lock(state).emit(payload)
553}
554
555fn read_body(request: &mut Request, limit: usize) -> Result<String, ServerError> {
562 if request.body_length().is_some_and(|len| len > limit) {
563 return Err(ServerError::BodyTooLarge { limit });
564 }
565 let mut body = Vec::new();
566 let capped = u64::try_from(limit).unwrap_or(u64::MAX).saturating_add(1);
569 request
570 .as_reader()
571 .take(capped)
572 .read_to_end(&mut body)
573 .map_err(|e| ServerError::MalformedJson(format!("reading the request body: {e}")))?;
574 if body.len() > limit {
575 return Err(ServerError::BodyTooLarge { limit });
576 }
577 String::from_utf8(body)
578 .map_err(|_| ServerError::MalformedJson("request body is not valid UTF-8".to_owned()))
579}
580
581fn completion_id() -> String {
588 static COUNTER: AtomicU64 = AtomicU64::new(0);
589 let n = COUNTER.fetch_add(1, Ordering::Relaxed);
590 format!("{COMPLETION_ID_PREFIX}{:x}{:04x}", unix_now(), n & 0xffff)
591}
592
593fn header(name: &str, value: &str) -> Option<Header> {
599 match Header::from_bytes(name.as_bytes(), value.as_bytes()) {
600 Ok(header) => Some(header),
601 Err(()) => {
602 tracing::error!(name, value, "non-ASCII header; dropped");
603 None
604 }
605 }
606}
607
608fn respond_json<T: serde::Serialize>(request: Request, status: u16, body: &T) {
614 match serde_json::to_string(body) {
615 Ok(json) => send(request, status, json, &[]),
616 Err(e) => {
617 let error = ServerError::Serialize(e);
618 respond_error(request, &error);
619 }
620 }
621}
622
623fn respond_error(request: Request, error: &ServerError) {
625 tracing::debug!(status = error.status(), error = %error, "refusing a request");
626 send(request, error.status(), error.body_json(), error.headers());
627}
628
629fn send(request: Request, status: u16, body: String, extra: &[(&str, &str)]) {
630 let mut response = Response::from_string(body).with_status_code(status);
631 if let Some(header) = header("Content-Type", "application/json") {
632 response = response.with_header(header);
633 }
634 for (name, value) in extra {
635 if let Some(header) = header(name, value) {
636 response = response.with_header(header);
637 }
638 }
639 if let Err(e) = request.respond(response) {
640 tracing::debug!(error = %e, "failed to send a response");
643 }
644}
645
646#[cfg(test)]
647mod tests {
648 use super::*;
649 use crate::engine::Completion;
650 use crate::response::{FinishReason, Usage};
651 use crate::wire::{SSE_HEAD, TERMINATOR};
652 use std::sync::atomic::AtomicUsize;
653 use std::time::Instant;
654
655 struct StubEngine {
662 prefill_steps: usize,
663 prefill_delay: Duration,
664 tokens: Vec<(u32, String)>,
668 token_delay: Duration,
669 finish_reason: FinishReason,
672 produced: Arc<AtomicUsize>,
675 }
676
677 const TEXT_ID: u32 = 7;
679
680 impl StubEngine {
681 fn new(tokens: &[&str]) -> Self {
682 StubEngine::with_ids(&tokens.iter().map(|t| (TEXT_ID, *t)).collect::<Vec<_>>())
683 }
684
685 fn with_ids(tokens: &[(u32, &str)]) -> Self {
686 StubEngine {
687 prefill_steps: 0,
688 prefill_delay: Duration::ZERO,
689 tokens: tokens
690 .iter()
691 .map(|(id, text)| (*id, (*text).to_owned()))
692 .collect(),
693 token_delay: Duration::ZERO,
694 finish_reason: FinishReason::Stop,
695 produced: Arc::new(AtomicUsize::new(0)),
696 }
697 }
698 }
699
700 impl Engine for StubEngine {
701 fn model_id(&self) -> &str {
702 "stub-model"
703 }
704
705 fn created(&self) -> u64 {
706 1_700_000_000
707 }
708
709 fn prepare(&mut self, request: &ChatCompletionRequest) -> Result<Plan, ServerError> {
710 request.validate()?;
711 Ok(Plan {
712 prompt_tokens: 11,
713 max_new: self.tokens.len(),
714 cached_prefix: 0,
715 prompt_ids: vec![1, 2, 3],
716 params: ramvamp_core::generate::GenerateParams {
717 max_new: self.tokens.len(),
718 temperature: 0.7,
719 top_k: Some(20),
720 top_p: 0.8,
721 seed: 42,
722 greedy: false,
723 },
724 })
725 }
726
727 fn run(&mut self, plan: Plan, sink: &mut dyn TokenSink) -> Result<Completion, ServerError> {
728 let mut text = String::new();
729 for step in 1..=self.prefill_steps {
730 std::thread::sleep(self.prefill_delay);
731 if sink.prefill(step, self.prefill_steps).is_err() {
732 return Ok(aborted(text, plan.prompt_tokens, 0));
733 }
734 }
735 for (index, (id, token)) in self.tokens.clone().iter().enumerate() {
736 std::thread::sleep(self.token_delay);
737 self.produced.store(index + 1, Ordering::Relaxed);
738 text.push_str(token);
739 if sink.token(*id, token).is_err() {
740 return Ok(aborted(text, plan.prompt_tokens, index + 1));
741 }
742 }
743 let generated = self.tokens.len();
744 Ok(Completion {
745 text,
746 usage: Usage::new(plan.prompt_tokens as u32, generated as u32),
747 finish_reason: self.finish_reason,
748 aborted: false,
749 })
750 }
751 }
752
753 fn aborted(text: String, prompt: usize, generated: usize) -> Completion {
754 Completion {
755 text,
756 usage: Usage::new(prompt as u32, generated as u32),
757 finish_reason: FinishReason::Stop,
758 aborted: true,
759 }
760 }
761
762 type Flushed = Arc<Mutex<Vec<(Duration, Vec<u8>)>>>;
764
765 #[derive(Clone)]
767 struct Recorder {
768 events: Flushed,
769 pending: Arc<Mutex<Vec<u8>>>,
770 flushes: Arc<AtomicUsize>,
771 start: Instant,
772 die_after_flushes: Option<usize>,
773 }
774
775 impl Recorder {
776 fn new() -> Self {
777 Recorder {
778 events: Arc::new(Mutex::new(Vec::new())),
779 pending: Arc::new(Mutex::new(Vec::new())),
780 flushes: Arc::new(AtomicUsize::new(0)),
781 start: Instant::now(),
782 die_after_flushes: None,
783 }
784 }
785
786 fn dying_after(mut self, flushes: usize) -> Self {
787 self.die_after_flushes = Some(flushes);
788 self
789 }
790
791 fn events(&self) -> Vec<(Duration, String)> {
793 self.events
794 .lock()
795 .unwrap_or_else(PoisonError::into_inner)
796 .iter()
797 .map(|(at, bytes)| (*at, String::from_utf8_lossy(bytes).into_owned()))
798 .collect()
799 }
800
801 fn body(&self) -> String {
802 self.events().into_iter().map(|(_, text)| text).collect()
803 }
804 }
805
806 impl Write for Recorder {
807 fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
808 self.pending
809 .lock()
810 .unwrap_or_else(PoisonError::into_inner)
811 .extend_from_slice(buf);
812 Ok(buf.len())
813 }
814
815 fn flush(&mut self) -> std::io::Result<()> {
816 let bytes =
817 std::mem::take(&mut *self.pending.lock().unwrap_or_else(PoisonError::into_inner));
818 let n = self.flushes.fetch_add(1, Ordering::Relaxed) + 1;
819 if self.die_after_flushes.is_some_and(|limit| n > limit) {
820 return Err(std::io::Error::from(std::io::ErrorKind::BrokenPipe));
821 }
822 self.events
823 .lock()
824 .unwrap_or_else(PoisonError::into_inner)
825 .push((self.start.elapsed(), bytes));
826 Ok(())
827 }
828 }
829
830 fn request(stream: bool) -> ChatCompletionRequest {
831 let body = format!(
832 r#"{{"model":"stub-model","messages":[{{"role":"user","content":"hi"}}],"stream":{stream}}}"#
833 );
834 ChatCompletionRequest::from_json(&body).expect("valid body")
835 }
836
837 fn request_with_tools() -> ChatCompletionRequest {
840 ChatCompletionRequest::from_json(
841 r#"{"model":"stub-model","messages":[{"role":"user","content":"list the rust files"}],
842 "stream":true,
843 "tools":[{"type":"function","function":{"name":"glob","parameters":{}}}]}"#,
844 )
845 .expect("valid body")
846 }
847
848 fn run_stream(engine: &mut StubEngine, recorder: Recorder, keepalive: Duration) {
849 run_stream_for(engine, &request(true), recorder, keepalive);
850 }
851
852 fn run_stream_for(
853 engine: &mut StubEngine,
854 request: &ChatCompletionRequest,
855 recorder: Recorder,
856 keepalive: Duration,
857 ) {
858 let plan = engine.prepare(request).expect("a plan");
859 stream_into(
860 engine,
861 request,
862 plan,
863 recorder,
864 "chatcmpl-test",
865 1_700_000_000,
866 keepalive,
867 );
868 }
869
870 fn chunks(body: &str) -> Vec<serde_json::Value> {
873 body.split("data: ")
874 .skip(1)
875 .map(|event| event.split("\n\n").next().unwrap_or_default())
876 .filter(|payload| *payload != crate::sse::DONE_SENTINEL)
877 .map(|payload| {
878 serde_json::from_str(payload)
879 .unwrap_or_else(|e| panic!("a chunk payload: {e}\n{payload}"))
880 })
881 .collect()
882 }
883
884 fn streamed_calls(body: &str) -> Vec<serde_json::Value> {
886 chunks(body)
887 .into_iter()
888 .filter_map(|chunk| {
889 chunk["choices"][0]["delta"]["tool_calls"]
890 .as_array()
891 .cloned()
892 })
893 .flatten()
894 .collect()
895 }
896
897 fn streamed_text(body: &str) -> String {
899 chunks(body)
900 .iter()
901 .filter_map(|chunk| chunk["choices"][0]["delta"]["content"].as_str())
902 .collect()
903 }
904
905 fn streamed_finish(body: &str) -> String {
907 chunks(body)
908 .iter()
909 .find_map(|chunk| {
910 chunk["choices"][0]["finish_reason"]
911 .as_str()
912 .map(str::to_owned)
913 })
914 .expect("a finish chunk")
915 }
916
917 fn glob_call_tokens() -> Vec<(u32, &'static str)> {
920 vec![
921 (crate::toolcall::OPEN_TAG_ID, "<tool_call>"),
922 (TEXT_ID, "\n{\"name\": \"glob\", "),
923 (TEXT_ID, "\"arguments\": {\"pattern\": \"*/.rs\"}}\n"),
924 (crate::toolcall::CLOSE_TAG_ID, "</tool_call>"),
925 ]
926 }
927
928 #[test]
932 fn events_reach_the_client_at_the_sources_pace() {
933 let mut engine = StubEngine::new(&["a", "b", "c", "d", "e"]);
934 engine.token_delay = Duration::from_millis(60);
935 let recorder = Recorder::new();
936 run_stream(&mut engine, recorder.clone(), Duration::from_secs(60));
937
938 let content: Vec<Duration> = recorder
939 .events()
940 .into_iter()
941 .filter(|(_, text)| text.contains(r#""content":"#) && !text.contains(r#""role""#))
942 .map(|(at, _)| at)
943 .collect();
944 assert_eq!(content.len(), 5, "{:?}", recorder.events());
945 for pair in content.windows(2) {
946 assert!(
947 pair[1] - pair[0] >= Duration::from_millis(40),
948 "events arrived in a burst: {content:?}"
949 );
950 }
951 assert!(
952 content[4] - content[0] >= Duration::from_millis(200),
953 "{content:?}"
954 );
955 }
956
957 #[test]
960 fn the_head_is_written_before_the_engine_runs() {
961 let mut engine = StubEngine::new(&["x"]);
962 engine.prefill_steps = 1;
963 engine.prefill_delay = Duration::from_millis(250);
964 let recorder = Recorder::new();
965 run_stream(&mut engine, recorder.clone(), Duration::from_secs(60));
966
967 let events = recorder.events();
968 assert_eq!(events[0].1, SSE_HEAD);
969 assert!(
970 events[0].0 < Duration::from_millis(100),
971 "the head waited for the model: {:?}",
972 events[0].0
973 );
974 assert!(events[1].1.contains(r#""role":"assistant""#));
976 assert!(events[1].0 < Duration::from_millis(100));
977 }
978
979 #[test]
982 fn a_slow_prefill_emits_keep_alive_comments() {
983 let mut engine = StubEngine::new(&["done"]);
984 engine.prefill_steps = 1;
985 engine.prefill_delay = Duration::from_millis(400);
986 let recorder = Recorder::new();
987 run_stream(&mut engine, recorder.clone(), Duration::from_millis(50));
988
989 let comments: Vec<Duration> = recorder
990 .events()
991 .into_iter()
992 .filter(|(_, text)| text.contains(KEEPALIVE))
993 .map(|(at, _)| at)
994 .collect();
995 assert!(
996 comments.len() >= 3,
997 "expected keep-alives during a 400ms prefill, got {comments:?}"
998 );
999 assert!(recorder.body().contains(r#""content":"done""#));
1001 }
1002
1003 #[test]
1006 fn a_dropped_client_surfaces_as_a_write_error_and_aborts_generation() {
1007 let mut engine = StubEngine::new(&["a", "b", "c", "d", "e", "f"]);
1008 let produced = Arc::clone(&engine.produced);
1009 let recorder = Recorder::new().dying_after(3);
1011 run_stream(&mut engine, recorder.clone(), Duration::from_secs(60));
1012
1013 let got = produced.load(Ordering::Relaxed);
1014 assert!(
1015 (1..=2).contains(&got),
1016 "generation should have stopped at the hangup, produced {got}"
1017 );
1018 assert!(!recorder.body().contains("[DONE]"), "{}", recorder.body());
1021 }
1022
1023 #[test]
1025 fn the_sentinel_precedes_the_zero_length_chunk() {
1026 let mut engine = StubEngine::new(&["he", "llo"]);
1027 let recorder = Recorder::new();
1028 run_stream(&mut engine, recorder.clone(), Duration::from_secs(60));
1029
1030 let body = recorder.body();
1031 let rest = body.strip_prefix(SSE_HEAD).expect("head first");
1032 assert!(rest.ends_with(TERMINATOR), "{rest:?}");
1033 let done = rest.find("data: [DONE]").expect("sentinel");
1034 let end = rest.rfind(TERMINATOR).expect("terminator");
1035 assert!(done < end, "the sentinel must precede the terminator");
1036 }
1037
1038 #[test]
1042 fn the_chunk_framing_decodes_back_to_the_reply() {
1043 let mut engine = StubEngine::new(&["Hé", "llo", "→!"]);
1044 let recorder = Recorder::new();
1045 run_stream(&mut engine, recorder.clone(), Duration::from_secs(60));
1046
1047 let body = recorder.body();
1048 let mut rest = body.strip_prefix(SSE_HEAD).expect("head first");
1049 let mut payloads = Vec::new();
1050 loop {
1051 let (head, tail) = rest.split_once("\r\n").expect("a chunk header");
1052 let len = usize::from_str_radix(head, 16).expect("hex length");
1053 if len == 0 {
1054 assert_eq!(tail, "\r\n", "the terminator ends the body");
1055 break;
1056 }
1057 assert!(tail.len() >= len + 2, "chunk shorter than its header says");
1058 payloads.push(tail[..len].to_owned());
1059 assert_eq!(&tail[len..len + 2], "\r\n", "chunk not CRLF-terminated");
1060 rest = &tail[len + 2..];
1061 }
1062
1063 let text: String = payloads
1064 .iter()
1065 .filter_map(|event| event.strip_prefix("data: "))
1066 .filter_map(|json| serde_json::from_str::<serde_json::Value>(json.trim_end()).ok())
1067 .filter_map(|chunk| {
1068 chunk["choices"][0]["delta"]["content"]
1069 .as_str()
1070 .map(str::to_owned)
1071 })
1072 .collect();
1073 assert_eq!(text, "Héllo→!");
1074 }
1075
1076 #[test]
1079 fn usage_is_emitted_only_when_asked_for_and_before_the_sentinel() {
1080 let mut engine = StubEngine::new(&["x"]);
1081 let recorder = Recorder::new();
1082 run_stream(&mut engine, recorder.clone(), Duration::from_secs(60));
1083 assert!(!recorder.body().contains(r#""usage""#));
1084
1085 let asked = ChatCompletionRequest::from_json(
1086 r#"{"model":"stub-model","messages":[{"role":"user","content":"hi"}],
1087 "stream":true,"stream_options":{"include_usage":true}}"#,
1088 )
1089 .expect("valid body");
1090 let plan = engine.prepare(&asked).expect("a plan");
1091 let recorder = Recorder::new();
1092 stream_into(
1093 &mut engine,
1094 &asked,
1095 plan,
1096 recorder.clone(),
1097 "chatcmpl-test",
1098 1_700_000_000,
1099 Duration::from_secs(60),
1100 );
1101 let body = recorder.body();
1102 let usage = body.find(r#""usage""#).expect("a usage chunk");
1103 let finish = body.find(r#""finish_reason":"stop""#).expect("a finish");
1104 let done = body.find("[DONE]").expect("the sentinel");
1105 assert!(finish < usage && usage < done, "{body}");
1106 assert!(body.contains(r#""prompt_tokens":11"#), "{body}");
1107 }
1108
1109 #[test]
1112 fn a_mid_stream_failure_becomes_an_error_event() {
1113 struct Failing;
1114 impl Engine for Failing {
1115 fn model_id(&self) -> &str {
1116 "stub-model"
1117 }
1118 fn created(&self) -> u64 {
1119 0
1120 }
1121 fn prepare(&mut self, _: &ChatCompletionRequest) -> Result<Plan, ServerError> {
1122 unreachable!("the test builds the plan itself")
1123 }
1124 fn run(&mut self, _: Plan, _: &mut dyn TokenSink) -> Result<Completion, ServerError> {
1125 Err(ServerError::Generate(
1126 ramvamp_core::generate::GenerateError::EmptyPrompt,
1127 ))
1128 }
1129 }
1130 let mut stub = StubEngine::new(&[]);
1131 let request = request(true);
1132 let plan = stub.prepare(&request).expect("a plan");
1133 let recorder = Recorder::new();
1134 stream_into(
1135 &mut Failing,
1136 &request,
1137 plan,
1138 recorder.clone(),
1139 "chatcmpl-test",
1140 0,
1141 Duration::from_secs(60),
1142 );
1143 let body = recorder.body();
1144 assert!(body.starts_with(SSE_HEAD));
1145 assert!(body.contains(r#""code":"generate_error""#), "{body}");
1146 assert!(body.contains("[DONE]"));
1147 assert!(body.ends_with(TERMINATOR));
1148 }
1149
1150 fn spawn_server() -> SocketAddr {
1153 spawn_server_with(&["ok"], FinishReason::Stop)
1154 }
1155
1156 fn spawn_server_with(tokens: &[&str], finish_reason: FinishReason) -> SocketAddr {
1157 let tokens: Vec<String> = tokens.iter().map(|t| (*t).to_owned()).collect();
1158 let (tx, rx) = std::sync::mpsc::channel();
1159 std::thread::spawn(move || {
1160 let borrowed: Vec<&str> = tokens.iter().map(String::as_str).collect();
1161 let mut engine = StubEngine::new(&borrowed);
1162 engine.finish_reason = finish_reason;
1163 let _ = serve_with(
1164 &mut engine,
1165 ServeConfig {
1166 port: 0,
1167 ..ServeConfig::default()
1168 },
1169 |address| {
1170 let _ = tx.send(address);
1171 },
1172 );
1173 });
1174 rx.recv_timeout(Duration::from_secs(10))
1175 .expect("the server binds")
1176 }
1177
1178 fn roundtrip(address: SocketAddr, raw: &str) -> String {
1179 let mut stream = std::net::TcpStream::connect(address).expect("connect");
1180 stream
1181 .set_read_timeout(Some(Duration::from_secs(10)))
1182 .expect("timeout");
1183 stream.write_all(raw.as_bytes()).expect("write");
1184 stream.flush().expect("flush");
1185 let mut out = Vec::new();
1186 let mut buffer = [0u8; 4096];
1187 loop {
1188 match stream.read(&mut buffer) {
1189 Ok(0) | Err(_) => break,
1190 Ok(n) => out.extend_from_slice(&buffer[..n]),
1191 }
1192 }
1193 String::from_utf8_lossy(&out).into_owned()
1194 }
1195
1196 fn get(address: SocketAddr, path: &str) -> String {
1197 roundtrip(
1198 address,
1199 &format!("GET {path} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"),
1200 )
1201 }
1202
1203 fn post(address: SocketAddr, path: &str, body: &str) -> String {
1204 roundtrip(
1205 address,
1206 &format!(
1207 "POST {path} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\
1208 Content-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}",
1209 body.len()
1210 ),
1211 )
1212 }
1213
1214 #[test]
1215 fn the_buffered_routes_answer_over_a_socket() {
1216 let address = spawn_server();
1217 assert!(address.ip().is_loopback(), "{address}");
1219
1220 let health = get(address, PATH_HEALTH);
1221 assert!(health.starts_with("HTTP/1.1 200"), "{health}");
1222 assert!(health.contains(r#"{"status":"ok"}"#), "{health}");
1223
1224 let models = get(address, PATH_MODELS);
1225 assert!(models.starts_with("HTTP/1.1 200"), "{models}");
1226 assert!(models.contains(r#""id":"stub-model""#), "{models}");
1227 assert!(models.contains(r#""object":"list""#), "{models}");
1228
1229 let versioned = get(address, "/v1/models?api-version=2024-02-01");
1231 assert!(versioned.contains(r#""id":"stub-model""#), "{versioned}");
1232
1233 let missing = get(address, "/v1/embeddings");
1234 assert!(missing.starts_with("HTTP/1.1 404"), "{missing}");
1235 assert!(
1236 missing.contains(r#""type":"invalid_request_error""#),
1237 "{missing}"
1238 );
1239 assert!(missing.contains(r#""code":"not_found""#), "{missing}");
1240
1241 let wrong = get(address, PATH_CHAT_COMPLETIONS);
1244 assert!(wrong.starts_with("HTTP/1.1 404"), "{wrong}");
1245 assert!(wrong.contains(r#""error""#), "{wrong}");
1246 }
1247
1248 #[test]
1249 fn a_non_streaming_completion_comes_back_as_one_json_body() {
1250 let address = spawn_server();
1251 let response = post(
1252 address,
1253 PATH_CHAT_COMPLETIONS,
1254 r#"{"model":"stub-model","messages":[{"role":"user","content":"hi"}]}"#,
1255 );
1256 assert!(response.starts_with("HTTP/1.1 200"), "{response}");
1257 assert!(
1258 response.contains(r#""object":"chat.completion""#),
1259 "{response}"
1260 );
1261 assert!(response.contains(r#""content":"ok""#), "{response}");
1262 assert!(response.contains(r#""finish_reason":"stop""#), "{response}");
1263 }
1264
1265 const TOOL_CALL_REPLY: &str = "Let me look.\n<tool_call>\n{\"name\": \"read\", \"arguments\": {\"path\": \"a.txt\"}}\n</tool_call>";
1268
1269 fn body_json(response: &str) -> serde_json::Value {
1271 let body = response
1272 .split_once("\r\n\r\n")
1273 .map_or(response, |(_, body)| body);
1274 serde_json::from_str(body).unwrap_or_else(|e| panic!("a JSON body: {e}\n{response}"))
1275 }
1276
1277 #[test]
1281 fn a_tool_call_reply_comes_back_as_structured_calls() {
1282 let address = spawn_server_with(&[TOOL_CALL_REPLY], FinishReason::Stop);
1283 let response = post(
1284 address,
1285 PATH_CHAT_COMPLETIONS,
1286 r#"{"model":"stub-model","messages":[{"role":"user","content":"read a.txt"}],
1287 "tools":[{"type":"function","function":{"name":"read","parameters":{}}}]}"#,
1288 );
1289 assert!(response.starts_with("HTTP/1.1 200"), "{response}");
1290 let json = body_json(&response);
1291 let choice = &json["choices"][0];
1292
1293 assert_eq!(choice["finish_reason"], "tool_calls");
1296
1297 let calls = choice["message"]["tool_calls"]
1298 .as_array()
1299 .unwrap_or_else(|| panic!("a tool_calls array: {json}"));
1300 assert_eq!(calls.len(), 1);
1301 assert_eq!(calls[0]["type"], "function");
1302 assert_eq!(calls[0]["function"]["name"], "read");
1303 assert_eq!(calls[0]["function"]["arguments"], r#"{"path": "a.txt"}"#);
1305 let call_id = calls[0]["id"].as_str().expect("an id");
1306 assert!(call_id.starts_with("call_"), "{call_id}");
1307 assert!(call_id.ends_with("_0"), "{call_id}");
1308 assert!(
1309 call_id.contains(
1310 json["id"]
1311 .as_str()
1312 .and_then(|id| id.strip_prefix("chatcmpl-"))
1313 .expect("a completion id")
1314 ),
1315 "the call id is derived from the completion id: {call_id}"
1316 );
1317
1318 assert_eq!(choice["message"]["content"], "Let me look.");
1320 assert!(
1321 !response.contains("<tool_call>"),
1322 "the raw block is still in the body: {response}"
1323 );
1324 }
1325
1326 #[test]
1333 fn a_block_in_a_reply_to_a_request_without_tools_stays_text() {
1334 let address = spawn_server_with(&[TOOL_CALL_REPLY], FinishReason::Stop);
1335 let response = post(
1336 address,
1337 PATH_CHAT_COMPLETIONS,
1338 r#"{"model":"stub-model","messages":[{"role":"user","content":"show me a tool call"}]}"#,
1339 );
1340 assert!(response.starts_with("HTTP/1.1 200"), "{response}");
1341 let json = body_json(&response);
1342 let choice = &json["choices"][0];
1343
1344 assert_eq!(choice["finish_reason"], "stop");
1345 assert!(
1346 choice["message"]["tool_calls"].is_null(),
1347 "a request without tools got tool_calls back: {json}"
1348 );
1349 assert_eq!(
1351 choice["message"]["content"].as_str().expect("content"),
1352 TOOL_CALL_REPLY
1353 );
1354 }
1355
1356 #[test]
1359 fn a_plain_reply_carries_no_tool_calls_key() {
1360 let address = spawn_server();
1361 let response = post(
1362 address,
1363 PATH_CHAT_COMPLETIONS,
1364 r#"{"model":"stub-model","messages":[{"role":"user","content":"hi"}]}"#,
1365 );
1366 let json = body_json(&response);
1367 assert_eq!(json["choices"][0]["message"]["content"], "ok");
1368 assert_eq!(json["choices"][0]["finish_reason"], "stop");
1369 assert!(
1370 json["choices"][0]["message"].get("tool_calls").is_none(),
1371 "{json}"
1372 );
1373 }
1374
1375 #[test]
1379 fn a_block_truncated_by_the_token_budget_keeps_the_length_reason() {
1380 let address = spawn_server_with(
1381 &["<tool_call>\n{\"name\": \"read\", \"argum"],
1382 FinishReason::Length,
1383 );
1384 let response = post(
1385 address,
1386 PATH_CHAT_COMPLETIONS,
1387 r#"{"model":"stub-model","messages":[{"role":"user","content":"hi"}],"max_tokens":8}"#,
1388 );
1389 let json = body_json(&response);
1390 assert_eq!(json["choices"][0]["finish_reason"], "length");
1391 assert!(
1392 json["choices"][0]["message"].get("tool_calls").is_none(),
1393 "{json}"
1394 );
1395 assert_eq!(
1396 json["choices"][0]["message"]["content"], "<tool_call>\n{\"name\": \"read\", \"argum",
1397 "half a block is text, and text is not dropped"
1398 );
1399 }
1400
1401 #[test]
1407 fn a_streamed_call_becomes_a_tool_calls_delta_and_no_raw_markup() {
1408 let mut engine = StubEngine::with_ids(&glob_call_tokens());
1409 let recorder = Recorder::new();
1410 run_stream_for(
1411 &mut engine,
1412 &request_with_tools(),
1413 recorder.clone(),
1414 Duration::from_secs(60),
1415 );
1416
1417 let body = recorder.body();
1418 assert!(
1419 !body.contains("tool_call>"),
1420 "the raw markup is still on the wire: {body}"
1421 );
1422
1423 let calls = streamed_calls(&body);
1424 assert_eq!(calls.len(), 1, "{body}");
1425 assert_eq!(calls[0]["index"], 0);
1426 assert_eq!(calls[0]["type"], "function");
1427 assert_eq!(calls[0]["function"]["name"], "glob");
1428 assert_eq!(
1429 calls[0]["function"]["arguments"], r#"{"pattern": "*/.rs"}"#,
1430 "the model's argument bytes reach the client unaltered"
1431 );
1432 assert_eq!(streamed_finish(&body), "tool_calls");
1434 assert_eq!(streamed_text(&body), "", "the block left no prose behind");
1435 assert!(body.contains("[DONE]"), "{body}");
1436 assert!(body.ends_with(TERMINATOR), "{body}");
1437 }
1438
1439 #[test]
1442 fn prose_before_between_and_after_streamed_calls_is_still_content() {
1443 let mut tokens = vec![(TEXT_ID, "Let me "), (TEXT_ID, "look.\n")];
1444 tokens.extend(glob_call_tokens());
1445 tokens.push((TEXT_ID, "\nand the tests\n"));
1446 tokens.extend(glob_call_tokens());
1447 tokens.push((TEXT_ID, "\nThen I will compare."));
1448
1449 let mut engine = StubEngine::with_ids(&tokens);
1450 let recorder = Recorder::new();
1451 run_stream_for(
1452 &mut engine,
1453 &request_with_tools(),
1454 recorder.clone(),
1455 Duration::from_secs(60),
1456 );
1457
1458 let body = recorder.body();
1459 assert_eq!(
1460 streamed_text(&body),
1461 "Let me look.\n\nand the tests\n\nThen I will compare."
1462 );
1463 assert!(!body.contains("tool_call>"), "{body}");
1464 }
1465
1466 #[test]
1468 fn two_streamed_calls_get_indices_zero_and_one() {
1469 let mut tokens = glob_call_tokens();
1470 tokens.extend(glob_call_tokens());
1471 let mut engine = StubEngine::with_ids(&tokens);
1472 let recorder = Recorder::new();
1473 run_stream_for(
1474 &mut engine,
1475 &request_with_tools(),
1476 recorder.clone(),
1477 Duration::from_secs(60),
1478 );
1479
1480 let body = recorder.body();
1481 let calls = streamed_calls(&body);
1482 assert_eq!(calls.len(), 2, "{body}");
1483 assert_eq!(calls[0]["index"], 0);
1484 assert_eq!(calls[1]["index"], 1);
1485 assert_ne!(calls[0]["id"], calls[1]["id"]);
1486 assert_eq!(streamed_finish(&body), "tool_calls");
1487 for call in &calls {
1490 assert_eq!(call["function"]["arguments"], r#"{"pattern": "*/.rs"}"#);
1491 }
1492 }
1493
1494 #[test]
1497 fn an_unterminated_streamed_block_flushes_as_content_and_keeps_length() {
1498 let mut engine = StubEngine::with_ids(&[
1499 (TEXT_ID, "Reading it.\n"),
1500 (crate::toolcall::OPEN_TAG_ID, "<tool_call>"),
1501 (TEXT_ID, "\n{\"name\": \"glob\", \"argum"),
1502 ]);
1503 engine.finish_reason = FinishReason::Length;
1504 let recorder = Recorder::new();
1505 run_stream_for(
1506 &mut engine,
1507 &request_with_tools(),
1508 recorder.clone(),
1509 Duration::from_secs(60),
1510 );
1511
1512 let body = recorder.body();
1513 assert!(streamed_calls(&body).is_empty(), "{body}");
1514 assert_eq!(streamed_finish(&body), "length");
1515 assert_eq!(
1516 streamed_text(&body),
1517 "Reading it.\n<tool_call>\n{\"name\": \"glob\", \"argum",
1518 "half a block is text, and text is not dropped"
1519 );
1520 }
1521
1522 #[test]
1526 fn a_streamed_block_without_tools_stays_ordinary_text() {
1527 let mut engine = StubEngine::with_ids(&glob_call_tokens());
1528 let recorder = Recorder::new();
1529 run_stream(&mut engine, recorder.clone(), Duration::from_secs(60));
1530
1531 let body = recorder.body();
1532 assert!(
1533 streamed_calls(&body).is_empty(),
1534 "a request without tools got tool_calls back: {body}"
1535 );
1536 assert_eq!(streamed_finish(&body), "stop");
1537 assert_eq!(
1538 streamed_text(&body),
1539 "<tool_call>\n{\"name\": \"glob\", \"arguments\": {\"pattern\": \"*/.rs\"}}\n</tool_call>",
1540 "byte for byte, markers and all"
1541 );
1542 }
1543
1544 #[test]
1547 fn malformed_json_in_a_streamed_block_still_yields_a_call() {
1548 let mut engine = StubEngine::with_ids(&[
1549 (crate::toolcall::OPEN_TAG_ID, "<tool_call>"),
1550 (
1551 TEXT_ID,
1552 "\n{\"name\": \"glob\", \"arguments\": {\"pattern\": \"*\",}}\n",
1553 ),
1554 (crate::toolcall::CLOSE_TAG_ID, "</tool_call>"),
1555 ]);
1556 let recorder = Recorder::new();
1557 run_stream_for(
1558 &mut engine,
1559 &request_with_tools(),
1560 recorder.clone(),
1561 Duration::from_secs(60),
1562 );
1563
1564 let body = recorder.body();
1565 let calls = streamed_calls(&body);
1566 assert_eq!(calls.len(), 1, "{body}");
1567 assert_eq!(calls[0]["function"]["name"], "glob");
1568 assert_eq!(calls[0]["function"]["arguments"], r#"{"pattern": "*",}"#);
1569 assert_eq!(streamed_finish(&body), "tool_calls");
1570 }
1571
1572 #[test]
1575 fn streamed_call_ids_match_the_buffered_paths_for_the_same_completion() {
1576 const ID: &str = "chatcmpl-18f3a0001";
1577 let mut engine = StubEngine::with_ids(&glob_call_tokens());
1578 let recorder = Recorder::new();
1579 let request = request_with_tools();
1580 let plan = engine.prepare(&request).expect("a plan");
1581 stream_into(
1582 &mut engine,
1583 &request,
1584 plan,
1585 recorder.clone(),
1586 ID,
1587 1_700_000_000,
1588 Duration::from_secs(60),
1589 );
1590
1591 let body = recorder.body();
1592 let streamed = streamed_calls(&body);
1593 let reply: String = glob_call_tokens().iter().map(|(_, text)| *text).collect();
1595 let buffered = crate::toolcall::wire_calls(ID, &crate::toolcall::extract(&reply).calls);
1596 assert_eq!(buffered.len(), 1);
1597 assert_eq!(streamed[0]["id"], buffered[0].id);
1598 assert_eq!(streamed[0]["id"], "call_18f3a0001_0");
1599 }
1600
1601 #[test]
1604 fn a_tool_call_delta_is_framed_like_every_other_event() {
1605 let mut engine = StubEngine::with_ids(&glob_call_tokens());
1606 let recorder = Recorder::new();
1607 run_stream_for(
1608 &mut engine,
1609 &request_with_tools(),
1610 recorder.clone(),
1611 Duration::from_secs(60),
1612 );
1613
1614 let body = recorder.body();
1615 let mut rest = body.strip_prefix(SSE_HEAD).expect("head first");
1616 let mut events = 0usize;
1617 loop {
1618 let (head, tail) = rest.split_once("\r\n").expect("a chunk header");
1619 let len = usize::from_str_radix(head, 16).expect("hex length");
1620 if len == 0 {
1621 assert_eq!(tail, "\r\n", "the terminator ends the body");
1622 break;
1623 }
1624 assert_eq!(&tail[len..len + 2], "\r\n", "chunk not CRLF-terminated");
1625 let payload = &tail[..len];
1626 assert_eq!(payload.matches("\n\n").count(), 1, "{payload:?}");
1627 events += 1;
1628 rest = &tail[len + 2..];
1629 }
1630 assert_eq!(events, 4, "{body}");
1632 }
1633
1634 #[test]
1637 fn the_array_form_content_reaches_the_engine() {
1638 let address = spawn_server();
1639 let response = post(
1640 address,
1641 PATH_CHAT_COMPLETIONS,
1642 r#"{"model":"stub-model","messages":[
1643 {"role":"user","content":[{"type":"text","text":"hi"}]}]}"#,
1644 );
1645 assert!(response.starts_with("HTTP/1.1 200"), "{response}");
1646 assert!(response.contains(r#""content":"ok""#), "{response}");
1647 }
1648
1649 #[test]
1650 fn a_malformed_body_is_a_400_with_the_openai_shape() {
1651 let address = spawn_server();
1652 let response = post(address, PATH_CHAT_COMPLETIONS, "{not json");
1653 assert!(response.starts_with("HTTP/1.1 400"), "{response}");
1654 assert!(response.contains(r#""code":"invalid_json""#), "{response}");
1655 assert!(!response.contains(r#""message":null"#), "{response}");
1656 }
1657
1658 #[test]
1661 fn an_oversized_body_is_refused_by_its_declared_length() {
1662 const BODY: &str = "0123456789abcdef0123456789abcdef";
1663 let mut request = tiny_http::TestRequest::new()
1664 .with_method(Method::Post)
1665 .with_path(PATH_CHAT_COMPLETIONS)
1666 .with_body(BODY)
1667 .into();
1668 let error = read_body(&mut request, 16).expect_err("over the cap");
1669 assert!(matches!(error, ServerError::BodyTooLarge { limit: 16 }));
1670 assert_eq!(error.status(), 413);
1671 }
1672
1673 #[test]
1674 fn a_body_at_the_cap_is_accepted() {
1675 const BODY: &str = "0123456789abcdef";
1676 let mut request = tiny_http::TestRequest::new()
1677 .with_method(Method::Post)
1678 .with_path(PATH_CHAT_COMPLETIONS)
1679 .with_body(BODY)
1680 .into();
1681 assert_eq!(
1682 read_body(&mut request, BODY.len()).expect("at the cap"),
1683 BODY
1684 );
1685 }
1686
1687 #[test]
1688 fn completion_ids_do_not_repeat() {
1689 let ids: std::collections::HashSet<String> = (0..64).map(|_| completion_id()).collect();
1690 assert_eq!(ids.len(), 64);
1691 assert!(ids.iter().all(|id| id.starts_with("chatcmpl-")));
1692 }
1693}