ramvamp_server/engine.rs
1//! The thing that turns a request into tokens, and the seam that lets the
2//! transport be tested without one.
3//!
4//! # Why there is a trait here
5//!
6//! Streaming behaviour is not verifiable by reading code. A response that is
7//! buffered to completion and one that is delivered live produce *byte
8//! identical* transcripts; they differ only in when the bytes arrive. The only
9//! way to assert the difference is to drive the transport from a source that
10//! emits known tokens at known intervals — which means the transport cannot be
11//! wired straight to a 30B model. Hence [`Engine`]: `ModelEngine` is one
12//! implementation, and the tests use another that sleeps.
13//!
14//! # The two-step shape
15//!
16//! [`Engine::prepare`] does everything that can fail *before* a byte of
17//! response is committed — parsing the conversation, encoding it, checking it
18//! against the context cap — and [`Engine::run`] does the part that cannot be
19//! taken back. That split is what lets a context overflow be a clean HTTP 400
20//! while a mid-generation failure is reported inside an already-200 stream.
21//!
22//! # Prefix caching, and the trap in it
23//!
24//! A Chat Completions client is stateless: every request carries the whole
25//! conversation, so a naive server re-prefills all of it every turn, which at
26//! this runtime's speeds is minutes of work to reach a state it already had.
27//! The fix is to keep the KV cache across requests and re-prefill only the
28//! part that changed — [`ForwardState::truncate`] plus a longest-common-prefix
29//! match.
30//!
31//! The trap is *what to match against*, and it fails silently. The client
32//! sends previous assistant turns back as **text**, and re-encoding that text
33//! does not reproduce the ids the model generated: the `\n` that closes
34//! `<|im_start|>assistant\n` and the first characters of the reply are
35//! candidates for the same BPE merge, so a generation-prompt render is not
36//! always a token prefix of the finished turn (measured; the CLI has a test
37//! named after it). Comparing against a re-render therefore produces a *wrong*
38//! cache hit — the positions are reused, the ids at them are not the ids the
39//! cache holds, `generate_from` cannot detect it because the arithmetic still
40//! adds up, and the model answers fluently from a context that never existed.
41//!
42//! So the comparison is against [`ModelEngine::history`], which holds the ids
43//! that were actually fed: prompt renders as encoded, and assistant turns as
44//! [`GenerateStats::generated_ids`], never as a re-encoding of the reply.
45//! Where the client's re-render diverges from what the model said, the common
46//! prefix simply ends there and the suffix is re-prefilled. Correct by
47//! construction rather than by luck.
48
49use std::cell::RefCell;
50use std::io::ErrorKind;
51use std::sync::Once;
52use std::time::{SystemTime, UNIX_EPOCH};
53
54use ramvamp_core::generate::{
55 GenerateParams, GenerateProgress, StopReason, generate_from_with_progress,
56};
57use ramvamp_core::model::{ForwardState, Model};
58use ramvamp_core::tokenizer::RvmpTokenizer;
59use thiserror::Error;
60
61use crate::error::ServerError;
62use crate::prompt;
63use crate::request::ChatCompletionRequest;
64use crate::response::{FinishReason, Usage};
65
66/// A failure writing to whoever is consuming the stream.
67///
68/// Deliberately `Copy`: the transport shares one of these between the thread
69/// running the model and the keep-alive thread, and the `io::Error` payload
70/// carries nothing worth the `Arc` — the kind is the whole story, and the
71/// response to every kind is the same.
72#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
73pub enum StreamError {
74 /// The client hung up. The ordinary way a stream ends early, and not an
75 /// error worth logging above `debug`.
76 #[error("the client closed the connection")]
77 Disconnected,
78 /// Any other write failure.
79 #[error("writing to the client failed: {0:?}")]
80 Io(ErrorKind),
81}
82
83impl From<std::io::Error> for StreamError {
84 /// Classify a write failure. The four kinds `tiny_http` itself treats as
85 /// "client went away" are the four treated as a disconnect here — the
86 /// difference being that here it is reported rather than swallowed.
87 fn from(error: std::io::Error) -> Self {
88 match error.kind() {
89 ErrorKind::BrokenPipe
90 | ErrorKind::ConnectionReset
91 | ErrorKind::ConnectionAborted
92 | ErrorKind::ConnectionRefused => StreamError::Disconnected,
93 other => StreamError::Io(other),
94 }
95 }
96}
97
98/// Where a running generation reports to.
99///
100/// Both methods return a `Result` because the usual implementation is a
101/// socket, and a socket that has gone away is the signal to stop generating —
102/// there is nobody left to generate for, and the session is worth minutes.
103pub trait TokenSink {
104 /// Prompt positions committed so far, out of the whole prompt.
105 ///
106 /// Counted against the *whole* prompt including the part served from
107 /// cache, so a cache hit shows up as progress already made rather than as
108 /// a prefill that starts at zero and finishes suspiciously fast.
109 fn prefill(&mut self, positions_done: usize, positions_total: usize)
110 -> Result<(), StreamError>;
111
112 /// One token: the id the model sampled, and the text it decoded to.
113 ///
114 /// The id is carried because `<tool_call>` and `</tool_call>` are single
115 /// added tokens and the decoder does not skip them, so a sink that has the
116 /// id can recognise a call block by an integer compare. The alternative —
117 /// scanning the decoded text — would have to be done at a layer that sees
118 /// only a stream of fragments, where a marker can be split across two of
119 /// them; an id cannot be split. See [`crate::toolcall::CallStream`].
120 fn token(&mut self, id: u32, text: &str) -> Result<(), StreamError>;
121}
122
123/// A sink that discards everything, for the non-streaming path.
124///
125/// The reply text comes back in [`Completion::text`] either way, so the
126/// buffered endpoint needs nothing from the sink but a place to not send it.
127#[derive(Debug, Default, Clone, Copy)]
128pub struct NullSink;
129
130impl TokenSink for NullSink {
131 fn prefill(&mut self, _: usize, _: usize) -> Result<(), StreamError> {
132 Ok(())
133 }
134
135 fn token(&mut self, _: u32, _: &str) -> Result<(), StreamError> {
136 Ok(())
137 }
138}
139
140/// One accepted request, ready to run.
141///
142/// Everything that could have been refused already has been, which is the
143/// point: holding a `Plan` means the response head can be committed.
144#[derive(Debug, Clone, PartialEq)]
145pub struct Plan {
146 /// The whole prompt, cached prefix included. This is the number the
147 /// client is billed in `usage.prompt_tokens`, not the suffix actually
148 /// prefilled — a cache hit is this server's business, not a discount the
149 /// client can observe and come to depend on.
150 pub prompt_tokens: usize,
151 /// Reply cap for this turn, after clamping to the context.
152 pub max_new: usize,
153 /// Leading positions the KV cache already holds and this turn reuses.
154 pub cached_prefix: usize,
155 /// The full sanitized prompt id sequence.
156 pub prompt_ids: Vec<u32>,
157 /// Sampling dials, merged from the request over the checkpoint defaults.
158 pub params: GenerateParams,
159}
160
161impl Plan {
162 /// How many positions this turn actually has to prefill.
163 pub fn to_prefill(&self) -> usize {
164 self.prompt_ids.len().saturating_sub(self.cached_prefix)
165 }
166}
167
168/// What a finished (or abandoned) generation produced.
169#[derive(Debug, Clone, PartialEq, Eq)]
170pub struct Completion {
171 /// The reply text, concatenated from what the sink was shown.
172 pub text: String,
173 /// Token accounting for the whole request.
174 pub usage: Usage,
175 /// Why it ended.
176 pub finish_reason: FinishReason,
177 /// The sink asked to stop — in practice, the client hung up. Nothing more
178 /// should be written; there is nowhere to write it.
179 pub aborted: bool,
180}
181
182/// The generator behind the HTTP surface.
183pub trait Engine {
184 /// The model id `/v1/models` advertises and every response echoes.
185 fn model_id(&self) -> &str;
186
187 /// Unix seconds to report as the model's creation time.
188 fn created(&self) -> u64;
189
190 /// Everything that can be refused before the response is committed.
191 ///
192 /// # Errors
193 ///
194 /// Any [`ServerError`] whose status is a 4xx: this is where a bad request
195 /// becomes an HTTP status rather than a broken stream.
196 fn prepare(&mut self, request: &ChatCompletionRequest) -> Result<Plan, ServerError>;
197
198 /// Run the plan, reporting through `sink`.
199 ///
200 /// A sink failure is not an error here: it produces a [`Completion`] with
201 /// [`Completion::aborted`] set, because "the client left" is an ordinary
202 /// outcome and the engine must be ready for the next request either way.
203 ///
204 /// # Errors
205 ///
206 /// [`ServerError::Generate`] or [`ServerError::Forward`] when the runtime
207 /// itself fails. The engine is left able to serve the next request.
208 fn run(&mut self, plan: Plan, sink: &mut dyn TokenSink) -> Result<Completion, ServerError>;
209}
210
211/// Payload of the deliberate unwind that stops a generation whose client has
212/// gone away.
213///
214/// The same device, for the same reason, as `ChatAbort` in the CLI:
215/// `generate_from`'s `on_token` returns `()`, so it has no way to report that
216/// it wants out, and an unwind is the only exit from a decode loop in
217/// progress. It is raised at a token boundary — between two forward passes,
218/// with no expert read in flight and the KV cache whole — which is what makes
219/// the state reusable afterwards.
220#[derive(Debug)]
221pub struct ServerAbort;
222
223/// Install, once per process, a panic hook that prints nothing for
224/// [`ServerAbort`] and forwards every other panic to the hook already in
225/// place.
226///
227/// Without it a client pressing Ctrl-C dumps a "thread panicked" backtrace
228/// into the server's log for what is an entirely routine event. Real panics,
229/// on any thread, still print exactly as before.
230pub fn hush_stream_aborts() {
231 static ONCE: Once = Once::new();
232 ONCE.call_once(|| {
233 let previous = std::panic::take_hook();
234 std::panic::set_hook(Box::new(move |info| {
235 if info.payload().downcast_ref::<ServerAbort>().is_none() {
236 previous(info);
237 }
238 }));
239 });
240}
241
242/// Length of the longest common prefix of two id sequences.
243pub fn common_prefix(a: &[u32], b: &[u32]) -> usize {
244 a.iter().zip(b).take_while(|(x, y)| x == y).count()
245}
246
247/// Unix seconds, or 0 if the clock is before the epoch.
248pub fn unix_now() -> u64 {
249 SystemTime::now()
250 .duration_since(UNIX_EPOCH)
251 .map_or(0, |d| d.as_secs())
252}
253
254/// How a [`ModelEngine`] was configured.
255#[derive(Debug, Clone)]
256pub struct EngineConfig {
257 /// The id to advertise and echo.
258 pub model_id: String,
259 /// The context cap the [`ForwardState`] was built with.
260 pub context_limit: usize,
261 /// Reply cap for a request that does not send one.
262 pub default_max_new: usize,
263 /// Sampling dials to start from; a request overrides field by field.
264 pub params: GenerateParams,
265}
266
267/// The real engine: a model, its state and its tokenizer, on one thread.
268///
269/// [`ForwardState`] is `Send` but not `Sync` — it owns the expert slot pool,
270/// the io_uring ring and the pinned compute pool — so it is moved onto the
271/// thread that serves requests and never shared. That is not a limitation
272/// being worked around: concurrency here is one, structurally, because a
273/// second decode loop would halve the token rate of both and there is exactly
274/// one KV cache.
275pub struct ModelEngine {
276 model: Model,
277 state: ForwardState,
278 tokenizer: RvmpTokenizer,
279 config: EngineConfig,
280 created: u64,
281 /// Every id the cached conversation consists of: prompt renders as
282 /// encoded, assistant turns as the ids the model actually generated.
283 ///
284 /// Runs *ahead* of the KV cache by design — the last id sampled is
285 /// emitted and never fed back, because there is nothing left to predict
286 /// from it — so [`ForwardState::seq_len`] is the authority on how much of
287 /// it the cache holds, never a count kept alongside. See the module docs
288 /// for why this is not a re-render of the transcript.
289 history: Vec<u32>,
290 /// Requests served, used to vary the seed when the client does not pin
291 /// one. Without it every repeat of a question gets a byte-identical
292 /// answer, which reads as a broken model rather than as determinism.
293 turn: u64,
294}
295
296impl ModelEngine {
297 /// Build an engine over an already-loaded model and state.
298 ///
299 /// The loading is the caller's, deliberately: `state` carries the cache
300 /// budget, thread pinning and prefill dials, and those belong to the
301 /// command line that set them rather than to this crate.
302 pub fn new(
303 model: Model,
304 state: ForwardState,
305 tokenizer: RvmpTokenizer,
306 config: EngineConfig,
307 ) -> Self {
308 ModelEngine {
309 model,
310 state,
311 tokenizer,
312 config,
313 created: unix_now(),
314 history: Vec::new(),
315 turn: 0,
316 }
317 }
318
319 /// The ids the cached conversation consists of.
320 pub fn history(&self) -> &[u32] {
321 &self.history
322 }
323
324 /// Merge a request's sampling fields over the configured defaults.
325 fn sampling(&self, request: &ChatCompletionRequest, max_new: usize) -> GenerateParams {
326 let mut params = self.config.params;
327 params.max_new = max_new;
328 if let Some(temperature) = request.temperature {
329 // <= 0 is greedy, which `generate` already implements; passing it
330 // through rather than translating keeps one rule in one place.
331 params.temperature = temperature;
332 }
333 if let Some(top_p) = request.top_p {
334 params.top_p = top_p;
335 }
336 if let Some(top_k) = request.top_k {
337 params.top_k = Some(top_k);
338 }
339 // An unpinned seed advances per request; a pinned one is honoured
340 // exactly, which is what `seed` means to a client.
341 params.seed = request
342 .seed
343 .unwrap_or_else(|| self.config.params.seed.wrapping_add(self.turn));
344 params
345 }
346
347 /// Recover from a runtime failure that may have left the cache ragged.
348 ///
349 /// Cheap — every allocation survives — and the alternative is an engine
350 /// that answers `PositionMismatch` to every request for the rest of its
351 /// life.
352 fn recover(&mut self) {
353 self.state.reset();
354 self.history.clear();
355 }
356}
357
358impl Engine for ModelEngine {
359 fn model_id(&self) -> &str {
360 &self.config.model_id
361 }
362
363 fn created(&self) -> u64 {
364 self.created
365 }
366
367 fn prepare(&mut self, request: &ChatCompletionRequest) -> Result<Plan, ServerError> {
368 request.validate()?;
369 // Sanitized here, before anything is compared: the ids that get
370 // prefilled are the sanitized ones, so a prefix match against the
371 // unsanitized stream would be a match against a sequence this server
372 // never feeds the model.
373 let built = prompt::build(&self.tokenizer, request)?;
374 if built.token_ids.is_empty() {
375 return Err(ServerError::EmptyMessages);
376 }
377
378 let max_new = request
379 .max_new_tokens()
380 .map_or(self.config.default_max_new, |n| n as usize)
381 .max(1);
382 prompt::check_context(built.token_ids.len(), max_new, self.config.context_limit)?;
383
384 // The cache says where it is; `history` says what is in it. The
385 // smaller of the two bounds what can be reused, because `history`
386 // runs one id ahead of the cache after every reply.
387 let fed = self.state.seq_len()?;
388 let shared = common_prefix(&self.history, &built.token_ids);
389 // At least one token has to be prefilled: `generate_with_stops`
390 // refuses an empty prompt with `EmptyPrompt`, and an identical resend
391 // (the incoming ids being a prefix of the history) would otherwise
392 // produce exactly that. Rewinding one position is the cheapest legal
393 // suffix.
394 let cached_prefix = shared.min(fed).min(built.token_ids.len() - 1);
395
396 tracing::debug!(
397 prompt_tokens = built.token_ids.len(),
398 cached = cached_prefix,
399 prefill = built.token_ids.len() - cached_prefix,
400 max_new,
401 "prepared a request"
402 );
403
404 Ok(Plan {
405 prompt_tokens: built.token_ids.len(),
406 max_new,
407 cached_prefix,
408 params: self.sampling(request, max_new),
409 prompt_ids: built.token_ids,
410 })
411 }
412
413 fn run(&mut self, plan: Plan, sink: &mut dyn TokenSink) -> Result<Completion, ServerError> {
414 // A truncate that does nothing still fails loudly downstream:
415 // `forward_token` re-validates `position == seq_len()` and raises
416 // `PositionMismatch`. So this is not papered over on any path.
417 if let Err(e) = self.state.truncate(plan.cached_prefix) {
418 self.recover();
419 return Err(ServerError::Forward(e));
420 }
421 self.turn = self.turn.wrapping_add(1);
422
423 let prompt_total = plan.prompt_ids.len();
424 let cached_prefix = plan.cached_prefix;
425 let new_ids = &plan.prompt_ids[cached_prefix..];
426
427 // One cell for everything both callbacks touch. They are never live
428 // at the same instant — `generate` calls one or the other, never both
429 // — but the borrow checker cannot see that through two closures held
430 // simultaneously, and a `RefCell` is the honest way to say so.
431 let live = RefCell::new(Live {
432 sink,
433 reply: String::new(),
434 spoken: Vec::new(),
435 abort: None,
436 });
437
438 let outcome = {
439 let mut on_progress = |event: GenerateProgress| {
440 let GenerateProgress::PrefillChunk { positions_done, .. } = event else {
441 return;
442 };
443 let mut live = live.borrow_mut();
444 if live.abort.is_some() {
445 return;
446 }
447 // Reported against the whole prompt, cached prefix included.
448 let done = cached_prefix + positions_done;
449 if let Err(e) = live.sink.prefill(done, prompt_total) {
450 // Recorded, *not* unwound. A prefill chunk boundary is
451 // inside `prefill_sweep`, which still owns a
452 // `PrefillSession` over the streamer's staging arena and
453 // has not called `finish()` on it; unwinding through that
454 // abandons the arena rather than releasing it. The decode
455 // loop's token boundary is the one place the runtime
456 // documents as safe to leave, so the abort waits for it.
457 // The cost is finishing a prefill nobody will read, which
458 // is bounded by the prompt.
459 live.abort = Some(e);
460 }
461 };
462
463 let mut on_token = |id: u32, text: &str| {
464 let mut live = live.borrow_mut();
465 live.reply.push_str(text);
466 live.spoken.push(id);
467 if live.abort.is_none()
468 && let Err(e) = live.sink.token(id, text)
469 {
470 live.abort = Some(e);
471 }
472 if live.abort.is_some() {
473 // Release the borrow before unwinding: the guard would
474 // drop anyway, but leaving it to the unwinder makes the
475 // catch site's `into_inner` depend on drop order.
476 drop(live);
477 std::panic::panic_any(ServerAbort);
478 }
479 };
480
481 std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
482 generate_from_with_progress(
483 &self.model,
484 &mut self.state,
485 &self.tokenizer,
486 new_ids,
487 cached_prefix,
488 &plan.params,
489 Some(&mut on_progress),
490 &mut on_token,
491 )
492 }))
493 };
494
495 let live = live.into_inner();
496 match outcome {
497 Ok(Ok(stats)) => {
498 // The cache now holds this prompt; the history is it plus what
499 // the model said, in the ids it actually produced.
500 self.history.clear();
501 self.history.extend_from_slice(&plan.prompt_ids);
502 self.history.extend_from_slice(&stats.generated_ids);
503 let finish_reason = match stats.stop {
504 StopReason::StopToken(_) => FinishReason::Stop,
505 StopReason::MaxNew => FinishReason::Length,
506 };
507 tracing::info!(
508 prompt_tokens = plan.prompt_tokens,
509 prefilled = new_ids.len(),
510 generated = stats.generated,
511 prefill_s = stats.prefill.as_secs_f64(),
512 decode_s = stats.decode.as_secs_f64(),
513 "served a completion"
514 );
515 Ok(Completion {
516 text: live.reply,
517 usage: Usage::new(
518 u32::try_from(plan.prompt_tokens).unwrap_or(u32::MAX),
519 u32::try_from(stats.generated).unwrap_or(u32::MAX),
520 ),
521 finish_reason,
522 aborted: false,
523 })
524 }
525 Ok(Err(e)) => {
526 // Mid-pass failure: the cache may be ragged, and a ragged one
527 // reports `PositionMismatch` forever.
528 self.recover();
529 Err(ServerError::Generate(e))
530 }
531 Err(payload) => {
532 if payload.downcast_ref::<ServerAbort>().is_none() {
533 // Somebody else's panic. Re-raise it untouched.
534 std::panic::resume_unwind(payload);
535 }
536 // The partial reply is in the cache up to its second-to-last
537 // id, so it has to be in the history too — otherwise the next
538 // turn would prefill from a position the model reached by a
539 // route the history no longer records. `spoken` is exactly
540 // what `generated_ids` would have held: the unwind leaves
541 // `on_token` before the id is fed, and the trailing flush
542 // call that would have repeated an id never happens.
543 self.history.clear();
544 self.history.extend_from_slice(&plan.prompt_ids);
545 self.history.extend_from_slice(&live.spoken);
546 tracing::debug!(
547 generated = live.spoken.len(),
548 reason = ?live.abort,
549 "client left; abandoned the reply"
550 );
551 Ok(Completion {
552 usage: Usage::new(
553 u32::try_from(plan.prompt_tokens).unwrap_or(u32::MAX),
554 u32::try_from(live.spoken.len()).unwrap_or(u32::MAX),
555 ),
556 text: live.reply,
557 finish_reason: FinishReason::Stop,
558 aborted: true,
559 })
560 }
561 }
562 }
563}
564
565/// The mutable half of a run, shared by the two callbacks.
566struct Live<'a> {
567 sink: &'a mut dyn TokenSink,
568 reply: String,
569 spoken: Vec<u32>,
570 abort: Option<StreamError>,
571}
572
573#[cfg(test)]
574mod tests {
575 use super::*;
576
577 #[test]
578 fn the_common_prefix_stops_at_the_first_divergent_id() {
579 assert_eq!(common_prefix(&[1, 2, 3], &[1, 2, 3]), 3);
580 assert_eq!(common_prefix(&[1, 2, 3], &[1, 2, 9, 3]), 2);
581 assert_eq!(common_prefix(&[1, 2, 3], &[]), 0);
582 assert_eq!(common_prefix(&[], &[1]), 0);
583 assert_eq!(common_prefix(&[1, 2], &[1, 2, 3, 4]), 2);
584 }
585
586 /// The identical-resend edge: the incoming ids are a prefix of the
587 /// history, so the naive suffix is empty and `generate_with_stops` answers
588 /// `EmptyPrompt`. Rewinding one position is the fix, and it costs one
589 /// forward pass.
590 #[test]
591 fn an_identical_resend_still_leaves_one_token_to_prefill() {
592 let history = [1u32, 2, 3, 4, 5];
593 let incoming = [1u32, 2, 3];
594 let fed = history.len();
595 let shared = common_prefix(&history, &incoming);
596 assert_eq!(shared, incoming.len());
597 let cached = shared.min(fed).min(incoming.len() - 1);
598 assert_eq!(cached, 2);
599 assert_eq!(incoming.len() - cached, 1);
600 }
601
602 /// The cache, not the history, bounds what can be reused: the history
603 /// runs one id ahead after every reply, because the last token sampled is
604 /// emitted and never fed back.
605 #[test]
606 fn the_cache_bounds_the_reusable_prefix_not_the_history() {
607 let history = [1u32, 2, 3, 4];
608 let incoming = [1u32, 2, 3, 4, 5, 6];
609 let fed = history.len() - 1;
610 let shared = common_prefix(&history, &incoming);
611 assert_eq!(shared, 4);
612 assert_eq!(shared.min(fed).min(incoming.len() - 1), 3);
613 }
614
615 /// The whole reason the history is not a re-render. A client sends the
616 /// assistant turn back as text; re-encoding it produces different ids
617 /// than the model generated, so the prefix ends at the divergence and the
618 /// rest is re-prefilled — instead of silently reusing positions whose ids
619 /// the cache does not hold.
620 #[test]
621 fn a_re_encoded_assistant_turn_diverges_and_shortens_the_prefix() {
622 // Ids up to and including the generation prompt, then the reply.
623 let generated_first = 5000u32;
624 let history = [
625 10,
626 11,
627 12,
628 /* <|im_start|>assistant\n */ 13,
629 generated_first,
630 77,
631 ];
632 // The client's re-render merges the closing newline with the reply's
633 // first characters, so position 3 differs.
634 let re_encoded = [10, 11, 12, 4242, 77, 30, 31];
635 assert_eq!(common_prefix(&history, &re_encoded), 3);
636 // Position 3 onwards is re-prefilled rather than wrongly reused.
637 assert_ne!(history[3], re_encoded[3]);
638 }
639
640 /// The abort payload round-trips through `catch_unwind` and is
641 /// distinguishable from a real panic, which is what keeps the engine
642 /// alive across a client hangup.
643 #[test]
644 fn a_stream_abort_unwinds_and_is_caught() {
645 hush_stream_aborts();
646 let outcome = std::panic::catch_unwind(|| std::panic::panic_any(ServerAbort));
647 let payload = outcome.expect_err("the abort unwinds");
648 assert!(payload.downcast_ref::<ServerAbort>().is_some());
649 }
650
651 #[test]
652 fn every_client_gone_io_kind_classifies_as_a_disconnect() {
653 for kind in [
654 ErrorKind::BrokenPipe,
655 ErrorKind::ConnectionReset,
656 ErrorKind::ConnectionAborted,
657 ErrorKind::ConnectionRefused,
658 ] {
659 assert_eq!(
660 StreamError::from(std::io::Error::from(kind)),
661 StreamError::Disconnected
662 );
663 }
664 assert_eq!(
665 StreamError::from(std::io::Error::from(ErrorKind::WouldBlock)),
666 StreamError::Io(ErrorKind::WouldBlock)
667 );
668 }
669
670 #[test]
671 fn a_plan_knows_how_much_it_has_to_prefill() {
672 let plan = Plan {
673 prompt_tokens: 6,
674 max_new: 8,
675 cached_prefix: 4,
676 prompt_ids: vec![1, 2, 3, 4, 5, 6],
677 params: GenerateParams {
678 max_new: 8,
679 temperature: 0.7,
680 top_k: Some(20),
681 top_p: 0.8,
682 seed: 42,
683 greedy: false,
684 },
685 };
686 assert_eq!(plan.to_prefill(), 2);
687 }
688}