Skip to main content

ramvamp_server/
engine.rs

1//! The thing that turns a request into tokens, and the seam that lets the
2//! transport be tested without one.
3//!
4//! # Why there is a trait here
5//!
6//! Streaming behaviour is not verifiable by reading code. A response that is
7//! buffered to completion and one that is delivered live produce *byte
8//! identical* transcripts; they differ only in when the bytes arrive. The only
9//! way to assert the difference is to drive the transport from a source that
10//! emits known tokens at known intervals — which means the transport cannot be
11//! wired straight to a 30B model. Hence [`Engine`]: `ModelEngine` is one
12//! implementation, and the tests use another that sleeps.
13//!
14//! # The two-step shape
15//!
16//! [`Engine::prepare`] does everything that can fail *before* a byte of
17//! response is committed — parsing the conversation, encoding it, checking it
18//! against the context cap — and [`Engine::run`] does the part that cannot be
19//! taken back. That split is what lets a context overflow be a clean HTTP 400
20//! while a mid-generation failure is reported inside an already-200 stream.
21//!
22//! # Prefix caching, and the trap in it
23//!
24//! A Chat Completions client is stateless: every request carries the whole
25//! conversation, so a naive server re-prefills all of it every turn, which at
26//! this runtime's speeds is minutes of work to reach a state it already had.
27//! The fix is to keep the KV cache across requests and re-prefill only the
28//! part that changed — [`ForwardState::truncate`] plus a longest-common-prefix
29//! match.
30//!
31//! The trap is *what to match against*, and it fails silently. The client
32//! sends previous assistant turns back as **text**, and re-encoding that text
33//! does not reproduce the ids the model generated: the `\n` that closes
34//! `<|im_start|>assistant\n` and the first characters of the reply are
35//! candidates for the same BPE merge, so a generation-prompt render is not
36//! always a token prefix of the finished turn (measured; the CLI has a test
37//! named after it). Comparing against a re-render therefore produces a *wrong*
38//! cache hit — the positions are reused, the ids at them are not the ids the
39//! cache holds, `generate_from` cannot detect it because the arithmetic still
40//! adds up, and the model answers fluently from a context that never existed.
41//!
42//! So the comparison is against [`ModelEngine::history`], which holds the ids
43//! that were actually fed: prompt renders as encoded, and assistant turns as
44//! [`GenerateStats::generated_ids`], never as a re-encoding of the reply.
45//! Where the client's re-render diverges from what the model said, the common
46//! prefix simply ends there and the suffix is re-prefilled. Correct by
47//! construction rather than by luck.
48
49use std::cell::RefCell;
50use std::io::ErrorKind;
51use std::sync::Once;
52use std::time::{SystemTime, UNIX_EPOCH};
53
54use ramvamp_core::generate::{
55    GenerateParams, GenerateProgress, StopReason, generate_from_with_progress,
56};
57use ramvamp_core::model::{ForwardState, Model};
58use ramvamp_core::tokenizer::RvmpTokenizer;
59use thiserror::Error;
60
61use crate::error::ServerError;
62use crate::prompt;
63use crate::request::ChatCompletionRequest;
64use crate::response::{FinishReason, Usage};
65
66/// A failure writing to whoever is consuming the stream.
67///
68/// Deliberately `Copy`: the transport shares one of these between the thread
69/// running the model and the keep-alive thread, and the `io::Error` payload
70/// carries nothing worth the `Arc` — the kind is the whole story, and the
71/// response to every kind is the same.
72#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
73pub enum StreamError {
74    /// The client hung up. The ordinary way a stream ends early, and not an
75    /// error worth logging above `debug`.
76    #[error("the client closed the connection")]
77    Disconnected,
78    /// Any other write failure.
79    #[error("writing to the client failed: {0:?}")]
80    Io(ErrorKind),
81}
82
83impl From<std::io::Error> for StreamError {
84    /// Classify a write failure. The four kinds `tiny_http` itself treats as
85    /// "client went away" are the four treated as a disconnect here — the
86    /// difference being that here it is reported rather than swallowed.
87    fn from(error: std::io::Error) -> Self {
88        match error.kind() {
89            ErrorKind::BrokenPipe
90            | ErrorKind::ConnectionReset
91            | ErrorKind::ConnectionAborted
92            | ErrorKind::ConnectionRefused => StreamError::Disconnected,
93            other => StreamError::Io(other),
94        }
95    }
96}
97
98/// Where a running generation reports to.
99///
100/// Both methods return a `Result` because the usual implementation is a
101/// socket, and a socket that has gone away is the signal to stop generating —
102/// there is nobody left to generate for, and the session is worth minutes.
103pub trait TokenSink {
104    /// Prompt positions committed so far, out of the whole prompt.
105    ///
106    /// Counted against the *whole* prompt including the part served from
107    /// cache, so a cache hit shows up as progress already made rather than as
108    /// a prefill that starts at zero and finishes suspiciously fast.
109    fn prefill(&mut self, positions_done: usize, positions_total: usize)
110    -> Result<(), StreamError>;
111
112    /// One token: the id the model sampled, and the text it decoded to.
113    ///
114    /// The id is carried because `<tool_call>` and `</tool_call>` are single
115    /// added tokens and the decoder does not skip them, so a sink that has the
116    /// id can recognise a call block by an integer compare. The alternative —
117    /// scanning the decoded text — would have to be done at a layer that sees
118    /// only a stream of fragments, where a marker can be split across two of
119    /// them; an id cannot be split. See [`crate::toolcall::CallStream`].
120    fn token(&mut self, id: u32, text: &str) -> Result<(), StreamError>;
121}
122
123/// A sink that discards everything, for the non-streaming path.
124///
125/// The reply text comes back in [`Completion::text`] either way, so the
126/// buffered endpoint needs nothing from the sink but a place to not send it.
127#[derive(Debug, Default, Clone, Copy)]
128pub struct NullSink;
129
130impl TokenSink for NullSink {
131    fn prefill(&mut self, _: usize, _: usize) -> Result<(), StreamError> {
132        Ok(())
133    }
134
135    fn token(&mut self, _: u32, _: &str) -> Result<(), StreamError> {
136        Ok(())
137    }
138}
139
140/// One accepted request, ready to run.
141///
142/// Everything that could have been refused already has been, which is the
143/// point: holding a `Plan` means the response head can be committed.
144#[derive(Debug, Clone, PartialEq)]
145pub struct Plan {
146    /// The whole prompt, cached prefix included. This is the number the
147    /// client is billed in `usage.prompt_tokens`, not the suffix actually
148    /// prefilled — a cache hit is this server's business, not a discount the
149    /// client can observe and come to depend on.
150    pub prompt_tokens: usize,
151    /// Reply cap for this turn, after clamping to the context.
152    pub max_new: usize,
153    /// Leading positions the KV cache already holds and this turn reuses.
154    pub cached_prefix: usize,
155    /// The full sanitized prompt id sequence.
156    pub prompt_ids: Vec<u32>,
157    /// Sampling dials, merged from the request over the checkpoint defaults.
158    pub params: GenerateParams,
159}
160
161impl Plan {
162    /// How many positions this turn actually has to prefill.
163    pub fn to_prefill(&self) -> usize {
164        self.prompt_ids.len().saturating_sub(self.cached_prefix)
165    }
166}
167
168/// What a finished (or abandoned) generation produced.
169#[derive(Debug, Clone, PartialEq, Eq)]
170pub struct Completion {
171    /// The reply text, concatenated from what the sink was shown.
172    pub text: String,
173    /// Token accounting for the whole request.
174    pub usage: Usage,
175    /// Why it ended.
176    pub finish_reason: FinishReason,
177    /// The sink asked to stop — in practice, the client hung up. Nothing more
178    /// should be written; there is nowhere to write it.
179    pub aborted: bool,
180}
181
182/// The generator behind the HTTP surface.
183pub trait Engine {
184    /// The model id `/v1/models` advertises and every response echoes.
185    fn model_id(&self) -> &str;
186
187    /// Unix seconds to report as the model's creation time.
188    fn created(&self) -> u64;
189
190    /// Everything that can be refused before the response is committed.
191    ///
192    /// # Errors
193    ///
194    /// Any [`ServerError`] whose status is a 4xx: this is where a bad request
195    /// becomes an HTTP status rather than a broken stream.
196    fn prepare(&mut self, request: &ChatCompletionRequest) -> Result<Plan, ServerError>;
197
198    /// Run the plan, reporting through `sink`.
199    ///
200    /// A sink failure is not an error here: it produces a [`Completion`] with
201    /// [`Completion::aborted`] set, because "the client left" is an ordinary
202    /// outcome and the engine must be ready for the next request either way.
203    ///
204    /// # Errors
205    ///
206    /// [`ServerError::Generate`] or [`ServerError::Forward`] when the runtime
207    /// itself fails. The engine is left able to serve the next request.
208    fn run(&mut self, plan: Plan, sink: &mut dyn TokenSink) -> Result<Completion, ServerError>;
209}
210
211/// Payload of the deliberate unwind that stops a generation whose client has
212/// gone away.
213///
214/// The same device, for the same reason, as `ChatAbort` in the CLI:
215/// `generate_from`'s `on_token` returns `()`, so it has no way to report that
216/// it wants out, and an unwind is the only exit from a decode loop in
217/// progress. It is raised at a token boundary — between two forward passes,
218/// with no expert read in flight and the KV cache whole — which is what makes
219/// the state reusable afterwards.
220#[derive(Debug)]
221pub struct ServerAbort;
222
223/// Install, once per process, a panic hook that prints nothing for
224/// [`ServerAbort`] and forwards every other panic to the hook already in
225/// place.
226///
227/// Without it a client pressing Ctrl-C dumps a "thread panicked" backtrace
228/// into the server's log for what is an entirely routine event. Real panics,
229/// on any thread, still print exactly as before.
230pub fn hush_stream_aborts() {
231    static ONCE: Once = Once::new();
232    ONCE.call_once(|| {
233        let previous = std::panic::take_hook();
234        std::panic::set_hook(Box::new(move |info| {
235            if info.payload().downcast_ref::<ServerAbort>().is_none() {
236                previous(info);
237            }
238        }));
239    });
240}
241
242/// Length of the longest common prefix of two id sequences.
243pub fn common_prefix(a: &[u32], b: &[u32]) -> usize {
244    a.iter().zip(b).take_while(|(x, y)| x == y).count()
245}
246
247/// Unix seconds, or 0 if the clock is before the epoch.
248pub fn unix_now() -> u64 {
249    SystemTime::now()
250        .duration_since(UNIX_EPOCH)
251        .map_or(0, |d| d.as_secs())
252}
253
254/// How a [`ModelEngine`] was configured.
255#[derive(Debug, Clone)]
256pub struct EngineConfig {
257    /// The id to advertise and echo.
258    pub model_id: String,
259    /// The context cap the [`ForwardState`] was built with.
260    pub context_limit: usize,
261    /// Reply cap for a request that does not send one.
262    pub default_max_new: usize,
263    /// Sampling dials to start from; a request overrides field by field.
264    pub params: GenerateParams,
265}
266
267/// The real engine: a model, its state and its tokenizer, on one thread.
268///
269/// [`ForwardState`] is `Send` but not `Sync` — it owns the expert slot pool,
270/// the io_uring ring and the pinned compute pool — so it is moved onto the
271/// thread that serves requests and never shared. That is not a limitation
272/// being worked around: concurrency here is one, structurally, because a
273/// second decode loop would halve the token rate of both and there is exactly
274/// one KV cache.
275pub struct ModelEngine {
276    model: Model,
277    state: ForwardState,
278    tokenizer: RvmpTokenizer,
279    config: EngineConfig,
280    created: u64,
281    /// Every id the cached conversation consists of: prompt renders as
282    /// encoded, assistant turns as the ids the model actually generated.
283    ///
284    /// Runs *ahead* of the KV cache by design — the last id sampled is
285    /// emitted and never fed back, because there is nothing left to predict
286    /// from it — so [`ForwardState::seq_len`] is the authority on how much of
287    /// it the cache holds, never a count kept alongside. See the module docs
288    /// for why this is not a re-render of the transcript.
289    history: Vec<u32>,
290    /// Requests served, used to vary the seed when the client does not pin
291    /// one. Without it every repeat of a question gets a byte-identical
292    /// answer, which reads as a broken model rather than as determinism.
293    turn: u64,
294}
295
296impl ModelEngine {
297    /// Build an engine over an already-loaded model and state.
298    ///
299    /// The loading is the caller's, deliberately: `state` carries the cache
300    /// budget, thread pinning and prefill dials, and those belong to the
301    /// command line that set them rather than to this crate.
302    pub fn new(
303        model: Model,
304        state: ForwardState,
305        tokenizer: RvmpTokenizer,
306        config: EngineConfig,
307    ) -> Self {
308        ModelEngine {
309            model,
310            state,
311            tokenizer,
312            config,
313            created: unix_now(),
314            history: Vec::new(),
315            turn: 0,
316        }
317    }
318
319    /// The ids the cached conversation consists of.
320    pub fn history(&self) -> &[u32] {
321        &self.history
322    }
323
324    /// Merge a request's sampling fields over the configured defaults.
325    fn sampling(&self, request: &ChatCompletionRequest, max_new: usize) -> GenerateParams {
326        let mut params = self.config.params;
327        params.max_new = max_new;
328        if let Some(temperature) = request.temperature {
329            // <= 0 is greedy, which `generate` already implements; passing it
330            // through rather than translating keeps one rule in one place.
331            params.temperature = temperature;
332        }
333        if let Some(top_p) = request.top_p {
334            params.top_p = top_p;
335        }
336        if let Some(top_k) = request.top_k {
337            params.top_k = Some(top_k);
338        }
339        // An unpinned seed advances per request; a pinned one is honoured
340        // exactly, which is what `seed` means to a client.
341        params.seed = request
342            .seed
343            .unwrap_or_else(|| self.config.params.seed.wrapping_add(self.turn));
344        params
345    }
346
347    /// Recover from a runtime failure that may have left the cache ragged.
348    ///
349    /// Cheap — every allocation survives — and the alternative is an engine
350    /// that answers `PositionMismatch` to every request for the rest of its
351    /// life.
352    fn recover(&mut self) {
353        self.state.reset();
354        self.history.clear();
355    }
356}
357
358impl Engine for ModelEngine {
359    fn model_id(&self) -> &str {
360        &self.config.model_id
361    }
362
363    fn created(&self) -> u64 {
364        self.created
365    }
366
367    fn prepare(&mut self, request: &ChatCompletionRequest) -> Result<Plan, ServerError> {
368        request.validate()?;
369        // Sanitized here, before anything is compared: the ids that get
370        // prefilled are the sanitized ones, so a prefix match against the
371        // unsanitized stream would be a match against a sequence this server
372        // never feeds the model.
373        let built = prompt::build(&self.tokenizer, request)?;
374        if built.token_ids.is_empty() {
375            return Err(ServerError::EmptyMessages);
376        }
377
378        let max_new = request
379            .max_new_tokens()
380            .map_or(self.config.default_max_new, |n| n as usize)
381            .max(1);
382        prompt::check_context(built.token_ids.len(), max_new, self.config.context_limit)?;
383
384        // The cache says where it is; `history` says what is in it. The
385        // smaller of the two bounds what can be reused, because `history`
386        // runs one id ahead of the cache after every reply.
387        let fed = self.state.seq_len()?;
388        let shared = common_prefix(&self.history, &built.token_ids);
389        // At least one token has to be prefilled: `generate_with_stops`
390        // refuses an empty prompt with `EmptyPrompt`, and an identical resend
391        // (the incoming ids being a prefix of the history) would otherwise
392        // produce exactly that. Rewinding one position is the cheapest legal
393        // suffix.
394        let cached_prefix = shared.min(fed).min(built.token_ids.len() - 1);
395
396        tracing::debug!(
397            prompt_tokens = built.token_ids.len(),
398            cached = cached_prefix,
399            prefill = built.token_ids.len() - cached_prefix,
400            max_new,
401            "prepared a request"
402        );
403
404        Ok(Plan {
405            prompt_tokens: built.token_ids.len(),
406            max_new,
407            cached_prefix,
408            params: self.sampling(request, max_new),
409            prompt_ids: built.token_ids,
410        })
411    }
412
413    fn run(&mut self, plan: Plan, sink: &mut dyn TokenSink) -> Result<Completion, ServerError> {
414        // A truncate that does nothing still fails loudly downstream:
415        // `forward_token` re-validates `position == seq_len()` and raises
416        // `PositionMismatch`. So this is not papered over on any path.
417        if let Err(e) = self.state.truncate(plan.cached_prefix) {
418            self.recover();
419            return Err(ServerError::Forward(e));
420        }
421        self.turn = self.turn.wrapping_add(1);
422
423        let prompt_total = plan.prompt_ids.len();
424        let cached_prefix = plan.cached_prefix;
425        let new_ids = &plan.prompt_ids[cached_prefix..];
426
427        // One cell for everything both callbacks touch. They are never live
428        // at the same instant — `generate` calls one or the other, never both
429        // — but the borrow checker cannot see that through two closures held
430        // simultaneously, and a `RefCell` is the honest way to say so.
431        let live = RefCell::new(Live {
432            sink,
433            reply: String::new(),
434            spoken: Vec::new(),
435            abort: None,
436        });
437
438        let outcome = {
439            let mut on_progress = |event: GenerateProgress| {
440                let GenerateProgress::PrefillChunk { positions_done, .. } = event else {
441                    return;
442                };
443                let mut live = live.borrow_mut();
444                if live.abort.is_some() {
445                    return;
446                }
447                // Reported against the whole prompt, cached prefix included.
448                let done = cached_prefix + positions_done;
449                if let Err(e) = live.sink.prefill(done, prompt_total) {
450                    // Recorded, *not* unwound. A prefill chunk boundary is
451                    // inside `prefill_sweep`, which still owns a
452                    // `PrefillSession` over the streamer's staging arena and
453                    // has not called `finish()` on it; unwinding through that
454                    // abandons the arena rather than releasing it. The decode
455                    // loop's token boundary is the one place the runtime
456                    // documents as safe to leave, so the abort waits for it.
457                    // The cost is finishing a prefill nobody will read, which
458                    // is bounded by the prompt.
459                    live.abort = Some(e);
460                }
461            };
462
463            let mut on_token = |id: u32, text: &str| {
464                let mut live = live.borrow_mut();
465                live.reply.push_str(text);
466                live.spoken.push(id);
467                if live.abort.is_none()
468                    && let Err(e) = live.sink.token(id, text)
469                {
470                    live.abort = Some(e);
471                }
472                if live.abort.is_some() {
473                    // Release the borrow before unwinding: the guard would
474                    // drop anyway, but leaving it to the unwinder makes the
475                    // catch site's `into_inner` depend on drop order.
476                    drop(live);
477                    std::panic::panic_any(ServerAbort);
478                }
479            };
480
481            std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
482                generate_from_with_progress(
483                    &self.model,
484                    &mut self.state,
485                    &self.tokenizer,
486                    new_ids,
487                    cached_prefix,
488                    &plan.params,
489                    Some(&mut on_progress),
490                    &mut on_token,
491                )
492            }))
493        };
494
495        let live = live.into_inner();
496        match outcome {
497            Ok(Ok(stats)) => {
498                // The cache now holds this prompt; the history is it plus what
499                // the model said, in the ids it actually produced.
500                self.history.clear();
501                self.history.extend_from_slice(&plan.prompt_ids);
502                self.history.extend_from_slice(&stats.generated_ids);
503                let finish_reason = match stats.stop {
504                    StopReason::StopToken(_) => FinishReason::Stop,
505                    StopReason::MaxNew => FinishReason::Length,
506                };
507                tracing::info!(
508                    prompt_tokens = plan.prompt_tokens,
509                    prefilled = new_ids.len(),
510                    generated = stats.generated,
511                    prefill_s = stats.prefill.as_secs_f64(),
512                    decode_s = stats.decode.as_secs_f64(),
513                    "served a completion"
514                );
515                Ok(Completion {
516                    text: live.reply,
517                    usage: Usage::new(
518                        u32::try_from(plan.prompt_tokens).unwrap_or(u32::MAX),
519                        u32::try_from(stats.generated).unwrap_or(u32::MAX),
520                    ),
521                    finish_reason,
522                    aborted: false,
523                })
524            }
525            Ok(Err(e)) => {
526                // Mid-pass failure: the cache may be ragged, and a ragged one
527                // reports `PositionMismatch` forever.
528                self.recover();
529                Err(ServerError::Generate(e))
530            }
531            Err(payload) => {
532                if payload.downcast_ref::<ServerAbort>().is_none() {
533                    // Somebody else's panic. Re-raise it untouched.
534                    std::panic::resume_unwind(payload);
535                }
536                // The partial reply is in the cache up to its second-to-last
537                // id, so it has to be in the history too — otherwise the next
538                // turn would prefill from a position the model reached by a
539                // route the history no longer records. `spoken` is exactly
540                // what `generated_ids` would have held: the unwind leaves
541                // `on_token` before the id is fed, and the trailing flush
542                // call that would have repeated an id never happens.
543                self.history.clear();
544                self.history.extend_from_slice(&plan.prompt_ids);
545                self.history.extend_from_slice(&live.spoken);
546                tracing::debug!(
547                    generated = live.spoken.len(),
548                    reason = ?live.abort,
549                    "client left; abandoned the reply"
550                );
551                Ok(Completion {
552                    usage: Usage::new(
553                        u32::try_from(plan.prompt_tokens).unwrap_or(u32::MAX),
554                        u32::try_from(live.spoken.len()).unwrap_or(u32::MAX),
555                    ),
556                    text: live.reply,
557                    finish_reason: FinishReason::Stop,
558                    aborted: true,
559                })
560            }
561        }
562    }
563}
564
565/// The mutable half of a run, shared by the two callbacks.
566struct Live<'a> {
567    sink: &'a mut dyn TokenSink,
568    reply: String,
569    spoken: Vec<u32>,
570    abort: Option<StreamError>,
571}
572
573#[cfg(test)]
574mod tests {
575    use super::*;
576
577    #[test]
578    fn the_common_prefix_stops_at_the_first_divergent_id() {
579        assert_eq!(common_prefix(&[1, 2, 3], &[1, 2, 3]), 3);
580        assert_eq!(common_prefix(&[1, 2, 3], &[1, 2, 9, 3]), 2);
581        assert_eq!(common_prefix(&[1, 2, 3], &[]), 0);
582        assert_eq!(common_prefix(&[], &[1]), 0);
583        assert_eq!(common_prefix(&[1, 2], &[1, 2, 3, 4]), 2);
584    }
585
586    /// The identical-resend edge: the incoming ids are a prefix of the
587    /// history, so the naive suffix is empty and `generate_with_stops` answers
588    /// `EmptyPrompt`. Rewinding one position is the fix, and it costs one
589    /// forward pass.
590    #[test]
591    fn an_identical_resend_still_leaves_one_token_to_prefill() {
592        let history = [1u32, 2, 3, 4, 5];
593        let incoming = [1u32, 2, 3];
594        let fed = history.len();
595        let shared = common_prefix(&history, &incoming);
596        assert_eq!(shared, incoming.len());
597        let cached = shared.min(fed).min(incoming.len() - 1);
598        assert_eq!(cached, 2);
599        assert_eq!(incoming.len() - cached, 1);
600    }
601
602    /// The cache, not the history, bounds what can be reused: the history
603    /// runs one id ahead after every reply, because the last token sampled is
604    /// emitted and never fed back.
605    #[test]
606    fn the_cache_bounds_the_reusable_prefix_not_the_history() {
607        let history = [1u32, 2, 3, 4];
608        let incoming = [1u32, 2, 3, 4, 5, 6];
609        let fed = history.len() - 1;
610        let shared = common_prefix(&history, &incoming);
611        assert_eq!(shared, 4);
612        assert_eq!(shared.min(fed).min(incoming.len() - 1), 3);
613    }
614
615    /// The whole reason the history is not a re-render. A client sends the
616    /// assistant turn back as text; re-encoding it produces different ids
617    /// than the model generated, so the prefix ends at the divergence and the
618    /// rest is re-prefilled — instead of silently reusing positions whose ids
619    /// the cache does not hold.
620    #[test]
621    fn a_re_encoded_assistant_turn_diverges_and_shortens_the_prefix() {
622        // Ids up to and including the generation prompt, then the reply.
623        let generated_first = 5000u32;
624        let history = [
625            10,
626            11,
627            12,
628            /* <|im_start|>assistant\n */ 13,
629            generated_first,
630            77,
631        ];
632        // The client's re-render merges the closing newline with the reply's
633        // first characters, so position 3 differs.
634        let re_encoded = [10, 11, 12, 4242, 77, 30, 31];
635        assert_eq!(common_prefix(&history, &re_encoded), 3);
636        // Position 3 onwards is re-prefilled rather than wrongly reused.
637        assert_ne!(history[3], re_encoded[3]);
638    }
639
640    /// The abort payload round-trips through `catch_unwind` and is
641    /// distinguishable from a real panic, which is what keeps the engine
642    /// alive across a client hangup.
643    #[test]
644    fn a_stream_abort_unwinds_and_is_caught() {
645        hush_stream_aborts();
646        let outcome = std::panic::catch_unwind(|| std::panic::panic_any(ServerAbort));
647        let payload = outcome.expect_err("the abort unwinds");
648        assert!(payload.downcast_ref::<ServerAbort>().is_some());
649    }
650
651    #[test]
652    fn every_client_gone_io_kind_classifies_as_a_disconnect() {
653        for kind in [
654            ErrorKind::BrokenPipe,
655            ErrorKind::ConnectionReset,
656            ErrorKind::ConnectionAborted,
657            ErrorKind::ConnectionRefused,
658        ] {
659            assert_eq!(
660                StreamError::from(std::io::Error::from(kind)),
661                StreamError::Disconnected
662            );
663        }
664        assert_eq!(
665            StreamError::from(std::io::Error::from(ErrorKind::WouldBlock)),
666            StreamError::Io(ErrorKind::WouldBlock)
667        );
668    }
669
670    #[test]
671    fn a_plan_knows_how_much_it_has_to_prefill() {
672        let plan = Plan {
673            prompt_tokens: 6,
674            max_new: 8,
675            cached_prefix: 4,
676            prompt_ids: vec![1, 2, 3, 4, 5, 6],
677            params: GenerateParams {
678                max_new: 8,
679                temperature: 0.7,
680                top_k: Some(20),
681                top_p: 0.8,
682                seed: 42,
683                greedy: false,
684            },
685        };
686        assert_eq!(plan.to_prefill(), 2);
687    }
688}