use std::cell::RefCell;
use std::io::ErrorKind;
use std::sync::Once;
use std::time::{SystemTime, UNIX_EPOCH};
use ramvamp_core::generate::{
GenerateParams, GenerateProgress, StopReason, generate_from_with_progress,
};
use ramvamp_core::model::{ForwardState, Model};
use ramvamp_core::tokenizer::RvmpTokenizer;
use thiserror::Error;
use crate::error::ServerError;
use crate::prompt;
use crate::request::ChatCompletionRequest;
use crate::response::{FinishReason, Usage};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
pub enum StreamError {
#[error("the client closed the connection")]
Disconnected,
#[error("writing to the client failed: {0:?}")]
Io(ErrorKind),
}
impl From<std::io::Error> for StreamError {
fn from(error: std::io::Error) -> Self {
match error.kind() {
ErrorKind::BrokenPipe
| ErrorKind::ConnectionReset
| ErrorKind::ConnectionAborted
| ErrorKind::ConnectionRefused => StreamError::Disconnected,
other => StreamError::Io(other),
}
}
}
pub trait TokenSink {
fn prefill(&mut self, positions_done: usize, positions_total: usize)
-> Result<(), StreamError>;
fn token(&mut self, id: u32, text: &str) -> Result<(), StreamError>;
}
#[derive(Debug, Default, Clone, Copy)]
pub struct NullSink;
impl TokenSink for NullSink {
fn prefill(&mut self, _: usize, _: usize) -> Result<(), StreamError> {
Ok(())
}
fn token(&mut self, _: u32, _: &str) -> Result<(), StreamError> {
Ok(())
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct Plan {
pub prompt_tokens: usize,
pub max_new: usize,
pub cached_prefix: usize,
pub prompt_ids: Vec<u32>,
pub params: GenerateParams,
}
impl Plan {
pub fn to_prefill(&self) -> usize {
self.prompt_ids.len().saturating_sub(self.cached_prefix)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Completion {
pub text: String,
pub usage: Usage,
pub finish_reason: FinishReason,
pub aborted: bool,
}
pub trait Engine {
fn model_id(&self) -> &str;
fn created(&self) -> u64;
fn prepare(&mut self, request: &ChatCompletionRequest) -> Result<Plan, ServerError>;
fn run(&mut self, plan: Plan, sink: &mut dyn TokenSink) -> Result<Completion, ServerError>;
}
#[derive(Debug)]
pub struct ServerAbort;
pub fn hush_stream_aborts() {
static ONCE: Once = Once::new();
ONCE.call_once(|| {
let previous = std::panic::take_hook();
std::panic::set_hook(Box::new(move |info| {
if info.payload().downcast_ref::<ServerAbort>().is_none() {
previous(info);
}
}));
});
}
pub fn common_prefix(a: &[u32], b: &[u32]) -> usize {
a.iter().zip(b).take_while(|(x, y)| x == y).count()
}
pub fn unix_now() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_or(0, |d| d.as_secs())
}
#[derive(Debug, Clone)]
pub struct EngineConfig {
pub model_id: String,
pub context_limit: usize,
pub default_max_new: usize,
pub params: GenerateParams,
}
pub struct ModelEngine {
model: Model,
state: ForwardState,
tokenizer: RvmpTokenizer,
config: EngineConfig,
created: u64,
history: Vec<u32>,
turn: u64,
}
impl ModelEngine {
pub fn new(
model: Model,
state: ForwardState,
tokenizer: RvmpTokenizer,
config: EngineConfig,
) -> Self {
ModelEngine {
model,
state,
tokenizer,
config,
created: unix_now(),
history: Vec::new(),
turn: 0,
}
}
pub fn history(&self) -> &[u32] {
&self.history
}
fn sampling(&self, request: &ChatCompletionRequest, max_new: usize) -> GenerateParams {
let mut params = self.config.params;
params.max_new = max_new;
if let Some(temperature) = request.temperature {
params.temperature = temperature;
}
if let Some(top_p) = request.top_p {
params.top_p = top_p;
}
if let Some(top_k) = request.top_k {
params.top_k = Some(top_k);
}
params.seed = request
.seed
.unwrap_or_else(|| self.config.params.seed.wrapping_add(self.turn));
params
}
fn recover(&mut self) {
self.state.reset();
self.history.clear();
}
}
impl Engine for ModelEngine {
fn model_id(&self) -> &str {
&self.config.model_id
}
fn created(&self) -> u64 {
self.created
}
fn prepare(&mut self, request: &ChatCompletionRequest) -> Result<Plan, ServerError> {
request.validate()?;
let built = prompt::build(&self.tokenizer, request)?;
if built.token_ids.is_empty() {
return Err(ServerError::EmptyMessages);
}
let max_new = request
.max_new_tokens()
.map_or(self.config.default_max_new, |n| n as usize)
.max(1);
prompt::check_context(built.token_ids.len(), max_new, self.config.context_limit)?;
let fed = self.state.seq_len()?;
let shared = common_prefix(&self.history, &built.token_ids);
let cached_prefix = shared.min(fed).min(built.token_ids.len() - 1);
tracing::debug!(
prompt_tokens = built.token_ids.len(),
cached = cached_prefix,
prefill = built.token_ids.len() - cached_prefix,
max_new,
"prepared a request"
);
Ok(Plan {
prompt_tokens: built.token_ids.len(),
max_new,
cached_prefix,
params: self.sampling(request, max_new),
prompt_ids: built.token_ids,
})
}
fn run(&mut self, plan: Plan, sink: &mut dyn TokenSink) -> Result<Completion, ServerError> {
if let Err(e) = self.state.truncate(plan.cached_prefix) {
self.recover();
return Err(ServerError::Forward(e));
}
self.turn = self.turn.wrapping_add(1);
let prompt_total = plan.prompt_ids.len();
let cached_prefix = plan.cached_prefix;
let new_ids = &plan.prompt_ids[cached_prefix..];
let live = RefCell::new(Live {
sink,
reply: String::new(),
spoken: Vec::new(),
abort: None,
});
let outcome = {
let mut on_progress = |event: GenerateProgress| {
let GenerateProgress::PrefillChunk { positions_done, .. } = event else {
return;
};
let mut live = live.borrow_mut();
if live.abort.is_some() {
return;
}
let done = cached_prefix + positions_done;
if let Err(e) = live.sink.prefill(done, prompt_total) {
live.abort = Some(e);
}
};
let mut on_token = |id: u32, text: &str| {
let mut live = live.borrow_mut();
live.reply.push_str(text);
live.spoken.push(id);
if live.abort.is_none()
&& let Err(e) = live.sink.token(id, text)
{
live.abort = Some(e);
}
if live.abort.is_some() {
drop(live);
std::panic::panic_any(ServerAbort);
}
};
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
generate_from_with_progress(
&self.model,
&mut self.state,
&self.tokenizer,
new_ids,
cached_prefix,
&plan.params,
Some(&mut on_progress),
&mut on_token,
)
}))
};
let live = live.into_inner();
match outcome {
Ok(Ok(stats)) => {
self.history.clear();
self.history.extend_from_slice(&plan.prompt_ids);
self.history.extend_from_slice(&stats.generated_ids);
let finish_reason = match stats.stop {
StopReason::StopToken(_) => FinishReason::Stop,
StopReason::MaxNew => FinishReason::Length,
};
tracing::info!(
prompt_tokens = plan.prompt_tokens,
prefilled = new_ids.len(),
generated = stats.generated,
prefill_s = stats.prefill.as_secs_f64(),
decode_s = stats.decode.as_secs_f64(),
"served a completion"
);
Ok(Completion {
text: live.reply,
usage: Usage::new(
u32::try_from(plan.prompt_tokens).unwrap_or(u32::MAX),
u32::try_from(stats.generated).unwrap_or(u32::MAX),
),
finish_reason,
aborted: false,
})
}
Ok(Err(e)) => {
self.recover();
Err(ServerError::Generate(e))
}
Err(payload) => {
if payload.downcast_ref::<ServerAbort>().is_none() {
std::panic::resume_unwind(payload);
}
self.history.clear();
self.history.extend_from_slice(&plan.prompt_ids);
self.history.extend_from_slice(&live.spoken);
tracing::debug!(
generated = live.spoken.len(),
reason = ?live.abort,
"client left; abandoned the reply"
);
Ok(Completion {
usage: Usage::new(
u32::try_from(plan.prompt_tokens).unwrap_or(u32::MAX),
u32::try_from(live.spoken.len()).unwrap_or(u32::MAX),
),
text: live.reply,
finish_reason: FinishReason::Stop,
aborted: true,
})
}
}
}
}
struct Live<'a> {
sink: &'a mut dyn TokenSink,
reply: String,
spoken: Vec<u32>,
abort: Option<StreamError>,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_common_prefix_stops_at_the_first_divergent_id() {
assert_eq!(common_prefix(&[1, 2, 3], &[1, 2, 3]), 3);
assert_eq!(common_prefix(&[1, 2, 3], &[1, 2, 9, 3]), 2);
assert_eq!(common_prefix(&[1, 2, 3], &[]), 0);
assert_eq!(common_prefix(&[], &[1]), 0);
assert_eq!(common_prefix(&[1, 2], &[1, 2, 3, 4]), 2);
}
#[test]
fn an_identical_resend_still_leaves_one_token_to_prefill() {
let history = [1u32, 2, 3, 4, 5];
let incoming = [1u32, 2, 3];
let fed = history.len();
let shared = common_prefix(&history, &incoming);
assert_eq!(shared, incoming.len());
let cached = shared.min(fed).min(incoming.len() - 1);
assert_eq!(cached, 2);
assert_eq!(incoming.len() - cached, 1);
}
#[test]
fn the_cache_bounds_the_reusable_prefix_not_the_history() {
let history = [1u32, 2, 3, 4];
let incoming = [1u32, 2, 3, 4, 5, 6];
let fed = history.len() - 1;
let shared = common_prefix(&history, &incoming);
assert_eq!(shared, 4);
assert_eq!(shared.min(fed).min(incoming.len() - 1), 3);
}
#[test]
fn a_re_encoded_assistant_turn_diverges_and_shortens_the_prefix() {
let generated_first = 5000u32;
let history = [
10,
11,
12,
13,
generated_first,
77,
];
let re_encoded = [10, 11, 12, 4242, 77, 30, 31];
assert_eq!(common_prefix(&history, &re_encoded), 3);
assert_ne!(history[3], re_encoded[3]);
}
#[test]
fn a_stream_abort_unwinds_and_is_caught() {
hush_stream_aborts();
let outcome = std::panic::catch_unwind(|| std::panic::panic_any(ServerAbort));
let payload = outcome.expect_err("the abort unwinds");
assert!(payload.downcast_ref::<ServerAbort>().is_some());
}
#[test]
fn every_client_gone_io_kind_classifies_as_a_disconnect() {
for kind in [
ErrorKind::BrokenPipe,
ErrorKind::ConnectionReset,
ErrorKind::ConnectionAborted,
ErrorKind::ConnectionRefused,
] {
assert_eq!(
StreamError::from(std::io::Error::from(kind)),
StreamError::Disconnected
);
}
assert_eq!(
StreamError::from(std::io::Error::from(ErrorKind::WouldBlock)),
StreamError::Io(ErrorKind::WouldBlock)
);
}
#[test]
fn a_plan_knows_how_much_it_has_to_prefill() {
let plan = Plan {
prompt_tokens: 6,
max_new: 8,
cached_prefix: 4,
prompt_ids: vec![1, 2, 3, 4, 5, 6],
params: GenerateParams {
max_new: 8,
temperature: 0.7,
top_k: Some(20),
top_p: 0.8,
seed: 42,
greedy: false,
},
};
assert_eq!(plan.to_prefill(), 2);
}
}