use std::collections::HashMap;
use crate::tokenizer::{BYTE_VOCAB_SIZE, FIRST_BYTE_ID};
pub trait ModelEngine: Send + Sync {
fn vocab_size(&self) -> usize;
fn forward(&self, context: &[u32]) -> Vec<f32>;
fn name(&self) -> &'static str;
fn session_forward_batch(
&self,
_sessions: &mut [Box<dyn ModelSession>],
_feeds: &[&[u32]],
) -> Option<Vec<Vec<f32>>> {
None
}
fn graft_session(&self, _prompt: &[u32], _max_len: usize) -> Option<Box<dyn ModelSession>> {
None
}
fn offer_session_kv(
&self,
_tokens: &[u32],
_prompt_len: usize,
_session: Box<dyn ModelSession>,
) {
}
fn begin_session(&self) -> Option<Box<dyn ModelSession>> {
None
}
}
pub trait ModelSession: Send {
fn context_len(&self) -> usize;
fn forward(&mut self, new_tokens: &[u32]) -> Vec<f32>;
fn as_any_mut(&mut self) -> Option<&mut dyn std::any::Any> {
None
}
fn as_any(&self) -> Option<&dyn std::any::Any> {
None
}
fn fork(&self) -> Option<Box<dyn ModelSession>> {
None
}
}
pub struct NGramModel {
order: usize,
vocab_size: usize,
table: HashMap<Vec<u8>, Vec<f32>>,
unigram: Vec<f32>,
}
impl NGramModel {
pub fn new(order: usize, corpus: &str) -> Self {
assert!(order >= 1);
let mut table: HashMap<Vec<u8>, Vec<f32>> = HashMap::new();
let bytes = corpus.as_bytes();
for k in 1..=order {
for window in bytes.windows(k + 1) {
let ctx = window[..k].to_vec();
let next = window[k];
let counts = table
.entry(ctx)
.or_insert_with(|| vec![1.0f32; BYTE_VOCAB_SIZE]); counts[next as usize] += 1.0;
}
}
let mut raw_counts = [1.0f32; BYTE_VOCAB_SIZE];
for &b in bytes {
raw_counts[b as usize] += 1.0;
}
let mut unigram: Vec<f32> = raw_counts.iter().map(|&c| c.ln()).collect();
for i in 0..=31 {
if i != b'\n' as usize && i != b'\t' as usize {
unigram[i] = -1e9;
}
}
for i in 127..BYTE_VOCAB_SIZE {
unigram[i] = -1e9;
}
Self {
order,
vocab_size: 3 + BYTE_VOCAB_SIZE,
table,
unigram,
}
}
pub fn order(&self) -> usize {
self.order
}
fn counts_for(&self, context: &[u32]) -> (Option<&Vec<f32>>, Option<u8>) {
let recover = |t: u32| -> Option<u8> {
if t >= FIRST_BYTE_ID && t < FIRST_BYTE_ID + BYTE_VOCAB_SIZE as u32 {
Some((t - FIRST_BYTE_ID) as u8)
} else {
None
}
};
for k in (1..=self.order).rev() {
let tail: Option<Vec<u8>> = context
.iter()
.rev()
.take(k)
.map(|&t| recover(t))
.collect::<Option<Vec<u8>>>()
.map(|mut v| {
v.reverse();
v
});
if let Some(key) = tail {
if let Some(counts) = self.table.get(&key) {
return (Some(counts), key.last().copied());
}
}
}
(None, None)
}
}
fn mask_non_printable(logits: &mut [f32]) {
for i in 0..BYTE_VOCAB_SIZE {
let keep = i == b'\n' as usize || i == b'\t' as usize || (32..127).contains(&i);
if !keep {
logits[FIRST_BYTE_ID as usize + i] = -1e9;
}
}
}
impl Default for NGramModel {
fn default() -> Self {
NGramModel::new(4, DEFAULT_CORPUS)
}
}
impl ModelEngine for NGramModel {
fn vocab_size(&self) -> usize {
self.vocab_size
}
fn forward(&self, context: &[u32]) -> Vec<f32> {
let mut logits = vec![-1e9f32; self.vocab_size];
match self.counts_for(context) {
(Some(counts), _) => {
for (b, &c) in counts.iter().enumerate() {
logits[FIRST_BYTE_ID as usize + b] = c.ln();
}
}
(None, _) => {
for (b, &c) in self.unigram.iter().enumerate() {
logits[FIRST_BYTE_ID as usize + b] = c;
}
}
}
mask_non_printable(&mut logits);
logits
}
fn name(&self) -> &'static str {
"ngram"
}
}
pub const DEFAULT_CORPUS: &str = "OpenAI serves large language models at scale \
with prefixed attention and paged memory to make inference fast and cheap. \
SGLang is a fast serving framework for large language models and vision \
language models. It uses a radix tree to cache the key and value tensors of \
repeated prompt prefixes, so common instructions are computed only once. \
The scheduler runs continuous batching to merge many requests into one forward \
pass and keep the hardware busy. A paged key value cache stores memory in fixed \
size blocks and uses copy on write when sequences diverge. The frontend language \
lets users write programs with generation, selection, and fork for parallel \
search. Sampling controls temperature and top p to trade quality and diversity. \
The server exposes OpenAI compatible chat completions and a native generate \
endpoint. Reliability comes from metrics and structured logging. \
";