use std::{
fmt::Debug,
sync::Arc,
};
use bunsen::{
blocks::transformers::models::language_decoder::{
LmDecoderMeta,
LmDecoderOps,
},
errors::{
BunsenError,
BunsenResult,
},
kits::{
language::generation::{
driver::{
CachedHistory,
CachedLm,
},
gemma::GemmaModel,
qwen3::Qwen3Model,
qwen3_5::Qwen3_5Model,
},
support::{
construct::LmConstruct,
decoding::{
CachedStep,
LogitFilter,
SuppressTokens,
},
model_config::GenerationPolicy,
tokens::{
BunsenTokenizer,
Detokenizer,
Tokenizer,
},
},
},
ops::transformers::attention::KVCacheStorage,
};
use burn::{
Tensor,
prelude::{
Int,
TensorData,
},
tensor::{
DType,
Device,
},
};
use crate::chat::{
ChatMessage,
ChatTemplate,
};
pub trait ChatBackend: Debug + Send + Sync + 'static {
type State: Debug + Send;
type Model: CachedStep<State = Self::State>;
fn model(&self) -> &Self::Model;
fn render(
&self,
messages: &[ChatMessage],
enable_thinking: Option<bool>,
) -> BunsenResult<Vec<i64>>;
fn snapshot_point(
&self,
_messages: &[ChatMessage],
_enable_thinking: Option<bool>,
) -> BunsenResult<Option<usize>> {
Ok(None)
}
fn snapshot(
&self,
_state: &mut Self::State,
) {
}
fn detokenizer(&self) -> Arc<dyn Detokenizer>;
fn generation(&self) -> &GenerationPolicy;
fn filters(&self) -> Vec<Arc<dyn LogitFilter>>;
fn trained_context(&self) -> Option<usize>;
fn float_dtype(&self) -> DType;
fn decode_state(
&self,
positions: usize,
storage: KVCacheStorage,
device: &Device,
) -> Self::State;
fn rewind<'a>(
&self,
history: &mut CachedHistory,
state: &mut Self::State,
next: &'a [i64],
device: &Device,
) -> &'a [i64];
fn prefill(
&self,
ids: &[i64],
state: &mut Self::State,
device: &Device,
) -> Tensor<2>;
}
pub fn row(
ids: &[i64],
device: &Device,
) -> Tensor<2, Int> {
Tensor::from_data(TensorData::new(ids.to_vec(), [1, ids.len()]), device)
}
pub trait ChatRenderer: Debug + Send + Sync {
fn render_prompt(
&self,
tokenizer: &dyn Tokenizer,
messages: &[ChatMessage],
enable_thinking: Option<bool>,
) -> BunsenResult<Vec<i64>>;
fn closed_len(
&self,
_tokenizer: &dyn Tokenizer,
_messages: &[ChatMessage],
_enable_thinking: Option<bool>,
) -> BunsenResult<Option<usize>> {
Ok(None)
}
}
impl ChatRenderer for ChatTemplate {
fn render_prompt(
&self,
tokenizer: &dyn Tokenizer,
messages: &[ChatMessage],
enable_thinking: Option<bool>,
) -> BunsenResult<Vec<i64>> {
tokenizer.encode(&self.render(messages, true, enable_thinking)?)
}
fn closed_len(
&self,
tokenizer: &dyn Tokenizer,
messages: &[ChatMessage],
enable_thinking: Option<bool>,
) -> BunsenResult<Option<usize>> {
let open = self.render(messages, true, enable_thinking)?;
let closed = self.render(messages, false, enable_thinking)?;
if !open.starts_with(&closed) || open.len() == closed.len() {
return Ok(None);
}
let (open, closed) = (tokenizer.encode(&open)?, tokenizer.encode(&closed)?);
Ok(open.starts_with(&closed).then_some(closed.len()))
}
}
#[derive(Debug)]
pub struct ChatParts {
pub tokenizer: Arc<dyn Tokenizer>,
pub renderer: Arc<dyn ChatRenderer>,
pub generation: GenerationPolicy,
}
pub trait ChatLm:
CachedLm + LmDecoderMeta + LmDecoderOps + Debug + Send + Sync + Sized + 'static
{
fn chat_parts(bundle: &LmConstruct<Self>) -> BunsenResult<ChatParts>;
}
pub fn template_chat_parts<M>(bundle: &LmConstruct<M>) -> BunsenResult<ChatParts> {
let tokenizer = BunsenTokenizer::from_files(
bundle
.tokenizer()
.ok_or_else(|| missing("tokenizer.json"))?,
Some(
bundle
.tokenizer_config()
.ok_or_else(|| missing("tokenizer_config.json"))?,
),
)?;
Ok(ChatParts {
tokenizer: Arc::new(tokenizer),
renderer: Arc::new(template_of(bundle)?),
generation: bundle.policy.generation.clone(),
})
}
impl ChatLm for Qwen3Model {
fn chat_parts(bundle: &LmConstruct<Self>) -> BunsenResult<ChatParts> {
template_chat_parts(bundle)
}
}
impl ChatLm for GemmaModel {
fn chat_parts(bundle: &LmConstruct<Self>) -> BunsenResult<ChatParts> {
template_chat_parts(bundle)
}
}
impl ChatLm for Qwen3_5Model {
fn chat_parts(bundle: &LmConstruct<Self>) -> BunsenResult<ChatParts> {
template_chat_parts(bundle)
}
}
#[derive(Debug)]
pub struct LmChat<M> {
bundle: Arc<LmConstruct<M>>,
tokenizer: Arc<dyn Tokenizer>,
renderer: Arc<dyn ChatRenderer>,
generation: GenerationPolicy,
unspelled: Arc<dyn LogitFilter>,
}
pub type Qwen3Chat = LmChat<Qwen3Model>;
pub type GemmaChat = LmChat<GemmaModel>;
pub type Qwen3_5Chat = LmChat<Qwen3_5Model>;
pub fn unspelled(
n_vocab: usize,
tokenizer: &dyn Tokenizer,
) -> Arc<dyn LogitFilter> {
Arc::new(SuppressTokens::new(
(0..n_vocab as i64).filter(|&id| !tokenizer.contains(id)),
))
}
pub(crate) fn missing(what: &str) -> BunsenError {
BunsenError::invalid_resource(format!("a checkpoint without {what}"))
}
pub fn no_chat_template(name: &str) -> BunsenError {
BunsenError::invalid_resource(format!(
"{name} has no chat template: none of its presets names one (it is a pretrained model, \
say); give one with `NAME{{generation: {{chat_template: ...}}}}`"
))
}
fn template_of<M>(bundle: &LmConstruct<M>) -> BunsenResult<ChatTemplate> {
let config = bundle
.tokenizer_config()
.ok_or_else(|| missing("tokenizer_config.json"))?;
let source = bundle
.policy
.generation
.chat_template
.as_ref()
.ok_or_else(|| no_chat_template("the checkpoint"))?;
ChatTemplate::from_source(source, |key| bundle.file(key), config)
}
impl<M: ChatLm> LmChat<M> {
pub fn new(
bundle: Arc<LmConstruct<M>>,
parts: ChatParts,
) -> BunsenResult<Self> {
let ChatParts {
tokenizer,
renderer,
generation,
} = parts;
if generation.stop_ids.is_empty() {
return Err(missing("a stop token (no generation_config.json)"));
}
Ok(Self {
unspelled: unspelled(bundle.model.n_logits(), tokenizer.as_ref()),
bundle,
tokenizer,
renderer,
generation,
})
}
pub fn from_construct(bundle: Arc<LmConstruct<M>>) -> BunsenResult<Self> {
let parts = M::chat_parts(&bundle)?;
Self::new(bundle, parts)
}
pub fn bundle(&self) -> &Arc<LmConstruct<M>> {
&self.bundle
}
pub fn tokenizer(&self) -> &Arc<dyn Tokenizer> {
&self.tokenizer
}
pub fn renderer(&self) -> &Arc<dyn ChatRenderer> {
&self.renderer
}
}
impl<M: ChatLm> ChatBackend for LmChat<M>
where
M::State: Debug + Send,
{
type Model = M;
type State = M::State;
fn model(&self) -> &M {
&self.bundle.model
}
fn render(
&self,
messages: &[ChatMessage],
enable_thinking: Option<bool>,
) -> BunsenResult<Vec<i64>> {
self.renderer
.render_prompt(self.tokenizer.as_ref(), messages, enable_thinking)
}
fn snapshot_point(
&self,
messages: &[ChatMessage],
enable_thinking: Option<bool>,
) -> BunsenResult<Option<usize>> {
if self.bundle.model.rewinds_anywhere() {
return Ok(None);
}
self.renderer
.closed_len(self.tokenizer.as_ref(), messages, enable_thinking)
}
fn snapshot(
&self,
state: &mut M::State,
) {
CachedLm::snapshot(&self.bundle.model, state);
}
fn detokenizer(&self) -> Arc<dyn Detokenizer> {
self.tokenizer.clone()
}
fn generation(&self) -> &GenerationPolicy {
&self.generation
}
fn filters(&self) -> Vec<Arc<dyn LogitFilter>> {
vec![self.unspelled.clone()]
}
fn trained_context(&self) -> Option<usize> {
self.bundle.max_positions
}
fn float_dtype(&self) -> DType {
LmDecoderOps::float_dtype(&self.bundle.model)
}
fn decode_state(
&self,
positions: usize,
storage: KVCacheStorage,
device: &Device,
) -> M::State {
CachedLm::decode_state(&self.bundle.model, 1, positions, storage, device)
}
fn rewind<'a>(
&self,
history: &mut CachedHistory,
state: &mut M::State,
next: &'a [i64],
device: &Device,
) -> &'a [i64] {
CachedLm::rewind(&self.bundle.model, history, state, next, device)
}
fn prefill(
&self,
ids: &[i64],
state: &mut M::State,
device: &Device,
) -> Tensor<2> {
self.bundle.model.prefill_ids(row(ids, device), state)
}
}