use burn::{
Tensor,
prelude::{
Int,
TensorData,
s,
},
};
use super::{
NanoChatGpt,
NanoChatGptMeta,
};
use crate::{
kits::{
language::driver::{
CachedHistory,
CachedLm,
},
support::decoding::CachedStep,
},
ops::transformers::attention::{
KVCache,
KVCacheModel,
KVCacheStorage,
},
};
#[derive(Debug)]
pub struct NanoChatDecodeState {
pub cache: KVCache,
pub last: Option<Tensor<2, Int>>,
}
impl NanoChatDecodeState {
pub fn rewind(
&mut self,
pos: usize,
last: Option<Tensor<2, Int>>,
) {
assert_eq!(
pos == 0,
last.is_none(),
"a decode rewound to {pos} needs the token before it exactly when past 0"
);
self.cache.truncate(pos);
self.last = last;
}
}
impl NanoChatGpt {
pub fn decode_state(
&self,
rows: usize,
positions: usize,
storage: KVCacheStorage,
) -> NanoChatDecodeState {
assert!(
positions <= self.max_seq_len(),
"{positions} positions exceed the rotary table's {}",
self.max_seq_len()
);
NanoChatDecodeState {
cache: self
.kv_cache_config()
.with_batch_size(rows)
.with_seq_len(positions)
.with_max_seq_len(Some(positions))
.with_storage(storage)
.init(),
last: None,
}
}
pub fn prefill(
&self,
ids: Tensor<2, Int>,
state: &mut NanoChatDecodeState,
) -> Tensor<2> {
let [rows, seq] = ids.dims();
let last = ids.clone().slice(s![.., seq - 1..seq]);
let logits = self.forward_after(ids, state.last.take(), &mut Some(&mut state.cache));
state.last = Some(last);
logits
.slice(s![.., seq - 1..seq])
.reshape([rows, self.vocab_size()])
}
}
impl CachedStep for NanoChatGpt {
type State = NanoChatDecodeState;
fn step(
&self,
ids: Tensor<2, Int>,
state: &mut NanoChatDecodeState,
) -> Tensor<2> {
self.prefill(ids, state)
}
fn reorder(
&self,
state: &mut NanoChatDecodeState,
sources: &[usize],
) {
state.cache.reorder(sources);
if let Some(last) = state.last.take() {
let indices = Tensor::<1, Int>::from_data(
TensorData::new(
sources.iter().map(|&s| s as i64).collect::<Vec<_>>(),
[sources.len()],
),
&last.device(),
);
state.last = Some(last.select(0, indices));
}
}
}
impl CachedLm for NanoChatGpt {
fn decode_state(
&self,
rows: usize,
positions: usize,
storage: KVCacheStorage,
_device: &burn::tensor::Device,
) -> NanoChatDecodeState {
NanoChatGpt::decode_state(self, rows, positions, storage)
}
fn prefill_ids(
&self,
ids: Tensor<2, Int>,
state: &mut NanoChatDecodeState,
) -> Tensor<2> {
NanoChatGpt::prefill(self, ids, state)
}
fn rewind<'a>(
&self,
history: &mut CachedHistory,
state: &mut NanoChatDecodeState,
next: &'a [i64],
device: &burn::tensor::Device,
) -> &'a [i64] {
let todo = history.rewind(&mut state.cache, next);
let shared = next.len() - todo.len();
state.last = (shared > 0).then(|| {
Tensor::from_data(
TensorData::new(next[shared - 1..shared].to_vec(), [1, 1]),
device,
)
});
todo
}
}
#[cfg(test)]
mod tests {
use burn::tensor::Device;
use serial_test::serial;
use super::*;
use crate::{
burner::module::ModuleInit,
kits::{
language::nanochat::NanoChatGptContractConfig,
support::decoding::{
SearchConfig,
decode_cached,
},
},
support::testing::{
DeviceMemoryGuard,
performance_device,
},
};
fn ids(
ids: &[i64],
rows: usize,
device: &Device,
) -> Tensor<2, Int> {
let data: Vec<i64> = (0..rows).flat_map(|_| ids.iter().copied()).collect();
Tensor::from_data(TensorData::new(data, [rows, ids.len()]), device)
}
fn model(device: &Device) -> NanoChatGpt {
let mut gpt: NanoChatGpt = NanoChatGptContractConfig::new()
.with_vocab_size(40)
.with_n_layer(2)
.with_n_head(2)
.with_n_kv_head(1)
.with_n_embed(32)
.with_init_seq_len(4)
.with_upstream_2026("SL")
.init(device);
gpt.set_smear_lambda_for_tests(0.8, device);
gpt
}
#[test]
#[serial]
fn test_decode_cached() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let model = model(&device);
let prompt = vec![1i64, 5, 7];
let budget = 6;
let mut state = model.decode_state(1, prompt.len() + budget, KVCacheStorage::Auto);
let first = model.prefill(ids(&prompt, 1, &device), &mut state);
let mut search = SearchConfig::new().init_search(&[39], 0);
let out = decode_cached(
&model,
&mut state,
vec![prompt.clone()],
first,
budget,
search.as_mut(),
&[],
);
let generated = out[0][0].tokens_with_stop();
assert!(!generated.is_empty() && generated.len() <= budget);
let mut seen = prompt.clone();
for &token in &generated {
let logits = model.forward(ids(&seen, 1, &device), &mut None);
let n = seen.len();
let expected = logits
.slice(s![.., n - 1..n])
.reshape([1, 40])
.argmax(1)
.into_scalar::<i64>();
assert_eq!(token, expected, "after {seen:?}");
seen.push(token);
}
let mut state = model.decode_state(3, prompt.len() + budget, KVCacheStorage::Auto);
let first = model.prefill(ids(&prompt, 3, &device), &mut state);
let mut beam = SearchConfig::new().with_beam_size(3).init_search(&[39], 0);
let out = decode_cached(
&model,
&mut state,
vec![prompt.clone(); 3],
first,
budget,
beam.as_mut(),
&[],
);
assert_eq!(out.len(), 1, "one group");
assert_eq!(out[0].len(), 3);
assert!(out[0].iter().all(|c| c.tokens.len() <= budget));
}
#[test]
#[serial]
fn test_rewind_by_history() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let model = model(&device);
let mut history = CachedHistory::new();
let mut state = CachedLm::decode_state(&model, 1, 12, KVCacheStorage::Auto, &device);
let first = [3, 1, 4, 1, 5];
let todo = CachedLm::rewind(&model, &mut history, &mut state, &first, &device);
model.prefill_ids(ids(todo, 1, &device), &mut state);
history.extend(todo);
let next = [3, 1, 4, 2, 6];
let todo = CachedLm::rewind(&model, &mut history, &mut state, &next, &device);
assert_eq!(todo, [2, 6], "the shared prefix kept");
let rewound = model.prefill_ids(ids(todo, 1, &device), &mut state);
let mut fresh = model.decode_state(1, 12, KVCacheStorage::Auto);
let expected = model.prefill(ids(&next, 1, &device), &mut fresh);
rewound.into_data().assert_approx_eq::<f32>(
&expected.into_data(),
burn::tensor::Tolerance::rel_abs(1e-3, 1e-4),
);
}
#[test]
#[serial]
fn test_rewind() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let model = model(&device);
let mut state = model.decode_state(1, 12, KVCacheStorage::Auto);
model.prefill(ids(&[3, 1, 4, 1, 5], 1, &device), &mut state);
state.rewind(3, Some(ids(&[4], 1, &device)));
let rewound = model.prefill(ids(&[2, 6], 1, &device), &mut state);
let mut fresh = model.decode_state(1, 12, KVCacheStorage::Auto);
let expected = model.prefill(ids(&[3, 1, 4, 2, 6], 1, &device), &mut fresh);
rewound.into_data().assert_approx_eq::<f32>(
&expected.into_data(),
burn::tensor::Tolerance::rel_abs(1e-3, 1e-4),
);
}
}