use std::path::Path;
use burn::{
store::{
ModuleSnapshot,
PathFilter,
bridge,
burn_pack::Tensor as PackTensor,
},
tensor::{
Device,
FloatDType,
TensorData,
},
};
use serde::Deserialize;
use super::{
NanoChatGpt,
NanoChatGptContractConfig,
};
use crate::{
burner::module::{
ModuleInit,
Rekey,
},
data::formats::{
NamedTensorsApplied,
read_pytorch_tensors,
stored_float_dtype,
},
errors::{
BunsenError,
BunsenResult,
ResultContext,
sys_at,
},
};
#[derive(Clone, Debug, PartialEq, Deserialize)]
pub struct NanoChatMeta {
#[serde(default)]
pub step: Option<u64>,
pub model_config: NanoChatModelConfig,
}
#[derive(Clone, Debug, PartialEq, Deserialize)]
pub struct NanoChatModelConfig {
pub sequence_len: usize,
pub vocab_size: usize,
pub n_layer: usize,
pub n_head: usize,
pub n_kv_head: usize,
pub n_embd: usize,
#[serde(default)]
pub window_pattern: Option<String>,
}
impl NanoChatMeta {
pub fn from_json(json: &str) -> BunsenResult<Self> {
serde_json::from_str(json).map_err(|e| {
BunsenError::invalid_resource("not a nanochat checkpoint meta").with_cause(e)
})
}
pub fn from_file(path: &Path) -> BunsenResult<Self> {
let json = std::fs::read_to_string(path).map_err(sys_at("read", path))?;
Self::from_json(&json).with_context(|| path.display().to_string())
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum NanoChatGeneration {
Upstream2025,
Upstream2026,
}
const UPSTREAM_2026_MARKERS: [&str; 5] = [
"resid_lambdas",
"x0_lambdas",
"smear_gate.weight",
"smear_lambda",
"backout_lambda",
];
impl NanoChatMeta {
pub fn generation<S: AsRef<str>>(
&self,
names: &[S],
) -> BunsenResult<NanoChatGeneration> {
let has = |name: &str| names.iter().any(|n| n.as_ref() == name);
let present: Vec<&str> = UPSTREAM_2026_MARKERS
.iter()
.copied()
.filter(|m| has(m))
.collect();
let value_embeds = names
.iter()
.any(|n| n.as_ref().starts_with("value_embeds."));
let windowed = self.model_config.window_pattern.is_some();
match (present.len(), value_embeds, windowed) {
(0, false, false) => Ok(NanoChatGeneration::Upstream2025),
(n, true, true) if n == UPSTREAM_2026_MARKERS.len() => {
Ok(NanoChatGeneration::Upstream2026)
}
_ => Err(BunsenError::unsupported(format!(
"a nanochat checkpoint between upstream's 2025 and 2026 models: it has {} of \
{UPSTREAM_2026_MARKERS:?}, {} value embeddings, and {} window pattern",
if present.is_empty() {
"none".to_string()
} else {
format!("{present:?}")
},
if value_embeds { "" } else { "no" },
if windowed { "a" } else { "no" },
))),
}
}
}
impl NanoChatMeta {
pub fn gpt_config(
&self,
generation: NanoChatGeneration,
) -> NanoChatGptContractConfig {
let m = &self.model_config;
let config = NanoChatGptContractConfig::new()
.with_init_seq_len(m.sequence_len)
.with_max_seq_len_factor(10)
.with_vocab_size(m.vocab_size)
.with_n_layer(m.n_layer)
.with_n_head(m.n_head)
.with_n_kv_head(m.n_kv_head)
.with_n_embed(m.n_embd);
match generation {
NanoChatGeneration::Upstream2025 => config,
NanoChatGeneration::Upstream2026 => {
config.with_upstream_2026(m.window_pattern.clone().unwrap_or_else(|| "L".into()))
}
}
}
}
pub fn pt_rekey() -> Rekey {
Rekey::keep_rest()
.rename("lm_head.", "lm.head.proj.")
.mount(
"transformer.",
"",
Rekey::keep_rest()
.rename("wte.", "lm.embedding.")
.mount_indexed(
"h.",
"lm.stack.layers.",
Rekey::keep_rest()
.rename("mlp.c_fc.", "mlp.linear1.")
.rename("mlp.c_proj.", "mlp.linear2.")
.rename("attn.ve_gate.", "attn.value_gate."),
),
)
}
pub fn load_pt_tensors(
model: &mut NanoChatGpt,
tensors: impl IntoIterator<Item = PackTensor>,
dtype: FloatDType,
) -> BunsenResult<NamedTensorsApplied> {
let mut tensors: Vec<PackTensor> = tensors.into_iter().collect();
let scales: Vec<PackTensor> = model
.collect(
Some(PathFilter::new().with_predicate(is_norm_scale)),
None,
true,
)
.into_iter()
.map(|t| {
let n: usize = t.shape.iter().product();
bridge::from_data(
TensorData::new(vec![1.0f32; n], t.shape.clone()),
t.name.clone(),
None,
)
})
.collect();
let last = scales
.iter()
.fold(Rekey::keep_rest(), |rekey, t| {
rekey.exact(t.name.clone(), t.name.clone())
})
.mount_canonical("", model);
tensors.extend(scales);
let rekey = pt_rekey().then(last);
let applied = rekey.apply_as(model, tensors, Some(dtype.into()))?;
if !applied.unused.is_empty() {
return Err(BunsenError::unsupported(format!(
"a nanochat state dict with tensors the model has no parameter for: {}",
applied.unused.join(", ")
)));
}
Ok(applied)
}
fn is_norm_scale(
path: &str,
containers: &str,
) -> bool {
path.ends_with(".gamma")
&& containers
.split('.')
.any(|c| c == "Enum:BunsenNormalization")
}
pub fn load_pt_checkpoint(
meta: &Path,
model: &Path,
device: &Device,
policy: crate::data::formats::LoadPolicy,
) -> BunsenResult<(NanoChatGpt, NanoChatGeneration)> {
let meta_data = NanoChatMeta::from_file(meta)?;
let tensors = read_pytorch_tensors(model)?;
let names: Vec<&str> = tensors.iter().map(|t| t.name.as_str()).collect();
let generation = meta_data
.generation(&names)
.with_context(|| format!("{} and {}", meta.display(), model.display()))?;
let stored = stored_float_dtype(&tensors).unwrap_or(FloatDType::F32);
let dtype = policy
.resolve(stored, device, |_| true)
.with_context(|| model.display().to_string())?;
let mut gpt: NanoChatGpt = meta_data.gpt_config(generation).try_init(device)?;
load_pt_tensors(&mut gpt, tensors, dtype).with_context(|| model.display().to_string())?;
Ok((gpt, generation))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::errors::{
BunsenErrorKind,
testing::ErrorMatcher,
};
fn meta(window: Option<&str>) -> NanoChatMeta {
NanoChatMeta {
step: Some(1),
model_config: NanoChatModelConfig {
sequence_len: 2048,
vocab_size: 65536,
n_layer: 20,
n_head: 10,
n_kv_head: 10,
n_embd: 1280,
window_pattern: window.map(str::to_string),
},
}
}
#[test]
fn test_meta_from_json() {
let m = NanoChatMeta::from_json(
r#"{"step": 650, "val_loss": 1.07, "model_config": {"sequence_len": 2048,
"vocab_size": 65536, "n_layer": 20, "n_head": 10, "n_kv_head": 10, "n_embd": 1280,
"bos_token": "<|bos|>", "chat_template": "..."}}"#,
)
.unwrap();
assert_eq!(m.step, Some(650));
assert_eq!(
m,
NanoChatMeta {
step: Some(650),
..meta(None)
}
);
let err = NanoChatMeta::from_json(r#"{"step": 1}"#).unwrap_err();
ErrorMatcher::kind(BunsenErrorKind::InvalidResource).assert(&err);
}
#[test]
fn test_generation() {
let v2025 = ["transformer.wte.weight", "lm_head.weight"];
assert_eq!(
meta(None).generation(&v2025).unwrap(),
NanoChatGeneration::Upstream2025
);
let mut v2026: Vec<&str> = v2025.to_vec();
v2026.extend(UPSTREAM_2026_MARKERS);
v2026.push("value_embeds.1.weight");
assert_eq!(
meta(Some("L")).generation(&v2026).unwrap(),
NanoChatGeneration::Upstream2026
);
let unsupported = ErrorMatcher::kind(BunsenErrorKind::Unsupported)
.message_contains("between upstream's 2025 and 2026 models");
unsupported.assert(&meta(Some("L")).generation(&v2025).unwrap_err());
unsupported.assert(&meta(None).generation(&v2026).unwrap_err());
let partial = ["transformer.wte.weight", "resid_lambdas", "x0_lambdas"];
unsupported.assert(&meta(Some("L")).generation(&partial).unwrap_err());
}
#[test]
fn test_config() {
let c = meta(None).gpt_config(NanoChatGeneration::Upstream2025);
assert_eq!(
(
c.init_seq_len,
c.max_seq_len_factor,
c.vocab_size,
c.n_layer,
c.n_embed
),
(2048, 10, 65536, 20, 1280)
);
assert!(!c.residual_lambdas && c.window_pattern.is_none() && c.rope_base == 10000);
let c = meta(Some("SSSL")).gpt_config(NanoChatGeneration::Upstream2026);
assert_eq!(c.window_pattern.as_deref(), Some("SSSL"));
assert!(c.residual_lambdas && c.value_embeds && c.backout && c.rope_base == 100000);
}
#[test]
fn test_pt_rekey() {
let r = pt_rekey();
for (from, to) in [
("transformer.wte.weight", "lm.embedding.weight"),
(
"transformer.h.3.attn.c_q.weight",
"lm.stack.layers.3.attn.c_q.weight",
),
(
"transformer.h.3.attn.c_proj.weight",
"lm.stack.layers.3.attn.c_proj.weight",
),
(
"transformer.h.0.mlp.c_fc.weight",
"lm.stack.layers.0.mlp.linear1.weight",
),
(
"transformer.h.0.mlp.c_proj.weight",
"lm.stack.layers.0.mlp.linear2.weight",
),
(
"transformer.h.17.attn.ve_gate.weight",
"lm.stack.layers.17.attn.value_gate.weight",
),
("lm_head.weight", "lm.head.proj.weight"),
("value_embeds.17.weight", "value_embeds.17.weight"),
("smear_gate.weight", "smear_gate.weight"),
("resid_lambdas", "resid_lambdas"),
] {
assert_eq!(r.apply(from).as_deref(), Some(to), "{from}");
}
}
}