use crate::{
data::pretrained::{
PrefabTable,
Prefabs,
},
kits::language::nanochat::NanoChatGptContractConfig,
};
fn upstream_geometry(
depth: usize,
vocab_size: usize,
) -> NanoChatGptContractConfig {
let n_embed = depth * 64;
NanoChatGptContractConfig::new()
.with_init_seq_len(2048)
.with_max_seq_len_factor(10)
.with_vocab_size(vocab_size)
.with_n_layer(depth)
.with_n_head(n_embed / 128)
.with_n_kv_head(n_embed / 128)
.with_n_embed(n_embed)
}
impl NanoChatGptContractConfig {
pub fn nanochat_d20() -> Self {
upstream_geometry(20, 65536)
}
pub fn nanochat_d18_2026() -> Self {
upstream_geometry(18, 32768).with_upstream_2026("L")
}
}
impl Prefabs for NanoChatGptContractConfig {
const PREFABS: PrefabTable<Self> = &[
("nanochat-d20", Self::nanochat_d20),
("nanochat-d18-2026", Self::nanochat_d18_2026),
];
const PREFAB_TABLE: &'static str = "nanochat";
}
#[cfg(test)]
mod tests {
use super::*;
use crate::kits::language::nanochat::NanoChatGptMeta;
#[test]
fn test_prefabs() {
let config = NanoChatGptContractConfig::expect_prefab("nanochat-d20");
assert_eq!(
(config.n_embed(), config.n_layer(), config.n_head()),
(1280, 20, 10)
);
assert!(!config.residual_lambdas);
let config = NanoChatGptContractConfig::expect_prefab("nanochat-d18-2026");
assert_eq!(
(config.n_embed(), config.n_layer(), config.n_head()),
(1152, 18, 9)
);
assert_eq!(config.window_pattern.as_deref(), Some("L"));
assert!(NanoChatGptContractConfig::try_prefab("nanochat-d32").is_err());
}
}