Portable LLM

Rust rewrite of llama2.c

Download testdata

make testdata

Example

let f = File::open("testdata/stories15M.bin").unwrap();
let mut reader = BufReader::new(f);
let config = Config::from_reader(&mut reader).unwrap();

let mut weights = Weights::new(config.clone());
weights.load_data(&mut reader).unwrap();

let tokenizer_file = File::open("testdata/tokenizer.bin").unwrap();
let tokenizer_reader = BufReader::new(tokenizer_file);
let tokenizer = Tokenizer::from_reader(config.vocab_size as usize, tokenizer_reader).unwrap();

let mut llm = LLM::new(config, tokenizer, weights);
llm.inference("a dog".to_string(), 0.8).unwrap();