ferrox-models 0.13.2

Model loaders and decoder stacks for the Ferrox inference engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14

use ferrox_gguf::GgufFile;
use ferrox_models::tokenizer::GgufSpmTokenizer;
use std::env;

fn main() {
    let path = env::args().nth(1).unwrap();
    let file = GgufFile::open(&path).unwrap();
    // Try SPM first
    let tok = GgufSpmTokenizer::from_gguf(&file).expect("spm");
    let prompt = "<start_of_turn>user\nWhat is the capital of France? Answer with one word.<end_of_turn>\n<start_of_turn>model\n";
    let ids = tok.encode(prompt);
    println!("n={}", ids.len());
    println!("{:?}", &ids[..ids.len().min(40)]);
}