Skip to main content

llama_rs_tokenize

Function llama_rs_tokenize 

Source
pub unsafe extern "C" fn llama_rs_tokenize(
    vocab: *const llama_vocab,
    text: *const c_char,
    text_len: i32,
    tokens: *mut llama_token,
    n_tokens_max: i32,
    add_special: bool,
    parse_special: bool,
    out_returned_count: *mut i32,
    out_error: *mut *mut c_char,
) -> llama_rs_tokenize_status