common_tokenize1

Function common_tokenize1 

Source
pub unsafe extern "C" fn common_tokenize1(
    vocab: *const llama_vocab,
    text: *const std_string,
    add_special: bool,
    parse_special: bool,
) -> __BindgenOpaqueArray<u64, 3usize>