vil_tokenizer 0.1.0

VIL Tokenizer Engine — native Rust BPE tokenization for LLM token counting and text splitting
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

//! SSE pipeline builders for tokenizer operations.

use vil_sdk::prelude::*;

pub fn tokenize_sink(port: u16, path: &str) -> HttpSinkBuilder {
    HttpSinkBuilder::new("TokenizeSink")
        .port(port)
        .path(path)
        .out_port("trigger_out")
        .in_port("response_data_in")
        .ctrl_in_port("response_ctrl_in")
}

pub fn tokenize_source(upstream_url: &str) -> HttpSourceBuilder {
    HttpSourceBuilder::new("TokenizeSource")
        .url(upstream_url)
        .format(HttpFormat::SSE)
        .json_tap("choices[0].delta.content")
        .in_port("trigger_in")
        .out_port("response_data_out")
        .ctrl_out_port("response_ctrl_out")
        .post_json(serde_json::json!({
            "model": "tokenizer",
            "messages": [{"role": "user", "content": "tokenize"}],
            "stream": true
        }))
}