pie 0.1.1

Programmable Inference Engine (PIE)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

interface tokenize {

    use core.{queue};                  // Import model definitions

    // Retrieves the tokenizer associated with the given model
    get-tokenizer: func(queue: borrow<queue>) -> tokenizer;

    // Tokenizer resource for encoding and decoding text
    resource tokenizer {

        // Converts input text into a list of token IDs
        tokenize: func(text: string) -> list<u32>;

        // Converts token IDs back into a decoded string
        detokenize: func(tokens: list<u32>) -> string;

        // Returns the tokenizer's vocabulary as a list of byte sequences (tokens)
        get-vocabs: func() -> tuple<list<u32>, list<list<u8>>>;
    }

}