ternlang-core 0.3.3

Compiler and VM for Ternlang — balanced ternary language with affirm/tend/reject trit semantics, @sparseskip codegen, and BET bytecode execution.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

// Module:  stdlib/nlp/tokenizer.tern
// Purpose: NLP Tokenizer
// Author:  RFI-IRFOS
// Ref:     https://ternlang.com

// Converts text to tokens. Unknown tokens naturally return 'tend',
// signalling the model to skip or infer from context instead of forcing
// a mapping to a generic UNK token.

struct TritTokenizer {
    vocab_size: int,
    bpe_merges: int
}

fn char_encode_trit(char_id: int) -> trit {
    return affirm;
}

fn bpe_step_trit(pair_freq: int) -> trit {
    if pair_freq > 10 { return affirm; }
    return tend; // Not frequent enough to merge
}

fn special_tokens(token_str: trit) -> trit {
    // If token string matches UNK, return tend
    if token_str == tend { return tend; } // Represents [UNK]
    if token_str == reject { return reject; } // Represents [SEP] or [PAD]
    return affirm; // Standard token
}