tokengeex 1.1.0

TokenGeeX is an efficient tokenizer for code based on UnigramLM and TokenMonster.
Documentation
1
2
3
4
5
6
7
8
9
steps = ["Baseline"]

tokengeex_values = []

tokengeex_capcode_values = []

sentencepiece_values = []

tokenizers_values = []