tiktokenx 0.1.0

A high-performance Rust implementation of OpenAI's tiktoken library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

[default]
# Extend the default configuration
extend-ignore-identifiers-re = [
    # Allow tiktoken-specific terms
    "tiktoken",
    "gpt",
    "openai",
    "bpe",
    # Allow common variable names
    "mut",
    "impl",
    "struct",
    "enum",
    "fn",
    "const",
    "static",
    "async",
    "await",
    # Allow hex values
    "[0-9a-fA-F]{8,}",
]

[default.extend-words]
# Add project-specific words that should not be flagged as typos
tiktoken = "tiktoken"
openai = "openai"
gpt = "gpt"
bpe = "bpe"
mergeable = "mergeable"
detokenization = "detokenization"
tokenization = "tokenization"
endoftext = "endoftext"
endofprompt = "endofprompt"
# BPE tokens that are legitimate but might be flagged as typos
nd = "nd"

[files]
extend-exclude = [
    "target/",
    "*.lock",
    "*.toml",
    "*.json",
    "*.md",
]