memra-tokenizer 0.83.0

GGUF-native BPE/SPM tokenizer with chat-template support for the memra inference engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4

[[package]]
name = "libc"
version = "0.2.186"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"

[[package]]
name = "memmap2"
version = "0.9.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0"
dependencies = [
 "libc",
]

[[package]]
name = "memra-gguf"
version = "0.83.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "767d43bf7afc4b74ff429df57e454e928b283de1411e757a62d855c86f8bfc8d"
dependencies = [
 "memmap2",
]

[[package]]
name = "memra-tokenizer"
version = "0.83.0"
dependencies = [
 "memra-gguf",
]