memra-tokenizer 0.87.1

GGUF-native BPE/SPM tokenizer with chat-template support for the memra inference engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4

[[package]]
name = "libc"
version = "0.2.186"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"

[[package]]
name = "memmap2"
version = "0.9.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0"
dependencies = [
 "libc",
]

[[package]]
name = "memra-gguf"
version = "0.87.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3bc5a1d61878c2c79921a5dcf1adff80edc6988dc124aa5bef36b95d79579e9"
dependencies = [
 "memmap2",
]

[[package]]
name = "memra-tokenizer"
version = "0.87.1"
dependencies = [
 "memra-gguf",
]