memra-tokenizer 0.84.1

GGUF-native BPE/SPM tokenizer with chat-template support for the memra inference engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4

[[package]]
name = "libc"
version = "0.2.186"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"

[[package]]
name = "memmap2"
version = "0.9.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0"
dependencies = [
 "libc",
]

[[package]]
name = "memra-gguf"
version = "0.84.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7adc33de99c3443572f5cca47fd9dd9aa404e3df006a5fab40171362a0ef1e3b"
dependencies = [
 "memmap2",
]

[[package]]
name = "memra-tokenizer"
version = "0.84.1"
dependencies = [
 "memra-gguf",
]