syntaxdot-tokenizers 0.2.0

Subword tokenizers
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

[dependencies.conllu]
version = "0.5"

[dependencies.ndarray]
version = "0.13"

[dependencies.sentencepiece]
version = "0.5"

[dependencies.thiserror]
version = "1"

[dependencies.wordpieces]
version = "0.4"

[features]
model-tests = []

[package]
authors = ["Daniël de Kok <me@danieldk.eu>"]
description = "Subword tokenizers"
documentation = "https://docs.rs/syntaxdot-tokenizers/"
edition = "2018"
homepage = "https://github.com/tensordot/syntaxdot"
license = "BlueOak-1.0.0"
name = "syntaxdot-tokenizers"
repository = "https://github.com/tensordot/syntaxdot.git"
version = "0.2.0"