syntaxdot-tokenizers 0.2.0

Subword tokenizers
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20

[package]
name = "syntaxdot-tokenizers"
version = "0.2.0"
authors = ["Daniël de Kok <me@danieldk.eu>"]
edition = "2018"
description = "Subword tokenizers"
homepage = "https://github.com/tensordot/syntaxdot"
repository = "https://github.com/tensordot/syntaxdot.git"
documentation = "https://docs.rs/syntaxdot-tokenizers/"
license = "BlueOak-1.0.0"

[dependencies]
conllu = "0.5"
ndarray = "0.13"
sentencepiece = "0.5"
thiserror = "1"
wordpieces = "0.4"

[features]
model-tests = []