bert_tokenizer 0.1.3

This crate is a Rust port of Google's BERT WordPiece tokenizer.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

[package]
name = "bert_tokenizer"
version = "0.1.3"
edition = "2021"
authors = ["Yiğit Bekir Kaya <kayayig@itu.edu.tr>"]
description = "This crate is a Rust port of Google's BERT WordPiece tokenizer."
license-file = "LICENSE"
repository = "https://github.com/yigit353/bert_tokenizer"

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
indexmap = "1.9.2"
unicode-normalization = "0.1.22"
unicode_categories = "0.1.1"