bert_tokenizer 0.1.3

This crate is a Rust port of Google's BERT WordPiece tokenizer.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

[dependencies.indexmap]
version = "1.9.2"

[dependencies.unicode-normalization]
version = "0.1.22"

[dependencies.unicode_categories]
version = "0.1.1"

[package]
authors = ["Yiğit Bekir Kaya <kayayig@itu.edu.tr>"]
description = "This crate is a Rust port of Google's BERT WordPiece tokenizer."
edition = "2021"
license-file = "LICENSE"
name = "bert_tokenizer"
readme = "README.md"
repository = "https://github.com/yigit353/bert_tokenizer"
version = "0.1.3"