token-align 0.1.0

Maps overlapping tokens and character positions between tokenized texts
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

[package]
name = "token-align"
version = "0.1.0"
edition = "2021"
rust-version = "1.63"
authors = ["Justin Jones <hey.jones@icloud.com>"]
license = "MIT AND Unicode-3.0"
description = "Maps overlapping tokens and character positions between tokenized texts"
keywords = ["nlp", "tokenization", "alignment", "unicode", "text"]
categories = ["algorithms", "no-std", "text-processing"]
readme = "README.md"
repository = "https://github.com/hey-jj/token-align"
documentation = "https://docs.rs/token-align"
exclude = ["/tools/**", "/devkit/**", "/fuzz/**", "/reviews/**"]

[dev-dependencies]
criterion = { version = "=0.5.1", default-features = false }
proptest = "=1.6.0"

[[bench]]
name = "growth"
harness = false

[[bench]]
name = "charmap"
harness = false