tokenizer_py 0.2.0

crate with a tokenizer that works like a Python tokenizer
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12

[dependencies]

[package]
description = "crate with a tokenizer that works like a Python tokenizer"
documentation = "https://docs.rs/tokenizer_py"
edition = "2021"
license = "MIT OR Apache-2.0"
name = "tokenizer_py"
readme = "README.md"
repository = "https://github.com/salam99823/tokenizer"
rust-version = "1.56"
version = "0.2.0"