fibpetokenizer 0.1.0

A blazing fast Byte Pair Encoding (BPE) tokenizer library with Python bindings
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

[build-system]

requires = ["maturin>=1.0,<2.0"]

build-backend = "maturin"



[project]

name = "fibpetokenizer"

version = "0.1.0"

description = "A blazing fast Byte Pair Encoding (BPE) tokenizer library with Python bindings"

authors = [

    {name = "Muhammad Faisal Ishfaq", email = "faisalishfaq59@gmail.com"}

]

readme = "README.md"

license = {text = "MIT OR Apache-2.0"}

keywords = ["tokenizer", "bpe", "nlp", "machine-learning", "text-processing"]

classifiers = [

    "Development Status :: 4 - Beta",

    "Intended Audience :: Developers",

    "Intended Audience :: Science/Research",

    "License :: OSI Approved :: MIT License",

    "License :: OSI Approved :: Apache Software License",

    "Programming Language :: Python :: 3",

    "Programming Language :: Python :: 3.7",

    "Programming Language :: Python :: 3.8",

    "Programming Language :: Python :: 3.9",

    "Programming Language :: Python :: 3.10",

    "Programming Language :: Python :: 3.11",

    "Programming Language :: Python :: 3.12",

    "Programming Language :: Rust",

    "Topic :: Scientific/Engineering :: Artificial Intelligence",

    "Topic :: Text Processing",

]

requires-python = ">=3.7"



[project.urls]

Homepage = "https://github.com/faisalishfaq2005/FIBpeTokenizer"

Repository = "https://github.com/faisalishfaq2005/FIBpeTokenizer"

Documentation = "https://github.com/faisalishfaq2005/FIBpeTokenizer"



[tool.maturin]

features = ["python"]

python-source = "python"

module-name = "fibpetokenizer"