chunk_norris 0.2.1

A Rust library for splitting large text into smaller batches for LLM input.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

[dependencies.unicode-segmentation]
version = "1.12.0"

[lib]
name = "chunk_norris"
path = "src/lib.rs"

[package]
authors = ["Valerio uberti valerio.uberti23@gmail.com"]
autobenches = false
autobins = false
autoexamples = false
autolib = false
autotests = false
build = false
categories = ["text-processing"]
description = "A Rust library for splitting large text into smaller batches for LLM input."
edition = "2021"
keywords = ["text", "batching", "llm", "nlp", "tokenizer"]
license = "MIT OR Apache-2.0"
name = "chunk_norris"
readme = "README.md"
repository = "https://github.com/valeriouberti/ChunkNorris"
version = "0.2.1"

[[test]]
name = "integration_tests"
path = "tests/tests.rs"