kibble 0.1.0

chew through any source into clean datasets — a fast ingestion, RAG & fine-tuning toolkit
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

[package]
name = "kibble"
version = "0.1.0"
edition = "2021"
description = "chew through any source into clean datasets — a fast ingestion, RAG & fine-tuning toolkit"
license = "MIT OR Apache-2.0"
repository = "https://github.com/femboyisp/kibble"
authors = ["Zoa Hickenlooper"]
readme = "README.md"
keywords = ["rag", "dataset", "ingestion", "embeddings", "cli"]
categories = ["command-line-utilities", "text-processing"]
exclude = ["assets/"]

[[bin]]
name = "kibble"
path = "src/main.rs"
required-features = ["full"]

[lib]
name = "kibble"
path = "src/lib.rs"

[dependencies]
clap = { version = "4", features = ["derive"] }
regex = "1"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
parquet = { version = "59", default-features = false, features = ["snap", "zstd", "brotli", "lz4", "flate2-rust_backend", "json"], optional = true }
serde_yaml = "0.9"
sha2 = "0.10"
md-5 = "0.10"
tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "time", "io-std", "io-util"] }
tokio-stream = "0.1"
axum = { version = "0.8", optional = true }
tower = { version = "0.5", features = ["util"], optional = true }
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "socks", "multipart"] }
toml = "0.8"
scraper = "0.20"
zip = { version = "2", default-features = false, features = ["deflate"], optional = true }
tar = { version = "0.4", optional = true }
flate2 = { version = "1", optional = true }
aes = { version = "0.9", optional = true }
base64 = "0.22"
pdf-extract = { version = "0.12", optional = true }

[features]
default = ["full"]
retrieval = []
full = ["retrieval", "dep:parquet", "dep:axum", "dep:tower", "dep:pdf-extract", "dep:tar", "dep:flate2", "dep:aes", "dep:zip"]

[dev-dependencies]
tokio = { version = "1", features = ["macros", "rt-multi-thread"] }

# The profile that 'dist' will build with
[profile.dist]
inherits = "release"
lto = "thin"