chunkshop-rs 0.3.2

Standalone ingest-to-pgvector: source -> chunker -> embedder -> extractor -> table. int8 BGE by default; bakeoff matrix evaluator built in. Cross-language wire-format compatible with the Python `chunkshop` package.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

//! chunkshop-rs — minimal Rust port of chunkshop.
//!
//! Implements one source (files), one chunker (sentence_aware), one embedder
//! (fastembed), and one pgvector sink. The YAML config schema and target table
//! shape match the Python reference so vectors are interchangeable across
//! implementations.

pub mod bakeoff;
pub mod chunker;
pub mod config;
pub mod embedder;
pub mod extractor;
pub mod framer;
pub(crate) mod hf_cache;
pub mod pipeline;
pub mod runner;
pub mod sentence_split;
pub mod sink;
pub mod source;
pub mod summarizer;

pub use bakeoff::{run_bakeoff, run_bakeoff_with_base, BakeoffConfig, BakeoffResults};
pub use chunker::{Chunk, SentenceAwareChunker};
pub use config::{load_config, CellConfig};
pub use embedder::FastembedEmbedder;
pub use pipeline::Pipeline;
pub use runner::{run_cell, CellResult};
pub use sink::PgVectorSink;
pub use source::{Document, FilesSource};