fornix 0.4.0

Knowledge storage, retrieval, and graph infrastructure for cognitive systems
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

//! Text chunking: the `Chunker` trait and all built-in implementations.
//!
//! Each chunker takes a `&str` and returns a `Vec<Chunk>`.

use crate::rag::types::Chunk;

/// The core chunking interface.
pub trait Chunker: Send + Sync {
    /// Split `text` into chunks.
    fn chunk(&self, text: &str) -> Vec<Chunk>;

    /// Human-readable name for this chunker.
    fn name(&self) -> &'static str;
}

pub mod fixed_length;
pub mod parent_child;
pub mod sentence_token;
pub mod token_count;

pub use fixed_length::FixedLength;
pub use parent_child::ParentChild;
pub use sentence_token::SentenceToken;
pub use token_count::TokenCount;