kalosm-language 0.3.3

A set of pretrained language models
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

// Retrieval Strategies:
// 1. Simple keyword search
// 2. Vector search
//  1. Search by sentence, return window around match
//  2. Search by summary, return document
//  3. Search through document tree
//  4. Search by questions that may be answered by the document
//  5. Classify documents, search by class
//
// Context extraction strategies:
// 1. Dump all sentences
// 2. Dump all sentences that mention an entity
// 3. Extract relevant sentences with an llm

use kalosm_language_model::Embedder;

use crate::context::Document;

use super::Chunk;

mod chunking;
pub use chunking::*;
mod hypothetical;
pub use hypothetical::*;
mod summary;
pub use summary::*;
mod sentence;
pub use sentence::*;
mod semantic;
pub use semantic::*;
mod html;
pub use html::*;

/// A strategy for chunking a document into smaller pieces.
pub trait Chunker {
    /// Chunk a document into embedded snippets.
    fn chunk<E: Embedder + Send>(
        &self,
        document: &Document,
        embedder: &E,
    ) -> impl std::future::Future<Output = anyhow::Result<Vec<Chunk<E::VectorSpace>>>> + Send;

    /// Chunk a batch of documents into embedded snippets.
    fn chunk_batch<'a, I, E: Embedder + Send>(
        &self,
        documents: I,
        embedder: &E,
    ) -> impl std::future::Future<Output = anyhow::Result<Vec<Vec<Chunk<E::VectorSpace>>>>> + Send
    where
        I: IntoIterator<Item = &'a Document> + Send,
        I::IntoIter: Send,
        Self: Sync,
    {
        async {
            let mut chunks = Vec::new();
            for document in documents {
                chunks.push(self.chunk(document, embedder).await?);
            }
            Ok(chunks)
        }
    }
}