markhor_extensions 0.1.0-alpha.0.2

Extensions for Markhor, a project connecting AI models, documents, and workflows for knowledge work
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

use markhor_core::extension::Extension;
use plaintext::PlainTextChunker;
use tracing::error;

pub mod plaintext;


pub struct Chunkers;

impl Extension for Chunkers {
    fn name(&self) -> &str {
        "chunkers"
    }

    fn description(&self) -> &str {
        "Chunking extension"
    }

    fn uri(&self) -> &str {
        "TODO chunker uri"
    }

    fn chunker(&self) -> Option<Box<dyn markhor_core::chunking::Chunker>> {
        // match markdown::chunker::MarkdownChunker::new(Default::default()) {
        //     Ok(chunker) => Some(std::sync::Arc::new(chunker)),
        //     Err(e) => {
        //         error!("Error creating chunker: {:?}", e);
        //         None
        //     }
        // }

        let chunker = PlainTextChunker::new(2000, 200).unwrap();
        Some(Box::new(chunker))
    }
}