swiftide 0.3.1

Blazing fast document and code indexation
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91

use std::fmt::Debug;

use crate::{ingestion::IngestionNode, ingestion::IngestionStream, Embeddings};
use anyhow::Result;
use async_trait::async_trait;

/// All traits are easilly mockable under tests
#[cfg(test)]
use mockall::{automock, predicate::*};

#[cfg_attr(test, automock)]
#[async_trait]
/// Transforms single nodes into single nodes
pub trait Transformer: Send + Sync + Debug {
    async fn transform_node(&self, node: IngestionNode) -> Result<IngestionNode>;

    /// Overrides the default concurrency of the pipeline
    fn concurrency(&self) -> Option<usize> {
        None
    }
}

#[cfg_attr(test, automock)]
#[async_trait]
/// Transforms batched single nodes into streams of nodes
pub trait BatchableTransformer: Send + Sync + Debug {
    /// Defines the batch size for the transformer
    fn batch_size(&self) -> Option<usize> {
        None
    }

    /// Transforms a batch of nodes into a stream of nodes
    async fn batch_transform(&self, nodes: Vec<IngestionNode>) -> IngestionStream;

    /// Overrides the default concurrency of the pipeline
    fn concurrency(&self) -> Option<usize> {
        None
    }
}

/// Starting point of a stream
#[cfg_attr(test, automock)]
pub trait Loader {
    fn into_stream(self) -> IngestionStream;
}

#[cfg_attr(test, automock)]
#[async_trait]
/// Turns one node into many nodes
pub trait ChunkerTransformer: Send + Sync + Debug {
    async fn transform_node(&self, node: IngestionNode) -> IngestionStream;

    /// Overrides the default concurrency of the pipeline
    fn concurrency(&self) -> Option<usize> {
        None
    }
}

#[cfg_attr(test, automock)]
#[async_trait]
/// Caches nodes, typically by their path and hash
/// Recommended to namespace on the storage
///
/// For now just bool return value for easy filter
pub trait NodeCache: Send + Sync + Debug {
    async fn get(&self, node: &IngestionNode) -> bool;
    async fn set(&self, node: &IngestionNode);
}

#[async_trait]
pub trait Embed: Debug + Send + Sync {
    async fn embed(&self, input: Vec<String>) -> Result<Embeddings>;
}

#[async_trait]
pub trait SimplePrompt: Debug + Send + Sync {
    // Takes a simple prompt, prompts the llm and returns the response
    async fn prompt(&self, prompt: &str) -> Result<String>;
}

#[cfg_attr(test, automock)]
#[async_trait]
/// Persists nodes
pub trait Persist: Send + Sync {
    async fn setup(&self) -> Result<()>;
    async fn store(&self, node: IngestionNode) -> Result<IngestionNode>;
    async fn batch_store(&self, nodes: Vec<IngestionNode>) -> IngestionStream;
    fn batch_size(&self) -> Option<usize> {
        None
    }
}