foxstash-core 0.5.0

High-performance local RAG library - SIMD-accelerated vector search, HNSW indexing
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94

//! Foxstash - Core library
//!
//! High-performance vector search and embedding generation for local-first AI.

// Allow some clippy lints for now - will address in future cleanup
#![allow(clippy::non_canonical_partial_ord_impl)]
#![allow(clippy::manual_is_multiple_of)]
#![allow(clippy::manual_div_ceil)]
#![allow(clippy::derivable_impls)]
#![allow(clippy::needless_range_loop)]
#![allow(clippy::unused_enumerate_index)]

pub mod embedding;
pub mod index;
pub mod storage;
pub mod vector;

use thiserror::Error;

/// Result type for RAG operations
pub type Result<T> = std::result::Result<T, RagError>;

/// Error types for RAG operations
#[derive(Debug, Error)]
pub enum RagError {
    #[error("Vector dimension mismatch: expected {expected}, got {actual}")]
    DimensionMismatch { expected: usize, actual: usize },

    #[error("Index error: {0}")]
    IndexError(String),

    #[error("Embedding error: {0}")]
    EmbeddingError(String),

    #[error("Storage error: {0}")]
    StorageError(String),

    #[error("IO error: {0}")]
    IoError(#[from] std::io::Error),

    #[error("Serialization error: {0}")]
    SerializationError(#[from] bincode::Error),

    #[error("Compression error: {0}")]
    CompressionError(#[from] storage::compression::CompressionError),

    #[error("Invalid input: {0}")]
    InvalidInput(String),
}

/// Document with embedding
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct Document {
    pub id: String,
    pub content: String,
    pub embedding: Vec<f32>,
    pub metadata: Option<serde_json::Value>,
}

/// Search result with similarity score
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct SearchResult {
    pub id: String,
    pub content: String,
    pub score: f32,
    pub metadata: Option<serde_json::Value>,
}

/// Configuration for RAG system
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct RagConfig {
    pub embedding_dim: usize,
    pub max_documents: usize,
    pub index_type: IndexType,
}

#[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize)]
pub enum IndexType {
    Flat,
    HNSW,
}

impl Default for RagConfig {
    fn default() -> Self {
        Self {
            embedding_dim: 384, // MiniLM-L6-v2
            max_documents: 10_000,
            index_type: IndexType::HNSW,
        }
    }
}

// Re-export commonly used items
pub use vector::{cosine_similarity, dot_product, l2_distance, normalize};