frankensearch-embed 0.1.3

Embedder implementations for frankensearch (hash, model2vec, fastembed)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120

//! Embedder implementations for the frankensearch hybrid search library.
//!
//! Provides three tiers of text embedding:
//! - **Hash** (`hash` feature, default): FNV-1a hash embedder, zero dependencies, always available.
//! - **`Model2Vec`** (`model2vec` feature): potion-128M static embedder, fast tier (~0.57ms).
//! - **`FastEmbed`** (`fastembed` feature): MiniLM-L6-v2 ONNX embedder, quality tier (~128ms).
//!
//! The `EmbedderStack` auto-detection probes for available models and configures
//! the best fast+quality pair automatically.

pub mod auto_detect;
pub mod batch_coalescer;
#[cfg(feature = "bundled-default-models")]
pub mod bundled_default_models;
pub mod cached_embedder;
pub mod model_cache;
pub mod model_manifest;
pub mod model_registry;
pub use auto_detect::{
    DimReduceEmbedder, EmbedderStack, ModelAvailabilityDiagnostic, ModelStatus, TwoTierAvailability,
};
pub use batch_coalescer::{
    BatchCoalescer, CoalescedBatch, CoalescerConfig, CoalescerMetrics, Priority,
};
#[cfg(feature = "bundled-default-models")]
pub use bundled_default_models::{EmbeddedModelInstallSummary, ensure_default_semantic_models};

// When bundled-default-models is disabled (lite build), provide a no-op
// `ensure_default_semantic_models` so downstream crates compile without
// feature-gating every call site.
#[cfg(not(feature = "bundled-default-models"))]
pub use lite_fallback::{EmbeddedModelInstallSummary, ensure_default_semantic_models};

#[cfg(not(feature = "bundled-default-models"))]
mod lite_fallback {
    use std::path::{Path, PathBuf};

    use frankensearch_core::error::SearchResult;

    /// Summary returned by the no-op lite-build materialization.
    #[derive(Debug, Clone, PartialEq, Eq)]
    pub struct EmbeddedModelInstallSummary {
        /// Effective model root (inherited from caller or platform default).
        pub model_root: PathBuf,
        /// Always 0 in the lite build -- no embedded models to write.
        pub models_written: usize,
        /// Always 0 in the lite build.
        pub bytes_written: u64,
    }

    /// No-op: lite builds have no embedded models to materialize.
    ///
    /// Returns a summary with zero writes. Callers should check for models
    /// on disk at the standard location (`~/.local/share/frankensearch/models/`)
    /// and prompt the user to run `fsfs download-models` if they are missing.
    pub fn ensure_default_semantic_models(
        model_root: Option<&Path>,
    ) -> SearchResult<EmbeddedModelInstallSummary> {
        let root = model_root.map(|p| p.to_path_buf()).unwrap_or_else(|| {
            crate::model_registry::ensure_model_storage_layout_checked()
                .unwrap_or_else(|_| PathBuf::from("models"))
        });
        Ok(EmbeddedModelInstallSummary {
            model_root: root,
            models_written: 0,
            bytes_written: 0,
        })
    }
}
pub use cached_embedder::{CacheStats, CachedEmbedder};
pub use model_cache::{
    ENV_DATA_DIR, ENV_MODEL_DIR, KnownModel, MODEL_CACHE_LAYOUT_VERSION, ModelCacheLayout,
    ModelDirEntry, ensure_cache_layout, ensure_default_cache, is_model_installed, known_models,
    model_file_path, resolve_cache_root,
};
pub use model_manifest::{
    ConsentSource, DOWNLOAD_CONSENT_ENV, DownloadConsent, MANIFEST_SCHEMA_VERSION, ModelFile,
    ModelLifecycle, ModelManifest, ModelManifestCatalog, ModelState, ModelTier,
    PLACEHOLDER_VERIFY_AFTER_DOWNLOAD, VerificationMarker, is_verification_cached,
    resolve_download_consent, verify_dir_cached, verify_file_sha256, write_verification_marker,
};
pub use model_registry::{
    BAKEOFF_CUTOFF_DATE, EmbedderRegistry, RegisteredEmbedder, RegisteredReranker,
    registered_embedders, registered_rerankers,
};

#[cfg(feature = "hash")]
pub mod hash_embedder;

#[cfg(feature = "hash")]
pub use hash_embedder::{HashAlgorithm, HashEmbedder};

#[cfg(feature = "model2vec")]
pub mod model2vec_embedder;

#[cfg(feature = "model2vec")]
pub use model2vec_embedder::{Model2VecEmbedder, find_model_dir};

#[cfg(feature = "fastembed")]
pub mod fastembed_embedder;

#[cfg(feature = "fastembed")]
pub use fastembed_embedder::{FastEmbedEmbedder, OnnxEmbedderConfig};

#[cfg(feature = "download")]
pub mod model_download;

#[cfg(feature = "download")]
pub use model_download::{DownloadConfig, DownloadProgress, ModelDownloader};

#[cfg(feature = "api")]
pub mod api_provider;

#[cfg(feature = "api")]
pub mod api_embedder;

#[cfg(feature = "api")]
pub use api_embedder::ApiEmbedder;
#[cfg(feature = "api")]
pub use api_provider::{ApiProvider, GeminiProvider, OpenAiProvider};