erio-embedding 0.1.0

Embedding engine for Erio
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

use std::path::Path;

pub(crate) struct Asset {
    pub(crate) asset_name: &'static str,
    pub(crate) relative_path: &'static str,
}

pub(crate) const ASSETS: &[Asset] = &[
    Asset {
        asset_name: "embeddinggemma-300M-Q8_0.gguf",
        relative_path: "embeddinggemma-300M-Q8_0.gguf",
    },
    Asset {
        asset_name: "tokenizer.json",
        relative_path: "tokenizer.json",
    },
    Asset {
        asset_name: "2_Dense_model.safetensors",
        relative_path: "2_Dense/model.safetensors",
    },
    Asset {
        asset_name: "3_Dense_model.safetensors",
        relative_path: "3_Dense/model.safetensors",
    },
];

pub(crate) fn validate_model_dir(model_dir: &Path) -> Result<(), String> {
    for asset in ASSETS {
        let expected = model_dir.join(asset.relative_path);
        if !expected.exists() {
            return Err(format!(
                "Missing required model file: {}\nSet ERIO_MODEL_DIR to a directory containing all model files.",
                expected.display()
            ));
        }
    }
    Ok(())
}

pub(crate) fn has_all_assets(model_dir: &Path) -> bool {
    ASSETS
        .iter()
        .all(|asset| model_dir.join(asset.relative_path).exists())
}