uni-xervo 0.4.0

Unified Rust runtime for local and remote embedding, reranking, and generation models
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

//! Quick-start example: load a local Candle embedding model and embed a sentence.
//!
//! Run with:
//! ```sh
//! cargo run --example quick_start --features provider-candle
//! ```

#[cfg(feature = "provider-candle")]
use uni_xervo::api::{ModelAliasSpec, ModelTask};
#[cfg(feature = "provider-candle")]
use uni_xervo::provider::candle::LocalCandleProvider;
#[cfg(feature = "provider-candle")]
use uni_xervo::runtime::ModelRuntime;

#[cfg(feature = "provider-candle")]
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    // 1. Define your model catalog
    let spec = ModelAliasSpec {
        alias: "embed/local".to_string(),
        task: ModelTask::Embed,
        provider_id: "local/candle".to_string(),
        model_id: "sentence-transformers/all-MiniLM-L6-v2".to_string(),
        revision: None,
        warmup: Default::default(), // Lazy by default
        required: true,
        timeout: None,
        load_timeout: None,
        retry: None,
        options: serde_json::Value::Null,
    };

    // 2. Build the runtime
    let runtime = ModelRuntime::builder()
        .register_provider(LocalCandleProvider::new())
        .catalog(vec![spec])
        .build()
        .await?;

    // 3. Get a typed handle to the model
    let model = runtime.embedding("embed/local").await?;

    // 4. Run inference
    let embeddings = model.embed(vec!["Hello, world!"]).await?;
    println!("Embedding vector length: {}", embeddings[0].len());

    Ok(())
}

#[cfg(not(feature = "provider-candle"))]
fn main() {
    eprintln!(
        "This example requires the `provider-candle` feature.\n\
         Run with: cargo run --example quick_start --features provider-candle"
    );
}