lc-embeddings 0.20.0

Embedding model implementations for langchainrust — OpenAI, DeepSeek, Qwen, local/ONNX
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105

use super::*;
use crate::cosine_similarity;

// ---- BagOfWordsEmbeddings tests ----

#[tokio::test]
async fn test_bow_dimension() {
    let e = BagOfWordsEmbeddings::new(128);
    let v = e.embed_query("hello world").await.unwrap();
    assert_eq!(v.len(), 128);
    assert_eq!(e.dimension(), 128);
}

#[tokio::test]
async fn test_bow_same_text_same_vector() {
    let e = BagOfWordsEmbeddings::new(64);
    let a = e.embed_query("rust programming").await.unwrap();
    let b = e.embed_query("rust programming").await.unwrap();
    assert_eq!(a, b);
}

#[tokio::test]
async fn test_bow_different_text_different_vector() {
    let e = BagOfWordsEmbeddings::new(64);
    let a = e.embed_query("rust programming").await.unwrap();
    let b = e.embed_query("cooking recipe pasta").await.unwrap();
    assert_ne!(a, b);
}

#[tokio::test]
async fn test_bow_shared_words_more_similar() {
    let e = BagOfWordsEmbeddings::new(256);
    let base = e.embed_query("rust programming language").await.unwrap();
    let similar = e.embed_query("rust programming tutorial").await.unwrap();
    let different = e.embed_query("cooking pasta recipe").await.unwrap();

    let sim_similar = cosine_similarity(&base, &similar).unwrap_or(0.0);
    let sim_different = cosine_similarity(&base, &different).unwrap_or(0.0);
    assert!(
        sim_similar > sim_different,
        "Shared words should be more similar: {} vs {}",
        sim_similar,
        sim_different
    );
}

#[tokio::test]
async fn test_bow_normalized() {
    let e = BagOfWordsEmbeddings::new(64);
    let v = e.embed_query("some text here").await.unwrap();
    let norm: f32 = v.iter().map(|x| x * x).sum::<f32>().sqrt();
    assert!((norm - 1.0).abs() < 1e-5, "norm = {}", norm);
}

#[tokio::test]
async fn test_bow_empty_text_returns_error() {
    let e = BagOfWordsEmbeddings::new(64);
    let result = e.embed_query("").await;
    assert!(result.is_err());
    assert!(matches!(result.unwrap_err(), EmbeddingError::EmptyInput));
}

#[tokio::test]
async fn test_bow_chinese_tokenize() {
    let e = BagOfWordsEmbeddings::new(128);
    let a = e.embed_query("机器学习").await.unwrap();
    let b = e.embed_query("机器学习").await.unwrap();
    assert_eq!(a, b);
    let c = e.embed_query("深度学习").await.unwrap();
    let sim = cosine_similarity(&a, &c).unwrap_or(0.0);
    assert!(
        sim > 0.0,
        "Shared '学习' should have positive similarity: {}",
        sim
    );
}

#[test]
fn test_bow_tokenize_english() {
    let t = BagOfWordsEmbeddings::tokenize("Hello, World! 123");
    assert!(t.contains(&"hello".to_string()));
    assert!(t.contains(&"world".to_string()));
    assert!(t.contains(&"123".to_string()));
}

#[test]
fn test_bow_tokenize_chinese() {
    let t = BagOfWordsEmbeddings::tokenize("机器学习");
    assert!(t.contains(&"机".to_string()));
    assert!(t.contains(&"学".to_string()));
    assert_eq!(t.len(), 4);
}

#[test]
fn test_bow_model_name() {
    let e = BagOfWordsEmbeddings::default_dim();
    assert_eq!(e.model_name(), "local-bow");
}

// ---- 1.0: the LocalEmbeddings fallback alias has been removed ----

// The old `test_local_embeddings_backward_compat` (verifying `LocalEmbeddings =
// BagOfWordsEmbeddings` without the feature) was removed along with the fallback alias:
// as of 1.0 the `LocalEmbeddings` name is unavailable without the feature, so there is no
// "backward-compatible alias" left to test.