laurus 0.7.0

Unified search library for lexical, vector, and semantic retrieval
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97

//! Simple example demonstrating synonym expansion with SynonymGraphFilter.
//!
//! This example shows how to:
//! - Create a synonym dictionary
//! - Apply synonym expansion to text
//! - Use boost to adjust synonym weights
//! - Observe how tokens are expanded with synonyms

use laurus::Result;
use laurus::analysis::synonym::dictionary::SynonymDictionary;
use laurus::analysis::token_filter::Filter;
use laurus::analysis::token_filter::synonym_graph::SynonymGraphFilter;
use laurus::analysis::tokenizer::Tokenizer;
use laurus::analysis::tokenizer::whitespace::WhitespaceTokenizer;

fn main() -> Result<()> {
    println!("=== SynonymGraphFilter Usage Example ===\n");

    // 1. Create a synonym dictionary
    println!("Step 1: Creating synonym dictionary");
    let mut dict = SynonymDictionary::new(None)?;

    // Add synonym groups (bidirectional)
    dict.add_synonym_group(vec!["ml".to_string(), "machine learning".to_string()]);
    dict.add_synonym_group(vec![
        "ai".to_string(),
        "artificial intelligence".to_string(),
    ]);

    println!("  ✓ Added synonyms:");
    println!("    - 'ml' ↔ 'machine learning'");
    println!("    - 'ai' ↔ 'artificial intelligence'\n");

    // 2. Example without boost
    println!("Step 2: Applying filter WITHOUT boost");
    let tokenizer = WhitespaceTokenizer;
    let synonym_filter = SynonymGraphFilter::new(dict.clone(), true); // keep_original=true

    let input_text = "ml tutorial";
    println!("  Input: \"{}\"\n", input_text);

    let tokens = tokenizer.tokenize(input_text)?;
    let result_tokens = synonym_filter.filter(tokens)?;

    println!("  Output tokens:");
    for (i, token) in result_tokens.enumerate() {
        println!(
            "    [{}] '{}' (pos={}, pos_inc={}, pos_len={}, boost={:.2})",
            i,
            token.text,
            token.position,
            token.position_increment,
            token.position_length,
            token.boost
        );
    }

    println!("\n  Explanation:");
    println!("    - All tokens have boost=1.0 (default)");
    println!("    - Synonyms have equal weight to original tokens\n");

    // 3. Example with boost
    println!("Step 3: Applying filter WITH boost=0.8");
    let synonym_filter_with_boost = SynonymGraphFilter::new(dict, true).with_boost(0.8); // Synonyms get 80% weight

    println!("  Input: \"{}\"\n", input_text);

    let tokens = tokenizer.tokenize(input_text)?;
    let result_tokens = synonym_filter_with_boost.filter(tokens)?;

    println!("  Output tokens:");
    for (i, token) in result_tokens.enumerate() {
        println!(
            "    [{}] '{}' (pos={}, pos_inc={}, pos_len={}, boost={:.2})",
            i,
            token.text,
            token.position,
            token.position_increment,
            token.position_length,
            token.boost
        );
    }

    println!("\n  Explanation:");
    println!("    - Original token 'ml' has boost=1.0");
    println!("    - 'machine' has boost=0.72 (0.9 × 0.8) - first token of multi-word synonym");
    println!("    - 'learning' has boost=0.64 (0.8 × 0.8) - second token");
    println!("    - Lower boost means synonyms contribute less to the final score");
    println!("    - This helps prioritize exact matches over synonym matches\n");

    println!("Use cases for boost:");
    println!("  - boost=0.8: Synonyms have 80% weight (common default)");
    println!("  - boost=0.5: Synonyms have 50% weight (conservative)");
    println!("  - boost=1.0: Synonyms equal to originals (no adjustment)\n");

    Ok(())
}