selfware 0.6.2

Your personal AI workshop — software you own, software that lasts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

use super::*;

#[test]
fn test_tokenize() {
    let text = "Hello world! How are you?";
    let tokens = tokenize(text);
    assert!(tokens.contains(&"hello".to_string()));
    assert!(tokens.contains(&"world".to_string()));
}

#[test]
fn test_extract_keywords() {
    let query = "How does the agent loop handle max iterations?";
    let keywords = extract_keywords(query);
    assert!(keywords.contains(&"agent".to_string()));
    assert!(keywords.contains(&"loop".to_string()));
    assert!(keywords.contains(&"iterations".to_string()));
    // Stop words should be removed
    assert!(!keywords.contains(&"how".to_string()));
}

#[test]
fn test_bm25_idf() {
    let engine = CodeQueryEngine {
        doc_freq: [("test".to_string(), 5)].into_iter().collect(),
        total_docs: 100,
        avg_doc_len: 100.0,
    };

    let idf = engine.idf("test");
    assert!(idf > 0.0);
}