use hipstr::HipStr;
use rapidhash::{RapidHashMap, RapidHashSet};
use wedb_embed::search::{
DEFAULT_STOP_WORDS, DistanceMetric, FtCreate, FtSearch, HnswGraph, IndexField, IndexFieldType,
IndexOnDataType, InvertedIndex, SearchIndexManager, SearchIndexSchema, SearchQueryNode,
SuggestionDict, VectorType, compute_vector_distance, decode_sortable_f64, decode_sortable_i64,
encode_sortable_f64, encode_sortable_i64, explain_search_query, explain_search_query_cli,
extract_doc_terms, levenshtein_distance, parse_search_query, parse_search_query_with_params,
parse_vector_from_slice, tokenize_tags, tokenize_text, tokenize_text_with_stopwords,
unescape_tag_string,
};
#[ctor::ctor(unsafe)]
fn _log_init() {
log_init::init();
}
#[test]
fn test_search_query_parsing_and_explain() {
let q = "@title:hello @tag:{rust | database} @age:[18 (30] -world";
let ast = parse_search_query(q);
let plan = explain_search_query(&ast);
assert!(plan.contains("INTERSECT"));
assert!(plan.contains("UNION <title:hello>"));
assert!(plan.contains("TAG <@tag:{rust | database}>"));
assert!(plan.contains("NUMERIC <@age:[18 30)>"));
assert!(plan.contains("NOT {"));
let cli_plan = explain_search_query_cli(&ast);
assert_eq!(plan, cli_plan);
}
#[test]
fn test_query_knn_and_vector_range_parsing() {
let mut params = RapidHashMap::default();
let vec = [1.0f64, 2.0, 3.0, 4.0];
let bytes: Vec<u8> = vec.iter().flat_map(|f| f.to_le_bytes()).collect();
params.insert("v".to_string(), unsafe {
String::from_utf8_unchecked(bytes)
});
params.insert("k_num".to_string(), "5".to_string());
params.insert("radius_val".to_string(), "0.75".to_string());
let q = "*=>[KNN $k_num @embedding $v]";
let ast = parse_search_query_with_params(q, ¶ms);
if let SearchQueryNode::VectorKnn {
field,
k,
vector_param,
vector,
} = ast
{
assert_eq!(field, "embedding");
assert_eq!(k, 5);
assert_eq!(vector_param, "v");
assert_eq!(vector, Some(vec![1.0, 2.0, 3.0, 4.0]));
} else {
panic!("expected VectorKnn node");
}
let q_range = "@embedding:[VECTOR_RANGE $radius_val $v]";
let ast_range = parse_search_query_with_params(q_range, ¶ms);
if let SearchQueryNode::VectorRange {
field,
radius,
vector_param,
vector,
} = ast_range
{
assert_eq!(field, "embedding");
assert!((radius - 0.75).abs() < 1e-6);
assert_eq!(vector_param, "v");
assert_eq!(vector, Some(vec![1.0, 2.0, 3.0, 4.0]));
} else {
panic!("expected VectorRange node");
}
}
#[test]
fn test_sortable_f64_and_i64_encoding() {
let numbers = vec![-100.5, -0.01, 0.0, 0.001, 42.0, 9999.99];
let encoded: Vec<String> = numbers.iter().map(|&n| encode_sortable_f64(n)).collect();
let mut sorted_encoded = encoded.clone();
sorted_encoded.sort();
assert_eq!(encoded, sorted_encoded);
for n in numbers {
let enc = encode_sortable_f64(n);
let dec = decode_sortable_f64(&enc).unwrap();
assert!((n - dec).abs() < 1e-9);
}
let ints = vec![-999999i64, -42, 0, 1, 100, 123456789];
let enc_ints: Vec<String> = ints.iter().map(|&i| encode_sortable_i64(i)).collect();
let mut sorted_enc_ints = enc_ints.clone();
sorted_enc_ints.sort();
assert_eq!(enc_ints, sorted_enc_ints);
for i in ints {
let enc = encode_sortable_i64(i);
let dec = decode_sortable_i64(&enc).unwrap();
assert_eq!(i, dec);
}
}
#[test]
fn test_tokenize_text_and_tags_and_stopwords() {
let words = tokenize_text("Hello, RediSearch 2.0_beta in Rust!");
assert_eq!(
words,
vec!["hello", "redisearch", "2", "0_beta", "in", "rust"]
);
let sw_set: RapidHashSet<String> = DEFAULT_STOP_WORDS.iter().map(|&s| s.to_string()).collect();
let filtered_words =
tokenize_text_with_stopwords("this is a test with stop words", Some(&sw_set));
assert_eq!(filtered_words, vec!["test", "stop", "words"]);
let tags = tokenize_tags("db, kv , redis, raft", ',', false);
assert_eq!(tags, vec!["db", "kv", "redis", "raft"]);
let case_tags = tokenize_tags("Redis, Raft, SQLite", ',', true);
assert_eq!(case_tags, vec!["Redis", "Raft", "SQLite"]);
assert_eq!(
unescape_tag_string(r"email\@example\.com"),
"email@example.com"
);
assert_eq!(unescape_tag_string(r"Hello\ World"), "Hello World");
}
#[test]
fn test_vector_distance_calculations() {
let v1 = vec![1.0, 0.0, 0.0];
let v2 = vec![0.0, 1.0, 0.0];
let v3 = vec![1.0, 0.0, 0.0];
let dist_l2 = compute_vector_distance(&v1, &v2, DistanceMetric::L2).unwrap();
assert!((dist_l2 - (2.0f64).sqrt()).abs() < 1e-6);
let dist_ip = compute_vector_distance(&v1, &v3, DistanceMetric::IP).unwrap();
assert!((dist_ip - (-1.0)).abs() < 1e-6);
let dist_cos_same = compute_vector_distance(&v1, &v3, DistanceMetric::Cosine).unwrap();
assert!(dist_cos_same.abs() < 1e-6);
let dist_cos_ortho = compute_vector_distance(&v1, &v2, DistanceMetric::Cosine).unwrap();
assert!((dist_cos_ortho - 1.0).abs() < 1e-6);
let bytes: Vec<u8> = [1.5f64, -2.5, 3.25]
.iter()
.flat_map(|f| f.to_le_bytes())
.collect();
let parsed = parse_vector_from_slice(&bytes, VectorType::Float64).unwrap();
assert_eq!(parsed, vec![1.5, -2.5, 3.25]);
let bytes_f32: Vec<u8> = [1.5f32, -2.5, 3.25]
.iter()
.flat_map(|f| f.to_le_bytes())
.collect();
let parsed_f32 = parse_vector_from_slice(&bytes_f32, VectorType::Float32).unwrap();
assert_eq!(parsed_f32, vec![1.5, -2.5, 3.25]);
}
#[test]
fn test_levenshtein_distance() {
assert_eq!(levenshtein_distance("kitten", "sitting"), 3);
assert_eq!(levenshtein_distance("rust", "rust"), 0);
assert_eq!(levenshtein_distance("redis", "reddis"), 1);
}
#[test]
fn test_inverted_index_indexing_and_search() {
let schema = SearchIndexSchema::with_full_spec(
HipStr::from("idx_books"),
IndexOnDataType::Json,
vec![HipStr::from("book:")],
vec![
IndexField::new("title", IndexFieldType::Text),
IndexField::with_tag("category", Some(','), false),
IndexField::with_numeric("price", true),
],
);
let mut idx = InvertedIndex::new();
let doc1 = sonic_rs::json!({
"title": "Rust Programming in Depth",
"category": "tech, programming",
"price": 49.9
});
let raw1 = sonic_rs::to_vec(&doc1).unwrap();
idx.index_doc(&schema, "book:1", &raw1, Some(1.0), None)
.unwrap();
let doc2 = sonic_rs::json!({
"title": "Distributed Databases and Consensus Algorithms",
"category": "tech, database",
"price": 89.0
});
let raw2 = sonic_rs::to_vec(&doc2).unwrap();
idx.index_doc(&schema, "book:2", &raw2, Some(2.0), None)
.unwrap();
let doc3 = sonic_rs::json!({
"title": "Cooking Masterclass Recipes",
"category": "lifestyle, food",
"price": 25.0
});
let raw3 = sonic_rs::to_vec(&doc3).unwrap();
idx.index_doc(&schema, "book:3", &raw3, Some(0.5), None)
.unwrap();
let res = idx.search(&schema, "rust", &FtSearch::default()).unwrap();
assert_eq!(res.total_results, 1);
assert_eq!(res.docs[0].id, HipStr::from("book:1"));
let res2 = idx
.search(
&schema,
"@category:{tech} @price:[40 100]",
&FtSearch::default(),
)
.unwrap();
assert_eq!(res2.total_results, 2);
let res3 = idx
.search(&schema, "@category:{tech} -consensus", &FtSearch::default())
.unwrap();
assert_eq!(res3.total_results, 1);
assert_eq!(res3.docs[0].id, HipStr::from("book:1"));
let search_opts = FtSearch {
sortby: Some(("price".to_string(), true)),
limit: Some((0, 2)),
..Default::default()
};
let res4 = idx.search(&schema, "*", &search_opts).unwrap();
assert_eq!(res4.total_results, 3);
assert_eq!(res4.docs.len(), 2);
assert_eq!(res4.docs[0].id, HipStr::from("book:3")); assert_eq!(res4.docs[1].id, HipStr::from("book:1"));
let search_return = FtSearch {
returns: vec![("title".to_string(), None)],
..Default::default()
};
let res5 = idx.search(&schema, "databases", &search_return).unwrap();
assert_eq!(res5.total_results, 1);
assert_eq!(res5.docs[0].fields.len(), 1);
assert_eq!(res5.docs[0].fields[0].0, HipStr::from("title"));
let tag_vals = idx.tag_vals("category");
assert!(tag_vals.contains(&"tech".to_string()));
assert!(tag_vals.contains(&"programming".to_string()));
assert!(tag_vals.contains(&"database".to_string()));
assert!(tag_vals.contains(&"lifestyle".to_string()));
let info = idx.info(&schema);
assert_eq!(info.index_name, "idx_books");
assert_eq!(info.num_docs, 3);
let deleted = idx.delete_doc(&schema, "book:3");
assert!(deleted);
assert_eq!(idx.docs.len(), 2);
}
#[test]
fn test_search_index_manager_and_aliases_and_config() {
let mut mgr = SearchIndexManager::new();
let schema = SearchIndexSchema::with_full_spec(
HipStr::from("users_idx"),
IndexOnDataType::Hash,
vec![HipStr::from("user:")],
vec![
IndexField::new("name", IndexFieldType::Text),
IndexField::with_numeric("age", true),
],
);
mgr.create_index(schema).unwrap();
assert_eq!(mgr.list_indexes(), vec!["users_idx".to_string()]);
let dup_schema = SearchIndexSchema::new(
HipStr::from("users_idx"),
vec![HipStr::from("user:")],
vec![HipStr::from("name")],
);
assert!(mgr.create_index(dup_schema).is_err());
mgr.add_alias("users_alias", "users_idx").unwrap();
assert_eq!(mgr.resolve_index_name("users_alias"), "users_idx");
mgr.update_alias("users_alias", "users_idx").unwrap();
assert_eq!(mgr.resolve_index_name("users_alias"), "users_idx");
mgr.del_alias("users_alias").unwrap();
assert_eq!(mgr.resolve_index_name("users_alias"), "users_alias");
let timeout = mgr.config_get("TIMEOUT").unwrap();
assert_eq!(timeout, "500");
mgr.config_set("TIMEOUT", "1000").unwrap();
assert_eq!(mgr.config_get("TIMEOUT").unwrap(), "1000");
let help = mgr.config_help("TIMEOUT").unwrap();
assert!(help.contains("timeout"));
mgr.drop_index("users_idx", false).unwrap();
assert!(mgr.list_indexes().is_empty());
}
#[test]
fn test_suggestions_dict() {
let mut dict = SuggestionDict::new();
assert_eq!(
dict.sug_add("redis", 10.0, false, Some("db".to_string())),
1
);
assert_eq!(dict.sug_add("rediss", 5.0, false, None), 2);
assert_eq!(dict.sug_add("redigo", 8.0, false, None), 3);
assert_eq!(
dict.sug_add("rust", 15.0, false, Some("lang".to_string())),
4
);
assert_eq!(dict.sug_len(), 4);
let res = dict.sug_get("redi", false, true, true, Some(10));
assert_eq!(res.len(), 3);
assert_eq!(res[0].string, "redis");
assert_eq!(res[0].score, 10.0);
assert_eq!(res[0].payload, Some("db".to_string()));
let fuzzy_res = dict.sug_get("radis", true, true, false, Some(5));
assert!(!fuzzy_res.is_empty());
assert_eq!(fuzzy_res[0].string, "redis");
assert!(dict.sug_del("redigo"));
assert_eq!(dict.sug_len(), 3);
assert!(!dict.sug_del("non_existent"));
}
#[test]
fn test_extract_doc_terms_and_schema_types() {
let schema = SearchIndexSchema::with_full_spec(
HipStr::from("idx"),
IndexOnDataType::Json,
vec![HipStr::from("user:")],
vec![
IndexField::new("title", IndexFieldType::Text),
IndexField::with_tag("tags", Some(','), false),
IndexField::with_numeric("score", true),
IndexField::with_vector("embedding", 4, DistanceMetric::Cosine),
],
);
let doc = sonic_rs::json!({
"title": "Distributed Database in Rust",
"tags": "db, kv, raft",
"score": 99.5
});
let raw = sonic_rs::to_vec(&doc).unwrap();
let terms = extract_doc_terms(&schema, "user:1", &raw);
let term_set: RapidHashSet<(String, String)> = terms.into_iter().collect();
assert!(term_set.contains(&("title".to_string(), "distributed".to_string())));
assert!(term_set.contains(&("title".to_string(), "database".to_string())));
assert!(term_set.contains(&("title".to_string(), "rust".to_string())));
assert!(term_set.contains(&("tags".to_string(), "db".to_string())));
assert!(term_set.contains(&("tags".to_string(), "kv".to_string())));
assert!(term_set.contains(&("tags".to_string(), "raft".to_string())));
assert!(term_set.contains(&("score".to_string(), encode_sortable_f64(99.5))));
assert!(schema.matches_key("user:100"));
assert!(!schema.matches_key("post:100"));
let field = schema.get_field("tags").unwrap();
assert_eq!(field.field_type, IndexFieldType::Tag);
}
#[test]
fn test_phrase_and_slop_search() {
let schema = SearchIndexSchema::with_full_spec(
HipStr::from("idx_phrases"),
IndexOnDataType::Hash,
vec![HipStr::from("doc:")],
vec![IndexField::new("content", IndexFieldType::Text)],
);
let mut idx = InvertedIndex::new();
let doc1 = sonic_rs::json!({ "content": "quick brown fox jumps over lazy dog" });
let raw1 = sonic_rs::to_vec(&doc1).unwrap();
idx.index_doc(&schema, "doc:1", &raw1, Some(1.0), None)
.unwrap();
let doc2 = sonic_rs::json!({ "content": "brown quick jumps dog over lazy" });
let raw2 = sonic_rs::to_vec(&doc2).unwrap();
idx.index_doc(&schema, "doc:2", &raw2, Some(1.0), None)
.unwrap();
let res1 = idx
.search(&schema, "\"quick brown\"", &FtSearch::default())
.unwrap();
assert_eq!(res1.total_results, 1);
assert_eq!(res1.docs[0].id, HipStr::from("doc:1"));
let res2 = idx
.search(&schema, "\"brown quick\"", &FtSearch::default())
.unwrap();
assert_eq!(res2.total_results, 1);
assert_eq!(res2.docs[0].id, HipStr::from("doc:2"));
}
#[test]
fn test_advanced_tag_and_escaping_and_numbers() {
let schema = SearchIndexSchema::with_full_spec(
HipStr::from("idx_tags"),
IndexOnDataType::Hash,
vec![HipStr::from("user:")],
vec![
IndexField::with_tag("email_tag", Some(','), false),
IndexField::with_tag("num_tag", Some(','), false),
],
);
let mut idx = InvertedIndex::new();
let doc1 = sonic_rs::json!({
"email_tag": "test\\@example.com, hello\\ world",
"num_tag": "3.1415926, 42"
});
let raw1 = sonic_rs::to_vec(&doc1).unwrap();
idx.index_doc(&schema, "user:1", &raw1, Some(1.0), None)
.unwrap();
let res1 = idx
.search(
&schema,
r"@email_tag:{test\@example\.com}",
&FtSearch::default(),
)
.unwrap();
assert_eq!(res1.total_results, 1);
let res2 = idx
.search(&schema, r"@email_tag:{hello\ world}", &FtSearch::default())
.unwrap();
assert_eq!(res2.total_results, 1);
let res3 = idx
.search(&schema, "@num_tag:{3.1415926}", &FtSearch::default())
.unwrap();
assert_eq!(res3.total_results, 1);
let res4 = idx
.search(&schema, "@email_tag:{test*}", &FtSearch::default())
.unwrap();
assert_eq!(res4.total_results, 1);
}
#[test]
fn test_hybrid_vector_knn_and_prefilter_search() {
let schema = SearchIndexSchema::with_full_spec(
HipStr::from("idx_vectors"),
IndexOnDataType::Json,
vec![HipStr::from("item:")],
vec![
IndexField::with_tag("genre", Some(','), false),
IndexField::with_numeric("price", true),
IndexField::with_vector("vec", 3, DistanceMetric::L2),
],
);
let mut idx = InvertedIndex::new();
let doc1 = sonic_rs::json!({
"genre": "scifi",
"price": 20.0,
"vec": [1.0, 0.0, 0.0]
});
idx.index_doc(
&schema,
"item:1",
&sonic_rs::to_vec(&doc1).unwrap(),
Some(1.0),
None,
)
.unwrap();
let doc2 = sonic_rs::json!({
"genre": "fantasy",
"price": 50.0,
"vec": [0.0, 1.0, 0.0]
});
idx.index_doc(
&schema,
"item:2",
&sonic_rs::to_vec(&doc2).unwrap(),
Some(1.0),
None,
)
.unwrap();
let doc3 = sonic_rs::json!({
"genre": "scifi",
"price": 80.0,
"vec": [0.9, 0.1, 0.0]
});
idx.index_doc(
&schema,
"item:3",
&sonic_rs::to_vec(&doc3).unwrap(),
Some(1.0),
None,
)
.unwrap();
let mut params = RapidHashMap::default();
let q_vec = [1.0f64, 0.0, 0.0];
let bytes: Vec<u8> = q_vec.iter().flat_map(|f| f.to_le_bytes()).collect();
params.insert("BLOB".to_string(), unsafe {
String::from_utf8_unchecked(bytes)
});
let opts = FtSearch {
params,
..Default::default()
};
let res_hybrid = idx
.search(&schema, "(@genre:{scifi})=>[KNN 1 @vec $BLOB]", &opts)
.unwrap();
assert_eq!(res_hybrid.total_results, 1);
assert_eq!(res_hybrid.docs[0].id, HipStr::from("item:1"));
let res_range = idx
.search(&schema, "@vec:[VECTOR_RANGE 0.5 $BLOB]", &opts)
.unwrap();
assert_eq!(res_range.total_results, 2); }
#[test]
fn test_ft_create_opts_conversion_and_full_lifecycle() {
let mut mgr = SearchIndexManager::new();
let create_opts = FtCreate {
index_name: "articles_idx".to_string(),
on_data_type: IndexOnDataType::Json,
prefixes: vec!["article:".to_string()],
filter: Some("@year > 2020".to_string()),
default_score: 1.5,
score_field: Some("score".to_string()),
payload_field: Some("payload".to_string()),
language: Some("english".to_string()),
language_field: None,
max_text_fields: true,
no_offsets: false,
no_hl: false,
no_fields: false,
no_freqs: false,
stop_words: vec!["the".to_string(), "is".to_string()],
fields: vec![
IndexField::with_text("title", 2.0, true).with_alias("t"),
IndexField::with_tag("tags", Some(','), false),
IndexField::with_numeric("year", true),
IndexField::with_vector("embedding", 128, DistanceMetric::Cosine),
],
};
mgr.create_index_from_opts(create_opts).unwrap();
let (schema, mut inverted) = mgr.indexes.remove("articles_idx").unwrap();
let doc = sonic_rs::json!({
"title": "State of the Art in Vector Search",
"tags": "db,ai",
"year": 2024
});
inverted
.index_doc(
&schema,
"article:1",
&sonic_rs::to_vec(&doc).unwrap(),
Some(1.0),
None,
)
.unwrap();
let res = inverted
.search(
&schema,
"@tags:{ai} @year:[2020 2025]",
&FtSearch::default(),
)
.unwrap();
assert_eq!(res.total_results, 1);
assert_eq!(res.docs[0].id, HipStr::from("article:1"));
let info = inverted.info(&schema);
assert_eq!(info.index_name, "articles_idx");
assert_eq!(info.num_docs, 1);
mgr.indexes
.insert(HipStr::from("articles_idx"), (schema, inverted));
let dropped_docs = mgr.drop_index("articles_idx", true).unwrap();
assert_eq!(dropped_docs, vec![HipStr::from("article:1")]);
}
#[test]
fn test_hnsw_vector_graph_operations() {
let mut graph = HnswGraph::new(3, DistanceMetric::L2, 4, 16, 8, 0.01);
let v1 = vec![0.0, 0.0, 0.0];
let v2 = vec![1.0, 0.0, 0.0];
let v3 = vec![0.0, 1.0, 0.0];
let v4 = vec![1.0, 1.0, 0.0];
let v5 = vec![10.0, 10.0, 10.0];
graph.insert(HipStr::from("doc1"), v1).unwrap();
graph.insert(HipStr::from("doc2"), v2).unwrap();
graph.insert(HipStr::from("doc3"), v3).unwrap();
graph.insert(HipStr::from("doc4"), v4).unwrap();
graph.insert(HipStr::from("doc5"), v5).unwrap();
assert_eq!(graph.nodes.len(), 5);
let query = vec![0.1, 0.1, 0.0];
let knn = graph.search_knn(&query, 2, None).unwrap();
assert_eq!(knn.len(), 2);
assert_eq!(knn[0].1, HipStr::from("doc1"));
let range_res = graph.search_range(&[0.0, 0.0, 0.0], 1.5, None).unwrap();
let range_ids: Vec<HipStr<'static>> = range_res.into_iter().map(|(_, id)| id).collect();
assert!(range_ids.contains(&HipStr::from("doc1")));
assert!(range_ids.contains(&HipStr::from("doc2")));
assert!(range_ids.contains(&HipStr::from("doc3")));
assert!(range_ids.contains(&HipStr::from("doc4")));
assert!(!range_ids.contains(&HipStr::from("doc5")));
assert!(graph.delete("doc1"));
assert_eq!(graph.nodes.len(), 4);
let knn_after_del = graph.search_knn(&query, 1, None).unwrap();
assert_ne!(knn_after_del[0].1, HipStr::from("doc1"));
}
#[test]
fn test_field_grouping_query_parsing() {
let q = "@title:(rust database)";
let ast = parse_search_query(q);
if let SearchQueryNode::And(nodes) = ast {
assert_eq!(nodes.len(), 2);
if let SearchQueryNode::Term { field, term, .. } = &nodes[0] {
assert_eq!(field.as_deref(), Some("title"));
assert_eq!(term, "rust");
} else {
panic!("expected Term node");
}
if let SearchQueryNode::Term { field, term, .. } = &nodes[1] {
assert_eq!(field.as_deref(), Some("title"));
assert_eq!(term, "database");
} else {
panic!("expected Term node");
}
} else {
panic!("expected And node for field grouping");
}
}