sonic-core 0.1.0

Fast, lightweight and schema-less search backend.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

// Sonic
//
// Fast, lightweight and schema-less search backend
// Copyright: 2026, Rémi Bardon <remi@remibardon.name>
// License: Mozilla Public License v2.0 (MPL v2.0)

//! Feature: Typo correction
//!
//! A “typo factor” is chosen based on the token’s length. Those tests ensure
//! typo correction works, and ensure no regression in the quality of the
//! results.
//! At the moment of this writing, the mapping function is the following:
//!
//! ```
//! let mut typo_factor = match word.len() {
//!     1..=3 => 0,
//!     4..=6 => 1,
//!     7..=9 => 2,
//!     _ => 3,
//! };
//! ```

mod common;

use crate::common::*;

/// This test sentence contains words that are 3–10 characters-long, while
/// not being stopwords. It’s not realistic but not having stopwords avoids
/// false positives in tests.
const ASTRONOMY_WORDS: &str = "sun moon comet nebula pulsars asteroid satellite spacecraft";

/// Search should allow a certain number of typos (depending on token length).
///
/// This covers missing letters, inverted letters and additional letters.
/// The underlying algorithm used is the Levenshtein distance, so this test is
/// based on that knowledge.
#[test]
fn test_search_allows_typos() {
    // NOTE: Need to make language explicit because of
    //   <https://github.com/valeriansaliou/sonic/issues/322#issuecomment-4638688602>.
    //   Will be fixed separately.
    #[rustfmt::skip]
    test_ingest_then_query!(push: ASTRONOMY_WORDS [ensure_no_stopword] LANG("eng"), query: [
        // 3-letter word, distance = 1.
        ("sum", false),
        // 4-letter word, distance = 1.
        ("ssun", true), // sun
        ("noon", true), // moon
        // 4-letter word, distance = 2.
        ("commit", false),
        // 6-letter word, distance = 1.
        ("nzbula", true), // nebula
        // 6-letter word, distance = 2.
        ("nzbala", false),
        // 7-letter word, distance = 2.
        ("plusars", true), // pulsars
        // 9-letter word, distance = 2.
        ("saetllite", true), // satellite
        // 9-letter word, distance = 3.
        ("satemmote", false),
        // 10-letter word, distance = 3.
        ("sapcecrzft", true), // spacecraft
        // 10-letter word, distance = 4.
        ("sapcecarft", false),
    ] LANG("eng"));
}

/// Ensures the order of words in search queries is insignificant.
#[test]
#[ignore = "Not supported yet (FIXME)"]
fn test_search_term_order_insignificant() {
    #[rustfmt::skip]
    test_ingest_then_query!(push: ASTRONOMY_WORDS [ensure_no_stopword], query: [
        ("satellite pulsars nebula", true),
        (&format!("missing {ASTRONOMY_WORDS}"), true),
    ]);
}

#[test]
#[ignore = "Not supported yet"]
fn test_chinese_typo_correction() {
    // We should make sure languages with shorter graphemes get typo correction
    // too.
    unimplemented!()
}