anda_db_tfs 0.8.2

A full-text search library using the BM25 ranking algorithm in Rust.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
/// Represents different types of boolean queries that can be parsed from a query string.
/// Supports Term, Or, And, and Not operations for building complex search expressions.
/// Operator precedence: OR < AND < NOT.
///
/// # Grammar (informal)
///
/// ```text
/// expr    := or_expr
/// or_expr := and_expr ( " OR " and_expr )*
/// and_expr := not_expr ( " AND " not_expr )*
/// not_expr := "NOT " term | term
/// term    := "(" or_expr ")" | word ( whitespace word )*
/// ```
///
/// Whitespace-separated words at the `term` level default to an implicit `OR`
/// between them, matching the behaviour of [`BM25Index::search`].
///
/// The parser is intentionally lenient: unbalanced parentheses are treated as
/// part of the surrounding text so that user input never causes a parse error.
///
/// # Examples
///
/// ```
/// use anda_db_tfs::QueryType;
///
/// let query = QueryType::parse("(hello AND world) OR (rust AND NOT java)");
/// ```
///
/// [`BM25Index::search`]: crate::BM25Index::search
#[derive(Debug, Clone, PartialEq)]
pub enum QueryType {
    /// A simple term query that matches a single word or phrase
    Term(String),

    /// A logical OR query that requires at least one subquery to match
    Or(Vec<Box<QueryType>>),

    /// A logical AND query that requires all subqueries to match
    And(Vec<Box<QueryType>>),

    /// A logical NOT query that negates the result of its subquery
    Not(Box<QueryType>),
}

impl QueryType {
    /// Parses a query string into a QueryType structure.
    ///
    /// This is the main entry point for converting a string query into a structured
    /// representation that can be used for searching.
    ///
    /// # Arguments
    ///
    /// * `query` - A string slice containing the query to parse
    ///
    /// # Returns
    ///
    /// A QueryType representing the parsed query
    ///
    /// # Examples
    ///
    /// ```
    /// use anda_db_tfs::QueryType;
    ///
    /// let query = QueryType::parse("(hello AND world) OR (rust AND NOT java)");
    /// ```
    pub fn parse(query: &str) -> Self {
        let query = query.trim();
        if query.is_empty() {
            return QueryType::Or(vec![]);
        }

        Self::parse_or_expression(query)
    }

    /// Parses an OR expression, which has the lowest precedence in the query grammar.
    ///
    /// # Arguments
    ///
    /// * `query` - A string slice containing the query to parse
    ///
    /// # Returns
    ///
    /// A QueryType representing the parsed OR expression
    fn parse_or_expression(query: &str) -> Self {
        let parts: Vec<&str> = Self::split_top_level(query, " OR ");

        if parts.len() == 1 {
            return Self::parse_and_expression(parts[0]);
        }

        let subqueries: Vec<Box<QueryType>> = parts
            .into_iter()
            .map(|p| Box::new(Self::parse_and_expression(p)))
            .collect();

        QueryType::Or(subqueries)
    }

    /// Parses an AND expression, which has medium precedence in the query grammar.
    ///
    /// # Arguments
    ///
    /// * `query` - A string slice containing the query to parse
    ///
    /// # Returns
    ///
    /// A QueryType representing the parsed AND expression
    fn parse_and_expression(query: &str) -> Self {
        let parts: Vec<&str> = Self::split_top_level(query, " AND ");

        if parts.len() == 1 {
            return Self::parse_not_expression(parts[0]);
        }

        let subqueries: Vec<Box<QueryType>> = parts
            .into_iter()
            .map(|p| Box::new(Self::parse_not_expression(p)))
            .collect();

        QueryType::And(subqueries)
    }

    /// Parses a NOT expression, which has high precedence in the query grammar.
    ///
    /// # Arguments
    ///
    /// * `query` - A string slice containing the query to parse
    ///
    /// # Returns
    ///
    /// A QueryType representing the parsed NOT expression
    fn parse_not_expression(query: &str) -> Self {
        let query = query.trim();

        if let Some(stripped) = query.strip_prefix("NOT ") {
            return QueryType::Not(Box::new(Self::parse_term(stripped)));
        }

        Self::parse_term(query)
    }

    /// Parses a term or parenthesized expression, which has the highest precedence.
    ///
    /// # Arguments
    ///
    /// * `query` - A string slice containing the query to parse
    ///
    /// # Returns
    ///
    /// A QueryType representing the parsed term or parenthesized expression
    fn parse_term(query: &str) -> Self {
        let query = query.trim();

        // Handle parenthesized expressions
        if let Some(stripped) = query.strip_prefix('(') {
            // 处理可能存在的非平衡括号
            if stripped.ends_with(')') && Self::is_balanced_parentheses(query) {
                // 完全平衡的括号表达式
                return Self::parse_or_expression(&stripped[..stripped.len() - 1]);
            } else {
                // 处理不平衡的括号
                // 1. 如果缺少右括号,尝试解析括号内的内容
                return Self::parse_or_expression(stripped);
            }
        } else if let Some(stripped) = query.strip_suffix(')') {
            // 处理只有右括号的情况
            return Self::parse_or_expression(stripped);
        }

        // Handle multiple terms (default to OR relationship)
        let terms: Vec<&str> = query.split_whitespace().collect();
        if terms.len() > 1 {
            let subqueries: Vec<Box<QueryType>> = terms
                .into_iter()
                .map(|t| Box::new(QueryType::Term(t.to_lowercase())))
                .collect();
            return QueryType::Or(subqueries);
        }

        // Handle single term
        if !query.is_empty() {
            return QueryType::Term(query.to_lowercase());
        }

        // Handle empty query
        QueryType::Or(vec![])
    }

    /// Checks if parentheses in a string are balanced.
    ///
    /// # Arguments
    ///
    /// * `s` - A string slice to check for balanced parentheses
    ///
    /// # Returns
    ///
    /// A boolean indicating whether the parentheses are balanced
    fn is_balanced_parentheses(s: &str) -> bool {
        let mut count = 0;

        for c in s.chars() {
            if c == '(' {
                count += 1;
            } else if c == ')' {
                count -= 1;
                if count < 0 {
                    return false;
                }
            }
        }

        count == 0
    }

    /// Splits a string at the top level by a delimiter, ignoring delimiters inside parentheses.
    /// Handles unbalanced parentheses by treating them as part of the text.
    ///
    /// This is a key function that enables proper parsing of nested expressions.
    ///
    /// # Arguments
    ///
    /// * `s` - A string slice to split
    /// * `delimiter` - The delimiter to split by
    ///
    /// # Returns
    ///
    /// A vector of string slices resulting from the split
    fn split_top_level<'a>(s: &'a str, delimiter: &str) -> Vec<&'a str> {
        // Delimiters (" OR ", " AND ") are pure ASCII, so byte-level comparison
        // is correct and inherently avoids UTF-8 char boundary issues.
        debug_assert!(delimiter.is_ascii());

        let mut result = Vec::new();
        let mut start = 0;
        let mut paren_count: u32 = 0;
        let bytes = s.as_bytes();
        let delim_bytes = delimiter.as_bytes();
        let delim_len = delim_bytes.len();
        let mut i = 0;

        while i < bytes.len() {
            match bytes[i] {
                b'(' => {
                    paren_count += 1;
                    i += 1;
                }
                b')' => {
                    paren_count = paren_count.saturating_sub(1);
                    i += 1;
                }
                _ if paren_count == 0
                    && i + delim_len <= bytes.len()
                    && bytes[i..i + delim_len] == *delim_bytes =>
                {
                    // Safety: start and i are always at ASCII boundaries,
                    // which are valid UTF-8 char boundaries.
                    result.push(s[start..i].trim());
                    i += delim_len;
                    start = i;
                }
                _ => {
                    i += 1;
                }
            }
        }

        result.push(s[start..].trim());
        result
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    /// Tests parsing a simple term query
    #[test]
    fn test_simple_term() {
        assert_eq!(
            QueryType::parse("hello"),
            QueryType::Term("hello".to_string())
        );
    }

    /// Tests parsing an AND query
    #[test]
    fn test_and_query() {
        assert_eq!(
            QueryType::parse("hello AND world"),
            QueryType::And(vec![
                Box::new(QueryType::Term("hello".to_string())),
                Box::new(QueryType::Term("world".to_string()))
            ])
        );
    }

    /// Tests parsing an OR query
    #[test]
    fn test_or_query() {
        assert_eq!(
            QueryType::parse("hello OR world"),
            QueryType::Or(vec![
                Box::new(QueryType::Term("hello".to_string())),
                Box::new(QueryType::Term("world".to_string()))
            ])
        );
    }

    /// Tests parsing a NOT query
    #[test]
    fn test_not_query() {
        assert_eq!(
            QueryType::parse("NOT hello"),
            QueryType::Not(Box::new(QueryType::Term("hello".to_string())))
        );
    }

    /// Tests parsing a complex query with nested expressions
    #[test]
    fn test_complex_query() {
        assert_eq!(
            QueryType::parse("(hello AND world) OR (rust AND NOT java)"),
            QueryType::Or(vec![
                Box::new(QueryType::And(vec![
                    Box::new(QueryType::Term("hello".to_string())),
                    Box::new(QueryType::Term("world".to_string()))
                ])),
                Box::new(QueryType::And(vec![
                    Box::new(QueryType::Term("rust".to_string())),
                    Box::new(QueryType::Not(Box::new(QueryType::Term(
                        "java".to_string()
                    ))))
                ]))
            ])
        );
    }

    /// Tests parsing queries with unbalanced parentheses
    #[test]
    fn test_unbalanced_parentheses() {
        // 缺少右括号
        assert_eq!(
            QueryType::parse("(hello AND world"),
            QueryType::And(vec![
                Box::new(QueryType::Term("hello".to_string())),
                Box::new(QueryType::Term("world".to_string()))
            ])
        );

        // 缺少左括号
        assert_eq!(
            QueryType::parse("hello AND world)"),
            QueryType::And(vec![
                Box::new(QueryType::Term("hello".to_string())),
                Box::new(QueryType::Term("world".to_string()))
            ])
        );

        // 嵌套括号不平衡
        assert_eq!(
            QueryType::parse("(hello AND (world OR rust)"),
            QueryType::And(vec![
                Box::new(QueryType::Term("hello".to_string())),
                Box::new(QueryType::Or(vec![
                    Box::new(QueryType::Term("world".to_string())),
                    Box::new(QueryType::Term("rust".to_string()))
                ]))
            ])
        );
    }

    /// Tests that multi-byte UTF-8 characters don't cause panic in split_top_level
    #[test]
    fn test_multibyte_utf8_query() {
        // 纯中文词,不应 panic
        assert_eq!(
            QueryType::parse("巨蟹"),
            QueryType::Term("巨蟹".to_string())
        );

        // 中文词 AND 英文词
        assert_eq!(
            QueryType::parse("巨蟹 AND rust"),
            QueryType::And(vec![
                Box::new(QueryType::Term("巨蟹".to_string())),
                Box::new(QueryType::Term("rust".to_string()))
            ])
        );

        // 中文词 OR 中文词
        assert_eq!(
            QueryType::parse("巨蟹 OR 天蝎"),
            QueryType::Or(vec![
                Box::new(QueryType::Term("巨蟹".to_string())),
                Box::new(QueryType::Term("天蝎".to_string()))
            ])
        );

        // 带括号的中文表达式
        assert_eq!(
            QueryType::parse("(巨蟹 AND 座) OR 天蝎"),
            QueryType::Or(vec![
                Box::new(QueryType::And(vec![
                    Box::new(QueryType::Term("巨蟹".to_string())),
                    Box::new(QueryType::Term("".to_string()))
                ])),
                Box::new(QueryType::Term("天蝎".to_string()))
            ])
        );

        // NOT + 中文词
        assert_eq!(
            QueryType::parse("NOT 巨蟹"),
            QueryType::Not(Box::new(QueryType::Term("巨蟹".to_string())))
        );

        // 多个中文词(默认 OR 关系)
        assert_eq!(
            QueryType::parse("巨蟹 天蝎 双鱼"),
            QueryType::Or(vec![
                Box::new(QueryType::Term("巨蟹".to_string())),
                Box::new(QueryType::Term("天蝎".to_string())),
                Box::new(QueryType::Term("双鱼".to_string()))
            ])
        );
    }
}