fathomdb_query/
lib.rs

1use std::collections::HashSet;
2
3#[derive(Clone, Debug, Eq, PartialEq)]
4pub struct QueryAst {
5    pub raw: String,
6}
7
8#[derive(Clone, Debug, Eq, PartialEq)]
9pub struct CompiledQuery {
10    pub match_expression: String,
11}
12
13/// Function words stripped from the content-OR query (they add false matches
14/// under OR semantics without carrying topical signal). Mirrors the IR-C
15/// `content-OR` experiment list (`dev/plans/runs/performance-output-and-compare.md`,
16/// 2026-06-10b): the smallest list that lifted exploratory recall with no
17/// exact_fact cost.
18const STOPWORDS: &[&str] = &[
19    "the", "and", "for", "are", "was", "were", "what", "when", "where", "who", "whom", "which",
20    "how", "why", "did", "does", "do", "is", "of", "to", "in", "on", "at", "by", "an", "a", "it",
21    "its", "this", "that", "these", "those", "with", "from", "as", "be", "or", "if", "about",
22    "into", "over", "than", "then", "they", "them", "their", "you", "your", "we", "our", "i",
23];
24
25/// Content tokens of a query: lowercased, split on non-alphanumeric, ≥3 chars,
26/// stopwords removed, de-duplicated in first-seen order. These are the OR terms
27/// of the compiled MATCH expression. Splitting on non-alphanumeric drops every
28/// FTS5 control character (`*`, `"`, `:`, `^`, `(`, `)`, `,`), so the emitted
29/// tokens are pure literals — the injection-safety property (AC-038).
30#[must_use]
31fn content_tokens(raw: &str) -> Vec<String> {
32    let stop: HashSet<&str> = STOPWORDS.iter().copied().collect();
33    let mut seen: HashSet<String> = HashSet::new();
34    let mut out: Vec<String> = Vec::new();
35    for token in raw.to_lowercase().split(|c: char| !c.is_alphanumeric()) {
36        if token.len() < 3 || stop.contains(token) {
37            continue;
38        }
39        if seen.insert(token.to_string()) {
40            out.push(token.to_string());
41        }
42    }
43    out
44}
45
46/// Compile a raw query into an FTS5 MATCH expression.
47///
48/// IR-C (2026-06-10b/c, `performance-output-and-compare.md`): the production
49/// arm was an **AND** of every whitespace token, which near-zeroes recall on
50/// natural-language questions (every token must be present). The validated
51/// recipe is **content-OR** — OR over the content tokens (stopwords stripped) —
52/// which any-token-matches and lets `bm25()` rank by overlap, the way the
53/// same-dataset BM25 baselines (EnronQA/QAConv) are run.
54///
55/// All-stopword / symbol-only / sub-3-char queries (no content tokens) fall back
56/// to an OR over the raw whitespace tokens as injection-safe quoted phrases, so
57/// such a query still searches instead of returning nothing.
58#[must_use]
59pub fn compile_text_query(raw: impl Into<String>) -> CompiledQuery {
60    let raw = raw.into();
61    let content = content_tokens(&raw);
62    let match_expression = if content.is_empty() {
63        raw.split_whitespace()
64            .filter(|token| !token.is_empty())
65            .map(|token| format!("\"{}\"", token.replace('"', "\"\"")))
66            .collect::<Vec<_>>()
67            .join(" OR ")
68    } else {
69        content.into_iter().map(|token| format!("\"{token}\"")).collect::<Vec<_>>().join(" OR ")
70    };
71
72    CompiledQuery { match_expression }
73}
74
75#[cfg(test)]
76mod tests {
77    use super::compile_text_query;
78
79    #[test]
80    fn content_tokens_are_or_joined() {
81        // IR-C content-OR: content tokens OR-joined (was AND).
82        let compiled = compile_text_query("alpha   beta");
83        assert_eq!(compiled.match_expression, "\"alpha\" OR \"beta\"");
84    }
85
86    #[test]
87    fn stopwords_and_short_tokens_are_dropped() {
88        // "of"/"the" are stopwords; "a" is sub-3-char — only content survives.
89        let compiled = compile_text_query("status of the alpha");
90        assert_eq!(compiled.match_expression, "\"status\" OR \"alpha\"");
91    }
92
93    #[test]
94    fn duplicate_content_tokens_collapse_in_order() {
95        let compiled = compile_text_query("alpha beta alpha");
96        assert_eq!(compiled.match_expression, "\"alpha\" OR \"beta\"");
97    }
98
99    #[test]
100    fn control_characters_are_stripped_to_literals() {
101        // FTS5 control syntax splits into literal content tokens — no operators
102        // reach SQLite (AC-038).
103        let compiled = compile_text_query("alpha* AND \"beta\" NEAR(gamma)");
104        assert_eq!(compiled.match_expression, "\"alpha\" OR \"beta\" OR \"near\" OR \"gamma\"");
105    }
106
107    #[test]
108    fn all_stopword_query_falls_back_to_raw_or() {
109        // No content tokens: OR over the raw whitespace tokens (still searches).
110        let compiled = compile_text_query("who are we");
111        assert_eq!(compiled.match_expression, "\"who\" OR \"are\" OR \"we\"");
112    }
113
114    #[test]
115    fn escapes_double_quotes_in_fallback_tokens() {
116        // The fallback path keeps injection-safe quote-escaping.
117        let compiled = compile_text_query("an \"of");
118        assert_eq!(compiled.match_expression, "\"an\" OR \"\"\"of\"");
119    }
120}
fathomdb_query/lib.rs

fathomdb_query/
lib.rs
