fathomdb_query/
lib.rs

1//! **FathomDB query** — the JSON-filter AST and its validation surface.
2//!
3//! An internal leaf crate of the FathomDB workspace, kept separate so the
4//! filter grammar can be validated without pulling in the engine. **Application
5//! code should depend on the `fathomdb` facade crate instead**, which exposes
6//! the supported filter types (`Predicate`, `ScalarValue`, `ComparisonOp`,
7//! `SearchFilter`) as governed surface.
8//!
9//! The filter grammar is deliberately **closed** — a fixed set of comparison
10//! operators over an allowlisted set of JSON paths — rather than an open DSL.
11//! Values are always bound as parameterized SQL and never interpolated.
12
13use std::collections::HashSet;
14
15#[derive(Clone, Debug, Eq, PartialEq)]
16pub struct QueryAst {
17    pub raw: String,
18}
19
20#[derive(Clone, Debug, Eq, PartialEq)]
21pub struct CompiledQuery {
22    pub match_expression: String,
23}
24
25/// Function words stripped from the content-OR query (they add false matches
26/// under OR semantics without carrying topical signal). Mirrors the IR-C
27/// `content-OR` experiment list (`dev/plans/runs/performance-output-and-compare.md`,
28/// 2026-06-10b): the smallest list that lifted exploratory recall with no
29/// exact_fact cost.
30const STOPWORDS: &[&str] = &[
31    "the", "and", "for", "are", "was", "were", "what", "when", "where", "who", "whom", "which",
32    "how", "why", "did", "does", "do", "is", "of", "to", "in", "on", "at", "by", "an", "a", "it",
33    "its", "this", "that", "these", "those", "with", "from", "as", "be", "or", "if", "about",
34    "into", "over", "than", "then", "they", "them", "their", "you", "your", "we", "our", "i",
35];
36
37/// Content tokens of a query: lowercased, split on non-alphanumeric, ≥3 chars,
38/// stopwords removed, de-duplicated in first-seen order. These are the OR terms
39/// of the compiled MATCH expression. Splitting on non-alphanumeric drops every
40/// FTS5 control character (`*`, `"`, `:`, `^`, `(`, `)`, `,`), so the emitted
41/// tokens are pure literals — the injection-safety property (AC-038).
42#[must_use]
43fn content_tokens(raw: &str) -> Vec<String> {
44    let stop: HashSet<&str> = STOPWORDS.iter().copied().collect();
45    let mut seen: HashSet<String> = HashSet::new();
46    let mut out: Vec<String> = Vec::new();
47    for token in raw.to_lowercase().split(|c: char| !c.is_alphanumeric()) {
48        if token.len() < 3 || stop.contains(token) {
49            continue;
50        }
51        if seen.insert(token.to_string()) {
52            out.push(token.to_string());
53        }
54    }
55    out
56}
57
58/// Compile a raw query into an FTS5 MATCH expression.
59///
60/// IR-C (2026-06-10b/c, `performance-output-and-compare.md`): the production
61/// arm was an **AND** of every whitespace token, which near-zeroes recall on
62/// natural-language questions (every token must be present). The validated
63/// recipe is **content-OR** — OR over the content tokens (stopwords stripped) —
64/// which any-token-matches and lets `bm25()` rank by overlap, the way the
65/// same-dataset BM25 baselines (EnronQA/QAConv) are run.
66///
67/// All-stopword / symbol-only / sub-3-char queries (no content tokens) fall back
68/// to an OR over the raw whitespace tokens as injection-safe quoted phrases, so
69/// such a query still searches instead of returning nothing.
70#[must_use]
71pub fn compile_text_query(raw: impl Into<String>) -> CompiledQuery {
72    let raw = raw.into();
73    let content = content_tokens(&raw);
74    let match_expression = if content.is_empty() {
75        raw.split_whitespace()
76            .filter(|token| !token.is_empty())
77            .map(|token| format!("\"{}\"", token.replace('"', "\"\"")))
78            .collect::<Vec<_>>()
79            .join(" OR ")
80    } else {
81        content.into_iter().map(|token| format!("\"{token}\"")).collect::<Vec<_>>().join(" OR ")
82    };
83
84    CompiledQuery { match_expression }
85}
86
87#[cfg(test)]
88mod tests {
89    use super::compile_text_query;
90
91    #[test]
92    fn content_tokens_are_or_joined() {
93        // IR-C content-OR: content tokens OR-joined (was AND).
94        let compiled = compile_text_query("alpha   beta");
95        assert_eq!(compiled.match_expression, "\"alpha\" OR \"beta\"");
96    }
97
98    #[test]
99    fn stopwords_and_short_tokens_are_dropped() {
100        // "of"/"the" are stopwords; "a" is sub-3-char — only content survives.
101        let compiled = compile_text_query("status of the alpha");
102        assert_eq!(compiled.match_expression, "\"status\" OR \"alpha\"");
103    }
104
105    #[test]
106    fn duplicate_content_tokens_collapse_in_order() {
107        let compiled = compile_text_query("alpha beta alpha");
108        assert_eq!(compiled.match_expression, "\"alpha\" OR \"beta\"");
109    }
110
111    #[test]
112    fn control_characters_are_stripped_to_literals() {
113        // FTS5 control syntax splits into literal content tokens — no operators
114        // reach SQLite (AC-038).
115        let compiled = compile_text_query("alpha* AND \"beta\" NEAR(gamma)");
116        assert_eq!(compiled.match_expression, "\"alpha\" OR \"beta\" OR \"near\" OR \"gamma\"");
117    }
118
119    #[test]
120    fn all_stopword_query_falls_back_to_raw_or() {
121        // No content tokens: OR over the raw whitespace tokens (still searches).
122        let compiled = compile_text_query("who are we");
123        assert_eq!(compiled.match_expression, "\"who\" OR \"are\" OR \"we\"");
124    }
125
126    #[test]
127    fn escapes_double_quotes_in_fallback_tokens() {
128        // The fallback path keeps injection-safe quote-escaping.
129        let compiled = compile_text_query("an \"of");
130        assert_eq!(compiled.match_expression, "\"an\" OR \"\"\"of\"");
131    }
132}
fathomdb_query/lib.rs

fathomdb_query/
lib.rs
