Skip to main content

fetchium_core/rank/
answer.rs

1//! Answer Extraction Engine (AXE) — extracts direct answers from search results.
2//!
3//! For factual queries like "What is the capital of France?", this engine
4//! extracts candidate answers from result snippets using pattern matching
5//! and cross-result voting.
6
7use crate::types::ResultItem;
8use std::collections::HashMap;
9
10/// Configuration for the answer extractor.
11#[derive(Debug, Clone)]
12pub struct AnswerConfig {
13    /// Minimum number of results that must agree on an answer.
14    pub min_agreement: usize,
15    /// Confidence threshold below which answers are not returned.
16    pub min_confidence: f64,
17    /// Maximum number of candidate answers to return.
18    pub max_answers: usize,
19}
20
21impl Default for AnswerConfig {
22    fn default() -> Self {
23        Self {
24            min_agreement: 1,
25            min_confidence: 0.3,
26            max_answers: 3,
27        }
28    }
29}
30
31/// A candidate answer extracted from results.
32#[derive(Debug, Clone)]
33pub struct ExtractedAnswer {
34    /// The answer text.
35    pub answer: String,
36    /// Confidence score (0.0–1.0).
37    pub confidence: f64,
38    /// Number of results that support this answer.
39    pub supporting_sources: usize,
40    /// Extraction method used.
41    pub method: ExtractionMethod,
42}
43
44/// How the answer was extracted.
45#[derive(Debug, Clone, Copy, PartialEq, Eq)]
46pub enum ExtractionMethod {
47    /// "X is Y" pattern matching.
48    DefinitionPattern,
49    /// Cross-result entity voting.
50    EntityVoting,
51    /// Direct snippet extraction.
52    SnippetExtraction,
53}
54
55/// Extract direct answers from search results for a given query.
56pub fn extract_answers(
57    query: &str,
58    results: &[ResultItem],
59    config: &AnswerConfig,
60) -> Vec<ExtractedAnswer> {
61    if results.is_empty() || query.is_empty() {
62        return vec![];
63    }
64
65    let mut candidates: Vec<ExtractedAnswer> = Vec::new();
66
67    // Method 1: Definition patterns ("X is Y", "X are Y")
68    let definitions = extract_definitions(query, results);
69    candidates.extend(definitions);
70
71    // Method 2: Entity voting (common entities across results)
72    let entities = entity_voting(query, results);
73    candidates.extend(entities);
74
75    // Filter by confidence and agreement
76    candidates.retain(|a| {
77        a.confidence >= config.min_confidence && a.supporting_sources >= config.min_agreement
78    });
79
80    // Sort by confidence descending
81    candidates.sort_by(|a, b| {
82        b.confidence
83            .partial_cmp(&a.confidence)
84            .unwrap_or(std::cmp::Ordering::Equal)
85    });
86
87    // Deduplicate similar answers
88    deduplicate_answers(&mut candidates);
89
90    candidates.truncate(config.max_answers);
91    candidates
92}
93
94/// Check if a query is answerable (factual question).
95pub fn is_answerable_query(query: &str) -> bool {
96    let lower = query.to_lowercase();
97    let words: Vec<&str> = lower.split_whitespace().collect();
98
99    // Check for question patterns
100    if let Some(first) = words.first() {
101        if QUESTION_WORDS.contains(first) {
102            return true;
103        }
104    }
105
106    // Check for "define X", "meaning of X" patterns
107    lower.starts_with("define ")
108        || lower.contains("meaning of")
109        || lower.contains("definition of")
110        || lower.ends_with('?')
111}
112
113// ─── Definition pattern extraction ────────────────────────
114
115fn extract_definitions(query: &str, results: &[ResultItem]) -> Vec<ExtractedAnswer> {
116    let query_lower = query.to_lowercase();
117    let query_subject = extract_subject(&query_lower);
118    let mut answers = Vec::new();
119
120    for result in results {
121        let snippet_lower = result.snippet.to_lowercase();
122        let sentences = split_sentences(&snippet_lower);
123
124        for sentence in &sentences {
125            // Look for "Subject is/are Definition" patterns
126            if let Some(definition) = extract_is_pattern(sentence, &query_subject) {
127                let clean = clean_answer(&definition);
128                if !clean.is_empty() && clean.len() < 200 {
129                    answers.push(ExtractedAnswer {
130                        answer: clean,
131                        confidence: 0.7,
132                        supporting_sources: 1,
133                        method: ExtractionMethod::DefinitionPattern,
134                    });
135                }
136            }
137        }
138    }
139
140    // Merge identical definitions
141    merge_answers(&mut answers);
142    answers
143}
144
145fn extract_subject(query: &str) -> String {
146    let mut subject = query.to_string();
147
148    // Strip question words
149    for prefix in &[
150        "what is ",
151        "what are ",
152        "who is ",
153        "who are ",
154        "where is ",
155        "when is ",
156        "define ",
157    ] {
158        if subject.starts_with(prefix) {
159            subject = subject[prefix.len()..].to_string();
160            break;
161        }
162    }
163
164    // Strip trailing "?"
165    if subject.ends_with('?') {
166        subject.pop();
167    }
168
169    subject.trim().to_string()
170}
171
172fn extract_is_pattern(sentence: &str, subject: &str) -> Option<String> {
173    if subject.is_empty() {
174        return None;
175    }
176
177    // Look for "subject is/are definition"
178    for connector in &[" is ", " are ", " was ", " refers to ", " means "] {
179        if let Some(pos) = sentence.find(connector) {
180            let before = &sentence[..pos];
181            // Check if subject appears before the connector
182            if before.contains(subject) || subject.contains(before.trim()) {
183                let after = &sentence[pos + connector.len()..];
184                let definition = after.trim().to_string();
185                if !definition.is_empty() {
186                    return Some(definition);
187                }
188            }
189        }
190    }
191    None
192}
193
194// ─── Entity voting ────────────────────────────────────────
195
196fn entity_voting(query: &str, results: &[ResultItem]) -> Vec<ExtractedAnswer> {
197    let query_terms: Vec<String> = query
198        .to_lowercase()
199        .split_whitespace()
200        .filter(|w| w.len() >= 3)
201        .map(|w| w.to_string())
202        .collect();
203
204    // Extract candidate entities from snippets (capitalized phrases, numbers, etc.)
205    let mut entity_counts: HashMap<String, usize> = HashMap::new();
206
207    for result in results {
208        let entities = extract_entities_from_snippet(&result.snippet, &query_terms);
209        for entity in entities {
210            *entity_counts.entry(entity).or_insert(0) += 1;
211        }
212    }
213
214    // Convert to answers, requiring multiple sources
215    let total = results.len().max(1);
216    entity_counts
217        .into_iter()
218        .filter(|(_, count)| *count >= 1)
219        .map(|(entity, count)| {
220            let confidence = (count as f64 / total as f64).min(0.95);
221            ExtractedAnswer {
222                answer: entity,
223                confidence,
224                supporting_sources: count,
225                method: ExtractionMethod::EntityVoting,
226            }
227        })
228        .collect()
229}
230
231fn extract_entities_from_snippet(snippet: &str, query_terms: &[String]) -> Vec<String> {
232    let mut entities = Vec::new();
233
234    // Extract numbers with units (e.g., "384,400 km", "100°C")
235    let words: Vec<&str> = snippet.split_whitespace().collect();
236    for window in words.windows(2) {
237        let first = window[0];
238        let second = window[1];
239        if first.chars().any(|c| c.is_ascii_digit()) && second.len() <= 10 {
240            let candidate = format!("{} {}", first, second);
241            if !has_overlap(&candidate.to_lowercase(), query_terms) {
242                entities.push(candidate);
243            }
244        }
245    }
246
247    // Extract capitalized phrases (potential named entities)
248    let mut current_phrase = Vec::new();
249    for word in &words {
250        if word.starts_with(|c: char| c.is_uppercase()) && word.len() >= 2 {
251            current_phrase.push(*word);
252        } else {
253            if !current_phrase.is_empty() {
254                let phrase = current_phrase.join(" ");
255                if phrase.len() >= 3 && !has_overlap(&phrase.to_lowercase(), query_terms) {
256                    entities.push(phrase);
257                }
258            }
259            current_phrase.clear();
260        }
261    }
262    if !current_phrase.is_empty() {
263        let phrase = current_phrase.join(" ");
264        if phrase.len() >= 3 && !has_overlap(&phrase.to_lowercase(), query_terms) {
265            entities.push(phrase);
266        }
267    }
268
269    entities
270}
271
272fn has_overlap(text: &str, terms: &[String]) -> bool {
273    terms.iter().any(|t| text.contains(t.as_str()))
274}
275
276// ─── Helpers ─────────────────────────────────────────────
277
278fn split_sentences(text: &str) -> Vec<String> {
279    text.split(['.', '!', '?'])
280        .map(|s| s.trim().to_string())
281        .filter(|s| !s.is_empty())
282        .collect()
283}
284
285fn clean_answer(answer: &str) -> String {
286    let trimmed = answer.trim();
287    // Strip trailing punctuation
288    let stripped = trimmed.trim_end_matches(['.', ',', ';']);
289    stripped.to_string()
290}
291
292fn merge_answers(answers: &mut Vec<ExtractedAnswer>) {
293    let mut merged: Vec<ExtractedAnswer> = Vec::new();
294
295    for answer in answers.drain(..) {
296        let lower = answer.answer.to_lowercase();
297        if let Some(existing) = merged.iter_mut().find(|a| a.answer.to_lowercase() == lower) {
298            existing.supporting_sources += answer.supporting_sources;
299            existing.confidence = (existing.confidence + answer.confidence) / 2.0;
300        } else {
301            merged.push(answer);
302        }
303    }
304
305    *answers = merged;
306}
307
308fn deduplicate_answers(answers: &mut Vec<ExtractedAnswer>) {
309    let mut seen: Vec<String> = Vec::new();
310    answers.retain(|a| {
311        let lower = a.answer.to_lowercase();
312        if seen
313            .iter()
314            .any(|s| s == &lower || s.contains(&lower) || lower.contains(s.as_str()))
315        {
316            false
317        } else {
318            seen.push(lower);
319            true
320        }
321    });
322}
323
324const QUESTION_WORDS: &[&str] = &["what", "who", "where", "when", "which", "how", "define"];
325
326#[cfg(test)]
327mod tests {
328    use super::*;
329    use crate::types::BackendId;
330
331    fn make_item(title: &str, snippet: &str) -> ResultItem {
332        ResultItem {
333            title: title.into(),
334            url: format!(
335                "https://{}.example.com",
336                title.to_lowercase().replace(' ', "-")
337            ),
338            snippet: snippet.into(),
339            rank: 0,
340            backend: BackendId::DuckDuckGo,
341            score: Some(0.5),
342            published_date: None,
343        }
344    }
345
346    #[test]
347    fn empty_input_no_answers() {
348        let answers = extract_answers("", &[], &AnswerConfig::default());
349        assert!(answers.is_empty());
350    }
351
352    #[test]
353    fn definition_pattern_extraction() {
354        let results = vec![
355            make_item(
356                "Rust",
357                "Rust is a systems programming language focused on safety and performance.",
358            ),
359            make_item(
360                "About Rust",
361                "Rust is a multi-paradigm systems programming language.",
362            ),
363        ];
364        let answers = extract_answers("What is Rust?", &results, &AnswerConfig::default());
365        assert!(
366            !answers.is_empty(),
367            "should extract definition for 'what is' query"
368        );
369    }
370
371    #[test]
372    fn answerable_query_detection() {
373        assert!(is_answerable_query("What is the capital of France?"));
374        assert!(is_answerable_query("Who invented the telephone?"));
375        assert!(is_answerable_query("define algorithm"));
376        assert!(!is_answerable_query("rust programming"));
377    }
378
379    #[test]
380    fn subject_extraction() {
381        assert_eq!(extract_subject("what is rust"), "rust");
382        assert_eq!(extract_subject("who is alan turing"), "alan turing");
383        assert_eq!(extract_subject("define algorithm"), "algorithm");
384    }
385
386    #[test]
387    fn entity_voting_finds_common_entities() {
388        let results = vec![
389            make_item(
390                "France Capital",
391                "The capital of France is Paris, a major European city.",
392            ),
393            make_item(
394                "Paris Info",
395                "Paris is the capital and largest city of France.",
396            ),
397            make_item(
398                "French Cities",
399                "Paris serves as the capital of France since centuries.",
400            ),
401        ];
402        let answers = extract_answers("capital of France", &results, &AnswerConfig::default());
403
404        let has_paris = answers.iter().any(|a| {
405            let lower = a.answer.to_lowercase();
406            lower.contains("paris")
407        });
408        assert!(
409            has_paris,
410            "Paris should be extracted as answer, got: {:?}",
411            answers
412        );
413    }
414
415    #[test]
416    fn confidence_sorted_descending() {
417        let results = vec![
418            make_item("A", "Rust is a systems programming language for safety."),
419            make_item("B", "Rust is a compiled language for performance."),
420        ];
421        let answers = extract_answers("What is Rust?", &results, &AnswerConfig::default());
422        if answers.len() >= 2 {
423            for i in 0..answers.len() - 1 {
424                assert!(answers[i].confidence >= answers[i + 1].confidence);
425            }
426        }
427    }
428
429    #[test]
430    fn max_answers_respected() {
431        let config = AnswerConfig {
432            max_answers: 1,
433            min_confidence: 0.0,
434            ..AnswerConfig::default()
435        };
436        let results = vec![make_item(
437            "A",
438            "Rust is safe. Rust is fast. Rust is concurrent.",
439        )];
440        let answers = extract_answers("What is Rust?", &results, &config);
441        assert!(answers.len() <= 1);
442    }
443
444    #[test]
445    fn is_pattern_extraction() {
446        let def = extract_is_pattern("rust is a systems programming language", "rust");
447        assert!(def.is_some());
448        assert!(def.unwrap().contains("systems programming language"));
449    }
450
451    #[test]
452    fn clean_answer_strips_punctuation() {
453        assert_eq!(clean_answer("  hello world.  "), "hello world");
454        assert_eq!(clean_answer("answer,"), "answer");
455    }
456}