Skip to main content

fetchium_core/search/
exa.rs

1//! Exa AI neural search backend — semantic search with content extraction.
2//!
3//! Uses next-link prediction for semantic relevance beyond keyword matching.
4//! Returns results with text content, highlights, author, and published date.
5//! API key required: set `EXA_API_KEY` env var or `search.exa_api_key` in config.
6
7use crate::error::{FetchiumError, FetchiumResult};
8use crate::http::HttpClient;
9use crate::search::{SearchBackend, SearchContext, TimeRange};
10use crate::types::{BackendId, ResultItem};
11use async_trait::async_trait;
12use serde::{Deserialize, Serialize};
13use tracing::debug;
14
15#[derive(Debug, Serialize)]
16#[serde(rename_all = "camelCase")]
17struct ExaRequest<'a> {
18    query: &'a str,
19    #[serde(rename = "type")]
20    search_type: &'a str,
21    num_results: u32,
22    contents: ExaContents,
23    #[serde(skip_serializing_if = "Option::is_none")]
24    start_published_date: Option<&'a str>,
25}
26
27#[derive(Debug, Serialize)]
28#[serde(rename_all = "camelCase")]
29struct ExaContents {
30    text: ExaTextConfig,
31    highlights: ExaHighlightConfig,
32}
33
34#[derive(Debug, Serialize)]
35#[serde(rename_all = "camelCase")]
36struct ExaTextConfig {
37    max_characters: u32,
38}
39
40#[derive(Debug, Serialize)]
41#[serde(rename_all = "camelCase")]
42struct ExaHighlightConfig {
43    num_sentences: u32,
44    highlights_per_url: u32,
45}
46
47#[derive(Debug, Deserialize)]
48#[serde(rename_all = "camelCase")]
49struct ExaResponse {
50    #[serde(default)]
51    results: Vec<ExaResult>,
52    #[serde(default)]
53    search_time: Option<f64>,
54}
55
56#[derive(Debug, Deserialize)]
57#[serde(rename_all = "camelCase")]
58struct ExaResult {
59    title: String,
60    url: String,
61    #[serde(default)]
62    text: Option<String>,
63    #[serde(default)]
64    highlights: Vec<String>,
65    #[serde(default)]
66    highlight_scores: Vec<f64>,
67    #[serde(default)]
68    published_date: Option<String>,
69    #[serde(default)]
70    score: Option<f64>,
71}
72
73use std::sync::atomic::{AtomicUsize, Ordering};
74
75/// Exa AI neural search backend — semantic search with content.
76pub struct ExaBackend {
77    http: HttpClient,
78    api_keys: Vec<String>,
79    current_key_index: AtomicUsize,
80}
81
82impl ExaBackend {
83    pub fn new(http: HttpClient, api_keys: Vec<String>) -> Self {
84        Self {
85            http,
86            api_keys,
87            current_key_index: AtomicUsize::new(0),
88        }
89    }
90
91    fn get_key(&self) -> &str {
92        if self.api_keys.is_empty() {
93            return "";
94        }
95        let idx = self.current_key_index.load(Ordering::Relaxed) % self.api_keys.len();
96        &self.api_keys[idx]
97    }
98
99    fn rotate_key(&self) {
100        if !self.api_keys.is_empty() {
101            self.current_key_index.fetch_add(1, Ordering::Relaxed);
102        }
103    }
104}
105
106impl ExaBackend {
107    async fn search_inner(
108        &self,
109        query: &str,
110        max_results: u32,
111        start_date: Option<&str>,
112    ) -> FetchiumResult<Vec<ResultItem>> {
113        let mut last_err = None;
114        let num_keys = self.api_keys.len().max(1);
115
116        for _ in 0..num_keys {
117            let api_key = self.get_key();
118            let request = ExaRequest {
119                query,
120                search_type: "auto",
121                num_results: max_results.min(10),
122                contents: ExaContents {
123                    text: ExaTextConfig {
124                        max_characters: 1500,
125                    },
126                    highlights: ExaHighlightConfig {
127                        num_sentences: 3,
128                        highlights_per_url: 3,
129                    },
130                },
131                start_published_date: start_date,
132            };
133
134            let body = serde_json::to_string(&request)
135                .map_err(|e| FetchiumError::Search(format!("Exa serialization: {e}")))?;
136
137            let result = self
138                .http
139                .post_json_with_header("https://api.exa.ai/search", &body, "x-api-key", api_key)
140                .await;
141
142            match result {
143                Ok(response) => {
144                    let parsed: ExaResponse = serde_json::from_str(&response)
145                        .map_err(|e| FetchiumError::Search(format!("Exa parse: {e}")))?;
146
147                    debug!(
148                        "Exa: {} results in {:.0}ms [start_date={:?}]",
149                        parsed.results.len(),
150                        parsed.search_time.unwrap_or(0.0),
151                        start_date,
152                    );
153
154                    let results = parsed
155                        .results
156                        .into_iter()
157                        .enumerate()
158                        .map(|(i, r)| {
159                            let snippet = if !r.highlights.is_empty() {
160                                r.highlights.join(" ... ")
161                            } else {
162                                r.text.unwrap_or_default()
163                            };
164
165                            let score = r.score.or_else(|| r.highlight_scores.first().copied());
166
167                            ResultItem {
168                                title: r.title,
169                                url: r.url,
170                                snippet,
171                                rank: (i + 1) as u32,
172                                backend: BackendId::Exa,
173                                score,
174                                published_date: r.published_date,
175                            }
176                        })
177                        .collect();
178
179                    return Ok(results);
180                }
181                Err(e) => {
182                    if let FetchiumError::Structured(ref se) = e {
183                        if se.message.contains("401")
184                            || se.message.contains("403")
185                            || se.message.contains("429")
186                            || se.message.contains("limit")
187                        {
188                            tracing::warn!(
189                                "Exa key exhausted or limited, rotating... Error: {}",
190                                se.message
191                            );
192                            self.rotate_key();
193                            last_err = Some(e);
194                            continue;
195                        }
196                    }
197                    return Err(e);
198                }
199            }
200        }
201
202        Err(last_err.unwrap_or(FetchiumError::Search("Exa: All keys exhausted".into())))
203    }
204
205    fn time_range_to_date(tr: Option<TimeRange>) -> Option<String> {
206        let now = chrono::Utc::now();
207        match tr {
208            Some(TimeRange::Day) => Some(
209                (now - chrono::Duration::days(1))
210                    .format("%Y-%m-%dT%H:%M:%SZ")
211                    .to_string(),
212            ),
213            Some(TimeRange::Week) => Some(
214                (now - chrono::Duration::days(7))
215                    .format("%Y-%m-%dT%H:%M:%SZ")
216                    .to_string(),
217            ),
218            Some(TimeRange::Month) => Some(
219                (now - chrono::Duration::days(30))
220                    .format("%Y-%m-%dT%H:%M:%SZ")
221                    .to_string(),
222            ),
223            Some(TimeRange::Year) => Some(
224                (now - chrono::Duration::days(365))
225                    .format("%Y-%m-%dT%H:%M:%SZ")
226                    .to_string(),
227            ),
228            None => None,
229        }
230    }
231}
232
233#[async_trait]
234impl SearchBackend for ExaBackend {
235    fn id(&self) -> BackendId {
236        BackendId::Exa
237    }
238
239    async fn search(&self, query: &str, max_results: u32) -> FetchiumResult<Vec<ResultItem>> {
240        self.search_inner(query, max_results, None).await
241    }
242
243    async fn search_with_context(
244        &self,
245        query: &str,
246        max_results: u32,
247        ctx: &SearchContext,
248    ) -> FetchiumResult<Vec<ResultItem>> {
249        let date_str = Self::time_range_to_date(ctx.time_range);
250        self.search_inner(query, max_results, date_str.as_deref())
251            .await
252    }
253}
254
255#[cfg(test)]
256mod tests {
257    use super::*;
258
259    #[test]
260    fn parse_exa_response() {
261        let json = r#"{
262            "results": [
263                {
264                    "title": "Test Article",
265                    "url": "https://example.com",
266                    "text": "Full text here",
267                    "highlights": ["Key finding 1", "Key finding 2"],
268                    "highlightScores": [0.95, 0.87],
269                    "publishedDate": "2025-06-15",
270                    "score": 0.92
271                }
272            ],
273            "searchTime": 800.5
274        }"#;
275        let parsed: ExaResponse = serde_json::from_str(json).unwrap();
276        assert_eq!(parsed.results.len(), 1);
277        assert_eq!(parsed.results[0].highlights.len(), 2);
278    }
279
280    #[test]
281    fn parse_exa_empty() {
282        let json = r#"{"results": [], "searchTime": 100.0}"#;
283        let parsed: ExaResponse = serde_json::from_str(json).unwrap();
284        assert!(parsed.results.is_empty());
285    }
286}