use async_trait::async_trait;
use reqwest::Client;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use crate::search::engine_trait::{SearchEngine, SearchRequest};
use crate::search::error::SearchError;
use crate::search::response::{Response, ResponseItem};
use crate::search::types::{EngineHealth, SearchEngineType};
mod text_fields;
const DEFAULT_EXA_ENDPOINT: &str = "https://mcp.exa.ai/mcp";
const EXA_TIMEOUT_SECS: u64 = 25;
const MAX_RESPONSE_SIZE: usize = 256 * 1024;
const MCP_PROTOCOL_VERSION: &str = "2025-03-26";
#[derive(Debug, Clone)]
pub struct ExaConfig {
pub api_key: String,
pub endpoint: String,
}
impl Default for ExaConfig {
fn default() -> Self {
Self {
api_key: String::new(),
endpoint: DEFAULT_EXA_ENDPOINT.to_string(),
}
}
}
struct SessionState {
session_id: Option<String>,
initialized: bool,
}
pub struct ExaSearchEngine {
client: Client,
config: ExaConfig,
session: Arc<parking_lot::Mutex<SessionState>>,
request_id: AtomicU64,
}
impl ExaSearchEngine {
pub fn new(client: Client, config: ExaConfig) -> Self {
Self {
client,
config,
session: Arc::new(parking_lot::Mutex::new(SessionState {
session_id: None,
initialized: false,
})),
request_id: AtomicU64::new(1),
}
}
fn next_id(&self) -> u64 {
self.request_id.fetch_add(1, Ordering::Relaxed)
}
fn request_url(&self) -> String {
if !self.config.api_key.is_empty() {
format!("{}?apiKey={}", self.config.endpoint, self.config.api_key)
} else {
self.config.endpoint.clone()
}
}
async fn send_mcp_request(
&self,
body: &Value,
session_id: Option<&str>,
) -> Result<(Value, Option<String>), SearchError> {
let url = self.request_url();
let mut req = self
.client
.post(&url)
.header("Content-Type", "application/json")
.header("Accept", "application/json, text/event-stream")
.json(body)
.timeout(std::time::Duration::from_secs(EXA_TIMEOUT_SECS));
if let Some(sid) = session_id {
req = req.header("Mcp-Session-Id", sid);
}
let response = req.send().await?;
let status = response.status();
if !status.is_success() {
return Err(SearchError::BadHttpStatus(
"Exa".to_string(),
status.as_u16(),
));
}
let new_session_id = response
.headers()
.get("mcp-session-id")
.and_then(|v| v.to_str().ok())
.map(|s| s.to_string());
let content_type = response
.headers()
.get("content-type")
.and_then(|v| v.to_str().ok())
.unwrap_or("")
.to_string();
let body_text = response.text().await?;
if body_text.len() > MAX_RESPONSE_SIZE {
return Err(SearchError::Parse(format!(
"Exa response exceeds 256KB limit: {} bytes",
body_text.len()
)));
}
let json_value = if content_type.contains("text/event-stream") {
Self::parse_sse_to_json(&body_text)?
} else {
serde_json::from_str::<Value>(&body_text).map_err(|e| {
SearchError::Parse(format!("Failed to parse Exa JSON response: {}", e))
})?
};
Ok((json_value, new_session_id))
}
async fn ensure_initialized(&self) -> Result<(), SearchError> {
{
let state = self.session.lock();
if state.initialized {
return Ok(());
}
}
let init_body = json!({
"jsonrpc": "2.0",
"id": self.next_id(),
"method": "initialize",
"params": {
"protocolVersion": MCP_PROTOCOL_VERSION,
"capabilities": {},
"clientInfo": {
"name": "crawlrs",
"version": env!("CARGO_PKG_VERSION")
}
}
});
let (response, new_session_id) = self.send_mcp_request(&init_body, None).await?;
if let Some(error) = response.get("error") {
return Err(SearchError::EngineFailed(format!(
"Exa MCP initialize error: {}",
error
)));
}
{
let mut state = self.session.lock();
state.session_id = new_session_id.or_else(|| state.session_id.take());
}
let session_id = self.session.lock().session_id.clone();
let notify_body = json!({
"jsonrpc": "2.0",
"method": "notifications/initialized"
});
let _ = self
.send_mcp_request(¬ify_body, session_id.as_deref())
.await;
self.session.lock().initialized = true;
Ok(())
}
fn parse_sse_to_json(body: &str) -> Result<Value, SearchError> {
let mut last_data = None;
for line in body.lines() {
if let Some(data) = line.strip_prefix("data: ") {
last_data = Some(data);
}
}
let data = last_data.ok_or_else(|| {
SearchError::Parse("No SSE data lines found in Exa response".to_string())
})?;
serde_json::from_str::<Value>(data)
.map_err(|e| SearchError::Parse(format!("Failed to parse SSE JSON: {}", e)))
}
fn extract_results_from_response(response: &Value) -> Vec<ExaSearchResult> {
let mut results = Vec::new();
let content_array = response
.get("result")
.and_then(|r| r.get("content"))
.and_then(|c| c.as_array());
let Some(content_array) = content_array else {
return results;
};
for content in content_array {
let text = content.get("text").and_then(|t| t.as_str()).unwrap_or("");
if text.is_empty() {
continue;
}
results.extend(Self::parse_content_results(text));
}
results
}
fn parse_content_results(text: &str) -> Vec<ExaSearchResult> {
if let Ok(mcp_result) = serde_json::from_str::<McpResult>(text) {
return mcp_result
.results
.iter()
.map(|item| ExaSearchResult {
title: item.title.clone(),
url: item.url.clone(),
description: item
.text
.as_deref()
.or(item.description.as_deref())
.or(item.snippet.as_deref())
.unwrap_or("")
.to_string(),
})
.collect();
}
if let Ok(Value::Array(items)) = serde_json::from_str::<Value>(text) {
return items.iter().filter_map(parse_generic_result_item).collect();
}
Self::parse_text_results(text)
}
fn parse_text_results(text: &str) -> Vec<ExaSearchResult> {
text.split("\n---\n")
.filter_map(Self::parse_text_block)
.collect()
}
fn parse_text_block(block: &str) -> Option<ExaSearchResult> {
let block = block.trim();
if block.is_empty() {
return None;
}
let (title, url, description) = text_fields::parse_text_fields(block);
if url.is_empty() {
return None;
}
Some(ExaSearchResult {
title,
url,
description,
})
}
fn build_search_request(&self, query: &str, limit: u32) -> Value {
json!({
"jsonrpc": "2.0",
"id": self.next_id(),
"method": "tools/call",
"params": {
"name": "web_search_exa",
"arguments": {
"query": query,
"numResults": limit,
}
}
})
}
}
#[async_trait]
impl SearchEngine for ExaSearchEngine {
fn name(&self) -> &'static str {
"Exa"
}
fn engine_type(&self) -> SearchEngineType {
SearchEngineType::Exa
}
fn health(&self) -> EngineHealth {
EngineHealth::Healthy
}
async fn search(&self, request: &SearchRequest) -> Result<Response<ResponseItem>, SearchError> {
self.ensure_initialized().await?;
let body = self.build_search_request(&request.query, request.limit);
let session_id = self.session.lock().session_id.clone();
let (response, new_session_id) =
self.send_mcp_request(&body, session_id.as_deref()).await?;
if let Some(new_sid) = new_session_id {
self.session.lock().session_id = Some(new_sid);
}
if let Some(error) = response.get("error") {
return Err(SearchError::EngineFailed(format!(
"Exa JSON-RPC error: {}",
error
)));
}
let results = Self::extract_results_from_response(&response);
let items: Vec<ResponseItem> = results
.into_iter()
.map(|r| ResponseItem {
title: r.title,
url: r.url,
description: r.description,
engine: SearchEngineType::Exa,
})
.collect();
Ok(Response {
total_results: Some(items.len() as u64),
items,
engine: SearchEngineType::Exa,
})
}
}
#[derive(Debug, Serialize, Deserialize)]
struct McpResult {
#[serde(default)]
results: Vec<ExaRawResult>,
}
#[derive(Debug, Serialize, Deserialize)]
struct ExaRawResult {
#[serde(default)]
title: String,
#[serde(default)]
url: String,
text: Option<String>,
description: Option<String>,
snippet: Option<String>,
}
struct ExaSearchResult {
title: String,
url: String,
description: String,
}
fn parse_generic_result_item(item: &Value) -> Option<ExaSearchResult> {
let url = item
.get("url")
.and_then(|v| v.as_str())
.unwrap_or_default()
.to_string();
if url.is_empty() {
return None;
}
let title = item
.get("title")
.and_then(|v| v.as_str())
.unwrap_or_default()
.to_string();
let description = item
.get("description")
.or_else(|| item.get("text"))
.or_else(|| item.get("snippet"))
.and_then(|v| v.as_str())
.unwrap_or_default()
.to_string();
Some(ExaSearchResult {
title,
url,
description,
})
}
#[cfg(test)]
mod tests {
use super::*;
fn test_config() -> ExaConfig {
ExaConfig::default()
}
fn test_engine() -> ExaSearchEngine {
ExaSearchEngine::new(Client::new(), test_config())
}
#[test]
fn test_build_request_structure() {
let engine = test_engine();
let req = engine.build_search_request("rust web scraping", 5);
assert_eq!(req["jsonrpc"], "2.0");
assert_eq!(req["method"], "tools/call");
assert_eq!(req["params"]["name"], "web_search_exa");
assert_eq!(req["params"]["arguments"]["query"], "rust web scraping");
assert_eq!(req["params"]["arguments"]["numResults"], 5);
assert!(req["id"].is_number());
}
#[test]
fn test_request_url_without_api_key() {
let engine = test_engine();
assert_eq!(engine.request_url(), "https://mcp.exa.ai/mcp");
}
#[test]
fn test_request_url_with_api_key() {
let config = ExaConfig {
api_key: "test-key-123".to_string(),
endpoint: "https://mcp.exa.ai/mcp".to_string(),
};
let engine = ExaSearchEngine::new(Client::new(), config);
assert_eq!(
engine.request_url(),
"https://mcp.exa.ai/mcp?apiKey=test-key-123"
);
}
#[test]
fn test_parse_text_results_single() {
let text = "Title: Rust Lang\nURL: https://rust-lang.org\nPublished: N/A\nAuthor: N/A\nHighlights:\nRust is fast\n...";
let results = ExaSearchEngine::parse_text_results(text);
assert_eq!(results.len(), 1);
assert_eq!(results[0].title, "Rust Lang");
assert_eq!(results[0].url, "https://rust-lang.org");
assert!(results[0].description.contains("Rust is fast"));
}
#[test]
fn test_parse_text_results_multiple() {
let text = "Title: First\nURL: https://first.com\nPublished: N/A\nAuthor: N/A\nHighlights:\nFirst desc\n...\n\n---\n\nTitle: Second\nURL: https://second.com\nPublished: N/A\nAuthor: N/A\nHighlights:\nSecond desc";
let results = ExaSearchEngine::parse_text_results(text);
assert_eq!(results.len(), 2);
assert_eq!(results[0].title, "First");
assert_eq!(results[0].url, "https://first.com");
assert_eq!(results[1].title, "Second");
assert_eq!(results[1].url, "https://second.com");
}
#[test]
fn test_parse_text_results_empty_url_skipped() {
let text = "Title: No URL\nURL: \nPublished: N/A";
let results = ExaSearchEngine::parse_text_results(text);
assert!(results.is_empty());
}
#[test]
fn test_parse_text_results_empty_input() {
let results = ExaSearchEngine::parse_text_results("");
assert!(results.is_empty());
}
#[test]
fn test_parse_sse_to_json() {
let body = "event: message\ndata: {\"result\":{\"content\":[]},\"id\":1}\n\n";
let value = ExaSearchEngine::parse_sse_to_json(body).unwrap();
assert!(value.get("result").is_some());
}
#[test]
fn test_parse_sse_no_data_lines() {
let body = "event: message\n\n";
let result = ExaSearchEngine::parse_sse_to_json(body);
assert!(result.is_err());
}
#[test]
fn test_extract_results_text_format() {
let response = json!({
"result": {
"content": [{
"type": "text",
"text": "Title: Test\nURL: https://test.com\nPublished: N/A\nAuthor: N/A\nHighlights:\nTest description"
}]
},
"id": 2
});
let results = ExaSearchEngine::extract_results_from_response(&response);
assert_eq!(results.len(), 1);
assert_eq!(results[0].title, "Test");
assert_eq!(results[0].url, "https://test.com");
}
#[test]
fn test_extract_results_json_format_fallback() {
let response = json!({
"result": {
"content": [{
"type": "text",
"text": "{\"results\": [{\"title\": \"JSON Result\", \"url\": \"https://json.com\", \"text\": \"desc\"}]}"
}]
},
"id": 2
});
let results = ExaSearchEngine::extract_results_from_response(&response);
assert_eq!(results.len(), 1);
assert_eq!(results[0].title, "JSON Result");
assert_eq!(results[0].url, "https://json.com");
}
#[test]
fn test_extract_results_empty_content() {
let response = json!({"result": {"content": []}, "id": 1});
let results = ExaSearchEngine::extract_results_from_response(&response);
assert!(results.is_empty());
}
#[test]
fn test_extract_results_no_result() {
let response = json!({"id": 1});
let results = ExaSearchEngine::extract_results_from_response(&response);
assert!(results.is_empty());
}
#[test]
fn test_engine_name() {
let engine = test_engine();
assert_eq!(engine.name(), "Exa");
}
#[test]
fn test_engine_type() {
let engine = test_engine();
assert_eq!(engine.engine_type(), SearchEngineType::Exa);
}
#[test]
fn test_engine_health() {
let engine = test_engine();
assert_eq!(engine.health(), EngineHealth::Healthy);
}
#[tokio::test]
async fn test_extract_results_from_real_exa_response() {
let response = json!({
"result": {
"content": [{
"type": "text",
"text": "Title: Rust Lang\nURL: https://rust-lang.org\nPublished: N/A\nAuthor: N/A\nHighlights:\nRust is blazingly fast\n...\n\n---\n\nTitle: Rust Book\nURL: https://doc.rust-lang.org/book/\nPublished: N/A\nAuthor: N/A\nHighlights:\nThe Rust Programming Language"
}]
},
"jsonrpc": "2.0",
"id": 2
});
let results = ExaSearchEngine::extract_results_from_response(&response);
assert_eq!(results.len(), 2);
assert_eq!(results[0].title, "Rust Lang");
assert_eq!(results[0].url, "https://rust-lang.org");
assert!(results[0].description.contains("Rust is blazingly fast"));
assert_eq!(results[1].title, "Rust Book");
assert_eq!(results[1].url, "https://doc.rust-lang.org/book/");
}
#[tokio::test]
async fn test_search_http_error() {
let mock_server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("POST"))
.respond_with(wiremock::ResponseTemplate::new(500))
.mount(&mock_server)
.await;
let config = ExaConfig {
api_key: String::new(),
endpoint: mock_server.uri(),
};
let engine = ExaSearchEngine::new(Client::new(), config);
let request = SearchRequest::new("test").with_limit(5);
let result = engine.search(&request).await;
assert!(result.is_err());
match result.unwrap_err() {
SearchError::BadHttpStatus(name, code) => {
assert_eq!(name, "Exa");
assert_eq!(code, 500);
}
other => panic!("expected BadHttpStatus, got {:?}", other),
}
}
#[test]
fn test_parse_generic_result_item_with_all_fields() {
let item = json!({
"title": "Test Title",
"url": "https://example.com",
"description": "Test description"
});
let result = parse_generic_result_item(&item);
assert!(result.is_some());
let r = result.unwrap();
assert_eq!(r.title, "Test Title");
assert_eq!(r.url, "https://example.com");
assert_eq!(r.description, "Test description");
}
#[test]
fn test_parse_generic_result_item_empty_url_returns_none() {
let item = json!({"title": "No URL", "url": ""});
assert!(parse_generic_result_item(&item).is_none());
}
#[test]
fn test_parse_generic_result_item_fallback_to_text_and_snippet() {
let item_text = json!({"url": "https://a.com", "text": "from text field"});
let r = parse_generic_result_item(&item_text).unwrap();
assert_eq!(r.description, "from text field");
let item_snippet = json!({"url": "https://b.com", "snippet": "from snippet"});
let r = parse_generic_result_item(&item_snippet).unwrap();
assert_eq!(r.description, "from snippet");
}
#[test]
fn test_extract_results_json_array_format() {
let response = json!({
"result": {
"content": [{
"type": "text",
"text": "[{\"title\": \"Arr Item\", \"url\": \"https://arr.com\", \"description\": \"arr desc\"}]"
}]
}
});
let results = ExaSearchEngine::extract_results_from_response(&response);
assert_eq!(results.len(), 1);
assert_eq!(results[0].title, "Arr Item");
assert_eq!(results[0].url, "https://arr.com");
}
#[test]
fn test_extract_results_empty_text_skipped() {
let response = json!({
"result": {
"content": [{"type": "text", "text": ""}]
}
});
let results = ExaSearchEngine::extract_results_from_response(&response);
assert!(results.is_empty());
}
#[test]
fn test_parse_text_results_description_truncated_at_500() {
let long_desc = "x".repeat(600);
let text = format!(
"Title: Truncated\nURL: https://trunc.com\nHighlights:\n{}",
long_desc
);
let results = ExaSearchEngine::parse_text_results(&text);
assert_eq!(results.len(), 1);
assert!(
results[0].description.len() <= 503,
"description should be truncated to ~500 chars + '...'"
);
assert!(results[0].description.ends_with("..."));
}
#[test]
fn test_parse_sse_multiple_data_lines_takes_last() {
let body = "data: {\"a\":1}\ndata: {\"b\":2}\n\n";
let value = ExaSearchEngine::parse_sse_to_json(body).unwrap();
assert!(value.get("b").is_some(), "should use last data line");
}
#[test]
fn test_parse_sse_invalid_json_returns_error() {
let body = "data: not-json\n\n";
assert!(ExaSearchEngine::parse_sse_to_json(body).is_err());
}
}