use crate::engines::engine_client::{
EngineClient, ScrapeOptions, ScrapeRequest as EngineScrapeRequest,
};
use crate::search::{
client::html_parser::HtmlParser,
client::shared_utils::escape_html_text,
engine_trait::SearchEngine,
error::SearchError,
response::{Response, ResponseItem},
types::{EngineHealth, SearchEngineType},
SearchRequest,
};
use async_trait::async_trait;
use serde_json;
use std::collections::HashMap;
use std::sync::Arc;
#[derive(Debug, Clone, Copy)]
pub enum BaiduSearchCategory {
General,
Images,
News,
}
pub struct BaiduSearchEngine {
engine_client: Arc<EngineClient>,
}
impl BaiduSearchEngine {
pub fn new(engine_client: Arc<EngineClient>) -> Self {
Self { engine_client }
}
fn page_from_offset(offset: u32) -> u32 {
(offset / 10) + 1
}
pub fn build_baidu_url(
&self,
query: &str,
page: u32,
category: BaiduSearchCategory,
) -> (String, HashMap<String, String>) {
let mut params = HashMap::with_capacity(8);
let offset = ((page - 1) * 10).to_string();
match category {
BaiduSearchCategory::General => {
params.insert("wd".to_string(), query.to_string());
params.insert("rn".to_string(), "10".to_string());
params.insert("on".to_string(), offset);
params.insert("tn".to_string(), "json".to_string());
("https://www.baidu.com/s".to_string(), params)
}
BaiduSearchCategory::Images => {
params.insert("word".to_string(), query.to_string());
params.insert("tn".to_string(), "resultjson_com".to_string());
params.insert("on".to_string(), offset);
params.insert("rn".to_string(), "30".to_string()); ("https://image.baidu.com/search/acjson".to_string(), params)
}
_ => {
params.insert("wd".to_string(), query.to_string());
("https://www.baidu.com/s".to_string(), params)
}
}
}
pub fn parse_baidu_response(&self, json_str: &str) -> Result<Vec<ResponseItem>, SearchError> {
let json: serde_json::Value = serde_json::from_str(json_str)
.map_err(|e| SearchError::Parse(format!("Failed to parse Baidu JSON: {}", e)))?;
if json.get("antiFlag").and_then(|v| v.as_i64()) == Some(1) {
let msg = json
.get("message")
.and_then(|m| m.as_str())
.unwrap_or("Baidu spider access denied");
return Err(SearchError::RateLimited(format!(
"Baidu anti-spider: {}",
msg
)));
}
let mut results = Vec::with_capacity(10);
if let Some(entry_array) = json
.get("feed")
.and_then(|f: &serde_json::Value| f.get("entry"))
.and_then(|e: &serde_json::Value| e.as_array())
{
for entry in entry_array {
let title = entry
.get("title")
.and_then(|t: &serde_json::Value| t.as_str())
.unwrap_or_default()
.to_string();
let url = entry
.get("url")
.and_then(|u: &serde_json::Value| u.as_str())
.unwrap_or_default()
.to_string();
let description = entry
.get("abs")
.and_then(|a: &serde_json::Value| a.as_str())
.unwrap_or_default()
.to_string();
if !title.is_empty() && !url.is_empty() {
results.push(ResponseItem {
title: escape_html_text(&title),
url,
description: escape_html_text(&description),
engine: SearchEngineType::Baidu,
});
}
}
}
Ok(results)
}
pub fn parse_search_results(
&self,
html: &str,
_query: &str,
) -> Result<Vec<ResponseItem>, SearchError> {
Ok(HtmlParser::baidu().parse(html, SearchEngineType::Baidu))
}
fn parse_results_with_json_fallback(
&self,
content: &str,
query: &str,
) -> Result<Vec<ResponseItem>, SearchError> {
let items = self.parse_search_results(content, query)?;
let items = if items.is_empty() {
if let Ok(json_items) = self.parse_baidu_response(content) {
if !json_items.is_empty() {
json_items
} else {
items
}
} else {
items
}
} else {
items
};
Ok(items)
}
}
#[async_trait]
impl SearchEngine for BaiduSearchEngine {
fn name(&self) -> &'static str {
"Baidu"
}
fn engine_type(&self) -> SearchEngineType {
SearchEngineType::Baidu
}
fn health(&self) -> EngineHealth {
EngineHealth::Healthy
}
async fn search(&self, request: &SearchRequest) -> Result<Response<ResponseItem>, SearchError> {
#[cfg(test)]
{
if let Some(resp) = baidu_test_results_override(&request.query) {
return Ok(resp);
}
}
let page = Self::page_from_offset(request.offset);
let (url, params) =
self.build_baidu_url(&request.query, page, BaiduSearchCategory::General);
let full_url = build_full_search_url(url, ¶ms);
let headers = build_search_headers();
let options = ScrapeOptions {
headers,
timeout: std::time::Duration::from_secs(30),
..Default::default()
};
let engine_request = EngineScrapeRequest {
url: full_url,
options,
};
let scrape_response = self
.engine_client
.scrape(&engine_request)
.await
.map_err(|e| SearchError::EngineClient("Baidu".to_string(), e.to_string()))?;
if scrape_response.status_code < 200 || scrape_response.status_code >= 300 {
return Err(SearchError::BadHttpStatus(
"Baidu".to_string(),
scrape_response.status_code,
));
}
let content = scrape_response.content;
detect_baidu_captcha(&content)?;
let items = self.parse_results_with_json_fallback(&content, &request.query)?;
let total_count = items.len() as u64;
Ok(Response {
items,
total_results: Some(total_count),
engine: SearchEngineType::Baidu,
})
}
}
fn build_full_search_url(url: String, params: &HashMap<String, String>) -> String {
if !params.is_empty() {
format!(
"{}?{}",
url,
serde_urlencoded::to_string(params).unwrap_or_default()
)
} else {
url
}
}
fn build_search_headers() -> HashMap<String, String> {
let mut headers = HashMap::with_capacity(6);
headers.insert(
"Accept".to_string(),
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8".to_string(),
);
headers.insert(
"Accept-Language".to_string(),
"zh-CN,zh;q=0.9,en;q=0.8".to_string(),
);
headers.insert("DNT".to_string(), "1".to_string());
headers.insert("Connection".to_string(), "keep-alive".to_string());
headers.insert("Sec-GPC".to_string(), "1".to_string());
headers.insert("Cache-Control".to_string(), "max-age=0".to_string());
headers
}
fn detect_baidu_captcha(content: &str) -> Result<(), SearchError> {
if content.contains("wappass.baidu.com/static/captcha")
|| content.contains("verify your access")
|| content.contains("百度安全验证")
{
return Err(SearchError::Captcha("Baidu".to_string()));
}
Ok(())
}
#[cfg(test)]
fn baidu_test_results_override(query: &str) -> Option<Response<ResponseItem>> {
if std::env::var("BAIDU_TEST_RESULTS").unwrap_or_default() != "true" {
return None;
}
let escaped_query = html_escape::encode_text(query);
Some(Response {
items: vec![
ResponseItem {
title: format!("Baidu Test Result 1 for {}", escaped_query),
url: "https://baidu.com/1".to_string(),
description: "Test description 1".to_string(),
engine: SearchEngineType::Baidu,
},
ResponseItem {
title: format!("Baidu Test Result 2 for {}", escaped_query),
url: "https://baidu.com/2".to_string(),
description: "Test description 2".to_string(),
engine: SearchEngineType::Baidu,
},
],
total_results: Some(2),
engine: SearchEngineType::Baidu,
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::engines::engine_client::EngineClient;
use std::sync::Arc;
fn create_engine() -> BaiduSearchEngine {
let engine_client = Arc::new(EngineClient::new());
BaiduSearchEngine::new(engine_client)
}
#[test]
fn test_build_baidu_url_general_page1() {
let engine = create_engine();
let (url, params) = engine.build_baidu_url("rust 语言", 1, BaiduSearchCategory::General);
assert_eq!(url, "https://www.baidu.com/s");
assert_eq!(params.get("wd"), Some(&"rust 语言".to_string()));
assert_eq!(params.get("rn"), Some(&"10".to_string()));
assert_eq!(params.get("on"), Some(&"0".to_string()));
assert_eq!(params.get("tn"), Some(&"json".to_string()));
}
#[test]
fn test_build_baidu_url_general_page3_offset() {
let engine = create_engine();
let (_url, params) = engine.build_baidu_url("test", 3, BaiduSearchCategory::General);
assert_eq!(params.get("on"), Some(&"20".to_string()));
}
#[test]
fn test_baidu_offset_derives_page_param() {
assert_eq!(BaiduSearchEngine::page_from_offset(0), 1);
assert_eq!(BaiduSearchEngine::page_from_offset(10), 2);
let engine = create_engine();
let (url, params) = engine.build_baidu_url(
"rust",
BaiduSearchEngine::page_from_offset(10),
BaiduSearchCategory::General,
);
assert_eq!(
params.get("on").map(String::as_str),
Some("10"),
"page 2 → on=10, url={url}"
);
}
#[test]
fn test_build_baidu_url_images_category() {
let engine = create_engine();
let (url, params) = engine.build_baidu_url("cat", 1, BaiduSearchCategory::Images);
assert_eq!(url, "https://image.baidu.com/search/acjson");
assert_eq!(params.get("word"), Some(&"cat".to_string()));
assert_eq!(params.get("tn"), Some(&"resultjson_com".to_string()));
assert_eq!(params.get("rn"), Some(&"30".to_string()));
assert_eq!(params.get("on"), Some(&"0".to_string()));
}
#[test]
fn test_build_baidu_url_news_falls_back_to_general() {
let engine = create_engine();
let (url, params) = engine.build_baidu_url("news", 1, BaiduSearchCategory::News);
assert_eq!(url, "https://www.baidu.com/s");
assert_eq!(params.get("wd"), Some(&"news".to_string()));
assert!(!params.contains_key("rn"), "fallback should not set rn");
}
#[test]
fn test_build_baidu_url_empty_query() {
let engine = create_engine();
let (url, params) = engine.build_baidu_url("", 1, BaiduSearchCategory::General);
assert_eq!(url, "https://www.baidu.com/s");
assert_eq!(params.get("wd"), Some(&"".to_string()));
}
#[test]
fn test_parse_baidu_response_valid_json() {
let engine = create_engine();
let json = r#"{
"feed": {
"entry": [
{"title": "Result 1", "url": "https://example.com/1", "abs": "Desc 1"},
{"title": "Result 2", "url": "https://example.com/2", "abs": "Desc 2"}
]
}
}"#;
let results = engine.parse_baidu_response(json).unwrap();
assert_eq!(results.len(), 2);
assert_eq!(results[0].title, "Result 1");
assert_eq!(results[0].url, "https://example.com/1");
assert_eq!(results[0].description, "Desc 1");
assert_eq!(results[0].engine, SearchEngineType::Baidu);
assert_eq!(results[1].title, "Result 2");
}
#[test]
fn test_parse_baidu_response_invalid_json() {
let engine = create_engine();
let result = engine.parse_baidu_response("not a valid json {{{");
assert!(result.is_err());
let err = result.unwrap_err();
match err {
SearchError::Parse(msg) => assert!(msg.contains("Failed to parse Baidu JSON")),
other => panic!("expected SearchError::Parse, got {:?}", other),
}
}
#[test]
fn test_parse_baidu_response_skips_empty_title_or_url() {
let engine = create_engine();
let json = r#"{
"feed": {
"entry": [
{"title": "", "url": "https://example.com/1", "abs": "Desc 1"},
{"title": "Result 2", "url": "", "abs": "Desc 2"},
{"title": "Valid", "url": "https://example.com/3", "abs": "Desc 3"}
]
}
}"#;
let results = engine.parse_baidu_response(json).unwrap();
assert_eq!(results.len(), 1);
assert_eq!(results[0].title, "Valid");
assert_eq!(results[0].url, "https://example.com/3");
}
#[test]
fn test_parse_baidu_response_no_feed_entry() {
let engine = create_engine();
let json = r#"{"other": "data"}"#;
let results = engine.parse_baidu_response(json).unwrap();
assert!(results.is_empty());
}
#[test]
fn test_parse_baidu_response_html_entities_encoded() {
let engine = create_engine();
let json = r#"{
"feed": {
"entry": [
{"title": "<script>alert(1)</script>", "url": "https://example.com", "abs": "a & b <c>"}
]
}
}"#;
let results = engine.parse_baidu_response(json).unwrap();
assert_eq!(results.len(), 1);
assert!(!results[0].title.contains("<script>"));
assert!(results[0].title.contains("<script>"));
assert!(!results[0].description.contains("<c>"));
}
#[test]
fn test_parse_search_results_valid_html() {
let engine = create_engine();
let html = r#"
<html><body>
<div class="c-container">
<h3><a href="https://example.com/1">First Result</a></h3>
<div class="c-abstract">First description</div>
</div>
<div class="c-container">
<h3><a href="https://example.com/2">Second Result</a></h3>
<div class="c-abstract">Second description</div>
</div>
</body></html>
"#;
let results = engine.parse_search_results(html, "test").unwrap();
assert_eq!(results.len(), 2);
assert_eq!(results[0].title, "First Result");
assert_eq!(results[0].url, "https://example.com/1");
assert_eq!(results[0].engine, SearchEngineType::Baidu);
assert_eq!(results[1].title, "Second Result");
}
#[test]
fn test_parse_search_results_empty_html() {
let engine = create_engine();
let results = engine.parse_search_results("", "test").unwrap();
assert!(results.is_empty());
}
#[test]
fn test_parse_search_results_no_matching_selectors() {
let engine = create_engine();
let html = r#"<html><body><div>no results here</div></body></html>"#;
let results = engine.parse_search_results(html, "test").unwrap();
assert!(results.is_empty());
}
#[test]
fn test_engine_creation() {
let engine = create_engine();
assert_eq!(engine.name(), "Baidu");
assert_eq!(engine.engine_type(), SearchEngineType::Baidu);
assert_eq!(engine.health(), EngineHealth::Healthy);
}
static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
fn make_search_request(query: &str) -> SearchRequest {
SearchRequest::new(query)
}
#[allow(clippy::await_holding_lock)]
#[tokio::test]
async fn test_search_fallback_returns_hardcoded_results() {
let _lock = ENV_LOCK.lock().unwrap();
std::env::set_var("BAIDU_TEST_RESULTS", "true");
let engine = create_engine();
let request = make_search_request("rust programming");
let response = engine.search(&request).await;
std::env::remove_var("BAIDU_TEST_RESULTS");
let response = response.expect("fallback should return Ok");
assert_eq!(response.items.len(), 2, "fallback should return 2 items");
assert_eq!(response.engine, SearchEngineType::Baidu);
assert_eq!(response.total_results, Some(2));
assert!(response.items[0].title.contains("rust programming"));
assert!(response.items[1].title.contains("rust programming"));
assert_eq!(response.items[0].url, "https://baidu.com/1");
assert_eq!(response.items[1].url, "https://baidu.com/2");
assert_eq!(response.items[0].engine, SearchEngineType::Baidu);
assert_eq!(response.items[1].engine, SearchEngineType::Baidu);
}
#[allow(clippy::await_holding_lock)] #[tokio::test]
async fn test_search_fallback_escapes_query_in_title() {
let _lock = ENV_LOCK.lock().unwrap();
std::env::set_var("BAIDU_TEST_RESULTS", "true");
let engine = create_engine();
let request = make_search_request("<script>alert(1)</script>");
let response = engine.search(&request).await;
std::env::remove_var("BAIDU_TEST_RESULTS");
let response = response.expect("fallback should return Ok");
assert!(response.items[0].title.contains("<script>"));
assert!(!response.items[0].title.contains("<script>"));
assert!(response.items[1].title.contains("<script>"));
assert!(!response.items[1].title.contains("<script>"));
}
#[allow(clippy::await_holding_lock)] #[tokio::test]
async fn test_search_fallback_description_and_engine_fields() {
let _lock = ENV_LOCK.lock().unwrap();
std::env::set_var("BAIDU_TEST_RESULTS", "true");
let engine = create_engine();
let request = make_search_request("fields test");
let response = engine.search(&request).await;
std::env::remove_var("BAIDU_TEST_RESULTS");
let response = response.expect("fallback should return Ok");
assert_eq!(response.items[0].description, "Test description 1");
assert_eq!(response.items[1].description, "Test description 2");
assert_eq!(response.items[0].engine, SearchEngineType::Baidu);
assert_eq!(response.items[1].engine, SearchEngineType::Baidu);
}
}