use crate::engines::engine_client::{
EngineClient, ScrapeOptions, ScrapeRequest as EngineScrapeRequest,
};
use crate::search::{
engine_trait::SearchEngine,
error::SearchError,
response::{Response, ResponseItem},
types::{EngineHealth, SearchEngineType},
SearchRequest,
};
use async_trait::async_trait;
use scraper::Html;
use std::collections::HashMap;
use std::sync::Arc;
use super::shared_utils::{escape_html_text, safe_parse_selector};
pub struct SogouSearchEngine {
engine_client: Arc<EngineClient>,
}
impl SogouSearchEngine {
pub fn new(engine_client: Arc<EngineClient>) -> Self {
Self { engine_client }
}
pub fn resolve_url(&self, url: &str) -> String {
if url.is_empty() {
return String::new();
}
if url.starts_with("http://") || url.starts_with("https://") {
return Self::validated_absolute_url(url);
}
if url.starts_with("/link?url=") {
return match Self::decode_sogou_redirect(url) {
Some(decoded) => self.resolve_url(decoded.as_ref()),
None => String::new(),
};
}
if url.starts_with("/") {
return format!("https://www.sogou.com{}", url);
}
String::new()
}
fn validated_absolute_url(url: &str) -> String {
if let Ok(parsed) = url::Url::parse(url) {
if parsed.scheme() == "http" || parsed.scheme() == "https" {
return url.to_string();
}
}
String::new()
}
fn decode_sogou_redirect(url: &str) -> Option<std::borrow::Cow<'_, str>> {
let encoded_url = url.trim_start_matches("/link?url=");
urlencoding::decode(encoded_url).ok()
}
pub fn parse_search_results(
&self,
html_content: &str,
) -> Result<Vec<ResponseItem>, SearchError> {
let document = Html::parse_document(html_content);
let selectors = SogouSelectors::new();
let mut results = Vec::with_capacity(10);
for element in document.select(&selectors.result) {
if let Some(item) = self.parse_result_element(element, &selectors) {
results.push(item);
}
}
Ok(results)
}
fn parse_result_element(
&self,
element: scraper::ElementRef<'_>,
selectors: &SogouSelectors,
) -> Option<ResponseItem> {
if element.select(&selectors.special_wrap).next().is_some() {
return None;
}
let title_node = element.select(&selectors.title).next()?;
let raw_title = title_node.text().collect::<String>();
let title = escape_html_text(raw_title.trim());
if title.is_empty() {
return None;
}
let url_node = element.select(&selectors.link).next()?;
let raw_url = url_node.value().attr("href").unwrap_or("").to_string();
let resolved_url = if raw_url.starts_with("/link?url=") {
element
.select(&selectors.data_url)
.next()
.and_then(|n| n.value().attr("data-url"))
.map(|s| s.to_string())
.filter(|s| !s.is_empty())
.unwrap_or_default()
} else {
self.resolve_url(&raw_url)
};
if resolved_url.is_empty() {
return None;
}
let description = element
.select(&selectors.snippet)
.next()
.map(|node| escape_html_text(&node.text().collect::<String>()))
.unwrap_or_default();
Some(ResponseItem {
title,
url: resolved_url,
description,
engine: SearchEngineType::Sogou,
})
}
}
struct SogouSelectors {
result: scraper::Selector,
title: scraper::Selector,
link: scraper::Selector,
data_url: scraper::Selector,
snippet: scraper::Selector,
special_wrap: scraper::Selector,
}
impl SogouSelectors {
fn new() -> Self {
Self {
result: safe_parse_selector(".vrwrap, .rb")
.expect("Failed to parse Sogou result selector"),
title: safe_parse_selector("h3").expect("Failed to parse Sogou title selector"),
link: safe_parse_selector("h3 a").expect("Failed to parse Sogou link selector"),
data_url: safe_parse_selector("[data-url]")
.expect("Failed to parse Sogou data-url selector"),
snippet: safe_parse_selector("div.text-layout p, div.ft p, p")
.expect("Failed to parse Sogou snippet selector"),
special_wrap: safe_parse_selector("div.special-wrap").expect("special-wrap selector"),
}
}
}
#[async_trait]
impl SearchEngine for SogouSearchEngine {
fn name(&self) -> &'static str {
"Sogou"
}
fn engine_type(&self) -> SearchEngineType {
SearchEngineType::Sogou
}
fn health(&self) -> EngineHealth {
EngineHealth::Healthy
}
async fn search(&self, request: &SearchRequest) -> Result<Response<ResponseItem>, SearchError> {
#[cfg(test)]
{
if let Some(resp) = sogou_test_results_override(&request.query) {
return Ok(resp);
}
}
let engine_request = build_engine_request(request);
let scrape_response = self
.engine_client
.scrape(&engine_request)
.await
.map_err(|e| SearchError::EngineClient("Sogou".to_string(), e.to_string()))?;
if scrape_response.status_code < 200 || scrape_response.status_code >= 300 {
return Err(SearchError::BadHttpStatus(
"Sogou".to_string(),
scrape_response.status_code,
));
}
let html_content = scrape_response.content;
if is_antispider_page(&html_content) {
return Err(SearchError::Captcha("Sogou".to_string()));
}
let items = self.parse_search_results(&html_content)?;
let total_count = items.len() as u64;
Ok(Response {
items,
total_results: Some(total_count),
engine: SearchEngineType::Sogou,
})
}
}
fn page_from_offset(offset: u32, limit: u32) -> u32 {
if offset > 0 {
(offset / limit.max(1)) + 1
} else {
1
}
}
fn build_search_headers() -> HashMap<String, String> {
let mut headers = HashMap::with_capacity(6);
headers.insert(
"Accept".to_string(),
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8".to_string(),
);
headers.insert(
"Accept-Language".to_string(),
"zh-CN,zh;q=0.9,en;q=0.8".to_string(),
);
headers.insert("DNT".to_string(), "1".to_string());
headers.insert("Connection".to_string(), "keep-alive".to_string());
headers.insert("Sec-GPC".to_string(), "1".to_string());
headers.insert("Cache-Control".to_string(), "max-age=0".to_string());
headers
}
fn build_engine_request(request: &SearchRequest) -> EngineScrapeRequest {
let base_url = "https://www.sogou.com/web";
let page = page_from_offset(request.offset, request.limit);
let full_url = format!(
"{}?query={}&page={}",
base_url,
urlencoding::encode(&request.query),
page,
);
EngineScrapeRequest {
url: full_url,
options: ScrapeOptions {
headers: build_search_headers(),
timeout: std::time::Duration::from_secs(30),
..Default::default()
},
}
}
fn is_antispider_page(html_content: &str) -> bool {
html_content.contains("www.sogou.com/antispider")
|| html_content.contains("antispider") && html_content.contains("验证")
}
#[cfg(test)]
fn sogou_test_results_override(query: &str) -> Option<Response<ResponseItem>> {
if std::env::var("SOGOU_TEST_RESULTS").unwrap_or_default() != "true" {
return None;
}
let escaped_query = html_escape::encode_text(query);
Some(Response {
items: vec![
ResponseItem {
title: format!("Test Result 1 for {}", escaped_query),
url: "https://sogou.com/1".to_string(),
description: "Test description 1".to_string(),
engine: SearchEngineType::Sogou,
},
ResponseItem {
title: format!("Test Result 2 for {}", escaped_query),
url: "https://sogou.com/2".to_string(),
description: "Test description 2".to_string(),
engine: SearchEngineType::Sogou,
},
],
total_results: Some(2),
engine: SearchEngineType::Sogou,
})
}
#[cfg(test)]
mod tests {
use super::*;
fn make_engine() -> SogouSearchEngine {
SogouSearchEngine::new(Arc::new(EngineClient::new()))
}
#[test]
fn test_resolve_url_empty() {
let engine = make_engine();
assert_eq!(engine.resolve_url(""), "");
}
#[test]
fn test_resolve_url_https() {
let engine = make_engine();
let url = "https://example.com/page";
assert_eq!(engine.resolve_url(url), url);
}
#[test]
fn test_resolve_url_http() {
let engine = make_engine();
let url = "http://example.com/page";
assert_eq!(engine.resolve_url(url), url);
}
#[test]
fn test_resolve_url_invalid_scheme_rejected() {
let engine = make_engine();
assert_eq!(engine.resolve_url("ftp://example.com"), "");
}
#[test]
fn test_resolve_url_malformed_http_rejected() {
let engine = make_engine();
assert_eq!(engine.resolve_url("http://"), "");
}
#[test]
fn test_resolve_url_sogou_link_redirect() {
let engine = make_engine();
let url = "/link?url=https%3A%2F%2Fwww.example.com%2Fpage";
let resolved = engine.resolve_url(url);
assert_eq!(resolved, "https://www.example.com/page");
}
#[test]
fn test_resolve_url_relative_path() {
let engine = make_engine();
let url = "/web?query=test";
let resolved = engine.resolve_url(url);
assert_eq!(resolved, "https://www.sogou.com/web?query=test");
}
#[test]
fn test_resolve_url_non_http_non_relative_rejected() {
let engine = make_engine();
assert_eq!(engine.resolve_url("just-some-text"), "");
}
#[test]
fn test_parse_search_results_valid_html() {
let engine = make_engine();
let html = r#"
<div class="vrwrap">
<h3><a href="https://example.com/result1">Example Result 1</a></h3>
</div>
<div class="vrwrap">
<h3><a href="https://example.com/result2">Example Result 2</a></h3>
</div>
"#;
let results = engine.parse_search_results(html).unwrap();
assert_eq!(results.len(), 2);
assert_eq!(results[0].title, "Example Result 1");
assert_eq!(results[0].url, "https://example.com/result1");
assert_eq!(results[0].engine, SearchEngineType::Sogou);
assert_eq!(results[1].title, "Example Result 2");
}
#[test]
fn test_parse_search_results_rb_class() {
let engine = make_engine();
let html = r#"
<div class="rb">
<h3><a href="https://example.com/rb-result">RB Result</a></h3>
</div>
"#;
let results = engine.parse_search_results(html).unwrap();
assert_eq!(results.len(), 1);
assert_eq!(results[0].title, "RB Result");
}
#[test]
fn test_parse_search_results_empty_html() {
let engine = make_engine();
let results = engine.parse_search_results("").unwrap();
assert!(results.is_empty());
}
#[test]
fn test_parse_search_results_no_results() {
let engine = make_engine();
let html = "<html><body><p>No results here</p></body></html>";
let results = engine.parse_search_results(html).unwrap();
assert!(results.is_empty());
}
#[test]
fn test_parse_search_results_missing_title_skipped() {
let engine = make_engine();
let html = r#"
<div class="vrwrap">
<a href="https://example.com/no-title">No Title</a>
</div>
<div class="vrwrap">
<h3><a href="https://example.com/with-title">With Title</a></h3>
</div>
"#;
let results = engine.parse_search_results(html).unwrap();
assert_eq!(results.len(), 1);
assert_eq!(results[0].title, "With Title");
}
#[test]
fn test_parse_search_results_empty_title_skipped() {
let engine = make_engine();
let html = r#"
<div class="vrwrap">
<h3> </h3>
<a href="https://example.com/empty-title">Empty Title</a>
</div>
"#;
let results = engine.parse_search_results(html).unwrap();
assert!(results.is_empty());
}
#[test]
fn test_parse_search_results_sogou_redirect_link() {
let engine = make_engine();
let html = r#"
<div class="vrwrap">
<h3><a href="/link?url=hedJjaC291N0Yt4ikBP2zHNFN13gvLUJQ3i3ukpLokg.">Redirect Result</a></h3>
<div class="img-layout" data-url="https://www.example.com/article"></div>
</div>
"#;
let results = engine.parse_search_results(html).unwrap();
assert_eq!(results.len(), 1);
assert_eq!(results[0].url, "https://www.example.com/article");
assert_eq!(results[0].title, "Redirect Result");
}
#[test]
fn test_parse_search_results_sogou_redirect_link_without_data_url_skipped() {
let engine = make_engine();
let html = r#"
<div class="vrwrap">
<h3><a href="/link?url=hedJjaC291InternalIdNoDataUrl">No Data URL</a></h3>
</div>
"#;
let results = engine.parse_search_results(html).unwrap();
assert!(
results.is_empty(),
"result without data-url should be skipped"
);
}
#[test]
fn test_parse_search_results_mixed_redirect_and_direct_urls() {
let engine = make_engine();
let html = r#"
<div class="vrwrap">
<h3><a href="/link?url=internalId1">Redirect Result</a></h3>
<div data-url="https://www.rust-lang.org/"></div>
</div>
<div class="vrwrap">
<h3><a href="https://baike.sogou.com/v12345.htm">Direct URL Result</a></h3>
</div>
"#;
let results = engine.parse_search_results(html).unwrap();
assert_eq!(results.len(), 2);
assert_eq!(results[0].url, "https://www.rust-lang.org/");
assert_eq!(results[1].url, "https://baike.sogou.com/v12345.htm");
}
#[test]
fn test_parse_search_results_empty_url_skipped() {
let engine = make_engine();
let html = r#"
<div class="vrwrap">
<h3><a>No href</a></h3>
</div>
"#;
let results = engine.parse_search_results(html).unwrap();
assert!(results.is_empty());
}
#[test]
fn test_parse_search_results_description_is_empty() {
let engine = make_engine();
let html = r#"
<div class="vrwrap">
<h3><a href="https://example.com/test">Test</a></h3>
</div>
"#;
let results = engine.parse_search_results(html).unwrap();
assert_eq!(results.len(), 1);
assert_eq!(results[0].description, "");
}
#[test]
fn test_name() {
let engine = make_engine();
assert_eq!(engine.name(), "Sogou");
}
#[test]
fn test_engine_type() {
let engine = make_engine();
assert_eq!(engine.engine_type(), SearchEngineType::Sogou);
}
#[test]
fn test_health() {
let engine = make_engine();
assert_eq!(engine.health(), EngineHealth::Healthy);
}
static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
fn make_search_request(query: &str) -> SearchRequest {
SearchRequest::new(query)
}
#[allow(clippy::await_holding_lock)]
#[tokio::test]
async fn test_search_with_test_results_env() {
let _lock = ENV_LOCK.lock().unwrap();
std::env::set_var("SOGOU_TEST_RESULTS", "true");
let engine = make_engine();
let request = make_search_request("test query");
let result = engine.search(&request).await;
std::env::remove_var("SOGOU_TEST_RESULTS");
assert!(result.is_ok());
let response = result.unwrap();
assert_eq!(response.items.len(), 2);
assert!(response.items[0].title.contains("test query"));
assert_eq!(response.engine, SearchEngineType::Sogou);
assert_eq!(response.total_results, Some(2));
}
#[allow(clippy::await_holding_lock)] #[tokio::test]
async fn test_search_fallback_returns_correct_urls_and_engine() {
let _lock = ENV_LOCK.lock().unwrap();
std::env::set_var("SOGOU_TEST_RESULTS", "true");
let engine = make_engine();
let request = make_search_request("url check");
let response = engine.search(&request).await;
std::env::remove_var("SOGOU_TEST_RESULTS");
let response = response.expect("fallback should return Ok");
assert_eq!(response.items.len(), 2);
assert_eq!(response.items[0].url, "https://sogou.com/1");
assert_eq!(response.items[1].url, "https://sogou.com/2");
assert_eq!(response.items[0].engine, SearchEngineType::Sogou);
assert_eq!(response.items[1].engine, SearchEngineType::Sogou);
assert_eq!(response.items[0].description, "Test description 1");
assert_eq!(response.items[1].description, "Test description 2");
}
#[allow(clippy::await_holding_lock)] #[tokio::test]
async fn test_search_fallback_escapes_query_in_title() {
let _lock = ENV_LOCK.lock().unwrap();
std::env::set_var("SOGOU_TEST_RESULTS", "true");
let engine = make_engine();
let request = make_search_request("<script>alert(1)</script>");
let response = engine.search(&request).await;
std::env::remove_var("SOGOU_TEST_RESULTS");
let response = response.expect("fallback should return Ok");
assert!(response.items[0].title.contains("<script>"));
assert!(!response.items[0].title.contains("<script>"));
assert!(response.items[1].title.contains("<script>"));
assert!(!response.items[1].title.contains("<script>"));
}
}