use super::{ResponseItem, SearchEngineType};
use once_cell::sync::Lazy;
use scraper::{ElementRef, Html, Selector};
use super::shared_utils::{escape_html_text, safe_parse_selector};
pub struct HtmlParser {
title_selector: Selector,
link_selector: Selector,
snippet_selectors: Vec<Selector>,
result_selectors: Vec<Selector>,
}
static GOOGLE_PARSER: Lazy<HtmlParser> = Lazy::new(|| {
HtmlParser::new(
vec!["div.g", "div[data-hveid]", "div:has(> a > h3)"],
"h3",
"a[href]",
vec!["[data-sncf]", "span.st", "div[class*='snippet']"],
)
});
static BING_PARSER: Lazy<HtmlParser> =
Lazy::new(|| HtmlParser::new(vec!["li.b_algo"], "h2", "a[href]", vec!["p"]));
static BAIDU_PARSER: Lazy<HtmlParser> = Lazy::new(|| {
HtmlParser::new(
vec![
"div.c-container",
"div.result",
"div.result-op",
".c-container",
],
"h3",
"a",
vec!["div.c-abstract", ".c-abstract", ".c-span18"],
)
});
static SOGOU_PARSER: Lazy<HtmlParser> =
Lazy::new(|| HtmlParser::new(vec![".vrwrap", ".rb"], "h3", "h3 > a", vec![]));
impl HtmlParser {
pub fn new(
result_selectors: Vec<&str>,
title_selector: &str,
link_selector: &str,
snippet_selectors: Vec<&str>,
) -> Self {
Self {
title_selector: safe_parse_selector(title_selector).unwrap_or_else(|| {
safe_parse_selector("h3").expect("Failed to parse fallback title selector")
}),
link_selector: safe_parse_selector(link_selector).unwrap_or_else(|| {
safe_parse_selector("a[href]").expect("Failed to parse fallback link selector")
}),
snippet_selectors: snippet_selectors
.iter()
.map(|s| {
safe_parse_selector(s).unwrap_or_else(|| {
safe_parse_selector("p").expect("Failed to parse fallback snippet selector")
})
})
.collect(),
result_selectors: result_selectors
.iter()
.map(|s| {
safe_parse_selector(s).unwrap_or_else(|| {
safe_parse_selector("div")
.expect("Failed to parse fallback result selector")
})
})
.collect(),
}
}
pub fn google() -> &'static Self {
&GOOGLE_PARSER
}
pub fn bing() -> &'static Self {
&BING_PARSER
}
pub fn baidu() -> &'static Self {
&BAIDU_PARSER
}
pub fn sogou() -> &'static Self {
&SOGOU_PARSER
}
pub fn parse(&self, html: &str, engine: SearchEngineType) -> Vec<ResponseItem> {
let document = Html::parse_document(html);
let mut results = Vec::with_capacity(20);
for element in self.find_result_elements(&document) {
let Some(mut item) = self.parse_result_element(&element, engine) else {
continue;
};
if results.iter().any(|r: &ResponseItem| r.url == item.url) {
continue;
}
item.title = escape_html_text(&item.title);
item.description = escape_html_text(&item.description);
results.push(item);
}
results
}
fn find_result_elements<'a>(&self, document: &'a Html) -> Vec<ElementRef<'a>> {
self.result_selectors
.iter()
.find_map(|s| {
let elements: Vec<_> = document.select(s).collect();
if !elements.is_empty() {
Some(elements)
} else {
None
}
})
.unwrap_or_default()
}
fn parse_result_element(
&self,
element: &ElementRef<'_>,
engine: SearchEngineType,
) -> Option<ResponseItem> {
let title = self.extract_title(element);
if title.is_empty() {
return None;
}
let url = self.extract_url(element, engine);
if url.is_empty() || !url.starts_with("http") {
return None;
}
let description = self.extract_description(element);
Some(ResponseItem {
title,
url,
description,
engine,
})
}
fn extract_title(&self, element: &ElementRef<'_>) -> String {
element
.select(&self.title_selector)
.next()
.map(|e| e.text().collect::<String>().trim().to_string())
.unwrap_or_default()
}
fn extract_url(&self, element: &ElementRef<'_>, engine: SearchEngineType) -> String {
let mut url = String::new();
if let Some(a) = element.select(&self.link_selector).next() {
let raw_href = a.value().attr("href").unwrap_or("");
if !raw_href.is_empty() {
url = if engine == SearchEngineType::Baidu {
self.resolve_baidu_url(raw_href, element)
} else {
Self::clean_url(raw_href)
};
}
}
url
}
fn extract_description(&self, element: &ElementRef<'_>) -> String {
self.snippet_selectors
.iter()
.find_map(|s| {
element
.select(s)
.next()
.map(|e| e.text().collect::<String>().trim().to_string())
.filter(|t| !t.is_empty())
})
.unwrap_or_default()
}
fn resolve_baidu_url(&self, raw_href: &str, element: &ElementRef) -> String {
let is_redirect =
raw_href.starts_with("/link?url=") || raw_href.contains("baidu.com/link?url=");
if !is_redirect {
return Self::clean_url(raw_href);
}
for attr in ["mu", "data-url"] {
if let Some(real_url) = element
.select(&self.link_selector)
.filter_map(|a| a.value().attr(attr))
.map(str::trim)
.find(|v| v.starts_with("http"))
{
return real_url.to_string();
}
}
if raw_href.starts_with("/link?url=") {
return format!("https://www.baidu.com{}", raw_href);
}
raw_href.to_string()
}
pub fn clean_url(url: &str) -> String {
if url.starts_with("/url?q=") {
url.trim_start_matches("/url?q=")
.split('&')
.next()
.unwrap_or(url)
.to_string()
} else if url.starts_with("/") && !url.starts_with("//") {
format!("https://www.google.com{}", url)
} else {
url.to_string()
}
}
pub fn escape_html(text: &str) -> String {
escape_html_text(text)
}
}
pub struct UserAgentManager {
user_agents: Vec<String>,
current_index: usize,
}
impl UserAgentManager {
pub fn new() -> Self {
let user_agents = vec![
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36".to_string(),
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36".to_string(),
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36".to_string(),
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0".to_string(),
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15".to_string(),
];
Self {
user_agents,
current_index: 0,
}
}
pub fn get(&mut self) -> &str {
let ua = &self.user_agents[self.current_index];
self.current_index = (self.current_index + 1) % self.user_agents.len();
ua
}
pub fn get_random(&self) -> &str {
use rand::RngExt;
let mut rng = rand::rng();
let index = rng.random_range(0..self.user_agents.len());
&self.user_agents[index]
}
}
impl Default for UserAgentManager {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_clean_url_google() {
let url = "/url?q=https://example.com&sa=t";
assert_eq!(HtmlParser::clean_url(url), "https://example.com");
}
#[test]
fn test_clean_url_normal() {
let url = "https://example.com";
assert_eq!(HtmlParser::clean_url(url), "https://example.com");
}
#[test]
fn test_clean_url_relative() {
let url = "/path/to/page";
assert_eq!(
HtmlParser::clean_url(url),
"https://www.google.com/path/to/page"
);
}
#[test]
fn test_escape_html() {
let text = "<script>alert(1)</script>";
let escaped = HtmlParser::escape_html(text);
assert!(!escaped.contains("<script>"));
}
#[test]
fn test_user_agent_manager() {
let mut manager = UserAgentManager::new();
let ua1 = manager.get().to_string();
let ua2 = manager.get().to_string();
assert!(!ua1.is_empty());
assert!(!ua2.is_empty());
}
#[test]
fn test_clean_url_google_url_no_extra_params() {
let url = "/url?q=https://example.com";
assert_eq!(HtmlParser::clean_url(url), "https://example.com");
}
#[test]
fn test_clean_url_protocol_relative_not_prefixed() {
let url = "//example.com/path";
assert_eq!(HtmlParser::clean_url(url), "//example.com/path");
}
#[test]
fn test_clean_url_empty_string() {
assert_eq!(HtmlParser::clean_url(""), "");
}
#[test]
fn test_clean_url_google_url_with_multiple_params() {
let url = "/url?q=https://example.com&sa=t&ved=123";
assert_eq!(HtmlParser::clean_url(url), "https://example.com");
}
#[test]
fn test_escape_html_plain_text_unchanged() {
let text = "Hello World Rust Programming";
assert_eq!(
HtmlParser::escape_html(text),
"Hello World Rust Programming"
);
}
#[test]
fn test_escape_html_special_chars_encoded() {
let text = "<div>a & b</div>";
let escaped = HtmlParser::escape_html(text);
assert!(!escaped.contains('<'), "should not contain raw <");
assert!(!escaped.contains('>'), "should not contain raw >");
assert!(escaped.contains("<"), "should contain <");
assert!(escaped.contains(">"), "should contain >");
assert!(escaped.contains("&"), "should contain &");
assert!(
!escaped.contains("& "),
"should not contain raw & followed by space"
);
}
#[test]
fn test_escape_html_empty_string() {
assert_eq!(HtmlParser::escape_html(""), "");
}
#[test]
fn test_escape_html_trims_whitespace() {
let text = " content with spaces ";
assert_eq!(HtmlParser::escape_html(text), "content with spaces");
}
#[test]
fn test_parse_empty_html_returns_empty() {
let parser = HtmlParser::google();
let results = parser.parse("", SearchEngineType::Bing);
assert!(results.is_empty());
}
#[test]
fn test_parse_google_valid_results() {
let parser = HtmlParser::google();
let html = r#"
<html><body>
<div class="g">
<a href="https://example.com/1"><h3>First Result</h3></a>
<span class="st">First snippet text</span>
</div>
<div class="g">
<a href="https://example.com/2"><h3>Second Result</h3></a>
<span class="st">Second snippet text</span>
</div>
</body></html>
"#;
let results = parser.parse(html, SearchEngineType::Bing);
assert_eq!(results.len(), 2);
assert_eq!(results[0].title, "First Result");
assert_eq!(results[0].url, "https://example.com/1");
assert_eq!(results[0].description, "First snippet text");
assert_eq!(results[0].engine, SearchEngineType::Bing);
}
#[test]
fn test_parse_no_matching_selectors_returns_empty() {
let parser = HtmlParser::google();
let html = r#"<html><body><div>nothing here</div></body></html>"#;
let results = parser.parse(html, SearchEngineType::Bing);
assert!(results.is_empty());
}
#[test]
fn test_parse_skips_results_without_title() {
let parser = HtmlParser::google();
let html = r#"
<html><body>
<div class="g">
<a href="https://example.com/1"></a>
</div>
<div class="g">
<a href="https://example.com/2"><h3>Valid Result</h3></a>
</div>
</body></html>
"#;
let results = parser.parse(html, SearchEngineType::Bing);
assert_eq!(results.len(), 1);
assert_eq!(results[0].title, "Valid Result");
}
#[test]
fn test_parse_skips_non_http_urls() {
let parser = HtmlParser::google();
let html = r#"
<html><body>
<div class="g">
<a href="javascript:void(0)"><h3>JS Link</h3></a>
</div>
<div class="g">
<a href="mailto:test@example.com"><h3>Mail Link</h3></a>
</div>
<div class="g">
<a href="https://example.com/valid"><h3>Valid</h3></a>
</div>
</body></html>
"#;
let results = parser.parse(html, SearchEngineType::Bing);
assert_eq!(results.len(), 1);
assert_eq!(results[0].url, "https://example.com/valid");
}
#[test]
fn test_parse_deduplicates_by_url() {
let parser = HtmlParser::google();
let html = r#"
<html><body>
<div class="g">
<a href="https://example.com/dup"><h3>First Title</h3></a>
</div>
<div class="g">
<a href="https://example.com/dup"><h3>Duplicate Title</h3></a>
</div>
<div class="g">
<a href="https://example.com/unique"><h3>Unique Title</h3></a>
</div>
</body></html>
"#;
let results = parser.parse(html, SearchEngineType::Bing);
assert_eq!(results.len(), 2);
assert_eq!(results[0].title, "First Title");
assert_eq!(results[1].title, "Unique Title");
}
#[test]
fn test_parse_bing_results() {
let parser = HtmlParser::bing();
let html = r#"
<html><body>
<li class="b_algo">
<h2><a href="https://example.com/1">Bing Result</a></h2>
<p>Bing snippet</p>
</li>
</body></html>
"#;
let results = parser.parse(html, SearchEngineType::Bing);
assert_eq!(results.len(), 1);
assert_eq!(results[0].title, "Bing Result");
assert_eq!(results[0].engine, SearchEngineType::Bing);
}
#[test]
fn test_parse_baidu_results() {
let parser = HtmlParser::baidu();
let html = r#"
<html><body>
<div class="c-container">
<h3><a href="https://example.com/1">Baidu Result</a></h3>
<div class="c-abstract">Baidu snippet</div>
</div>
</body></html>
"#;
let results = parser.parse(html, SearchEngineType::Baidu);
assert_eq!(results.len(), 1);
assert_eq!(results[0].title, "Baidu Result");
assert_eq!(results[0].engine, SearchEngineType::Baidu);
}
#[test]
fn test_parse_baidu_redirect_unwraps_mu_attribute() {
let parser = HtmlParser::baidu();
let html = r#"
<html><body>
<div class="c-container">
<h3><a href="https://www.baidu.com/link?url=abc123" mu="https://example.com/real">Baidu Result</a></h3>
<div class="c-abstract">snippet</div>
</div>
</body></html>
"#;
let results = parser.parse(html, SearchEngineType::Baidu);
assert_eq!(results.len(), 1, "unwrapped result must not be dropped");
assert_eq!(results[0].url, "https://example.com/real");
assert_eq!(results[0].title, "Baidu Result");
}
#[test]
fn test_parse_baidu_redirect_unwraps_mu_on_sibling_anchor() {
let parser = HtmlParser::baidu();
let html = r#"
<html><body>
<div class="c-container">
<h3><a href="https://www.baidu.com/link?url=abc123">Baidu Result</a></h3>
<a href="https://www.baidu.com/link?url=abc123" mu="https://example.com/sibling">site</a>
</div>
</body></html>
"#;
let results = parser.parse(html, SearchEngineType::Baidu);
assert_eq!(results.len(), 1);
assert_eq!(results[0].url, "https://example.com/sibling");
}
#[test]
fn test_parse_baidu_redirect_falls_back_to_data_url() {
let parser = HtmlParser::baidu();
let html = r#"
<html><body>
<div class="c-container">
<h3><a href="https://www.baidu.com/link?url=abc123" data-url="https://example.com/from-data-url">Baidu Result</a></h3>
</div>
</body></html>
"#;
let results = parser.parse(html, SearchEngineType::Baidu);
assert_eq!(results.len(), 1);
assert_eq!(results[0].url, "https://example.com/from-data-url");
}
#[test]
fn test_parse_baidu_redirect_kept_when_no_attributes() {
let parser = HtmlParser::baidu();
let html = r#"
<html><body>
<div class="c-container">
<h3><a href="https://www.baidu.com/link?url=abc123">Baidu Result</a></h3>
</div>
</body></html>
"#;
let results = parser.parse(html, SearchEngineType::Baidu);
assert_eq!(results.len(), 1, "redirect link must not be dropped");
assert_eq!(results[0].url, "https://www.baidu.com/link?url=abc123");
}
#[test]
fn test_parse_baidu_redirect_ignores_non_http_attribute() {
let parser = HtmlParser::baidu();
let html = r#"
<html><body>
<div class="c-container">
<h3><a href="https://www.baidu.com/link?url=abc123" mu="javascript:alert(1)">Baidu Result</a></h3>
</div>
</body></html>
"#;
let results = parser.parse(html, SearchEngineType::Baidu);
assert_eq!(results.len(), 1);
assert_eq!(results[0].url, "https://www.baidu.com/link?url=abc123");
}
#[test]
fn test_parse_baidu_relative_redirect_gets_baidu_host() {
let parser = HtmlParser::baidu();
let html = r#"
<html><body>
<div class="c-container">
<h3><a href="/link?url=abc123">Baidu Result</a></h3>
</div>
</body></html>
"#;
let results = parser.parse(html, SearchEngineType::Baidu);
assert_eq!(results.len(), 1);
assert_eq!(results[0].url, "https://www.baidu.com/link?url=abc123");
}
#[test]
fn test_parse_baidu_normal_url_untouched() {
let parser = HtmlParser::baidu();
let html = r#"
<html><body>
<div class="c-container">
<h3><a href="https://example.com/plain">Baidu Result</a></h3>
</div>
<div class="c-container">
<h3><a href="http://example.org/another" mu="https://ignored.example/">Baidu Result 2</a></h3>
</div>
</body></html>
"#;
let results = parser.parse(html, SearchEngineType::Baidu);
assert_eq!(results.len(), 2);
assert_eq!(results[0].url, "https://example.com/plain");
assert_eq!(results[1].url, "http://example.org/another");
}
#[test]
fn test_user_agent_manager_round_robin_wraps_around() {
let mut manager = UserAgentManager::new();
let ua_count = 5;
let mut uas = Vec::new();
for _ in 0..(ua_count + 1) {
uas.push(manager.get().to_string());
}
for ua in &uas {
assert!(!ua.is_empty());
}
assert_eq!(uas[ua_count], uas[0], "round-robin should wrap around");
}
#[test]
fn test_user_agent_manager_get_random_returns_valid_ua() {
let manager = UserAgentManager::new();
let ua = manager.get_random();
assert!(!ua.is_empty());
assert!(ua.contains("Mozilla"), "UA should look like a browser UA");
}
#[test]
fn test_user_agent_manager_default_equals_new() {
let mut m1 = UserAgentManager::new();
let mut m2 = UserAgentManager::default();
assert_eq!(m1.get(), m2.get());
}
}