use crate::engines::engine_client::{
EngineClient, ScrapeOptions, ScrapeRequest as EngineScrapeRequest,
};
use crate::search::{
client::html_parser::HtmlParser,
engine_trait::SearchEngine,
error::SearchError,
response::{Response, ResponseItem},
types::{EngineHealth, SearchEngineType},
SearchRequest,
};
use async_trait::async_trait;
use base64::{engine::general_purpose::URL_SAFE, Engine as _};
use std::collections::HashMap;
use std::sync::Arc;
use url::Url;
pub struct BingSearchEngine {
engine_client: Arc<EngineClient>,
}
impl BingSearchEngine {
pub fn new(engine_client: Arc<EngineClient>) -> Self {
Self { engine_client }
}
pub fn get_bing_cookies(&self, lang: &str, region: &str) -> HashMap<String, String> {
let mut cookies = HashMap::with_capacity(4);
cookies.insert("_EDGE_CD".to_string(), format!("m={}&u={}", region, lang));
cookies.insert("_EDGE_S".to_string(), format!("mkt={}&ui={}", region, lang));
cookies
}
pub fn build_params(&self, query: &str, page: u32) -> HashMap<String, String> {
let mut params = HashMap::with_capacity(8);
params.insert("q".to_string(), query.to_string());
params.insert("pq".to_string(), query.to_string());
if page > 1 {
params.insert("first".to_string(), ((page - 1) * 10 + 1).to_string());
let form_value = if page == 2 {
"PERE".to_string()
} else {
format!("PERE{}", page - 2)
};
params.insert("FORM".to_string(), form_value);
}
params
}
pub fn decode_bing_url(&self, url: &str) -> String {
if !url.starts_with("https://www.bing.com/ck/a?") {
return url.to_string();
}
let parsed_url = match Url::parse(url) {
Ok(url) => url,
Err(_) => return url.to_string(),
};
let u_param = match parsed_url.query_pairs().find(|(key, _)| key == "u") {
Some(param) => param,
None => return url.to_string(),
};
let encoded: &str = match u_param.1.strip_prefix("a1") {
Some(rest) if !rest.is_empty() => rest,
_ => return url.to_string(),
};
let padding = "=".repeat((4 - encoded.len() % 4) % 4);
let padded_encoded = format!("{}{}", encoded, padding);
let decoded_bytes = match URL_SAFE.decode(padded_encoded) {
Ok(bytes) => bytes,
Err(_) => return url.to_string(),
};
match String::from_utf8(decoded_bytes) {
Ok(decoded_str) => decoded_str,
Err(_) => url.to_string(),
}
}
fn page_from_offset(offset: u32, limit: u32) -> u32 {
(offset / limit.max(1)) + 1
}
pub fn build_bing_url(&self, query: &str, page: u32) -> String {
self.build_bing_url_with_locale(query, page, None, None)
}
pub fn build_bing_url_with_locale(
&self,
query: &str,
page: u32,
lang: Option<&str>,
country: Option<&str>,
) -> String {
let base_url = "https://cn.bing.com/search";
let mut params = vec![("q", query.to_string()), ("pq", query.to_string())];
if let (Some(lang_code), Some(country_code)) = (lang, country) {
let mkt = format!("{}-{}", lang_code, country_code);
params.push(("mkt", mkt));
}
if page > 1 {
let first_value = ((page - 1) * 10 + 1).to_string();
params.push(("first", first_value));
let form_value = if page == 2 {
"PERE".to_string()
} else {
format!("PERE{}", page - 2)
};
params.push(("FORM", form_value));
}
format!(
"{}?{}",
base_url,
serde_urlencoded::to_string(¶ms).unwrap_or_default()
)
}
pub fn parse_search_results(&self, html: &str) -> Result<Vec<ResponseItem>, SearchError> {
if html.is_empty() {
return Err(SearchError::Parse(
"Empty HTML response received".to_string(),
));
}
let mut items = HtmlParser::bing().parse(html, SearchEngineType::Bing);
if items.is_empty() {
let probe_end = html
.char_indices()
.nth(32 * 1024)
.map(|(i, _)| i)
.unwrap_or(html.len());
let lower = html[..probe_end].to_lowercase();
const CHALLENGE_MARKERS: &[&str] = &[
"bnp.bing.com", "verify you are human", ];
if CHALLENGE_MARKERS.iter().any(|m| lower.contains(m)) {
return Err(SearchError::Captcha("Bing".to_string()));
}
}
for item in &mut items {
item.url = self.decode_bing_url(&item.url);
}
Ok(items)
}
}
#[async_trait]
impl SearchEngine for BingSearchEngine {
fn name(&self) -> &'static str {
"Bing"
}
fn engine_type(&self) -> SearchEngineType {
SearchEngineType::Bing
}
fn health(&self) -> EngineHealth {
EngineHealth::Healthy
}
async fn search(&self, request: &SearchRequest) -> Result<Response<ResponseItem>, SearchError> {
#[cfg(test)]
{
if let Some(resp) = bing_test_results_override(&request.query) {
return Ok(resp);
}
}
if request.query.trim().is_empty() {
return Err(SearchError::Parse(
"Search query cannot be empty".to_string(),
));
}
let page = Self::page_from_offset(request.offset, request.limit);
let url = self.build_bing_url_with_locale(
&request.query,
page,
request.lang.as_deref(),
request.country.as_deref(),
);
let headers = build_bing_headers(request.lang.as_deref(), request.country.as_deref());
let options = ScrapeOptions {
headers,
timeout: std::time::Duration::from_secs(30),
..Default::default()
};
let engine_request = EngineScrapeRequest { url, options };
let scrape_response = self
.engine_client
.scrape(&engine_request)
.await
.map_err(|e| SearchError::EngineClient("Bing".to_string(), e.to_string()))?;
if scrape_response.status_code < 200 || scrape_response.status_code >= 300 {
return Err(SearchError::BadHttpStatus(
"Bing".to_string(),
scrape_response.status_code,
));
}
let html = scrape_response.content;
let items = self.parse_search_results(&html)?;
let total_count = items.len() as u64;
Ok(Response {
items,
total_results: Some(total_count),
engine: SearchEngineType::Bing,
})
}
}
fn build_bing_headers(lang: Option<&str>, country: Option<&str>) -> HashMap<String, String> {
let mut headers = HashMap::with_capacity(8);
headers.insert(
"Accept".to_string(),
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8".to_string(),
);
let accept_lang = match (lang, country) {
(Some(lang), Some(country)) => format!("{}-{},{};q=0.9,en;q=0.5", lang, country, lang),
(Some(lang), None) => format!("{},{};q=0.9,en;q=0.5", lang, lang),
_ => "zh-CN,zh;q=0.9,en;q=0.8".to_string(),
};
headers.insert("Accept-Language".to_string(), accept_lang);
headers.insert("DNT".to_string(), "1".to_string());
headers.insert("Connection".to_string(), "keep-alive".to_string());
headers.insert("Upgrade-Insecure-Requests".to_string(), "1".to_string());
headers.insert("Sec-GPC".to_string(), "1".to_string());
headers.insert("Cache-Control".to_string(), "max-age=0".to_string());
headers
}
#[cfg(test)]
fn bing_test_results_override(query: &str) -> Option<Response<ResponseItem>> {
if std::env::var("BING_TEST_RESULTS").unwrap_or_default() != "true" {
return None;
}
Some(Response {
items: vec![
ResponseItem {
title: format!("Bing Test Result 1 for {}", query),
url: "https://bing.com/1".to_string(),
description: "Test description 1".to_string(),
engine: SearchEngineType::Bing,
},
ResponseItem {
title: format!("Bing Test Result 2 for {}", query),
url: "https://bing.com/2".to_string(),
description: "Test description 2".to_string(),
engine: SearchEngineType::Bing,
},
],
total_results: Some(2),
engine: SearchEngineType::Bing,
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::engines::engine_client::EngineClient;
fn create_engine() -> BingSearchEngine {
let engine_client = Arc::new(EngineClient::new());
BingSearchEngine::new(engine_client)
}
fn build_bing_redirect_url(target_url: &str) -> String {
let encoded = URL_SAFE.encode(target_url);
format!("https://www.bing.com/ck/a?!&p=0&u=a1{}&t=1", encoded)
}
#[test]
fn test_get_bing_cookies_contains_edge_keys() {
let engine = create_engine();
let cookies = engine.get_bing_cookies("en", "US");
assert_eq!(cookies.len(), 2);
assert!(cookies.contains_key("_EDGE_CD"));
assert!(cookies.contains_key("_EDGE_S"));
}
#[test]
fn test_get_bing_cookies_embeds_lang_and_region() {
let engine = create_engine();
let cookies = engine.get_bing_cookies("zh", "CN");
let edge_cd = cookies.get("_EDGE_CD").unwrap();
assert!(edge_cd.contains("m=CN"), "_EDGE_CD should contain region");
assert!(edge_cd.contains("u=zh"), "_EDGE_CD should contain lang");
let edge_s = cookies.get("_EDGE_S").unwrap();
assert!(edge_s.contains("mkt=CN"), "_EDGE_S should contain region");
assert!(edge_s.contains("ui=zh"), "_EDGE_S should contain lang");
}
#[test]
fn test_get_bing_cookies_empty_values() {
let engine = create_engine();
let cookies = engine.get_bing_cookies("", "");
let edge_cd = cookies.get("_EDGE_CD").unwrap();
assert!(edge_cd.contains("m="));
assert!(edge_cd.contains("u="));
}
#[test]
fn test_build_params_page1() {
let engine = create_engine();
let params = engine.build_params("rust", 1);
assert_eq!(params.get("q"), Some(&"rust".to_string()));
assert_eq!(params.get("pq"), Some(&"rust".to_string()));
assert!(
!params.contains_key("first"),
"page 1 should not have 'first'"
);
assert!(
!params.contains_key("FORM"),
"page 1 should not have 'FORM'"
);
}
#[test]
fn test_build_params_page2_form_pere() {
let engine = create_engine();
let params = engine.build_params("rust", 2);
assert_eq!(params.get("first"), Some(&"11".to_string()));
assert_eq!(params.get("FORM"), Some(&"PERE".to_string()));
}
#[test]
fn test_build_params_page3_form_pere1() {
let engine = create_engine();
let params = engine.build_params("rust", 3);
assert_eq!(params.get("first"), Some(&"21".to_string()));
assert_eq!(params.get("FORM"), Some(&"PERE1".to_string()));
}
#[test]
fn test_build_params_page5_offset_calculation() {
let engine = create_engine();
let params = engine.build_params("test", 5);
assert_eq!(params.get("first"), Some(&"41".to_string()));
assert_eq!(params.get("FORM"), Some(&"PERE3".to_string()));
}
#[test]
fn test_build_bing_url_page1() {
let engine = create_engine();
let url = engine.build_bing_url("rust language", 1);
assert!(url.starts_with("https://cn.bing.com/search?"));
assert!(
url.contains("q=rust+language"),
"URL should contain encoded query"
);
assert!(url.contains("pq=rust+language"));
assert!(!url.contains("first="), "page 1 should not have 'first'");
}
#[test]
fn test_build_bing_url_page2_includes_pagination() {
let engine = create_engine();
let url = engine.build_bing_url("test", 2);
assert!(url.contains("first=11"), "page 2 should have first=11");
assert!(url.contains("FORM=PERE"), "page 2 should have FORM=PERE");
}
#[test]
fn test_build_bing_url_special_chars_encoded() {
let engine = create_engine();
let url = engine.build_bing_url("rust & go", 1);
assert!(url.contains("q=rust+%26+go") || url.contains("q=rust+%26+go"));
}
#[test]
fn test_decode_bing_url_non_bing_url_returned_as_is() {
let engine = create_engine();
let original = "https://example.com/page";
let decoded = engine.decode_bing_url(original);
assert_eq!(decoded, original);
}
#[test]
fn test_decode_bing_url_valid_redirect_decoded() {
let engine = create_engine();
let target = "https://example.com/real-target";
let redirect_url = build_bing_redirect_url(target);
let decoded = engine.decode_bing_url(&redirect_url);
assert_eq!(decoded, target);
}
#[test]
fn test_decode_bing_url_missing_u_param_returned_as_is() {
let engine = create_engine();
let url = "https://www.bing.com/ck/a?!&p=0&t=1";
let decoded = engine.decode_bing_url(url);
assert_eq!(decoded, url);
}
#[test]
fn test_decode_bing_url_invalid_base64_returned_as_is() {
let engine = create_engine();
let url = "https://www.bing.com/ck/a?u=a1!!!invalid_base64!!!";
let decoded = engine.decode_bing_url(url);
assert_eq!(decoded, url);
}
#[test]
fn test_decode_bing_url_unparsable_url_returned_as_is() {
let engine = create_engine();
let url = "https://www.bing.com/ck/a?u=a1[invalid";
let decoded = engine.decode_bing_url(url);
assert!(!decoded.is_empty());
}
#[test]
fn test_parse_search_results_empty_html_returns_error() {
let engine = create_engine();
let result = engine.parse_search_results("");
assert!(result.is_err());
let err = result.unwrap_err();
match err {
SearchError::Parse(msg) => {
assert!(
msg.contains("Empty HTML"),
"error should mention empty HTML"
)
}
other => panic!("expected SearchError::Parse, got {:?}", other),
}
}
#[test]
fn test_parse_search_results_valid_html() {
let engine = create_engine();
let html = r#"
<html><body>
<li class="b_algo">
<h2><a href="https://example.com/1">First Result</a></h2>
<p>First description text</p>
</li>
<li class="b_algo">
<h2><a href="https://example.com/2">Second Result</a></h2>
<p>Second description text</p>
</li>
</body></html>
"#;
let results = engine.parse_search_results(html).unwrap();
assert_eq!(results.len(), 2);
assert_eq!(results[0].title, "First Result");
assert_eq!(results[0].url, "https://example.com/1");
assert_eq!(results[0].engine, SearchEngineType::Bing);
assert_eq!(results[1].title, "Second Result");
}
#[test]
fn test_bing_offset_derives_first_param() {
let engine = create_engine();
assert_eq!(BingSearchEngine::page_from_offset(10, 10), 2);
assert_eq!(BingSearchEngine::page_from_offset(0, 10), 1);
assert_eq!(BingSearchEngine::page_from_offset(10, 0), 11);
let page = BingSearchEngine::page_from_offset(10, 10);
let url = engine.build_bing_url_with_locale("rust", page, None, None);
assert!(
url.contains("first=11"),
"page 2 must carry first=11, got: {url}"
);
}
#[test]
fn test_bing_mkt_requires_lang_and_country() {
let engine = create_engine();
let both = engine.build_bing_url_with_locale("rust", 1, Some("en"), Some("US"));
assert!(both.contains("mkt=en-US"), "got: {both}");
let lang_only = engine.build_bing_url_with_locale("rust", 1, Some("en"), None);
assert!(!lang_only.contains("mkt="), "got: {lang_only}");
}
#[test]
fn test_parse_search_results_no_matching_selectors() {
let engine = create_engine();
let html = r#"<html><body><div>no bing results here</div></body></html>"#;
let results = engine.parse_search_results(html).unwrap();
assert!(results.is_empty());
}
#[test]
fn test_parse_search_results_captcha_page_returns_captcha_error() {
let engine = create_engine();
let html = r#"
<html><head><title>Verify to continue</title></head>
<body><div>Please verify you are human to continue browsing.
Challenge endpoint: https://bnp.bing.com/challenge</div></body></html>
"#;
match engine.parse_search_results(html) {
Err(SearchError::Captcha(engine_name)) => {
assert_eq!(engine_name, "Bing");
}
other => panic!(
"expected SearchError::Captcha, got {:?}",
other.map(|r| r.len())
),
}
}
#[test]
fn test_parse_search_results_decodes_bing_redirect_urls() {
let engine = create_engine();
let redirect =
"https://www.bing.com/ck/a?!&&p=abc123&u=a1aHR0cHM6Ly9leGFtcGxlLmNvbS9wYWdlMQ&ntb=1";
let html = format!(
r#"
<html><body>
<li class="b_algo">
<h2><a href="{redirect}">Encoded Result</a></h2>
<p>Description with direct link <a href="https://example.com/direct">direct</a></p>
</li>
</body></html>
"#
);
let results = engine.parse_search_results(&html).unwrap();
assert_eq!(results.len(), 1);
assert_eq!(
results[0].url, "https://example.com/page1",
"ck/a redirect must be unwrapped to the real target URL, got: {}",
results[0].url
);
}
#[test]
fn test_engine_creation() {
let engine = create_engine();
assert_eq!(engine.name(), "Bing");
assert_eq!(engine.engine_type(), SearchEngineType::Bing);
assert_eq!(engine.health(), EngineHealth::Healthy);
}
static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
fn make_search_request(query: &str) -> SearchRequest {
SearchRequest::new(query)
}
#[allow(clippy::await_holding_lock)]
#[tokio::test]
async fn test_search_fallback_returns_hardcoded_results() {
let _lock = ENV_LOCK.lock().unwrap();
std::env::set_var("BING_TEST_RESULTS", "true");
let engine = create_engine();
let request = make_search_request("rust programming");
let response = engine.search(&request).await;
std::env::remove_var("BING_TEST_RESULTS");
let response = response.expect("fallback should return Ok");
assert_eq!(response.items.len(), 2, "fallback should return 2 items");
assert_eq!(response.engine, SearchEngineType::Bing);
assert_eq!(response.total_results, Some(2));
assert!(response.items[0].title.contains("rust programming"));
assert!(response.items[1].title.contains("rust programming"));
assert_eq!(response.items[0].url, "https://bing.com/1");
assert_eq!(response.items[1].url, "https://bing.com/2");
assert_eq!(response.items[0].engine, SearchEngineType::Bing);
assert_eq!(response.items[1].engine, SearchEngineType::Bing);
}
#[allow(clippy::await_holding_lock)] #[tokio::test]
async fn test_search_fallback_description_fields() {
let _lock = ENV_LOCK.lock().unwrap();
std::env::set_var("BING_TEST_RESULTS", "true");
let engine = create_engine();
let request = make_search_request("description check");
let response = engine.search(&request).await;
std::env::remove_var("BING_TEST_RESULTS");
let response = response.expect("fallback should return Ok");
assert_eq!(response.items[0].description, "Test description 1");
assert_eq!(response.items[1].description, "Test description 2");
}
#[allow(clippy::await_holding_lock)] #[tokio::test]
async fn test_search_empty_query_returns_error_without_fallback() {
let _lock = ENV_LOCK.lock().unwrap();
std::env::remove_var("BING_TEST_RESULTS");
let engine = create_engine();
let request = make_search_request(" ");
let result = engine.search(&request).await;
assert!(result.is_err(), "empty query should return Err");
let err = result.unwrap_err();
match err {
SearchError::Parse(msg) => {
assert!(
msg.contains("empty"),
"error should mention empty query: {}",
msg
);
}
other => panic!("expected SearchError::Parse, got {:?}", other),
}
}
#[test]
fn test_decode_bing_url_non_utf8_payload_returned_as_is() {
let engine = create_engine();
let invalid_utf8_bytes: [u8; 3] = [0xFF, 0xFE, 0xFD];
let encoded = URL_SAFE.encode(invalid_utf8_bytes);
let url = format!("https://www.bing.com/ck/a?u=a1{}", encoded);
let decoded = engine.decode_bing_url(&url);
assert_eq!(
decoded, url,
"non-UTF-8 Base64 payload should return original URL"
);
}
}