use anyhow::{Context, Result};
use rquest::StatusCode;
use rquest_util::{Emulation, EmulationOption};
use std::time::Duration;
const IMPERSONATION_DOMAINS: &[&str] = &[
"linkedin.com",
"www.linkedin.com",
];
pub fn needs_impersonation(url: &str) -> bool {
let lower = url.to_lowercase();
IMPERSONATION_DOMAINS
.iter()
.any(|domain| lower.contains(domain))
}
fn select_emulation(_url: &str) -> Emulation {
Emulation::Chrome136
}
pub struct ImpersonatedResponse {
pub status: StatusCode,
pub content_type: String,
pub body: String,
}
pub async fn fetch_impersonated(
url: &str,
cookies: Option<&str>,
extra_headers: Option<&[(String, String)]>,
) -> Result<ImpersonatedResponse> {
let emulation = select_emulation(url);
let emulation_option = EmulationOption::builder().emulation(emulation).build();
let client = rquest::Client::builder()
.emulation(emulation_option)
.cookie_store(true)
.redirect(rquest::redirect::Policy::limited(10))
.connect_timeout(Duration::from_secs(10))
.timeout(Duration::from_secs(30))
.build()
.context("Failed to build impersonating HTTP client")?;
let mut request = client
.get(url)
.header(
"Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
)
.header("Accept-Language", "en-US,en;q=0.9");
if let Some(cookie_val) = cookies {
request = request.header("Cookie", cookie_val);
}
if let Some(headers) = extra_headers {
for (name, value) in headers {
request = request.header(name.as_str(), value.as_str());
}
}
let response = request
.send()
.await
.context("Impersonated request failed")?;
let status = response.status();
let content_type = response
.headers()
.get("content-type")
.and_then(|v| v.to_str().ok())
.unwrap_or("text/html")
.to_string();
let body = response
.text()
.await
.context("Failed to read impersonated response body")?;
Ok(ImpersonatedResponse {
status,
content_type,
body,
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn detects_linkedin_domains() {
assert!(needs_impersonation("https://www.linkedin.com/in/someuser"));
assert!(needs_impersonation(
"https://linkedin.com/company/somecompany"
));
assert!(needs_impersonation(
"https://www.linkedin.com/posts/user_topic-123"
));
}
#[test]
fn ignores_non_impersonation_domains() {
assert!(!needs_impersonation("https://example.com"));
assert!(!needs_impersonation("https://twitter.com/user/status/123"));
assert!(!needs_impersonation("https://github.com/user/repo"));
}
#[test]
fn selects_chrome_136() {
let emulation = select_emulation("https://www.linkedin.com/in/user");
assert_eq!(emulation, Emulation::Chrome136);
}
}