use crate::js_runtime::state::DomState;
use deno_core::op2;
use deno_core::OpState;
use serde::Serialize;
use std::cell::{Cell, RefCell};
use std::collections::HashMap;
use url::Url;
const MAX_SYNC_FETCH_PER_PAGE: usize = 30;
thread_local! {
static SYNC_FETCH_COUNT: Cell<usize> = const { Cell::new(0) };
}
pub fn reset_sync_fetch_count() {
SYNC_FETCH_COUNT.with(|c| c.set(0));
}
pub fn record_resource_timing(state: &mut OpState, timings: crate::net::TimingStats) {
if let Some(dom_state) = state.try_borrow_mut::<DomState>() {
dom_state.resource_timings.push(timings);
}
}
pub struct FetchState {
pub client: Option<crate::net::HttpClient>,
}
impl FetchState {
pub fn new(client: Option<crate::net::HttpClient>) -> Self {
Self { client }
}
pub fn with_profile(profile: &crate::stealth::StealthProfile) -> Self {
Self {
client: crate::net::HttpClient::new(profile).ok(),
}
}
}
thread_local! {
static FETCH_CLIENT: RefCell<Option<crate::net::HttpClient>> = const { RefCell::new(None) };
}
thread_local! {
static ACTIVE_CSP: RefCell<Option<ActiveCsp>> = const { RefCell::new(None) };
}
#[derive(Clone)]
struct ActiveCsp {
policy: std::sync::Arc<crate::net::csp::PolicySet>,
origin: Url,
enforce: bool,
}
pub fn set_csp_policy(
policy: std::sync::Arc<crate::net::csp::PolicySet>,
origin: Url,
enforce: bool,
) {
CSP_VIOLATIONS.with(|q| q.borrow_mut().clear());
ACTIVE_CSP.with(|c| {
*c.borrow_mut() = Some(ActiveCsp {
policy,
origin,
enforce,
});
});
}
pub fn clear_csp_policy() {
CSP_VIOLATIONS.with(|q| q.borrow_mut().clear());
ACTIVE_CSP.with(|c| *c.borrow_mut() = None);
}
pub fn check_csp(
directive: crate::net::csp::Directive,
url: &Url,
nonce: Option<&str>,
parser_inserted: bool,
) -> Result<(), &'static str> {
let decision = ACTIVE_CSP.with(|c| {
let guard = c.borrow();
let active = guard.as_ref()?;
if !active.enforce {
return None;
}
let ctx = crate::net::csp::CheckCtx {
directive,
url,
page_origin: &active.origin,
nonce,
parser_inserted,
};
Some(active.policy.allows(&ctx))
});
let Some(decision) = decision else {
return Ok(());
};
if decision.allowed {
Ok(())
} else {
let dir_name = decision.matched_directive.as_str();
push_csp_violation(CspViolation {
blocked_uri: url.as_str().to_string(),
effective_directive: dir_name.to_string(),
violated_directive: dir_name.to_string(),
disposition: "enforce".to_string(),
});
Err(dir_name)
}
}
#[derive(Clone, serde::Serialize)]
pub struct CspViolation {
#[serde(rename = "blockedURI")]
pub blocked_uri: String,
#[serde(rename = "effectiveDirective")]
pub effective_directive: String,
#[serde(rename = "violatedDirective")]
pub violated_directive: String,
pub disposition: String,
}
thread_local! {
static CSP_VIOLATIONS: RefCell<Vec<CspViolation>> = const { RefCell::new(Vec::new()) };
}
fn push_csp_violation(v: CspViolation) {
CSP_VIOLATIONS.with(|q| {
let mut q = q.borrow_mut();
if q.len() < 256 {
q.push(v);
}
});
}
#[op2]
#[serde]
pub fn op_drain_csp_violations() -> Vec<CspViolation> {
CSP_VIOLATIONS.with(|q| std::mem::take(&mut *q.borrow_mut()))
}
pub fn init_fetch_client(profile: &crate::stealth::StealthProfile) {
if let Ok(client) = crate::net::HttpClient::new(profile) {
FETCH_CLIENT.with(|c| *c.borrow_mut() = Some(client));
}
}
pub fn set_fetch_client(client: crate::net::HttpClient) {
FETCH_CLIENT.with(|c| *c.borrow_mut() = Some(client));
}
pub fn fetch_client() -> Option<crate::net::HttpClient> {
FETCH_CLIENT.with(|c| c.borrow().clone())
}
#[derive(Serialize)]
pub struct FetchResponse {
pub status: u16,
pub status_text: String,
pub headers: HashMap<String, String>,
pub body: String,
pub url: String,
pub ok: bool,
}
#[op2(async(deferred))]
#[serde]
pub async fn op_fetch(
#[string] url: String,
#[string] method: String,
#[serde] headers: HashMap<String, String>,
#[string] body: String,
) -> Result<FetchResponse, deno_error::JsErrorBox> {
if let Ok(parsed) = Url::parse(&url) {
if let Err(violated) =
check_csp(crate::net::csp::Directive::ConnectSrc, &parsed, None, false)
{
eprintln!(
"[csp] Refused to connect to '{}' because it violates the following Content Security Policy directive: \"{}\".",
url, violated
);
return Ok(FetchResponse {
status: 0,
status_text: "".to_string(),
headers: HashMap::new(),
body: String::new(),
url: url.clone(),
ok: false,
});
}
}
let request_type = crate::net::blocker::classify_request_type(
&url,
headers
.get("x-browser-oxide-request-type")
.map(|s| s.as_str()),
);
if crate::net::blocker::should_block(&url, "", request_type) {
return Ok(FetchResponse {
status: 200,
status_text: "OK".to_string(),
headers: HashMap::new(),
body: String::new(),
url: url.clone(),
ok: true,
});
}
let installed_client = FETCH_CLIENT.with(|c| c.borrow().clone());
let default_client;
let client = match installed_client.as_ref() {
Some(c) => c,
None => {
let profile = crate::stealth::chrome_148_linux();
default_client = crate::net::HttpClient::new(&profile)
.map_err(|e| deno_error::JsErrorBox::generic(e.to_string()))?;
&default_client
}
};
let mut extra_headers: Vec<(String, String)> = Vec::with_capacity(headers.len());
let mut origin: Option<String> = None;
for (k, v) in headers.into_iter() {
let lk = k.to_ascii_lowercase();
if lk == "x-browser-oxide-origin" {
origin = Some(v);
continue;
}
extra_headers.push((lk, v));
}
let body_bytes: Vec<u8> = if let Some(rest) = body.strip_prefix("b:") {
use base64::Engine as _;
base64::engine::general_purpose::STANDARD
.decode(rest.as_bytes())
.unwrap_or_default()
} else if let Some(rest) = body.strip_prefix("s:") {
rest.as_bytes().to_vec()
} else {
body.as_bytes().to_vec()
};
let method_upper = method.to_uppercase();
let fetch_timeout = std::time::Duration::from_secs(30);
let resp_result = tokio::time::timeout(fetch_timeout, async {
match method_upper.as_str() {
"POST" | "PUT" | "PATCH" => {
client
.fetch_post_bytes(&url, &body_bytes, &extra_headers, origin.as_deref())
.await
}
_ => {
client
.fetch_get(&url, &extra_headers, origin.as_deref())
.await
}
}
})
.await;
let resp = match resp_result {
Ok(r) => r,
Err(_) => {
return Err(deno_error::JsErrorBox::generic(format!(
"fetch timeout after {}s: {}",
fetch_timeout.as_secs(),
url
)));
}
};
let resp = match resp {
Ok(r) => r,
Err(e) => return Err(deno_error::JsErrorBox::generic(e.to_string())),
};
let ok = resp.ok();
let body_text = resp.text();
let final_resp = FetchResponse {
status: resp.status,
status_text: resp.status_text.clone(),
headers: resp.headers.clone(),
body: body_text,
url: resp.url.clone(),
ok,
};
Ok(final_resp)
}
#[op2(async(lazy), fast)]
#[string]
pub async fn op_cookie_get(#[string] url: String) -> String {
let Some(client) = FETCH_CLIENT.with(|c| c.borrow().clone()) else {
return String::new();
};
let Ok(parsed) = Url::parse(&url) else {
return String::new();
};
client.cookies_for_url(&parsed).await.unwrap_or_default()
}
#[op2(async(lazy), fast)]
pub async fn op_cookie_set(#[string] url: String, #[string] cookie: String) {
let Some(client) = FETCH_CLIENT.with(|c| c.borrow().clone()) else {
return;
};
let Ok(parsed) = Url::parse(&url) else { return };
client.set_cookie_str(&parsed, &cookie).await;
}
#[op2(fast)]
pub fn op_cookie_set_sync(#[string] url: String, #[string] cookie: String) {
let Some(client) = FETCH_CLIENT.with(|c| c.borrow().clone()) else {
tracing::debug!("op_cookie_set_sync: no FETCH_CLIENT");
return;
};
let Ok(parsed) = Url::parse(&url) else {
tracing::debug!(url = %url, "op_cookie_set_sync: bad url");
return;
};
let synced = client.set_cookie_str_sync(&parsed, &cookie);
let shared_synced = crate::net::set_shared_cookie_sync(&parsed, &cookie);
if std::env::var("BROWSER_OXIDE_COOKIE_TRACE").is_ok() {
let for_url = crate::net::shared_session()
.cookies
.try_lock()
.ok()
.and_then(|j| j.cookies_for(&parsed))
.unwrap_or_default();
eprintln!(
"[cookie-set-sync] url={url} synced={synced} shared={shared_synced} shared_for_url='{}' cookie={}",
for_url.chars().take(60).collect::<String>(),
cookie.chars().take(50).collect::<String>()
);
}
if !synced {
tokio::task::spawn(async move {
client.set_cookie_str(&parsed, &cookie).await;
});
}
}
#[op2]
#[string]
pub fn op_net_fetch_sync(#[string] url: String, #[string] referer: String) -> String {
if let Ok(parsed) = Url::parse(&url) {
if let Err(violated) = check_csp(
crate::net::csp::Directive::ScriptSrcElem,
&parsed,
None,
false,
) {
eprintln!(
"[csp] Refused to load the script '{}' (sync-fetch) — violates: \"{}\".",
url, violated
);
return String::new();
}
}
if crate::net::blocker::should_block(
&url,
&referer,
crate::net::blocker::classify_request_type(&url, Some("script")),
) {
return String::new();
}
let n = SYNC_FETCH_COUNT.with(|c| {
let v = c.get();
c.set(v + 1);
v
});
if n >= MAX_SYNC_FETCH_PER_PAGE {
eprintln!(
"[op_net_fetch_sync] CHAIN LIMIT ({}) exceeded — returning empty for {}",
MAX_SYNC_FETCH_PER_PAGE, url
);
return String::new();
}
tracing::debug!("[op_net_fetch_sync] fetching {}", url);
let main_client = FETCH_CLIENT.with(|c| c.borrow().clone());
let (_profile, client_res) = match main_client.as_ref() {
Some(main) => (
main.profile().clone(),
crate::net::HttpClient::new_with_shared_state(
main.profile(),
main.cookies(),
main.accept_ch_origins(),
main.dns_cache(),
main.alt_svc_cache(),
),
),
None => {
let p = crate::stealth::presets::chrome_148_ru();
(p.clone(), crate::net::HttpClient::new(&p))
}
};
let client = match client_res {
Ok(c) => c,
Err(_) => return String::new(),
};
let mut extra_headers = vec![
("referer".to_string(), referer.clone()),
("sec-fetch-dest".to_string(), "script".to_string()),
("sec-fetch-mode".to_string(), "no-cors".to_string()),
("sec-fetch-site".to_string(), "same-origin".to_string()),
];
if let Ok(parsed) = Url::parse(&referer) {
if let Some(origin) = parsed.origin().ascii_serialization().into() {
extra_headers.push(("origin".to_string(), origin));
}
}
let url_clone = url.clone();
let result = std::thread::spawn(move || {
let rt = match tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
{
Ok(rt) => rt,
Err(e) => {
eprintln!("[op_net_fetch_sync] runtime build error: {e}");
return String::new();
}
};
rt.block_on(async move {
match tokio::time::timeout(
std::time::Duration::from_secs(30),
client.get_with_headers(&url_clone, &extra_headers),
)
.await
{
Ok(Ok(resp)) => {
let body = resp.text();
if body.is_empty() {
eprintln!(
"[op_net_fetch_sync] empty body for {} (status={})",
url_clone, resp.status
);
} else if url_clone.ends_with(".js") && body.len() > 10000 {
let filename = format!("/tmp/fetched_script_{}.js", body.len());
let _ = std::fs::write(&filename, &body);
eprintln!("[op_net_fetch_sync] saved script to {}", filename);
}
body
}
Ok(Err(e)) => {
eprintln!("[op_net_fetch_sync] FAILED fetch {}: {}", url_clone, e);
String::new()
}
Err(_) => {
eprintln!("[op_net_fetch_sync] TIMEOUT fetching {}", url_clone);
String::new()
}
}
})
})
.join()
.unwrap_or_default();
eprintln!(
"[op_net_fetch_sync] fetched {} bytes from {}",
result.len(),
url
);
result
}
#[op2]
#[string]
pub fn op_net_xhr_sync(
#[string] url: String,
#[string] method: String,
#[string] headers_json: String,
#[string] body: String,
#[string] origin: String,
) -> String {
let extra_headers: Vec<(String, String)> =
serde_json::from_str(&headers_json).unwrap_or_default();
let body_bytes: Vec<u8> = if let Some(rest) = body.strip_prefix("b:") {
use base64::Engine as _;
base64::engine::general_purpose::STANDARD
.decode(rest.as_bytes())
.unwrap_or_default()
} else if let Some(rest) = body.strip_prefix("s:") {
rest.as_bytes().to_vec()
} else if body.is_empty() {
Vec::new()
} else {
body.as_bytes().to_vec()
};
let url = if url::Url::parse(&url).is_ok() {
url
} else if let Ok(base) = url::Url::parse(&origin) {
base.join(&url).map(|u| u.to_string()).unwrap_or(url)
} else {
url
};
let url_clone = url.clone();
let method_upper = method.to_uppercase();
let origin_str = if origin.is_empty() {
None
} else {
Some(origin)
};
let main_client = FETCH_CLIENT.with(|c| c.borrow().clone());
let result = std::thread::spawn(move || {
let rt = match tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
{
Ok(rt) => rt,
Err(_) => return "{}".to_string(),
};
rt.block_on(async move {
let client = match main_client.as_ref() {
Some(main) => {
crate::net::HttpClient::new_with_shared_state(
main.profile(),
main.cookies(),
main.accept_ch_origins(),
main.dns_cache(),
main.alt_svc_cache(),
).unwrap_or_else(|_| crate::net::HttpClient::new(main.profile()).unwrap())
}
None => {
let p = crate::stealth::presets::chrome_148_ru();
crate::net::HttpClient::new(&p).unwrap()
}
};
let resp_result = match method_upper.as_str() {
"GET" | "HEAD" => {
client.get_with_headers(&url_clone, &extra_headers).await
}
_ => {
let hdrs = crate::net::headers::chrome_headers_fetch(
client.profile(),
&url_clone,
origin_str.as_deref(),
);
let mut merged = hdrs;
for h in &extra_headers { merged.push(h.clone()); }
client.post_bytes_with_exact_headers(&url_clone, &body_bytes, &merged).await
}
};
match tokio::time::timeout(
std::time::Duration::from_secs(15),
async { resp_result },
).await {
Ok(Ok(resp)) => {
if let Some(main) = main_client.as_ref() {
if let Ok(parsed) = url::Url::parse(&url_clone) {
for ck in &resp.set_cookies {
main.set_cookie_str(&parsed, ck).await;
}
}
}
let status = resp.status;
let resp_url = resp.url.clone();
let body_text = resp.text();
let headers_arr: Vec<[String; 2]> = resp.headers
.into_iter()
.map(|(k, v)| [k, v])
.collect();
serde_json::json!({
"status": status,
"url": resp_url,
"headers": headers_arr,
"body": body_text,
}).to_string()
}
Ok(Err(e)) => {
eprintln!("[op_net_xhr_sync] error {}: {e}", url_clone);
serde_json::json!({"status": 0, "url": url_clone, "headers": [], "body": "", "error": e.to_string()}).to_string()
}
Err(_) => {
eprintln!("[op_net_xhr_sync] timeout {}", url_clone);
serde_json::json!({"status": 0, "url": url_clone, "headers": [], "body": "", "error": "timeout"}).to_string()
}
}
})
})
.join()
.unwrap_or_else(|_| "{}".to_string());
result
}
deno_core::extension!(
fetch_extension,
ops = [
op_fetch,
op_cookie_get,
op_cookie_set,
op_cookie_set_sync,
op_net_fetch_sync,
op_net_xhr_sync,
op_drain_csp_violations
],
);