use crate::domain::id::Id;
use crate::engines::types::ScrapeOptions;
use crate::utils::coalesce::{
CoalesceGuard as SingleFlightGuard, CoalesceResult as SingleFlightResult,
RequestCoalescer as SingleFlight,
};
use crate::workers::request_identity::coalesce_fingerprint;
pub type RequestCoalescer = SingleFlight<CoalesceKey, Id>;
pub type CoalesceGuard = SingleFlightGuard<CoalesceKey, Id>;
pub type CoalesceResult = SingleFlightResult<CoalesceKey, Id>;
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct CoalesceKey {
team_id: Id,
url: String,
fingerprint: u64,
}
impl CoalesceKey {
pub fn new(team_id: Id, url: &str, options: &ScrapeOptions) -> Self {
Self {
team_id,
url: normalize_coalesce_url(url),
fingerprint: coalesce_fingerprint(options),
}
}
pub fn team_id(&self) -> Id {
self.team_id
}
pub fn url(&self) -> &str {
&self.url
}
#[cfg(test)]
pub fn fingerprint(&self) -> u64 {
self.fingerprint
}
}
fn normalize_coalesce_url(raw: &str) -> String {
match url::Url::parse(raw.trim()) {
Ok(mut parsed) => {
parsed.set_fragment(None);
parsed.to_string()
}
Err(_) => raw.trim().to_string(),
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::common::{CacheMode, HttpMethod};
use crate::engines::types::{PageAction, ScreenshotConfig, WaitFor};
use std::collections::HashMap;
use std::time::Duration;
fn test_team() -> Id {
Id::from_u128(0x7e57_0000_0000_0000_0000_0000_0000_0001)
}
fn key_with(url: &str, options: &ScrapeOptions) -> CoalesceKey {
CoalesceKey::new(test_team(), url, options)
}
fn options_with_headers(pairs: &[(&str, &str)]) -> ScrapeOptions {
ScrapeOptions {
headers: pairs
.iter()
.map(|(name, value)| ((*name).to_string(), (*value).to_string()))
.collect(),
..Default::default()
}
}
#[test]
fn same_tenant_same_fingerprint_coalesces() {
let coalescer = RequestCoalescer::new();
let team = Id::generate();
let url = "https://example.com/coalesce-hit";
let mut options_a = options_with_headers(&[("X-Api-Key", "k1"), ("Accept", "text/html")]);
options_a.proxy = Some("http://proxy:8080".to_string());
let mut options_b = options_with_headers(&[("Accept", "text/html"), ("X-Api-Key", "k1")]);
options_b.proxy = Some("http://proxy:8080".to_string());
let leader_key = CoalesceKey::new(team, url, &options_a);
let waiter_key = CoalesceKey::new(team, url, &options_b);
assert_eq!(
leader_key, waiter_key,
"header insertion order must not change the key"
);
let _guard = match coalescer.try_start(&leader_key, Id::generate()) {
CoalesceResult::Proceed(g) => g,
_ => panic!("expected Proceed"),
};
assert!(
matches!(
coalescer.try_start(&waiter_key, Id::generate()),
CoalesceResult::Wait(..)
),
"same tenant + same fingerprint must coalesce"
);
assert_eq!(coalescer.in_flight_count(), 1);
}
#[test]
fn different_team_id_does_not_coalesce() {
let coalescer = RequestCoalescer::new();
let url = "https://example.com/tenant-split";
let key_a = CoalesceKey::new(Id::generate(), url, &ScrapeOptions::default());
let key_b = CoalesceKey::new(Id::generate(), url, &ScrapeOptions::default());
assert_ne!(key_a, key_b);
let _guard_a = match coalescer.try_start(&key_a, Id::generate()) {
CoalesceResult::Proceed(g) => g,
_ => panic!("expected Proceed for tenant A"),
};
let _guard_b = match coalescer.try_start(&key_b, Id::generate()) {
CoalesceResult::Proceed(g) => g,
CoalesceResult::Wait(..) => {
panic!("tenant B must not wait on tenant A's in-flight entry")
}
};
assert_eq!(coalescer.in_flight_count(), 2);
}
#[test]
fn cross_tenant_different_headers_never_adopts_leader_result() {
let coalescer = RequestCoalescer::new();
let url = "https://example.com/private";
let options_a = options_with_headers(&[("Authorization", "Bearer tenant-a")]);
let options_b = options_with_headers(&[("Authorization", "Bearer tenant-b")]);
let key_a = CoalesceKey::new(Id::generate(), url, &options_a);
let key_b = CoalesceKey::new(Id::generate(), url, &options_b);
let _guard_a = match coalescer.try_start(&key_a, Id::generate()) {
CoalesceResult::Proceed(g) => g,
_ => panic!("expected Proceed for tenant A"),
};
let _guard_b = match coalescer.try_start(&key_b, Id::generate()) {
CoalesceResult::Proceed(g) => g,
CoalesceResult::Wait(..) => {
panic!("tenant B adopted tenant A's leader instead of proceeding")
}
};
}
#[test]
fn different_headers_do_not_coalesce() {
let coalescer = RequestCoalescer::new();
let team = Id::generate();
let url = "https://example.com/headers-split";
let options_a = options_with_headers(&[("X-Trace", "a")]);
let options_b = options_with_headers(&[("X-Trace", "b")]);
let key_a = CoalesceKey::new(team, url, &options_a);
let key_b = CoalesceKey::new(team, url, &options_b);
assert_ne!(key_a.fingerprint(), key_b.fingerprint());
let _guard_a = match coalescer.try_start(&key_a, Id::generate()) {
CoalesceResult::Proceed(g) => g,
_ => panic!("expected Proceed"),
};
let _guard_b = match coalescer.try_start(&key_b, Id::generate()) {
CoalesceResult::Proceed(g) => g,
CoalesceResult::Wait(..) => panic!("different headers must not coalesce"),
};
assert_eq!(coalescer.in_flight_count(), 2);
}
#[test]
fn different_proxy_do_not_coalesce() {
let coalescer = RequestCoalescer::new();
let team = Id::generate();
let url = "https://example.com/proxy-split";
let options_a = ScrapeOptions {
proxy: Some("http://proxy-a:8080".to_string()),
..Default::default()
};
let options_b = ScrapeOptions {
proxy: Some("http://proxy-b:8080".to_string()),
..Default::default()
};
let options_empty = ScrapeOptions {
proxy: Some(String::new()),
..Default::default()
};
let no_proxy = CoalesceKey::new(team, url, &ScrapeOptions::default());
let proxy_a = CoalesceKey::new(team, url, &options_a);
let proxy_b = CoalesceKey::new(team, url, &options_b);
let empty_proxy = CoalesceKey::new(team, url, &options_empty);
assert_ne!(no_proxy, proxy_a);
assert_ne!(proxy_a, proxy_b);
assert_ne!(no_proxy, empty_proxy);
let _guard_a = match coalescer.try_start(&proxy_a, Id::generate()) {
CoalesceResult::Proceed(g) => g,
_ => panic!("expected Proceed"),
};
let _guard_b = match coalescer.try_start(&proxy_b, Id::generate()) {
CoalesceResult::Proceed(g) => g,
CoalesceResult::Wait(..) => panic!("different proxy must not coalesce"),
};
assert_eq!(coalescer.in_flight_count(), 2);
}
#[test]
fn empty_headers_and_proxy_fingerprint_is_stable() {
let team = test_team();
let url = "https://example.com/empty";
let first = CoalesceKey::new(team, url, &ScrapeOptions::default());
let second = CoalesceKey::new(team, url, &ScrapeOptions::default());
assert_eq!(first.fingerprint(), second.fingerprint());
assert_eq!(first, second);
let explicit_empty = ScrapeOptions {
headers: HashMap::with_capacity(4),
..Default::default()
};
assert_eq!(
first,
CoalesceKey::new(team, url, &explicit_empty),
"empty header maps must produce the same fingerprint"
);
}
#[test]
fn url_normalization_groups_equivalent_forms() {
let team = test_team();
let options = ScrapeOptions::default();
let base = CoalesceKey::new(team, "https://example.com/page", &options);
assert_eq!(
base,
CoalesceKey::new(team, "HTTPS://Example.COM/page#section", &options),
"scheme/host case and fragment must not change the key"
);
assert_eq!(
base,
CoalesceKey::new(team, "https://example.com:443/page", &options),
"default port must not change the key"
);
assert_ne!(
base,
CoalesceKey::new(team, "https://example.com/page?a=1", &options),
"query strings must stay part of the key"
);
assert_ne!(
base,
CoalesceKey::new(team, "http://example.com/page", &options),
"scheme must stay part of the key"
);
}
#[test]
fn header_name_case_is_normalized() {
let team = test_team();
let url = "https://example.com/case";
let lower = options_with_headers(&[("x-token", "v")]);
let mixed = options_with_headers(&[("X-Token", "v")]);
assert_eq!(
CoalesceKey::new(team, url, &lower),
CoalesceKey::new(team, url, &mixed),
"header names are case-insensitive per RFC 9110"
);
let lower_value = options_with_headers(&[("X-Token", "V")]);
assert_ne!(
CoalesceKey::new(team, url, &mixed),
CoalesceKey::new(team, url, &lower_value),
"header values must stay case-sensitive"
);
}
#[test]
fn mixed_case_header_names_are_order_independent() {
let url = "https://example.com/case-duplicates";
let first = options_with_headers(&[
("Accept", "text/html"),
("accept", "application/json"),
("X-Trace", "1"),
]);
let second = options_with_headers(&[
("x-trace", "1"),
("accept", "application/json"),
("ACCEPT", "text/html"),
]);
assert_eq!(
CoalesceKey::new(test_team(), url, &first),
CoalesceKey::new(test_team(), url, &second),
"lowercased header names must be sorted deterministically"
);
}
#[test]
fn fingerprint_encoding_is_unambiguous() {
let team = test_team();
let url = "https://example.com/encoding";
let split_a = options_with_headers(&[("ab", "c")]);
let split_b = options_with_headers(&[("a", "bc")]);
assert_ne!(
CoalesceKey::new(team, url, &split_a).fingerprint(),
CoalesceKey::new(team, url, &split_b).fingerprint()
);
}
#[test]
fn fingerprint_is_stable_within_process() {
let team = Id::from_u128(1);
let options = ScrapeOptions {
proxy: Some("http://p:1".to_string()),
..Default::default()
};
let key = CoalesceKey::new(team, "https://example.com/fixed", &options);
let again = CoalesceKey::new(team, "https://example.com/fixed", &options);
assert_eq!(key.fingerprint(), again.fingerprint());
assert_eq!(key, again);
}
#[test]
fn fingerprint_covers_all_request_affecting_fields() {
fn fp(mutate: impl FnOnce(&mut ScrapeOptions)) -> u64 {
let mut options = ScrapeOptions::default();
mutate(&mut options);
key_with("https://example.com/full-identity", &options).fingerprint()
}
let base = fp(|_| {});
assert_ne!(base, fp(|o| o.method = HttpMethod::Post), "method");
assert_ne!(base, fp(|o| o.needs_js = true), "needs_js");
assert_ne!(base, fp(|o| o.needs_screenshot = true), "needs_screenshot");
assert_ne!(base, fp(|o| o.mobile = true), "mobile");
assert_ne!(base, fp(|o| o.timeout = Duration::from_secs(5)), "timeout");
assert_ne!(base, fp(|o| o.body = Some("payload".into())), "body");
assert_ne!(base, fp(|o| o.sync_wait_ms = 1), "sync_wait_ms");
assert_ne!(
base,
fp(|o| o.actions = vec![PageAction::Wait { milliseconds: 1 }]),
"actions"
);
assert_ne!(
base,
fp(|o| o.screenshot_config = Some(ScreenshotConfig::default())),
"screenshot_config"
);
assert_ne!(
base,
fp(|o| o.proxy = Some("http://proxy:8080".into())),
"proxy"
);
assert_ne!(
base,
fp(|o| o.skip_tls_verification = true),
"skip_tls_verification"
);
assert_ne!(
base,
fp(|o| {
o.headers.insert("X-Token".into(), "v".into());
}),
"headers"
);
assert_ne!(
base,
fp(|o| o.needs_tls_fingerprint = true),
"needs_tls_fingerprint"
);
assert_ne!(base, fp(|o| o.use_fire_engine = true), "use_fire_engine");
assert_ne!(base, fp(|o| o.block_ads = true), "block_ads");
assert_ne!(base, fp(|o| o.block_media = true), "block_media");
assert_ne!(
base,
fp(|o| o.session_id = Some("session-1".into())),
"session_id"
);
assert_ne!(
base,
fp(|o| o.wait_for = Some(WaitFor::Selector("main".into()))),
"wait_for"
);
assert_ne!(base, fp(|o| o.needs_mllm = true), "needs_mllm");
}
#[test]
fn merge_key_projects_cache_read_allowed() {
fn fp(cache_mode: Option<CacheMode>) -> u64 {
let options = ScrapeOptions {
cache_mode,
..Default::default()
};
key_with("https://example.com/cache-mode", &options).fingerprint()
}
let readable = fp(Some(CacheMode::Enabled));
assert_eq!(fp(None), readable, "None defaults to Enabled");
assert_eq!(fp(Some(CacheMode::ReadOnly)), readable, "ReadOnly reads");
let bypass = fp(Some(CacheMode::Bypass));
let disabled = fp(Some(CacheMode::Disabled));
assert_ne!(bypass, readable, "Bypass must not merge with Enabled");
assert_ne!(disabled, readable, "Disabled must not merge with Enabled");
assert_eq!(bypass, disabled, "both modes forbid cache reads");
let mut with_header = ScrapeOptions {
cache_mode: Some(CacheMode::Bypass),
..Default::default()
};
with_header
.headers
.insert("Authorization".into(), "Bearer token".into());
assert_ne!(
readable,
key_with("https://example.com/cache-mode", &with_header).fingerprint(),
"headers must stay part of request identity"
);
let with_js = ScrapeOptions {
cache_mode: Some(CacheMode::ReadOnly),
needs_js: true,
..Default::default()
};
assert_ne!(
readable,
key_with("https://example.com/cache-mode", &with_js).fingerprint(),
"needs_js must stay part of request identity"
);
}
#[test]
fn read_only_waiter_coalesces_with_enabled_leader() {
let coalescer = RequestCoalescer::new();
let team = Id::generate();
let url = "https://example.com/read-only-merge";
let enabled = CoalesceKey::new(
team,
url,
&ScrapeOptions {
cache_mode: Some(CacheMode::Enabled),
..Default::default()
},
);
let read_only = CoalesceKey::new(
team,
url,
&ScrapeOptions {
cache_mode: Some(CacheMode::ReadOnly),
..Default::default()
},
);
assert_eq!(enabled, read_only);
let _guard = match coalescer.try_start(&enabled, Id::generate()) {
CoalesceResult::Proceed(g) => g,
_ => panic!("expected Proceed"),
};
assert!(
matches!(
coalescer.try_start(&read_only, Id::generate()),
CoalesceResult::Wait(..)
),
"ReadOnly waiter must coalesce with Enabled leader"
);
}
#[test]
fn bypass_waiter_does_not_coalesce_with_enabled_leader() {
let coalescer = RequestCoalescer::new();
let team = Id::generate();
let url = "https://example.com/bypass-merge";
let enabled = CoalesceKey::new(team, url, &ScrapeOptions::default());
let bypass = CoalesceKey::new(
team,
url,
&ScrapeOptions {
cache_mode: Some(CacheMode::Bypass),
..Default::default()
},
);
let disabled = CoalesceKey::new(
team,
url,
&ScrapeOptions {
cache_mode: Some(CacheMode::Disabled),
..Default::default()
},
);
assert_ne!(enabled, bypass);
assert_ne!(enabled, disabled);
assert_eq!(bypass, disabled);
let _guard = match coalescer.try_start(&enabled, Id::generate()) {
CoalesceResult::Proceed(g) => g,
_ => panic!("expected Proceed"),
};
let _bypass_guard = match coalescer.try_start(&bypass, Id::generate()) {
CoalesceResult::Proceed(g) => g,
CoalesceResult::Wait(..) => {
panic!("Bypass waiter must not adopt a cache-reading leader's result")
}
};
match coalescer.try_start(&disabled, Id::generate()) {
CoalesceResult::Wait(..) => (),
CoalesceResult::Proceed(_) => {
panic!("Disabled and Bypass share the same no-read merge key")
}
};
assert_eq!(coalescer.in_flight_count(), 2);
}
#[test]
fn same_tenant_different_scrape_options_do_not_coalesce() {
let coalescer = RequestCoalescer::new();
let team = Id::generate();
let url = "https://example.com/needs-js-split";
let plain = ScrapeOptions::default();
let rendered = ScrapeOptions {
needs_js: true,
..Default::default()
};
let key_plain = CoalesceKey::new(team, url, &plain);
let key_rendered = CoalesceKey::new(team, url, &rendered);
assert_ne!(key_plain, key_rendered);
let _plain_guard = match coalescer.try_start(&key_plain, Id::generate()) {
CoalesceResult::Proceed(g) => g,
_ => panic!("expected Proceed"),
};
let _rendered_guard = match coalescer.try_start(&key_rendered, Id::generate()) {
CoalesceResult::Proceed(g) => g,
CoalesceResult::Wait(..) => panic!("different needs_js must not coalesce"),
};
assert_eq!(coalescer.in_flight_count(), 2);
}
}