use std::collections::HashMap;
use std::hash::Hasher;
use crate::common::CacheMode;
use crate::engines::types::ScrapeOptions;
#[must_use]
pub(crate) fn request_fingerprint(options: &ScrapeOptions) -> u64 {
hash_identity(request_identity(options))
}
#[must_use]
pub(crate) fn coalesce_fingerprint(options: &ScrapeOptions) -> u64 {
const CACHE_READ_ALLOWED_FIELD: &str = "cache_read_allowed";
let mut identity = request_identity(options);
if !cache_read_allowed(options.cache_mode) {
if let serde_json::Value::Object(map) = &mut identity {
map.insert(
CACHE_READ_ALLOWED_FIELD.to_string(),
serde_json::Value::Bool(false),
);
}
}
hash_identity(identity)
}
fn cache_read_allowed(mode: Option<CacheMode>) -> bool {
!matches!(mode, Some(CacheMode::Bypass | CacheMode::Disabled))
}
fn request_identity(options: &ScrapeOptions) -> serde_json::Value {
const HEADERS_FIELD: &str = "headers";
const CACHE_MODE_FIELD: &str = "cache_mode";
let mut identity =
serde_json::to_value(options).expect("ScrapeOptions JSON serialization is infallible");
if let serde_json::Value::Object(map) = &mut identity {
map.insert(
HEADERS_FIELD.to_string(),
canonical_headers(&options.headers),
);
map.insert(CACHE_MODE_FIELD.to_string(), serde_json::Value::Null);
}
identity
}
fn hash_identity(identity: serde_json::Value) -> u64 {
let canonical = canonicalize_json(identity);
let bytes =
serde_json::to_vec(&canonical).expect("canonical JSON value serialization is infallible");
let mut hasher = std::collections::hash_map::DefaultHasher::new();
hasher.write(&bytes);
hasher.finish()
}
fn canonical_headers(headers: &HashMap<String, String>) -> serde_json::Value {
use serde_json::Value;
let mut pairs: Vec<(String, &str)> = headers
.iter()
.map(|(name, value)| (name.to_ascii_lowercase(), value.as_str()))
.collect();
pairs.sort_unstable_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(b.1)));
Value::Array(
pairs
.into_iter()
.map(|(name, value)| {
Value::Array(vec![Value::String(name), Value::String(value.to_owned())])
})
.collect(),
)
}
fn canonicalize_json(value: serde_json::Value) -> serde_json::Value {
use serde_json::Value;
match value {
Value::Object(map) => {
let mut entries: Vec<(String, Value)> = map.into_iter().collect();
entries.sort_unstable_by(|a, b| a.0.cmp(&b.0));
let mut sorted = serde_json::Map::with_capacity(entries.len());
for (key, value) in entries {
sorted.insert(key, canonicalize_json(value));
}
Value::Object(sorted)
}
Value::Array(items) => Value::Array(items.into_iter().map(canonicalize_json).collect()),
scalar => scalar,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn request_fingerprint_is_stable() {
let first = request_fingerprint(&ScrapeOptions::default());
let second = request_fingerprint(&ScrapeOptions::default());
assert_eq!(first, second);
}
#[test]
fn coalesce_fingerprint_groups_cache_modes() {
fn fp(mode: Option<CacheMode>) -> u64 {
coalesce_fingerprint(&ScrapeOptions {
cache_mode: mode,
..Default::default()
})
}
let readable = fp(None);
assert_eq!(fp(Some(CacheMode::Enabled)), readable);
assert_eq!(fp(Some(CacheMode::ReadOnly)), readable);
assert_eq!(fp(Some(CacheMode::Bypass)), fp(Some(CacheMode::Disabled)));
assert_ne!(fp(Some(CacheMode::Bypass)), readable);
}
#[test]
fn coalesce_fingerprint_omits_default_true_projection() {
assert_eq!(
coalesce_fingerprint(&ScrapeOptions::default()),
request_fingerprint(&ScrapeOptions::default()),
"readable mode must not add a default projection field"
);
assert_ne!(
coalesce_fingerprint(&ScrapeOptions {
cache_mode: Some(CacheMode::Bypass),
..Default::default()
}),
request_fingerprint(&ScrapeOptions::default()),
"non-readable mode must add the false projection field"
);
}
#[test]
fn coalesce_fingerprint_handles_headers_deterministically() {
let mut a = ScrapeOptions::default();
a.headers.insert("X-Token".into(), "V".into());
a.headers.insert("accept".into(), "text/html".into());
let mut b = ScrapeOptions::default();
b.headers.insert("ACCEPT".into(), "text/html".into());
b.headers.insert("x-token".into(), "V".into());
assert_eq!(coalesce_fingerprint(&a), coalesce_fingerprint(&b));
let mut lower_value = ScrapeOptions::default();
lower_value.headers.insert("X-Token".into(), "v".into());
assert_ne!(coalesce_fingerprint(&a), coalesce_fingerprint(&lower_value));
}
}