use std::collections::VecDeque;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, OnceLock};
use dashmap::DashMap;
use parking_lot::Mutex;
use sha2::{Digest, Sha256};
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct Soft404Fingerprint {
pub status_code: u16,
pub title_hash: u64,
pub content_length: usize,
pub body_hash: [u8; 16],
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
struct ContentKey {
title_hash: u64,
content_length: usize,
body_hash: [u8; 16],
}
impl Soft404Fingerprint {
pub fn of(status_code: u16, content: &str) -> Self {
let title = extract_html_title(content);
Self {
status_code,
title_hash: sha256_prefix8(title.as_bytes()),
content_length: content.len(),
body_hash: sha256_prefix16(content.as_bytes()),
}
}
fn content_key(&self) -> ContentKey {
ContentKey {
title_hash: self.title_hash,
content_length: self.content_length,
body_hash: self.body_hash,
}
}
}
#[derive(Debug)]
struct Learning {
status_code: u16,
learned_at: std::time::Instant,
hits: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Soft404Verdict {
Learned,
Soft404,
Keep,
}
pub struct Soft404Detector {
learned: DashMap<ContentKey, Learning>,
insertion_order: parking_lot::Mutex<VecDeque<ContentKey>>,
max_fingerprints: usize,
hit_count: AtomicU64,
}
pub const DEFAULT_MAX_FINGERPRINTS: usize = 10_000;
static SHARED_DETECTOR: OnceLock<Arc<Soft404Detector>> = OnceLock::new();
pub fn shared() -> Arc<Soft404Detector> {
SHARED_DETECTOR
.get_or_init(|| Arc::new(Soft404Detector::new(DEFAULT_MAX_FINGERPRINTS)))
.clone()
}
pub fn init_shared(max_fingerprints: usize) {
if SHARED_DETECTOR
.set(Arc::new(Soft404Detector::new(max_fingerprints)))
.is_err()
{
log::warn!(
"{}",
crate::i18n::tr_log_args(
"soft404-already-initialized",
&[("max_fingerprints", (max_fingerprints as i64).to_string(),)],
)
);
}
}
impl Soft404Detector {
pub fn new(max_fingerprints: usize) -> Self {
Self {
learned: DashMap::new(),
insertion_order: Mutex::new(VecDeque::new()),
max_fingerprints: max_fingerprints.max(1),
hit_count: AtomicU64::new(0),
}
}
pub fn len(&self) -> usize {
self.learned.len()
}
pub fn is_empty(&self) -> bool {
self.learned.is_empty()
}
pub fn hit_count(&self) -> u64 {
self.hit_count.load(Ordering::Relaxed)
}
pub fn learn(&self, fingerprint: Soft404Fingerprint) {
let key = fingerprint.content_key();
if let Some(mut entry) = self.learned.get_mut(&key) {
entry.status_code = fingerprint.status_code;
entry.learned_at = std::time::Instant::now();
return;
}
while self.learned.len() >= self.max_fingerprints {
let evicted = self
.insertion_order
.lock()
.pop_front()
.and_then(|stale| self.learned.remove(&stale).map(|_| stale));
if evicted.is_none() {
break;
}
}
self.insertion_order.lock().push_back(key.clone());
self.learned.insert(
key,
Learning {
status_code: fingerprint.status_code,
learned_at: std::time::Instant::now(),
hits: 0,
},
);
}
pub fn is_soft404(&self, fingerprint: &Soft404Fingerprint) -> bool {
if self.learned.is_empty() {
return false;
}
let key = fingerprint.content_key();
let Some(mut learning) = self.learned.get_mut(&key) else {
return false;
};
let hits = learning.hits + 1;
learning.hits = hits;
let learned_from = learning.status_code;
let total = self.hit_count.fetch_add(1, Ordering::Relaxed) + 1;
log::info!(
"{}",
crate::i18n::tr_log_args(
"soft404-hit",
&[
("status", (fingerprint.status_code as i64).to_string(),),
("learned_from", (learned_from as i64).to_string()),
(
"content_length",
(fingerprint.content_length as i64).to_string(),
),
("body_hash", format!("{:02x?}", fingerprint.body_hash),),
("hits", (hits as i64).to_string()),
("total", (total as i64).to_string()),
],
)
);
true
}
pub fn observe(&self, status_code: u16, content: &str) -> Soft404Verdict {
match status_code {
404 | 410 => {
self.learn(Soft404Fingerprint::of(status_code, content));
Soft404Verdict::Learned
}
200 => {
if self.learned.is_empty() {
return Soft404Verdict::Keep;
}
let fingerprint = Soft404Fingerprint::of(status_code, content);
if self.is_soft404(&fingerprint) {
Soft404Verdict::Soft404
} else {
Soft404Verdict::Keep
}
}
_ => Soft404Verdict::Keep,
}
}
}
pub fn observe_crawl_response(enabled: bool, status_code: u16, content: &str) -> Soft404Verdict {
if !enabled {
return Soft404Verdict::Keep;
}
shared().observe(status_code, content)
}
fn extract_html_title(html: &str) -> &str {
const HTML_PROBE_BYTES: usize = 64 * 1024;
let probe = if html.len() <= HTML_PROBE_BYTES {
html
} else {
let mut end = HTML_PROBE_BYTES;
while end > 0 && !html.is_char_boundary(end) {
end -= 1;
}
&html[..end]
};
let lower = probe.to_ascii_lowercase();
let start_tag = match lower.find("<title") {
Some(i) => i,
None => return "",
};
let content_start = match lower[start_tag..].find('>') {
Some(rel) => start_tag + rel + 1,
None => return "",
};
let end_rel = match lower[content_start..].find("</title") {
Some(e) => e,
None => return "",
};
probe[content_start..content_start + end_rel].trim()
}
fn sha256_prefix8(data: &[u8]) -> u64 {
let digest = Sha256::digest(data);
u64::from_be_bytes(digest[..8].try_into().expect("sha256 digest >= 8 bytes"))
}
fn sha256_prefix16(data: &[u8]) -> [u8; 16] {
let digest = Sha256::digest(data);
digest[..16].try_into().expect("sha256 digest >= 16 bytes")
}
#[cfg(test)]
mod tests {
use super::*;
fn page(title: &str, body: &str) -> String {
format!("<html><head><title>{title}</title></head><body>{body}</body></html>")
}
#[test]
fn fingerprint_is_deterministic_and_discriminating() {
let a = Soft404Fingerprint::of(404, &page("Not Found", "nothing here"));
let b = Soft404Fingerprint::of(404, &page("Not Found", "nothing here"));
assert_eq!(a, b, "同内容指纹必须一致");
let diff_body = Soft404Fingerprint::of(404, &page("Not Found", "different"));
assert_ne!(a.body_hash, diff_body.body_hash);
assert_ne!(a.content_length, diff_body.content_length);
let diff_title = Soft404Fingerprint::of(404, &page("Other", "nothing here"));
assert_ne!(a.title_hash, diff_title.title_hash);
}
#[test]
fn extract_html_title_semantics() {
assert_eq!(
extract_html_title("<TITLE> Big Case </TITLE>"),
"Big Case"
);
assert_eq!(extract_html_title("<title attr=\"x\">v</title>"), "v");
assert_eq!(extract_html_title("<html><body>no title</body></html>"), "");
let filler = "字".repeat(40_000);
let html = format!("<html>{filler}<title>t</title></html>");
let _ = extract_html_title(&html);
}
#[test]
fn learn_from_404_then_200_hits() {
let detector = Soft404Detector::new(100);
let shell = page("Not Found", "empty shell");
assert_eq!(
detector.observe(404, &shell),
Soft404Verdict::Learned,
"404 响应触发学习"
);
assert_eq!(detector.len(), 1);
assert_eq!(
detector.observe(200, &shell),
Soft404Verdict::Soft404,
"同指纹的 200 壳必须命中"
);
assert_eq!(detector.hit_count(), 1);
}
#[test]
fn learn_from_410() {
let detector = Soft404Detector::new(100);
assert_eq!(
detector.observe(410, &page("Gone", "gone forever")),
Soft404Verdict::Learned
);
}
#[test]
fn unlearned_200_keeps() {
let detector = Soft404Detector::new(100);
let shell = page("Not Found", "empty shell");
assert_eq!(detector.observe(200, &shell), Soft404Verdict::Keep);
assert_eq!(detector.hit_count(), 0);
}
#[test]
fn different_content_does_not_hit() {
let detector = Soft404Detector::new(100);
detector.observe(404, &page("Not Found", "shell v1"));
let probe = Soft404Fingerprint::of(200, &page("Not Found", "shell v2"));
assert!(!detector.is_soft404(&probe));
}
#[test]
fn empty_library_fast_path_keeps() {
let detector = Soft404Detector::new(100);
assert!(detector.is_empty());
assert_eq!(
detector.observe(200, &page("Not Found", "anything")),
Soft404Verdict::Keep
);
}
#[test]
fn max_fingerprints_evicts_oldest() {
let detector = Soft404Detector::new(2);
let first = page("T1", "body-1");
let second = page("T2", "body-2");
let third = page("T3", "body-3");
detector.observe(404, &first);
detector.observe(404, &second);
assert_eq!(detector.len(), 2);
detector.observe(404, &third);
assert_eq!(detector.len(), 2, "容量上限 2");
let first_fp = Soft404Fingerprint::of(200, &first);
assert!(
!detector.is_soft404(&first_fp),
"被淘汰的最旧指纹不得再命中"
);
let third_fp = Soft404Fingerprint::of(200, &third);
assert!(detector.is_soft404(&third_fp), "新指纹仍可命中");
}
#[test]
fn relearn_same_fingerprint_keeps_single_entry() {
let detector = Soft404Detector::new(2);
let shell = page("T", "same");
detector.observe(404, &shell);
detector.observe(404, &shell);
detector.observe(404, &shell);
assert_eq!(detector.len(), 1, "同指纹重复学习只保留一条");
}
#[test]
fn other_status_codes_keep() {
let detector = Soft404Detector::new(100);
assert_eq!(
detector.observe(500, &page("Err", "boom")),
Soft404Verdict::Keep
);
assert_eq!(
detector.observe(301, &page("Move", "x")),
Soft404Verdict::Keep
);
assert!(detector.is_empty(), "非学习状态码不得入库");
}
#[test]
fn observe_crawl_response_disabled_keeps() {
let shell = page("Not Found", "shell");
assert_eq!(
observe_crawl_response(false, 404, &shell),
Soft404Verdict::Keep,
"关闭时 404 也不学习"
);
assert_eq!(
observe_crawl_response(false, 200, &shell),
Soft404Verdict::Keep,
"关闭时 200 壳照常落库"
);
}
#[test]
fn zero_capacity_clamps_to_one() {
let detector = Soft404Detector::new(0);
detector.observe(404, &page("A", "a"));
assert_eq!(detector.len(), 1, "容量下限为 1");
}
}