use std::collections::HashMap;
use std::fs;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
const MAX_TRACKED_KEYS: usize = 10_000;
use base64::engine::general_purpose::STANDARD as BASE64;
use base64::Engine as _;
use dashmap::DashMap;
use sha2::{Digest, Sha256};
use crate::common::metrics_shim::counter;
use crate::config::settings::ObservabilitySettings;
pub struct SnapshotInput<'a> {
pub url: &'a str,
pub status_code: u16,
pub reason: &'a str,
pub content: String,
pub headers: &'a HashMap<String, String>,
pub screenshot: Option<String>,
}
#[derive(Debug)]
pub struct FailureSnapshotter {
dir: PathBuf,
counts: DashMap<String, u64>,
total_written: AtomicU64,
max_total: usize,
persist_lock: Arc<parking_lot::Mutex<()>>,
}
impl FailureSnapshotter {
pub fn new(dir: impl Into<PathBuf>, max_total: usize) -> Self {
Self {
dir: dir.into(),
counts: DashMap::new(),
total_written: AtomicU64::new(0),
max_total: max_total.max(1),
persist_lock: Arc::new(parking_lot::Mutex::new(())),
}
}
pub fn from_config(settings: &ObservabilitySettings) -> Option<Arc<Self>> {
if settings.failure_snapshot_dir.is_empty() {
return None;
}
Some(Arc::new(Self::new(
settings.failure_snapshot_dir.clone(),
settings.failure_snapshot_max_total,
)))
}
pub async fn snapshot(&self, input: SnapshotInput<'_>) {
let key = Self::signature_key(input.url, input.status_code, input.reason);
let Some(count) = self.track_hit(&key) else {
return;
};
let meta = serde_json::json!({
"url": input.url,
"status": input.status_code,
"reason": input.reason,
"count": count,
"timestamp": chrono::Utc::now().to_rfc3339(),
"headers": input.headers,
});
let dir = self.dir.clone();
let key_closure = key.clone();
let content = input.content;
let screenshot_b64 = input.screenshot;
let max_total = self.max_total;
let persist_lock = self.persist_lock.clone();
let outcome = tokio::task::spawn_blocking(move || {
Self::persist_hit(
dir,
key_closure,
count,
content,
screenshot_b64,
meta,
max_total,
persist_lock,
)
})
.await;
self.record_persist_outcome(outcome);
}
fn record_persist_outcome(&self, outcome: Result<(u64, u64), tokio::task::JoinError>) {
match outcome {
Ok((written, evicted)) => {
if written > 0 || evicted > 0 {
let _ = self.total_written.try_update(
Ordering::Relaxed,
Ordering::Relaxed,
|cur| Some(cur.saturating_add(written).saturating_sub(evicted)),
);
}
}
Err(e) => log::debug!(
"{}",
crate::i18n::tr_log_args("snapshot-join-failed", &[("error", e.to_string())],)
),
}
}
fn track_hit(&self, key: &str) -> Option<u64> {
let count = match self.counts.get_mut(key) {
Some(mut c) => {
*c += 1;
*c
}
None => {
if self.counts.len() >= MAX_TRACKED_KEYS {
counter!("crawlrs_failure_snapshot_total").increment(1);
return None;
}
self.counts.insert(key.to_string(), 1);
1
}
};
counter!("crawlrs_failure_snapshot_total").increment(1);
Some(count)
}
#[allow(clippy::too_many_arguments)]
fn persist_hit(
dir: PathBuf,
key: String,
count: u64,
content: String,
screenshot_b64: Option<String>,
meta: serde_json::Value,
max_total: usize,
persist_lock: Arc<parking_lot::Mutex<()>>,
) -> (u64, u64) {
let screenshot = if count == 1 {
screenshot_b64
.as_deref()
.map(|s| BASE64.decode(s))
.transpose()
.ok()
.flatten()
} else {
None
};
let _guard = (count == 1).then(|| persist_lock.lock());
persist_snapshot(
&dir,
&key,
count,
&content,
screenshot.as_deref(),
&meta,
max_total,
)
}
fn signature_key(url: &str, status_code: u16, reason: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(url.as_bytes());
hasher.update(b"|");
hasher.update(status_code.to_string().as_bytes());
hasher.update(b"|");
hasher.update(reason.as_bytes());
let hex = hex::encode(hasher.finalize());
let mut key = String::with_capacity(60);
key.push_str(&hex[..30]);
key.extend(
reason
.chars()
.filter(|c| c.is_ascii_alphanumeric() || *c == '_' || *c == '-')
.take(30),
);
key
}
pub fn hit_count(&self, key: &str) -> Option<u64> {
self.counts.get(key).map(|c| *c)
}
pub fn total_written(&self) -> u64 {
self.total_written.load(Ordering::Relaxed)
}
}
fn persist_snapshot(
dir: &Path,
key: &str,
count: u64,
content: &str,
screenshot: Option<&[u8]>,
meta: &serde_json::Value,
max_total: usize,
) -> (u64, u64) {
let mut written = 0u64;
let mut evicted = 0u64;
if let Err(e) = fs::create_dir_all(dir) {
log::debug!(
"{}",
crate::i18n::tr_log_args(
"snapshot-dir-create-failed",
&[("dir", dir.display().to_string()), ("error", e.to_string()),],
)
);
return (0, 0);
}
if count == 1 {
let needed: u64 = 1 + u64::from(screenshot.is_some()) + 1; evicted += evict_oldest(dir, needed, max_total);
let html_path = dir.join(format!("{key}.html"));
match write_private(&html_path, content.as_bytes()) {
Ok(()) => written += 1,
Err(e) => log_snapshot_write_failed(&html_path, &e),
}
if let Some(bytes) = screenshot {
let jpg_path = dir.join(format!("{key}.jpg"));
match write_private(&jpg_path, bytes) {
Ok(()) => written += 1,
Err(e) => log_snapshot_write_failed(&jpg_path, &e),
}
}
}
let meta_path = dir.join(format!("{key}.meta.json"));
match serde_json::to_string_pretty(meta)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))
.and_then(|raw| write_private(&meta_path, raw.as_bytes()))
{
Ok(()) => written += 1,
Err(e) => log_snapshot_write_failed(&meta_path, &e),
}
(written, evicted)
}
fn log_snapshot_write_failed(path: &Path, error: &std::io::Error) {
log::debug!(
"{}",
crate::i18n::tr_log_args(
"snapshot-write-failed",
&[
("path", path.display().to_string()),
("error", error.to_string()),
],
)
);
}
fn evict_oldest(dir: &Path, needed: u64, max_total: usize) -> u64 {
let entries = match fs::read_dir(dir) {
Ok(rd) => rd,
Err(e) => {
log::debug!(
"{}",
crate::i18n::tr_log_args(
"snapshot-read-dir-failed",
&[("dir", dir.display().to_string()), ("error", e.to_string()),],
)
);
return 0;
}
};
let mut files: Vec<(std::time::SystemTime, PathBuf)> = entries
.filter_map(|e| e.ok())
.filter_map(|e| {
let meta = e.metadata().ok()?;
if !meta.is_file() {
return None;
}
Some((meta.modified().ok()?, e.path()))
})
.collect();
let target = (max_total as u64).saturating_sub(needed);
if (files.len() as u64) <= target {
return 0;
}
files.sort_by_key(|(mtime, _)| *mtime);
let total = files.len() as u64;
let mut evicted = 0u64;
for (_, path) in files {
if total.saturating_sub(evicted) <= target {
break;
}
match fs::remove_file(&path) {
Ok(()) => evicted += 1,
Err(e) => log::debug!(
"{}",
crate::i18n::tr_log_args(
"snapshot-evict-failed",
&[
("path", path.display().to_string()),
("error", e.to_string()),
],
)
),
}
}
evicted
}
#[cfg(unix)]
fn write_private(path: &Path, bytes: &[u8]) -> std::io::Result<()> {
use std::io::Write;
use std::os::unix::fs::OpenOptionsExt;
let mut file = fs::OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.mode(0o600)
.open(path)?;
file.write_all(bytes)
}
#[cfg(not(unix))]
fn write_private(path: &Path, bytes: &[u8]) -> std::io::Result<()> {
fs::write(path, bytes)
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::HashMap;
use std::fs;
use std::path::Path;
use std::sync::LazyLock;
static EMPTY_HEADERS: LazyLock<HashMap<String, String>> = LazyLock::new(HashMap::new);
fn temp_snapshotter(max_total: usize) -> (FailureSnapshotter, PathBuf) {
let tmp = tempfile::tempdir().expect("tempdir");
let dir = tmp.path().join("snapshots");
(FailureSnapshotter::new(dir.clone(), max_total), dir)
}
fn sample_input<'a>(
url: &'a str,
status: u16,
reason: &'a str,
content: &'a str,
screenshot: Option<&'a str>,
) -> SnapshotInput<'a> {
SnapshotInput {
url,
status_code: status,
reason,
content: content.to_string(),
headers: &EMPTY_HEADERS,
screenshot: screenshot.map(str::to_owned),
}
}
fn dir_file_count(dir: &Path) -> usize {
fs::read_dir(dir).expect("read_dir").count()
}
fn read_meta(dir: &Path, key: &str) -> serde_json::Value {
let raw = fs::read_to_string(dir.join(format!("{key}.meta.json"))).expect("meta.json");
serde_json::from_str(&raw).expect("meta json parse")
}
fn set_mtime(path: &Path, t: std::time::SystemTime) {
std::fs::File::options()
.write(true)
.open(path)
.unwrap_or_else(|e| panic!("open {}: {e}", path.display()))
.set_modified(t)
.unwrap_or_else(|e| panic!("set_modified {}: {e}", path.display()));
}
fn stamped_time(i: u64) -> std::time::SystemTime {
std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000 + i)
}
fn stamp_signature(dir: &Path, key: &str, i: u64) {
let t = stamped_time(i);
set_mtime(&dir.join(format!("{key}.html")), t);
set_mtime(&dir.join(format!("{key}.meta.json")), t);
}
#[tokio::test]
async fn rolling_eviction_keeps_dir_bounded_and_newest_observable() {
let dir = std::env::temp_dir().join(format!("crawlrs-snap-test-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
let snap = FailureSnapshotter::new(&dir, 6); let headers = HashMap::new();
for i in 0..8 {
let url = format!("https://x.com/err/{i}");
let reason = format!("rate limited {i}");
snap.snapshot(SnapshotInput {
url: &url,
status_code: 429,
reason: &reason,
content: "429".to_string(),
headers: &headers,
screenshot: None,
})
.await;
let key = FailureSnapshotter::signature_key(&url, 429, &reason);
stamp_signature(&dir, &key, i);
}
let file_count = std::fs::read_dir(&dir).unwrap().count();
assert!(
file_count <= 6,
"目录文件数应 ≤ max_total=6,实际 {file_count}"
);
let oldest_key =
FailureSnapshotter::signature_key("https://x.com/err/0", 429, "rate limited 0");
assert!(
!dir.join(format!("{oldest_key}.meta.json")).exists(),
"最旧签名的 meta 应被滚动删除"
);
let newest_key =
FailureSnapshotter::signature_key("https://x.com/err/7", 429, "rate limited 7");
assert!(
dir.join(format!("{newest_key}.meta.json")).exists(),
"最新签名的 meta 应保留"
);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn different_reason_yields_different_key() {
let k1 = FailureSnapshotter::signature_key("https://x.com/a", 403, "cloudflare challenge");
let k2 = FailureSnapshotter::signature_key("https://x.com/a", 403, "rate limited");
assert_ne!(k1, k2);
assert_eq!(
FailureSnapshotter::signature_key("https://x.com/a", 403, "cloudflare challenge"),
k1
);
}
#[test]
fn signature_key_sanitizes_path_separators_and_control_chars() {
let key = FailureSnapshotter::signature_key(
"https://x.com/a",
403,
"bad/../../etc\\passwd\n\t chall§enge",
);
assert!(!key.contains('/'), "key must not contain /: {key}");
assert!(!key.contains('\\'), "key must not contain \\: {key}");
assert!(
key.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-'),
"key must be filename-safe: {key}"
);
assert!(key.chars().count() <= 60);
}
#[tokio::test]
async fn same_signature_second_hit_keeps_file_count_and_updates_meta_count() {
let (snapshotter, dir) = temp_snapshotter(200);
let url = "https://x.com/challenge";
let reason = "cloudflare challenge detected";
snapshotter
.snapshot(sample_input(url, 403, reason, "<html>cf</html>", None))
.await;
let key = FailureSnapshotter::signature_key(url, 403, reason);
let count_after_first = dir_file_count(&dir);
assert!(count_after_first >= 2, "first hit writes html + meta");
snapshotter
.snapshot(sample_input(url, 403, reason, "<html>cf</html>", None))
.await;
assert_eq!(
dir_file_count(&dir),
count_after_first,
"second hit must not create new files"
);
assert_eq!(snapshotter.hit_count(&key), Some(2));
assert_eq!(read_meta(&dir, &key)["count"], 2, "meta count must be 2");
}
#[tokio::test]
async fn different_reason_writes_distinct_snapshots() {
let (snapshotter, dir) = temp_snapshotter(200);
snapshotter
.snapshot(sample_input(
"https://x.com/a",
403,
"reason-one",
"<html>1</html>",
None,
))
.await;
snapshotter
.snapshot(sample_input(
"https://x.com/a",
403,
"reason-two",
"<html>2</html>",
None,
))
.await;
assert!(dir
.join(format!(
"{}.html",
FailureSnapshotter::signature_key("https://x.com/a", 403, "reason-one")
))
.exists());
assert!(dir
.join(format!(
"{}.html",
FailureSnapshotter::signature_key("https://x.com/a", 403, "reason-two")
))
.exists());
}
#[tokio::test]
async fn meta_json_carries_response_metadata() {
let (snapshotter, dir) = temp_snapshotter(200);
let mut headers = HashMap::new();
headers.insert("server".to_string(), "cloudflare".to_string());
let input = SnapshotInput {
url: "https://x.com/meta",
status_code: 429,
reason: "rate limited",
content: "<html>429</html>".to_string(),
headers: &headers,
screenshot: None,
};
snapshotter.snapshot(input).await;
let key = FailureSnapshotter::signature_key("https://x.com/meta", 429, "rate limited");
let meta = read_meta(&dir, &key);
assert_eq!(meta["url"], "https://x.com/meta");
assert_eq!(meta["status"], 429);
assert_eq!(meta["reason"], "rate limited");
assert_eq!(meta["headers"]["server"], "cloudflare");
assert_eq!(meta["count"], 1);
}
#[tokio::test]
async fn max_total_rolls_oldest_files_to_admit_new_snapshot() {
let (snapshotter, dir) = temp_snapshotter(2);
let key_one = FailureSnapshotter::signature_key("https://x.com/one", 403, "r1");
snapshotter
.snapshot(sample_input(
"https://x.com/one",
403,
"r1",
"<html>1</html>",
None,
))
.await;
assert_eq!(snapshotter.total_written(), 2, "html + meta = 2 files");
assert!(dir.join(format!("{key_one}.html")).exists());
stamp_signature(&dir, &key_one, 0);
let key_two = FailureSnapshotter::signature_key("https://x.com/two", 403, "r2");
snapshotter
.snapshot(sample_input(
"https://x.com/two",
403,
"r2",
"<html>2</html>",
None,
))
.await;
stamp_signature(&dir, &key_two, 1);
assert_eq!(
dir_file_count(&dir),
2,
"rolling keeps directory bounded at max_total"
);
assert!(
!dir.join(format!("{key_one}.html")).exists(),
"oldest snapshot files evicted"
);
assert!(
dir.join(format!("{key_two}.html")).exists(),
"new snapshot admitted after eviction"
);
assert_eq!(snapshotter.hit_count(&key_two), Some(1), "new key counted");
assert_eq!(
snapshotter.total_written(),
2,
"cursor nets writes/evictions"
);
}
#[cfg(unix)]
#[tokio::test]
async fn snapshot_files_written_with_owner_only_permissions() {
use std::os::unix::fs::PermissionsExt;
let (snapshotter, dir) = temp_snapshotter(200);
snapshotter
.snapshot(sample_input(
"https://x.com/perm",
403,
"waf",
"<html/>",
None,
))
.await;
let key = FailureSnapshotter::signature_key("https://x.com/perm", 403, "waf");
for name in [format!("{key}.html"), format!("{key}.meta.json")] {
let mode = fs::metadata(dir.join(&name))
.expect("snapshot file exists")
.permissions()
.mode();
assert_eq!(
mode & 0o777,
0o600,
"{name} must be owner-only, got {:o}",
mode & 0o777
);
}
}
#[tokio::test]
async fn counts_map_caps_distinct_keys() {
let (snapshotter, _dir) = temp_snapshotter(1);
let first_key = FailureSnapshotter::signature_key("https://x.com/k0", 403, "r");
for i in 0..=MAX_TRACKED_KEYS {
snapshotter
.snapshot(sample_input(
&format!("https://x.com/k{i}"),
403,
"r",
"x",
None,
))
.await;
}
assert_eq!(
snapshotter.hit_count(&first_key),
Some(1),
"early key still tracked"
);
let overflow_key = FailureSnapshotter::signature_key(
&format!("https://x.com/k{MAX_TRACKED_KEYS}"),
403,
"r",
);
assert_eq!(
snapshotter.hit_count(&overflow_key),
None,
"key beyond cap must not enter the map"
);
}
#[tokio::test]
async fn screenshot_some_writes_decoded_jpg() {
let (snapshotter, dir) = temp_snapshotter(200);
snapshotter
.snapshot(sample_input(
"https://x.com/shot",
403,
"waf",
"<html>shot</html>",
Some("aGVsbG8="),
))
.await;
let key = FailureSnapshotter::signature_key("https://x.com/shot", 403, "waf");
let jpg = fs::read(dir.join(format!("{key}.jpg"))).expect("jpg file");
assert_eq!(jpg, b"hello", "jpg must be base64-decoded bytes");
assert_eq!(snapshotter.total_written(), 3, "html + meta + jpg");
}
#[tokio::test]
async fn screenshot_none_writes_no_jpg() {
let (snapshotter, dir) = temp_snapshotter(200);
snapshotter
.snapshot(sample_input(
"https://x.com/noshot",
403,
"waf",
"<html>noshot</html>",
None,
))
.await;
let key = FailureSnapshotter::signature_key("https://x.com/noshot", 403, "waf");
assert!(!dir.join(format!("{key}.jpg")).exists());
assert!(dir.join(format!("{key}.html")).exists());
}
#[tokio::test]
async fn invalid_base64_screenshot_is_silently_skipped() {
let (snapshotter, dir) = temp_snapshotter(200);
snapshotter
.snapshot(sample_input(
"https://x.com/b64",
403,
"waf",
"<html>b64</html>",
Some("!!!not-base64!!!"),
))
.await;
let key = FailureSnapshotter::signature_key("https://x.com/b64", 403, "waf");
assert!(
!dir.join(format!("{key}.jpg")).exists(),
"bad base64: no jpg"
);
assert!(
dir.join(format!("{key}.html")).exists(),
"html still written"
);
assert_eq!(snapshotter.total_written(), 2);
}
#[tokio::test]
async fn unwritable_dir_returns_silently_no_panic() {
let tmp = tempfile::tempdir().expect("tempdir");
let file_path = tmp.path().join("not-a-dir");
fs::write(&file_path, b"occupied").expect("write blocker file");
let snapshotter = FailureSnapshotter::new(file_path.clone(), 200);
snapshotter
.snapshot(sample_input("https://x.com/x", 403, "r", "<html/>", None))
.await;
assert_eq!(snapshotter.total_written(), 0, "nothing written");
assert!(file_path.is_file(), "blocker file untouched");
}
#[test]
fn from_config_empty_dir_is_none_and_populated_is_some() {
let disabled = ObservabilitySettings {
failure_snapshot_dir: String::new(),
failure_snapshot_max_total: 200,
};
assert!(FailureSnapshotter::from_config(&disabled).is_none());
let enabled = ObservabilitySettings {
failure_snapshot_dir: "data/failure_snapshots".to_string(),
failure_snapshot_max_total: 50,
};
let s = FailureSnapshotter::from_config(&enabled).expect("some");
assert_eq!(s.total_written(), 0);
}
#[tokio::test]
async fn concurrent_snapshots_keep_dir_bounded_and_accounted() {
let (snapshotter, dir) = temp_snapshotter(4);
let snapshotter = Arc::new(snapshotter);
let mut handles = Vec::new();
for i in 0..16 {
let snap = snapshotter.clone();
handles.push(tokio::spawn(async move {
let url = format!("https://x.com/c{i}");
let reason = format!("rate limited {i}");
snap.snapshot(SnapshotInput {
url: &url,
status_code: 429,
reason: &reason,
content: format!("<html>{i}</html>"),
headers: &EMPTY_HEADERS,
screenshot: None,
})
.await;
}));
}
for handle in handles {
handle.await.expect("snapshot task");
}
let files = fs::read_dir(&dir).expect("read_dir").count() as u64;
assert!(
files <= 4,
"dir must stay bounded by max_total, got {files}"
);
assert_eq!(
snapshotter.total_written(),
files,
"for unique keys the net write counter must equal on-disk file count"
);
}
}