pagesift_store/
snapshot.rs

1//! 原始 HTML 快照（F-8.3）：zstd 压缩，按内容的 BLAKE3 哈希去重，按 URL 和时间查询。
2//!
3//! 每个 URL 一个追加列表，元素是 (时间, 内容哈希, 原始长度)；内容本身按哈希只存一份。
4//! 内容可能被 LRU 淘汰，这时 [`SnapshotMeta::available`] 为 false。
5
6use std::fmt;
7use std::ops::RangeBounds;
8use std::time::SystemTime;
9
10use crate::engine::ns;
11use crate::tasks::{list_push, list_scan};
12use crate::{Error, Result, Store, from_ms, to_ms};
13
14#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
15pub struct ContentHash(pub [u8; 32]);
16
17impl ContentHash {
18    pub fn of(data: &[u8]) -> Self {
19        Self(*blake3::hash(data).as_bytes())
20    }
21}
22
23impl fmt::Display for ContentHash {
24    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
25        self.0.iter().try_for_each(|b| write!(f, "{b:02x}"))
26    }
27}
28
29impl fmt::Debug for ContentHash {
30    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
31        write!(f, "ContentHash({self})")
32    }
33}
34
35#[derive(Debug, Clone, PartialEq, Eq)]
36pub struct SnapshotMeta {
37    pub at: SystemTime,
38    pub hash: ContentHash,
39    pub raw_len: u32,
40    pub available: bool,
41}
42
43#[derive(Debug, Clone, PartialEq, Eq)]
44pub struct SnapshotPut {
45    pub hash: ContentHash,
46    /// 同样的内容已经存过，这次没有再写内容。
47    pub deduped: bool,
48    /// 压缩后的大小（去重时为 0）。
49    pub stored_bytes: usize,
50}
51
52pub struct Snapshots<'a> {
53    store: &'a Store,
54}
55
56impl<'a> Snapshots<'a> {
57    pub(crate) fn new(store: &'a Store) -> Self {
58        Self { store }
59    }
60
61    pub fn put(&self, url: &str, html: &[u8]) -> Result<SnapshotPut> {
62        self.put_at(url, html, SystemTime::now())
63    }
64
65    pub fn put_at(&self, url: &str, html: &[u8], at: SystemTime) -> Result<SnapshotPut> {
66        let cfg = self.store.config();
67        if html.len() > cfg.snapshot_max_bytes {
68            return Err(Error::ValueTooLarge {
69                size: html.len(),
70                limit: cfg.snapshot_max_bytes,
71            });
72        }
73        let hash = ContentHash::of(html);
74        // 先压缩，避免持有写锁时做 CPU 密集的工作；去重命中时浪费一次压缩。
75        let deduped = self.store.contains(ns::SNAP_BLOB, &hash.0)?;
76        let compressed = if deduped {
77            Vec::new()
78        } else {
79            zstd::bulk::compress(html, cfg.snapshot_zstd_level)?
80        };
81        let mut item = Vec::with_capacity(36);
82        item.extend_from_slice(&hash.0);
83        item.extend_from_slice(&(html.len() as u32).to_le_bytes());
84
85        let mut tx = self.store.tx();
86        let mut stored_bytes = 0;
87        if !tx.contains(ns::SNAP_BLOB, &hash.0)? {
88            let data = if deduped {
89                zstd::bulk::compress(html, cfg.snapshot_zstd_level)?
90            } else {
91                compressed
92            };
93            stored_bytes = data.len();
94            tx.put(ns::SNAP_BLOB, &hash.0, &data, to_ms(at))?;
95        }
96        list_push(&mut tx, ns::SNAP_ITEM, url.as_bytes(), &item, to_ms(at))?;
97        drop(tx);
98        self.store.finish();
99        Ok(SnapshotPut {
100            hash,
101            deduped: stored_bytes == 0,
102            stored_bytes,
103        })
104    }
105
106    /// 按哈希取回原始 HTML。
107    pub fn get(&self, hash: &ContentHash) -> Result<Option<Vec<u8>>> {
108        let Some((_, data)) = self.store.get(ns::SNAP_BLOB, &hash.0)? else {
109            return Ok(None);
110        };
111        let html = zstd::stream::decode_all(data.as_slice())
112            .map_err(|e| Error::Corrupt(format!("snapshot {hash}: {e}")))?;
113        Ok(Some(html))
114    }
115
116    /// 某个 URL 在时间范围内的快照，按时间先后排列。
117    pub fn list(
118        &self,
119        url: &str,
120        range: impl RangeBounds<SystemTime>,
121    ) -> Result<Vec<SnapshotMeta>> {
122        let mut out = Vec::new();
123        for (_, ts, v) in list_scan(self.store, ns::SNAP_ITEM, url.as_bytes(), usize::MAX)? {
124            let at = from_ms(ts);
125            if !range.contains(&at) || v.len() != 36 {
126                continue;
127            }
128            let hash = ContentHash(v[..32].try_into().expect("32 bytes"));
129            out.push(SnapshotMeta {
130                at,
131                hash,
132                raw_len: u32::from_le_bytes(v[32..36].try_into().expect("4 bytes")),
133                available: self.store.contains(ns::SNAP_BLOB, &hash.0)?,
134            });
135        }
136        out.sort_by_key(|m| m.at);
137        Ok(out)
138    }
139
140    /// 最新的一份快照。
141    pub fn latest(&self, url: &str) -> Result<Option<SnapshotMeta>> {
142        Ok(self.list(url, ..)?.pop())
143    }
144}
pagesift_store/snapshot.rs

pagesift_store/
snapshot.rs
