web_simhash/
lib.rs

1//! Web-oriented SimHash.
2//!
3//! This crate implements the public pieces of the Google WWW 2007 near-duplicate
4//! detection paper: weighted-feature SimHash, 64-bit Hamming comparison, and a
5//! multi-table online lookup index. It does not attempt to reproduce Google's
6//! private crawler feature pipeline.
7
8mod fingerprint;
9mod index;
10mod web;
11
12pub use fingerprint::{
13    fnv1a64, FeatureHash, SimHash64, SimHashOptions, TieBreaker, WeightedFeature,
14    DEFAULT_NEAR_DUP_DISTANCE,
15};
16pub use index::{IndexConfig, Match, SimHashIndex, TableSpec};
17pub use web::{WebFeatureExtractor, WebFeatureOptions};
18
19#[cfg(test)]
20mod tests {
21    use super::*;
22
23    #[test]
24    fn end_to_end_html_near_duplicate_lookup() {
25        let extractor = WebFeatureExtractor::default();
26        let canonical = SimHash64::from_html(
27            r#"
28            <html>
29              <head><title>Ignored title chrome</title></head>
30              <body>
31                <nav>Home About Contact</nav>
32                <main>
33                  <h1>Rust SimHash for web crawling</h1>
34                  <p>Near duplicate web documents differ mostly in counters and ads.</p>
35                  <p>The crawler should avoid fetching the same useful content twice.</p>
36                </main>
37                <aside>Advertisement 12345</aside>
38              </body>
39            </html>
40            "#,
41            &extractor,
42        );
43        let duplicate = SimHash64::from_html(
44            r#"
45            <html>
46              <body>
47                <main>
48                  <h1>Rust SimHash for web crawling</h1>
49                  <p>Near duplicate web documents differ mostly in counters and ads.</p>
50                  <p>The crawler should avoid fetching the same useful content twice.</p>
51                </main>
52                <aside>Advertisement 99999</aside>
53              </body>
54            </html>
55            "#,
56            &extractor,
57        );
58
59        let mut index = SimHashIndex::new_google64_k3();
60        index.insert("canonical", canonical);
61        let matches = index.query(duplicate);
62
63        assert_eq!(matches.len(), 1);
64        assert_eq!(matches[0].id, "canonical");
65        assert!(matches[0].distance <= DEFAULT_NEAR_DUP_DISTANCE);
66    }
67}
web_simhash/lib.rs

web_simhash/
lib.rs
