use crate::utils::dedup::{DedupError, DedupResult, UrlInterner};
use crate::utils::url_normalizer::UrlNormalizer;
#[derive(Debug)]
pub struct Deduplicator {
normalizer: UrlNormalizer,
interner: UrlInterner,
}
impl Default for Deduplicator {
fn default() -> Self {
Self {
normalizer: UrlNormalizer::with_default(),
interner: UrlInterner::new(),
}
}
}
impl Deduplicator {
pub fn new() -> Self {
Self::default()
}
pub fn with_config(strip_query: bool, capacity: usize) -> Self {
Self {
normalizer: UrlNormalizer::new(strip_query),
interner: UrlInterner::with_capacity(capacity),
}
}
pub fn check(&self, url: &str) -> Result<DedupResult, DedupError> {
let normalized = self.normalizer.normalize(url)?;
let mut variants = self.normalizer.permutations(&normalized);
if !variants.contains(&normalized) {
variants.push(normalized.clone());
}
for v in &variants {
if !self.interner.definitely_absent(v) {
return Ok(DedupResult::MaybeExisting {
normalized,
variants,
});
}
}
Ok(DedupResult::DefinitelyNew { normalized })
}
pub fn check_and_insert(&mut self, url: &str) -> Result<DedupResult, DedupError> {
let result = self.check(url)?;
if let DedupResult::DefinitelyNew { ref normalized } = result {
self.interner.insert(normalized);
}
Ok(result)
}
pub fn insert(&mut self, url: &str) {
self.interner.insert(url);
}
pub fn extend<I, S>(&mut self, urls: I)
where
I: IntoIterator<Item = S>,
S: AsRef<str>,
{
self.interner.extend(urls);
}
pub fn len(&self) -> usize {
self.interner.len()
}
pub fn is_empty(&self) -> bool {
self.interner.is_empty()
}
pub fn clear(&mut self) {
self.interner.clear();
}
}
impl Clone for Deduplicator {
fn clone(&self) -> Self {
Self {
normalizer: self.normalizer,
interner: self.interner.clone(),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::utils::dedup::{DedupError, DedupResult};
#[test]
fn test_new_is_empty() {
let d = Deduplicator::new();
assert!(d.is_empty());
assert_eq!(d.len(), 0);
}
#[test]
fn test_with_config_strip_query() {
let d = Deduplicator::with_config(true, 1000);
let r1 = d.check("https://example.com/path?a=1").unwrap();
let r2 = d.check("https://example.com/path?b=2").unwrap();
assert!(matches!(r1, DedupResult::DefinitelyNew { .. }));
assert!(matches!(r2, DedupResult::DefinitelyNew { .. }));
if let (
DedupResult::DefinitelyNew { normalized: n1 },
DedupResult::DefinitelyNew { normalized: n2 },
) = (r1, r2)
{
assert_eq!(n1, n2);
assert_eq!(n1, "https://example.com/path");
}
}
#[test]
fn test_check_definitely_new_for_unseen_url() {
let d = Deduplicator::new();
let result = d.check("https://example.com/path").unwrap();
assert!(matches!(result, DedupResult::DefinitelyNew { .. }));
}
#[test]
fn test_check_returns_normalized_in_definitely_new() {
let d = Deduplicator::new();
let result = d.check("HTTPS://Example.COM/Path/?b=2&a=1#frag").unwrap();
if let DedupResult::DefinitelyNew { normalized } = result {
assert_eq!(normalized, "https://example.com/Path?a=1&b=2");
} else {
panic!("expected DefinitelyNew");
}
}
#[test]
fn test_check_maybe_existing_after_insert() {
let mut d = Deduplicator::new();
d.insert("https://example.com/path");
let result = d.check("https://example.com/path").unwrap();
assert!(matches!(result, DedupResult::MaybeExisting { .. }));
}
#[test]
fn test_check_maybe_existing_includes_variants() {
let mut d = Deduplicator::new();
d.insert("https://example.com/path");
let result = d.check("https://example.com/path").unwrap();
if let DedupResult::MaybeExisting {
normalized,
variants,
} = result
{
assert_eq!(normalized, "https://example.com/path");
assert!(variants.contains(&normalized));
assert!(variants.iter().any(|v| v.contains("www.example.com")));
} else {
panic!("expected MaybeExisting");
}
}
#[test]
fn test_check_www_and_non_www_are_equivalent() {
let mut d = Deduplicator::new();
d.insert("https://example.com/path");
let result = d.check("https://www.example.com/path").unwrap();
assert!(matches!(result, DedupResult::MaybeExisting { .. }));
}
#[test]
fn test_check_http_and_https_are_equivalent() {
let mut d = Deduplicator::new();
d.insert("https://example.com/path");
let result = d.check("http://example.com/path").unwrap();
assert!(matches!(result, DedupResult::MaybeExisting { .. }));
}
#[test]
fn test_check_index_html_equivalent() {
let mut d = Deduplicator::new();
d.insert("https://example.com/path");
let result = d.check("https://example.com/path/index.html").unwrap();
assert!(matches!(result, DedupResult::MaybeExisting { .. }));
}
#[test]
fn test_check_invalid_url_returns_err() {
let d = Deduplicator::new();
assert!(d.check("not a url").is_err());
}
#[test]
fn test_check_empty_url_returns_err() {
let d = Deduplicator::new();
assert!(d.check("").is_err());
}
#[test]
fn test_check_unsupported_scheme_returns_err() {
let d = Deduplicator::new();
let result = d.check("ftp://example.com/file");
match result {
Err(DedupError::Normalize(_)) => {}
Ok(_) => panic!("expected error for ftp scheme"),
}
}
#[test]
fn test_insert_then_check_is_maybe_existing() {
let mut d = Deduplicator::new();
let normalized = "https://example.com/path";
d.insert(normalized);
let result = d.check(normalized).unwrap();
assert!(matches!(result, DedupResult::MaybeExisting { .. }));
}
#[test]
fn test_insert_normalized_form_not_raw_form() {
let mut d = Deduplicator::new();
d.insert("HTTPS://Example.COM/Path");
let result = d.check("https://example.com/Path").unwrap();
match result {
DedupResult::DefinitelyNew { .. } | DedupResult::MaybeExisting { .. } => {}
}
}
#[test]
fn test_check_and_insert_atomic_for_definitely_new() {
let mut d = Deduplicator::new();
let result = d.check_and_insert("https://example.com/path").unwrap();
assert!(matches!(result, DedupResult::DefinitelyNew { .. }));
let result2 = d.check("https://example.com/path").unwrap();
assert!(matches!(result2, DedupResult::MaybeExisting { .. }));
}
#[test]
fn test_check_and_insert_returns_maybe_existing_without_insert() {
let mut d = Deduplicator::new();
d.insert("https://example.com/path");
let result = d.check_and_insert("https://example.com/path").unwrap();
assert!(matches!(result, DedupResult::MaybeExisting { .. }));
assert_eq!(d.len(), 1);
}
#[test]
fn test_check_and_insert_concurrent_safety() {
let mut d = Deduplicator::new();
let url = "https://example.com/path";
let result_a = d.check_and_insert(url).unwrap();
assert!(matches!(result_a, DedupResult::DefinitelyNew { .. }));
let result_b = d.check_and_insert(url).unwrap();
assert!(matches!(result_b, DedupResult::MaybeExisting { .. }));
}
#[test]
fn test_clear_resets_state() {
let mut d = Deduplicator::new();
d.insert("https://example.com/path");
assert!(!d.is_empty());
d.clear();
assert!(d.is_empty());
let result = d.check("https://example.com/path").unwrap();
assert!(matches!(result, DedupResult::DefinitelyNew { .. }));
}
#[test]
fn test_clone_independence() {
let mut d = Deduplicator::new();
d.insert("https://example.com/path");
let mut cloned = d.clone();
cloned.insert("https://other.com/path");
let result_orig = d.check("https://other.com/path").unwrap();
assert!(matches!(result_orig, DedupResult::DefinitelyNew { .. }));
let result_clone = cloned.check("https://other.com/path").unwrap();
assert!(matches!(result_clone, DedupResult::MaybeExisting { .. }));
}
#[test]
fn test_scrape_flow_simulation() {
let mut d = Deduplicator::new();
let urls = vec![
"https://example.com/page1",
"https://example.com/page2",
"https://example.com/page3",
];
let mut to_enqueue: Vec<String> = Vec::new();
let mut to_db_check: Vec<String> = Vec::new();
for url in &urls {
match d.check_and_insert(url).unwrap() {
DedupResult::DefinitelyNew { normalized } => {
to_enqueue.push(normalized.clone());
}
DedupResult::MaybeExisting { normalized, .. } => {
to_db_check.push(normalized);
}
}
}
assert_eq!(to_enqueue.len(), 3);
assert!(to_db_check.is_empty());
let urls2 = vec![
"https://example.com/page1", "https://example.com/page4", ];
for url in &urls2 {
match d.check_and_insert(url).unwrap() {
DedupResult::DefinitelyNew { normalized } => {
to_enqueue.push(normalized.clone());
}
DedupResult::MaybeExisting { normalized, .. } => {
to_db_check.push(normalized);
}
}
}
assert!(to_enqueue.iter().any(|u| u == "https://example.com/page4"));
assert!(to_db_check.iter().any(|u| u.contains("example.com/page1")));
}
}