argus-dedupe 0.1.0

Content deduplication utilities for web crawling
Documentation 
1
2
3
4
5
6
7

use async_trait::async_trait;

#[async_trait]
pub trait SeenSet: Send + Sync {
    /// Returns true if the URL was newly inserted, false if it was already seen.
    async fn insert_if_new(&self, url: String) -> bool;
}