spire-core 0.2.0

Core types and traits for Spire web scraping framework
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

//! Seen sets: what a [`Dedup`](crate::frontier::facade::Dedup) frontier remembers.

mod bloom;
mod memory;

use std::future::Future;

pub use bloom::{BloomSeen, BloomSnapshot};
pub use memory::InMemSeen;

use crate::Result;
use crate::frontier::seed::Fingerprint;

/// Remembers which [`Fingerprint`]s a [`Dedup`](crate::frontier::facade::Dedup) frontier has seen.
pub trait SeenSet: Send + Sync + 'static {
    /// Records `fingerprint`, returning `true` if it was not seen before.
    ///
    /// # Errors
    ///
    /// Returns an error if the underlying storage fails.
    fn insert(&self, fingerprint: Fingerprint) -> impl Future<Output = Result<bool>> + Send;
}

impl<S: SeenSet> SeenSet for std::sync::Arc<S> {
    fn insert(&self, fingerprint: Fingerprint) -> impl Future<Output = Result<bool>> + Send {
        S::insert(self, fingerprint)
    }
}