Struct InMemoryFrontier 

pub struct InMemoryFrontier { /* private fields */ }

A simple in-memory URL frontier with deduplication.

Uses BFS ordering (FIFO) by default.

Implementations

impl InMemoryFrontier

pub fn new() -> Self

Create a new empty InMemoryFrontier.

Trait Implementations

impl Debug for InMemoryFrontier

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for InMemoryFrontier

fn default() -> Self

Returns the “default value” for a type.

impl Frontier for InMemoryFrontier

fn push<'life0, 'async_trait>( &'life0 self, entry: FrontierEntry, ) -> Pin<Box<dyn Future<Output = Result<(), CrawlError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Push a new entry onto the frontier.

fn pop<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<Option<FrontierEntry>, CrawlError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Pop the next entry from the frontier.

fn pop_batch<'life0, 'async_trait>( &'life0 self, n: usize, ) -> Pin<Box<dyn Future<Output = Result<Vec<FrontierEntry>, CrawlError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Pop up to n entries from the frontier.

fn is_seen<'life0, 'life1, 'async_trait>( &'life0 self, url: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<bool, CrawlError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Check whether a URL has already been seen.

fn mark_seen<'life0, 'life1, 'async_trait>( &'life0 self, url: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<(), CrawlError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Mark a URL as seen.

fn len<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<usize, CrawlError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Return the number of entries in the frontier.

fn is_empty<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<bool, CrawlError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Check whether the frontier is empty.