Struct Frontier 

pub struct Frontier { /* private fields */ }

Thread-safe frontier

Implementations

impl Frontier

pub fn new( strategy: CrawlStrategy, max_depth: u32, max_per_domain: u32, max_total: u32, ) -> Self

New frontier

pub fn push(&self, entry: CrawlEntry) -> bool

Add a URL to the frontier

pub fn push_many(&self, entries: Vec<CrawlEntry>) -> usize

Add multiple URLs

pub fn pop(&self) -> Option<CrawlEntry>

Get the next URL

pub fn len(&self) -> usize

Queue size

pub fn is_empty(&self) -> bool

Is queue empty?

pub fn seen_count(&self) -> usize

Number of seen URLs

pub fn has_seen(&self, url: &Url) -> bool

URL already seen?

pub fn mark_seen(&self, url: &Url)

Mark a URL as seen without adding it to the queue

pub fn clear(&self)

Clear the frontier

pub fn domain_stats(&self) -> HashMap<String, u32>

Stats per domain