Struct RequestQueueWithSitemap 

pub struct RequestQueueWithSitemap { /* private fields */ }

A request queue that lazily feeds sitemap entries into another queue.

The wrapper lets the crawler consume already-queued work first, then drains the sitemap in bounded batches whenever the inner queue becomes empty.

Example

use std::sync::Arc;
use millipede_core::{
    sitemap::{RequestQueueWithSitemap, SitemapRequestList},
    storage::RequestQueue,
};

let queue: Arc<dyn RequestQueue> =
    Arc::new(RequestQueueWithSitemap::new(inner, list).batch_size(64));

Implementations

impl RequestQueueWithSitemap

pub fn new(inner: Arc<dyn RequestQueue>, list: SitemapRequestList) -> Self

Wraps inner with a sitemap source, draining at most 32 entries per pass.

pub fn batch_size(self, n: usize) -> Self

Sets the maximum number of sitemap entries drained per pass.

Trait Implementations

impl RequestQueue for RequestQueueWithSitemap

fn is_empty<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = StorageResult<bool>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Returns false while sitemap entries remain undrained, preventing premature engine termination; once the sitemap is drained, delegates to the inner queue.

fn is_finished<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = StorageResult<bool>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Returns false while sitemap entries remain undrained, preventing premature engine termination; once the sitemap is drained, delegates to the inner queue.

fn pending_count<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = StorageResult<u64>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Returns only the inner queue’s pending count. Undrained sitemap entries are not included because their count is unknown until the sitemap is streamed.

fn add<'life0, 'async_trait>( &'life0 self, req: Request, opts: AddOptions, ) -> Pin<Box<dyn Future<Output = StorageResult<QueueOpInfo>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Adds a request and returns its deduplication status.

fn add_batch<'life0, 'async_trait>( &'life0 self, reqs: Vec<RequestSource>, opts: AddOptions, ) -> Pin<Box<dyn Future<Output = StorageResult<BatchAddHandle>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Adds multiple request sources and returns a handle for deferred completion.

fn fetch_next<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = StorageResult<Option<Lease>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Hands temporary ownership of the next request to the caller as a lease.

fn mark_handled<'life0, 'async_trait>( &'life0 self, lease: Lease, ) -> Pin<Box<dyn Future<Output = StorageResult<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Marks a leased request as successful and consumes its lease.

fn reclaim<'life0, 'async_trait>( &'life0 self, lease: Lease, opts: ReclaimOptions, ) -> Pin<Box<dyn Future<Output = StorageResult<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Re-queues a lease, incrementing retry count unless opts disables it.

fn renew<'life0, 'life1, 'async_trait>( &'life0 self, lease_id: &'life1 LeaseId, extend_by: Duration, ) -> Pin<Box<dyn Future<Output = StorageResult<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Extends an active lease deadline, returning LeaseNotFound if it is unknown or completed.

fn abandon<'life0, 'async_trait>( &'life0 self, lease: Lease, ) -> Pin<Box<dyn Future<Output = StorageResult<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Re-queues and consumes a lease without incrementing its request retry count.

fn handled_count<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = StorageResult<u64>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Returns the number of successfully handled requests.