Struct HttpKind 

pub struct HttpKind { /* private fields */ }

HTTP fetching behavior used by HttpCrawler.

End-to-end example

use std::sync::Arc;
use millipede_core::{crawler::Crawler, request::Request};
use millipede_http::{HttpContext, HttpKind};
use millipede_storage_memory::MemoryStorageClient;

let crawler = Crawler::builder(HttpKind::builder().build()?)
    .request_handler(|ctx: HttpContext| async move {
        println!("{} {}", ctx.response.status, ctx.request.url);
        Ok(())
    })
    .storage_client(Arc::new(MemoryStorageClient::new()))
    .build()
    .await?;

let stats = crawler.run([Request::get("https://example.com").build()?]).await?;
assert_eq!(stats.requests_finished, 1);

Implementations

impl HttpKind

pub fn builder() -> HttpKindBuilder

Starts configuring an HTTP crawler kind.

pub fn new() -> Result<Self, HttpClientError>

Creates an HTTP kind with all defaults.

Trait Implementations

impl CrawlerKind for HttpKind

type Context = HttpContext

The context passed to user handlers and lifecycle hooks.

fn start<'a>( &'a self, env: &'a CrawlerEnv, ) -> BoxFuture<'a, Result<(), CrawlError>>

Runs once before the crawler fetches any request.

fn execute<'a>( &'a self, env: RequestEnv<'a>, ) -> BoxFuture<'a, Result<Self::Context, CrawlError>>

Executes one request attempt and constructs its handler context.

fn observe(&self, ctx: &Self::Context) -> AttemptObservation

Called once after execute() succeeds; feeds statistics, HandledRequest, and RetryStrategy metadata.

fn cleanup( &self, outcome: RequestOutcome<Self::Context>, ) -> BoxFuture<'_, Result<(), CrawlError>>

Runs after every attempt concludes, regardless of its outcome.

fn stop<'a>( &'a self, _env: &'a CrawlerEnv, ) -> BoxFuture<'a, Result<(), CrawlError>>

Runs once when crawler shutdown begins.

fn before_request<'a>( &'a self, prep: &'a mut RequestPrep, ) -> Pin<Box<dyn Future<Output = Result<(), CrawlError>> + Send + 'a>>

Mutates a request before an attempt executes.

fn after_success<'a>( &'a self, ctx: &'a mut Self::Context, ) -> Pin<Box<dyn Future<Output = Result<(), CrawlError>> + Send + 'a>>

Runs after the user handler succeeds.