Crate spider_util 

spider-util

Utility types and traits for the spider-lib framework.

Example

use spider_util::{request::Request, item::ScrapedItem};
use url::Url;

let url = Url::parse("https://example.com").unwrap();
let request = Request::new(url);

#[spider_macro::scraped_item]
struct Article {
    title: String,
    content: String,
}

Re-exports

pub use formatters::format_bytes;

pub use formatters::format_duration;

pub use formatters::format_request_time;

pub use formatters::ByteFormatter;

pub use formatters::DefaultByteFormatter;

pub use formatters::DefaultDurationFormatter;

pub use formatters::DefaultRateCalculator;

pub use formatters::DurationFormatter;

pub use formatters::RateCalculator;

pub use http_client::HttpClient;

pub use constants::*;

Modules

bloom

Bloom Filter Module

constants

Global constants used across the spider-lib workspace.

error

Custom error types for the spider-core framework.

formatters

Formatting utilities for the spider framework.

http_client

HTTP Client trait for fetching web content.

item

Data structures for scraped items in spider-lib.

metrics

Metrics Utilities

request

Data structures for representing HTTP requests in spider-lib.

response

Data structures and utilities for handling HTTP responses in spider-lib.

selector

Selector Cache Module

util

Utility functions for the spider-lib framework.