pub async fn crawl_http(
seed: &str,
robots: RobotsPolicy,
opts: &ScrapeOpts,
limit: usize,
max_depth: usize,
same_host: bool,
) -> Result<Value, CliError>Expand description
BFS crawl from seed URL (HTTP), limited by limit and max_depth.