crawly 0.1.9

A lightweight async Web crawler in Rust, optimized for concurrent scraping while respecting `robots.txt` rules.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

[package]
name = "crawly"
description = "A lightweight async Web crawler in Rust, optimized for concurrent scraping while respecting `robots.txt` rules."
version = "0.1.9"
authors = ["Dario Cancelliere <dario@ai-chat.it>"]
edition = "2021"
repository = "https://github.com/aichat-bot/crawly"
homepage = "https://ai-chat.it"
license-file = "LICENSE.md"
readme = "README.md"

[features]
rustls-tls = ["reqwest/rustls-tls"]
native-tls = ["reqwest/native-tls"]
default-tls = ["reqwest/default-tls"]
default = ["rustls-tls"]

[dependencies]
# Tokio and Futures
tokio = { version = "^1.38", default-features = false, features = ["sync", "time"] }
futures = { version = "^0.3", default-features = false, features = ["alloc"] }

# Tracing
tracing = { version = "^0.1", default-features = false, features = ["attributes"] }

# Utils
anyhow = { version = "^1.0", default-features = false, features = ["std"] }
reqwest = { version = "^0.12", default-features = false }
scraper = { version = "^0.19", default-features = false }
async-recursion = { version = "^1.1", default-features = false }
robotstxt = { version = "^0.3", default-features = false }
indexmap = { version = "^2.2", default-features = false, features = ["std"] }
mime = { version = "^0.3", default-features = false }
infer = { version = "^0.16", default-features = false, features = ["std"] }