recursive_scraper 0.3.0

Constant-frequency recursive CLI web scraper with frequency, filtering, file directory, and many other options for scraping HTML, images and other files.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

[package]
name = "recursive_scraper"
authors = ["Steven Hé (Sīchàng)"]
version = "0.3.0"

categories = [
    "accessibility",
    "asynchronous",
    "command-line-utilities",
    "web-programming::http-client",
]
description = "Constant-frequency recursive CLI web scraper with frequency, filtering, file directory, and many other options for scraping HTML, images and other files."
edition = "2021"
keywords = ["crawler", "recursive", "scraper", "spider", "web"]
license = "MIT OR Apache-2.0"
readme = "README.md"
repository = "https://github.com/SichangHe/scraper"

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
anyhow = "1.0.58"
bytes = "1.2.0"
clap = { version = "3.2.15", features = ["derive"] }
regex = "1.6.0"
reqwest = "0.11.11"
select = "0.5.0"
serde = { version = "1.0.140", features = ["derive"] }
tokio = { version = "1.20.0", features = [
    "rt-multi-thread",
    "macros",
    "fs",
    "time",
    "parking_lot",
] }
toml = "0.5.9"

[profile.release]
opt-level = 2
lto = "thin"