millipede 0.1.0

An idiomatic Rust web-crawling library inspired by Crawlee. Umbrella crate re-exporting the Millipede workspace.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101

[package]
name = "millipede"
description = "An idiomatic Rust web-crawling library inspired by Crawlee. Umbrella crate re-exporting the Millipede workspace."
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
repository.workspace = true
readme = "README.md"
keywords = ["crawler", "scraper", "spider", "crawlee", "web-crawler"]
categories = ["web-programming", "asynchronous"]

[package.metadata.docs.rs]
all-features = true

[lints]
workspace = true

[dependencies]
millipede-core = { path = "../crates/millipede-core", version = "0.1.0" }
millipede-storage-memory = { path = "../crates/millipede-storage-memory", version = "0.1.0", optional = true }
millipede-storage-fs = { path = "../crates/millipede-storage-fs", version = "0.1.0", optional = true }
millipede-http = { path = "../crates/millipede-http", version = "0.1.0", optional = true }
millipede-html = { path = "../crates/millipede-html", version = "0.1.0", optional = true }
millipede-browser = { path = "../crates/millipede-browser", version = "0.1.0", optional = true }
millipede-browser-chromiumoxide = { path = "../crates/millipede-browser-chromiumoxide", version = "0.1.0", optional = true }
millipede-fingerprint = { path = "../crates/millipede-fingerprint", version = "0.1.0", optional = true }
serde = { workspace = true }

[dev-dependencies]
anyhow = { workspace = true }
async-trait = { workspace = true }
flate2 = { workspace = true }
tokio = { workspace = true }
wiremock = { workspace = true }
millipede-storage-fs = { path = "../crates/millipede-storage-fs" }
tempfile = { workspace = true }
serde_json = { workspace = true }
url = { workspace = true }

[features]
default = ["http", "html", "storage-memory"]
http = ["dep:millipede-http"]
html = ["dep:millipede-html"]
storage-memory = ["dep:millipede-storage-memory"]
storage-fs = ["dep:millipede-storage-fs"]
browser = ["dep:millipede-browser"]
browser-chromiumoxide = ["dep:millipede-browser-chromiumoxide", "browser"]
fingerprint = ["dep:millipede-fingerprint"]

[[example]]
name = "phase1_queue_demo"
required-features = ["storage-memory"]

[[example]]
name = "basic_engine"
required-features = ["storage-memory"]

[[example]]
name = "http_crawl"
required-features = ["http", "storage-memory"]

[[example]]
name = "autoscale_demo"
required-features = ["http", "storage-memory"]

[[example]]
name = "scrape_books"
required-features = ["http", "html", "storage-fs"]

[[example]]
name = "browser_crawl"
required-features = ["browser-chromiumoxide", "storage-memory"]

[[example]]
name = "smart_crawl"
required-features = ["browser-chromiumoxide", "html", "storage-memory"]

[[example]]
name = "fingerprint_crawl"
required-features = ["http", "fingerprint", "storage-memory"]

[[example]]
name = "basic"
required-features = ["http", "html", "storage-memory"]

[[example]]
name = "error_handling"
required-features = ["http", "storage-memory"]

[[example]]
name = "rate_limit"
required-features = ["http", "storage-memory"]

[[example]]
name = "proxy_switcher"
required-features = ["http", "storage-memory"]

[[example]]
name = "hackernews"
required-features = ["http", "html", "storage-memory"]