crawlkit-engine 2.0.0

High-performance Rust web crawler and SEO analysis toolkit with 28 analyzers, WASM plugin system, and enterprise features
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

[package]
name = "crawlkit-engine"
version.workspace = true
edition.workspace = true
authors.workspace = true
license.workspace = true
description.workspace = true
repository.workspace = true
keywords.workspace = true
categories.workspace = true
readme.workspace = true
homepage.workspace = true
documentation.workspace = true

[lints]
workspace = true

[[bench]]
name = "crawlkit_benchmarks"
harness = false

[dependencies]
tokio.workspace = true
tokio-stream.workspace = true
reqwest.workspace = true
bytes.workspace = true
serde.workspace = true
serde_json.workspace = true
thiserror.workspace = true
url.workspace = true
chrono.workspace = true
tracing.workspace = true
dashmap.workspace = true
parking_lot.workspace = true
rusqlite = { version = "0.31", features = ["bundled"] }
uuid = { version = "1", features = ["v4"] }
scraper.workspace = true
ego-tree.workspace = true
csv.workspace = true
lru.workspace = true
sha2.workspace = true
futures = "0.3"
toml = "0.8"
async-trait = "0.1"
urlencoding = "2.1"
aes-gcm = "0.10"
getrandom = "0.2"
dirs = "5.0"
wasmtime.workspace = true
quick-xml = "0.41"
rayon = "1"

[dev-dependencies]
tempfile = "3"
criterion.workspace = true
tokio = { workspace = true, features = ["test-util", "macros"] }