stygian-graph 0.2.0

High-performance graph-based web scraping engine with AI extraction, multi-modal support, and anti-bot capabilities
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114

[package]
name = "stygian-graph"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
authors.workspace = true
license.workspace = true
repository.workspace = true
documentation = "https://greysquirr3l.github.io/stygian"
description = "High-performance graph-based web scraping engine with AI extraction, multi-modal support, and anti-bot capabilities"
keywords = ["scraping", "web-scraping", "graph", "ai", "extraction"]
categories = ["web-programming", "network-programming", "asynchronous"]
readme = "README.md"

[dependencies]
# Core async runtime
tokio = { workspace = true, features = ["full"] }
tokio-util = { workspace = true }
futures.workspace = true
async-trait.workspace = true

# Concurrency
rayon.workspace = true

# Error handling
thiserror.workspace = true
anyhow.workspace = true

# Serialization
serde = { workspace = true, features = ["derive"] }
serde_json.workspace = true
toml.workspace = true

# HTTP client
reqwest = { workspace = true, features = ["rustls", "json", "query", "cookies", "gzip", "brotli"] }

# HTML parsing
scraper.workspace = true

# Graph algorithms
petgraph.workspace = true

# Logging & tracing
tracing.workspace = true
tracing-subscriber = { workspace = true, features = ["env-filter", "json"] }

# Utilities
ulid.workspace = true
parking_lot.workspace = true
dashmap.workspace = true
lru.workspace = true
figment.workspace = true
clap.workspace = true
indicatif.workspace = true
prometheus-client.workspace = true

# API server
axum.workspace = true
tower-http.workspace = true
uuid.workspace = true

# Output formats
csv.workspace = true

# OpenAPI spec parsing (always included — pure Rust)
openapiv3.workspace = true
serde_yaml.workspace = true

# Browser automation (optional)
stygian-browser = { path = "../stygian-browser", version = "0.2", optional = true }

# WASM plugins (optional)
wasmtime = { workspace = true, optional = true }

# Database (optional)
sqlx = { workspace = true, optional = true }

[dev-dependencies]
tokio = { workspace = true, features = ["test-util", "macros"] }
tokio-test = "0.4"
tower = { version = "0.5", features = ["util"] }
proptest.workspace = true
mockall.workspace = true
tempfile = "3"
criterion.workspace = true

[[bench]]
name = "pipeline_bench"
harness = false

[features]
default = ["browser"]
# Include browser automation support
browser = ["dep:stygian-browser"]
# Full feature set
full = ["browser", "api", "wasm-plugins", "postgres", "cloudflare-crawl"]
# REST API server
api = []
# WASM plugin system (requires wasmtime — significant compile time)
wasm-plugins = ["dep:wasmtime"]
# PostgreSQL storage adapter
postgres = ["dep:sqlx"]
# Cloudflare Browser Rendering crawl adapter
cloudflare-crawl = []

[[bin]]
name = "stygian"
path = "src/bin/stygian.rs"
required-features = []

[[bin]]
name = "stygian-api"
path = "src/bin/stygian-api.rs"
required-features = []