[package]
edition = "2024"
rust-version = "1.94.0"
name = "stygian-graph"
version = "0.4.0"
authors = ["Nick Campbell <s0ma@protonmail.com>"]
build = false
autolib = false
autobins = false
autoexamples = false
autotests = false
autobenches = false
description = "High-performance graph-based web scraping engine with AI extraction, multi-modal support, and anti-bot capabilities"
documentation = "https://greysquirr3l.github.io/stygian"
readme = "README.md"
keywords = [
"scraping",
"web-scraping",
"graph",
"ai",
"extraction",
]
categories = [
"web-programming",
"network-programming",
"asynchronous",
]
license = "AGPL-3.0-only OR LicenseRef-Commercial"
repository = "https://github.com/greysquirr3l/stygian"
resolver = "2"
[features]
api = []
browser = ["dep:stygian-browser"]
cloudflare-crawl = []
default = ["browser"]
escalation = []
full = [
"browser",
"api",
"wasm-plugins",
"postgres",
"cloudflare-crawl",
"redis",
"object-storage",
]
object-storage = ["dep:rust-s3"]
postgres = ["dep:sqlx"]
redis = [
"dep:redis",
"dep:deadpool-redis",
]
wasm-plugins = ["dep:wasmtime"]
[lib]
name = "stygian_graph"
path = "src/lib.rs"
[[bin]]
name = "stygian"
path = "src/bin/stygian.rs"
required-features = []
[[bin]]
name = "stygian-api"
path = "src/bin/stygian-api.rs"
required-features = []
[[test]]
name = "chaos"
path = "tests/chaos.rs"
[[test]]
name = "crawllab"
path = "tests/crawllab.rs"
[[test]]
name = "integration"
path = "tests/integration.rs"
[[test]]
name = "openapi"
path = "tests/openapi.rs"
[[test]]
name = "property_tests"
path = "tests/property_tests.rs"
[[bench]]
name = "pipeline_bench"
path = "benches/pipeline_bench.rs"
harness = false
[dependencies.anyhow]
version = "1.0"
[dependencies.async-trait]
version = "0.1"
[dependencies.axum]
version = "0.8"
features = ["macros"]
[dependencies.base64]
version = "0.22"
[dependencies.clap]
version = "4"
features = ["derive"]
[dependencies.csv]
version = "1.3"
[dependencies.dashmap]
version = "6"
[dependencies.deadpool-redis]
version = "0.20"
optional = true
[dependencies.feed-rs]
version = "2.3"
[dependencies.figment]
version = "0.10"
features = [
"toml",
"env",
]
[dependencies.flate2]
version = "1.1"
[dependencies.futures]
version = "0.3"
[dependencies.hmac]
version = "0.12"
[dependencies.indexmap]
version = "1"
[dependencies.indicatif]
version = "0.18"
[dependencies.lru]
version = "0.16"
[dependencies.openapiv3]
version = "1"
[dependencies.parking_lot]
version = "0.12"
[dependencies.petgraph]
version = "0.8"
[dependencies.prometheus-client]
version = "0.24"
[dependencies.quick-xml]
version = "0.37"
features = ["serialize"]
[dependencies.rayon]
version = "1.10"
[dependencies.redis]
version = "0.29"
features = [
"tokio-comp",
"connection-manager",
]
optional = true
[dependencies.reqwest]
version = "0.13"
features = [
"rustls",
"json",
"cookies",
"gzip",
"brotli",
"rustls",
"json",
"query",
"cookies",
"gzip",
"brotli",
]
[dependencies.rust-s3]
version = "0.37"
features = ["tokio-rustls-tls"]
optional = true
default-features = false
[dependencies.scraper]
version = "0.25"
[dependencies.serde]
version = "1.0"
features = [
"derive",
"derive",
]
[dependencies.serde_json]
version = "1.0"
[dependencies.serde_yaml]
version = "0.9"
[dependencies.sha2]
version = "0.10"
[dependencies.sqlx]
version = "0.8"
features = [
"runtime-tokio-rustls",
"postgres",
"uuid",
"json",
"migrate",
]
optional = true
[dependencies.stygian-browser]
version = "0.4"
optional = true
[dependencies.thiserror]
version = "2.0"
[dependencies.tokio]
version = "1.49"
features = [
"full",
"full",
]
[dependencies.tokio-tungstenite]
version = "0.29"
features = ["rustls-tls-webpki-roots"]
[dependencies.tokio-util]
version = "0.7"
features = ["rt"]
[dependencies.toml]
version = "0.8"
[dependencies.tower-http]
version = "0.6"
features = [
"cors",
"trace",
"compression-full",
]
[dependencies.tracing]
version = "0.1"
[dependencies.tracing-subscriber]
version = "0.3"
features = [
"env-filter",
"json",
"env-filter",
"json",
]
[dependencies.ulid]
version = "1.2"
[dependencies.uuid]
version = "1"
features = [
"v4",
"serde",
]
[dependencies.wasmtime]
version = "42"
features = [
"runtime",
"cranelift",
"component-model",
]
optional = true
default-features = false
[dev-dependencies.criterion]
version = "0.8"
features = ["async_tokio"]
[dev-dependencies.mockall]
version = "0.14"
[dev-dependencies.proptest]
version = "1.5"
[dev-dependencies.tempfile]
version = "3"
[dev-dependencies.tokio]
version = "1.49"
features = [
"full",
"test-util",
"macros",
]
[dev-dependencies.tokio-test]
version = "0.4"
[dev-dependencies.tower]
version = "0.5"
features = ["util"]