stygian-graph 0.9.2

High-performance graph-based web scraping engine with AI extraction, multi-modal support, and anti-bot capabilities
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119

//! Adapter implementations - infrastructure concerns
//!
//! Concrete implementations of port traits:
//! - HTTP client with anti-bot features
//! - AI providers (Claude, GPT, Gemini, Ollama, Copilot)
//! - Storage backends
//! - Cache backends

/// HTTP scraping adapter with anti-bot capabilities
pub mod http;

/// REST API adapter — JSON APIs with auth, pagination, and data extraction
pub mod rest_api;

/// AI provider adapters
pub mod ai;

/// Storage adapters (file, S3, database)
pub mod storage;

/// Cache adapters (memory, Redis)
pub mod cache;

/// Resilience adapters (circuit breaker, retry)
pub mod resilience;

/// No-op service for testing
pub mod noop;

/// JavaScript rendering adapter (headless browser via stygian-browser)
#[cfg(feature = "browser")]
pub mod browser;

/// Multi-modal content extraction (CSV, JSON, XML, images, PDFs)
pub mod multimodal;

/// Mock AI provider for testing
pub mod mock_ai;

/// GraphQL API adapter — generic ScrapingService for any GraphQL endpoint
pub mod graphql;

/// OpenAPI 3.x introspection adapter — resolves operations from an OpenAPI spec and delegates to RestApiAdapter
pub mod openapi;

pub mod graphql_rate_limit;
/// Proactive cost-throttle management for GraphQL APIs
pub mod graphql_throttle;

/// Distributed work queue and executor adapters
pub mod distributed;

/// GraphQL target plugin implementations (one file per API target)
pub mod graphql_plugins;

/// WASM plugin adapter (feature = "wasm-plugins")
pub mod wasm_plugin;

/// Cloudflare Browser Rendering crawl adapter (feature = "cloudflare-crawl")
#[cfg(feature = "cloudflare-crawl")]
pub mod cloudflare_crawl;

/// Output format helpers — CSV, JSONL, JSON
pub mod output_format;

/// Request signing adapters — Noop passthrough and HTTP sidecar bridge.
/// Covers Frida RPC, AWS Sig V4, OAuth 1.0a, custom HMAC, and device attestation.
pub mod signing;

/// OpenAPI spec generator from API discovery reports
pub mod openapi_gen;

/// PostgreSQL database source adapter (feature = "postgres")
#[cfg(feature = "postgres")]
pub mod database;

/// File system / document source adapter
pub mod document;

/// Server-Sent Events stream source adapter
pub mod stream;

/// LLM agent source adapter — wraps AIProvider as a pipeline node
pub mod agent_source;

/// Redis / Valkey cache adapter (feature = "redis")
#[cfg(feature = "redis")]
pub mod cache_redis;

/// Redis Streams work queue adapter (feature = "redis")
#[cfg(feature = "redis")]
pub mod distributed_redis;

/// Sitemap / sitemap-index source adapter
pub mod sitemap;

/// RSS/Atom feed source adapter
pub mod rss_feed;

/// WebSocket stream source adapter
pub mod websocket;

/// CSV/TSV data source adapter
pub mod csv_source;

/// S3-compatible object storage adapter (feature = "object-storage")
#[cfg(feature = "object-storage")]
pub mod storage_s3;

/// Axum-based webhook trigger adapter (feature = "api")
#[cfg(feature = "api")]
pub mod webhook;

/// Default tiered escalation policy with challenge detection (feature = "escalation")
#[cfg(feature = "escalation")]
pub mod escalation;

#[cfg(feature = "scrape-exchange")]
pub mod scrape_exchange;