spider_agent 2.44.21

A concurrent-safe multimodal agent for web automation and research.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128

[package]
name = "spider_agent"
version = "2.44.21"
authors = ["j-mendez <jeff@spider.cloud>"]
description = "A concurrent-safe multimodal agent for web automation and research."
repository = "https://github.com/spider-rs/spider"
readme = "README.md"
keywords = ["agent", "llm", "automation", "web", "search"]
categories = ["web-programming", "asynchronous"]
license = "MIT"
documentation = "https://docs.rs/spider_agent"
edition = "2021"

[badges]
maintenance = { status = "actively-developed" }

[dependencies]
# Core
tokio = { version = "1", features = ["sync", "rt"] }
async-trait = "0.1"
reqwest = { version = "0.12", features = ["json"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
log = "0.4"
thiserror = "1"
dashmap = "6"
parking_lot = "0.12"
base64 = "0.22"

# HTML processing
lol_html = "2"
aho-corasick = "1"

# Model capabilities detection
llm_models_spider = "0.1"

# LLM Providers (optional)
async-openai = { version = "0.29", optional = true }
tiktoken-rs = { version = "0.7", optional = true }

# Browser (optional) - uses chromey (spider's chrome library)
chromey = { version = "2", optional = true, default-features = false, features = [
    "bytes",
    "stream",
] }

# WebDriver (optional) - uses thirtyfour
thirtyfour = { version = "0.36", optional = true }

# Temp filesystem for large operations
tempfile = { version = "3", optional = true }

# Stream utilities for event handling
futures = { version = "0.3", optional = true }

# Long-term experience memory via semantic search
memvid-rs = { version = "1.2", optional = true }

# Skill definitions (optional)
spider_skills = { version = "0.1", optional = true, default-features = false }

# S3 skill loading (optional)
aws-sdk-s3 = { version = "1", optional = true }
aws-config = { version = "1", optional = true }

[dev-dependencies]
tokio = { version = "1", features = ["full"] }
env_logger = "0.11"
fastrand = "2"
futures = "0.3"

[features]
default = []

# LLM Providers
openai = ["dep:async-openai", "dep:tiktoken-rs"]

# Browser automation
chrome = ["dep:chromey", "dep:futures"]

# WebDriver automation
webdriver = ["dep:thirtyfour"]

# Temp filesystem support
fs = ["dep:tempfile"]

# Skills - dynamic skill loading for web challenge solving
skills = ["dep:spider_skills"]

# Long-term experience memory
memvid = ["dep:memvid-rs"]

# S3 skill loading
skills_s3 = ["skills", "dep:aws-sdk-s3", "dep:aws-config"]

# Search providers
search = []
search_serper = ["search"]
search_brave = ["search"]
search_bing = ["search"]
search_tavily = ["search"]

# Full feature set
full = ["openai", "chrome", "webdriver", "fs", "skills", "memvid", "search_serper", "search_brave", "search_bing", "search_tavily", "skills_s3"]

[[example]]
name = "basic_search"
required-features = ["search_serper"]

[[example]]
name = "research"
required-features = ["openai", "search_serper"]

[[example]]
name = "extract"
required-features = ["openai"]

[[example]]
name = "concurrent"
required-features = ["openai", "search_serper"]

[[example]]
name = "multimodal"
required-features = ["openai", "search_serper"]

[[example]]
name = "open_page_concurrent"
required-features = ["chrome"]