crw-cli 0.19.0

crw — Unified CLI for web scraping, crawling, search, and serving
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95

[package]
name = "crw-cli"
version.workspace = true
edition.workspace = true
license.workspace = true
repository.workspace = true
homepage.workspace = true
keywords.workspace = true
categories.workspace = true
description = "crw — Unified CLI for web scraping, crawling, search, and serving"

[[bin]]
name = "crw"
path = "src/main.rs"

[features]
default = ["serve", "mcp-embedded", "browse"]

# Enable serve subcommand (REST API server)
serve = ["dep:axum", "dep:tower", "dep:tower-http"]

# Enable embedded MCP mode (self-contained, no server needed)
mcp-embedded = ["dep:crw-server"]

# Enable browse subcommand (browser automation MCP)
browse = ["dep:crw-browse", "dep:rmcp", "dep:anyhow"]

# Opt-in Camoufox stealth renderer tier (REST). Default OFF so the shipped
# `crw` binary stays lean. Build with `--features camoufox` to enable. Forwards
# to crw-renderer/camoufox and, when the embedded server is present, to
# crw-server/camoufox.
camoufox = ["crw-renderer/camoufox", "crw-server?/camoufox"]

[dependencies]
# Core crates
crw-core = { workspace = true }
crw-renderer = { workspace = true, features = ["auto-browser", "cdp"] }
crw-extract = { workspace = true }
crw-crawl = { workspace = true }
crw-search = { workspace = true }

# Server (for serve + mcp-embedded + setup commands)
crw-server = { workspace = true, optional = true, features = ["cdp"] }

# Browse (for browse command)
crw-browse = { workspace = true, optional = true }

# Web framework (for serve command)
axum = { workspace = true, optional = true }
tower = { workspace = true, optional = true }
tower-http = { workspace = true, optional = true }

# MCP (for browse command)
rmcp = { version = "1.5", features = ["server", "macros", "transport-io"], optional = true }

# Async runtime
tokio = { workspace = true }
futures = { workspace = true }

# CLI
clap = { workspace = true }

# Serialization
serde = { workspace = true }
serde_json = { workspace = true }

# HTTP client
reqwest = { workspace = true }
# Base64 decode for crw_parse_file proxy forwarding (multipart upload)
base64 = { workspace = true }

# Logging
tracing = { workspace = true }
tracing-subscriber = { workspace = true }

# Error handling
anyhow = { version = "1", optional = true }

# UUID (for crawl job IDs)
uuid = { workspace = true }

# Interactive setup wizard
dialoguer = { version = "0.11", features = ["fuzzy-select"] }
console = "0.15"
indicatif = "0.17"

# Crypto (for checksum verification)
sha2 = { workspace = true }
hex = { workspace = true }

# SearXNG settings file (random secret_key)
rand = { workspace = true }

# User config (~/.config/crw/config.toml) written by setup wizard
toml = { workspace = true }