use clap::{Args, Parser, Subcommand, ValueEnum};
pub const KEY_RESOLUTION: &str = "The key comes from SPIDER_API_KEY, then SPIDER_CLOUD_API_KEY, then the keychain, when the binary was built with the keyring feature, then ~/.spider/credentials.";
#[derive(Debug, Parser)]
#[command(
name = "spider-agent",
version,
about = "Fetch, crawl, search and extract with Spider Cloud.",
long_about = "Fetch, crawl, search and extract with Spider Cloud.
Transport is chosen locally before a call goes out, escalation answers the \
status the site returned, and a budget you set stops the run. No large model \
is called and no model key is read.
Results go to stdout and diagnostics go to stderr, so a pipe carries the \
payload and nothing else. --ndjson writes one object per line, flushed as each \
arrives, and that schema is a contract: `spider-agent schema` prints it.
Exit codes: 0 done, 1 failed, 2 usage, 3 auth, 4 budget, 5 the site refused, \
6 transport, 7 output refused.
There is no flag for the key, because an argument is \
readable in the process list.",
after_long_help = KEY_RESOLUTION,
args_conflicts_with_subcommands = true,
subcommand_negates_reqs = true,
disable_help_subcommand = true
)]
pub struct Cli {
#[command(flatten)]
pub global: Global,
#[command(flatten)]
pub default: ScrapeArgs,
#[command(subcommand)]
pub command: Option<Command>,
}
#[derive(Debug, Subcommand)]
pub enum Command {
Scrape(ScrapeArgs),
#[command(long_about = "Read one path under the config the service holds for it.
This is not scrape. Scrape sends an address and the settings you chose. Fetch \
names a host and a path, and the service answers with a config it already \
worked out for that path: which fields to pull, whether the page needs a \
browser, what to wait for.
The first call on a path nobody has asked for yet goes away and works one out, \
which takes far longer than a scrape and can fail while it tries. The query \
string is not part of the target, so a path carrying one reaches the page \
without it. Reach for scrape unless you want the stored config.")]
Fetch(FetchArgs),
Crawl(CrawlArgs),
Extract(ExtractArgs),
Links(LinksArgs),
Search(SearchArgs),
Screenshot(ScreenshotArgs),
Transform(TransformArgs),
Run(RunArgs),
Credits,
Logs(RowsArgs),
#[command(long_about = "The sites configured on the account.
One row per site, with the settings the service holds for it. Columns: \
anti_bot, blacklist, cache, crawl_budget, created_at, domain, fts, \
full_resources, gpt_config, headless, id, last_checked_at, metadata, \
proxy_enabled, return_format, scheme, smart_mode, subdomains, tld, \
updated_at, url, user_id, whitelist.
The columns belong to the service and change without a release of this tool, \
so a row is passed through as it arrived.")]
Sites(RowsArgs),
#[command(long_about = "The API keys on the account, as metadata.
No key is returned. There is no token, secret or hash column in this reply, \
and token_name is the label you gave a key rather than any part of it. \
Columns: created_at, id, last_used, team_id, team_member_id, token_name, \
updated_at, user_id.
Use it to find a key that has not been used in months, or to count what is \
issued against the account. Rotating or revoking one is done on the \
dashboard, not here.")]
Keys(RowsArgs),
#[command(
long_about = "The account itself: plan limits, totals and billing caps.
One row. Columns include ai_function_calls_limit, ai_requests_limit, \
ai_tokens_limit, approved_usage, auto_recharge, auto_recharge_threshold, \
billing_allowed, billing_limit, billing_limit_soft, disabled_at, \
disabled_reason, email, email_alert_percent, has_stripe, has_subscription, \
id, provider, total_crawl_count, total_credits_spent, usage and user_role.
For what is left to spend right now, `credits` is the shorter answer."
)]
Profile,
#[command(long_about = "Sign in and store the key.
With no flag a browser opens on this machine. On a machine with no browser, \
such as one reached over SSH, a container or an agent's sandbox, pass --device: \
the command prints a short code and a web address, a person opens the address \
on any device and types the code, and the key arrives here.
A caller that cannot wait while a person finds a browser splits it in two. \
--device --no-wait prints the code and leaves. --device --resume collects the \
key afterwards, and leaves with code 3 while the code is still unapproved.")]
Login(LoginArgs),
#[command(
long_about = "Store a provider fallback that every page request carries.
The request parameter router names an outside scraping provider and how it \
takes part: fallback puts it behind Spider's own fetch, tried only when that \
fails; first puts it in front, and needs a provider and its key; off keeps the \
request away from outside providers. funding own allows only routes your own \
keys pay for. Providers that take one token: firecrawl, zenrows, scrapingbee, \
scraperapi, brightdata, zyte, apify, crawlbase, diffbot. oxylabs and \
dataforseo take a username and password as credentials instead.
The config lives in ~/.spider/router.json, readable by its owner only. Every \
command that fetches pages sends it unless the request already names a router. \
Pass --no-router, or set SPIDER_AGENT_NO_ROUTER to any value, to skip it for \
one run."
)]
Router(RouterArgs),
#[command(
long_about = "Store how the parameter optimizer runs, in ~/.spider/optimize.json.
The optimizer is a second decision layer. After the router picks the mode, the \
pool and the wait, it lists the valid edits to the request, scores each one \
against leaving the request alone, and writes at most one set that clears the \
gate. It never edits a field you set, it only touches the first attempt, and a \
request that fixes the mode, the pool or the country is not passed to it.
shadow scores and changes nothing. apply writes the pick onto the fields you \
left unset. Without weights the scorer abstains, so apply edits nothing: name \
a weights file with --model, or with --optimize-model for one run.
Every command that fetches pages reads this file. --optimize and \
--optimize-model override it for one run, and --no-optimize, or \
SPIDER_AGENT_NO_OPTIMIZE set to any value, leaves the optimizer out. Local, no \
call."
)]
Optimize(OptimizeArgs),
Route(RouteArgs),
Schema,
#[command(long_about = "Add the MCP server to the agent clients on this machine.
Looks for Claude Code, Codex, Cursor, Windsurf, Claude Desktop and VS Code, and \
registers spider-agent mcp with each one it finds, under the name spider. \
Claude Code, Codex and VS Code are asked through their own commands. The \
others keep a JSON file, and one entry is added to it. Every other server a \
client has is left alone, and a second run changes nothing.
--dry-run says what would change and writes nothing. --client names the \
clients to set up, instead of every one found.")]
Setup(SetupArgs),
#[command(
long_about = "Serve these commands as tools over the Model Context Protocol, on stdin and stdout.
An agent client starts this once and calls it many times. The process stays \
up, so every call after the first reuses the connection the first one opened \
and pays for no process start. Calls that arrive together run together.
Tools: spider_scrape, spider_crawl, spider_search, spider_links, \
spider_transform and spider_get_credits. Each runs the command of the same \
name and answers with the JSON document that command writes under --json, \
closing report included. A call that fails answers with isError set and the \
reason in words.
spider_crawl reads 10 pages unless the call names a limit, because a crawl \
with no limit reads the whole site. Every tool that fetches takes a budget in \
credits, and it caps that one call.
Nothing but protocol messages is written to stdout. Register it with a client \
as the command `spider-agent mcp`."
)]
Mcp,
#[command(
long_about = "Install the newest release now instead of on a later run.
Once a day, a run checks for a newer release in the background, downloads it, \
checks it against the release's SHA256SUMS.txt and the minisign signature \
on that file, and leaves it beside the \
binary. The next run moves it into place and carries on under the new \
version. This command does all of that at once and says what it did on \
stderr.
Exit codes: 0 installed or already the newest, 1 the release was refused or \
the install failed, 2 self update is turned off, 6 the release host could not \
be reached, 7 this binary belongs to cargo, a package manager, or a directory \
this user cannot write, so it is left alone.
Set SPIDER_AGENT_NO_UPDATE to any value, or pass --no-update, to turn self \
update off, including the background check and an update already waiting. \
With CI set, runs skip the background check and the waiting update, and this \
command still works."
)]
Update,
}
#[derive(Debug, Args)]
pub struct Global {
#[arg(long, global = true, conflicts_with_all = ["ndjson", "format"])]
pub json: bool,
#[arg(long, global = true, conflicts_with = "format")]
pub ndjson: bool,
#[arg(long, global = true, value_enum)]
pub format: Option<Format>,
#[arg(short = 'o', long, global = true, value_name = "FILE")]
pub output: Option<String>,
#[arg(
short = 'd',
long,
global = true,
value_name = "DIR",
conflicts_with = "output"
)]
pub output_dir: Option<String>,
#[arg(long, global = true, requires = "output")]
pub append: bool,
#[arg(long, global = true)]
pub mkdir: bool,
#[arg(long, global = true)]
pub force: bool,
#[arg(long, global = true, value_name = "CREDITS", value_parser = credits)]
pub budget: Option<f64>,
#[arg(long, global = true, value_name = "CREDITS", value_parser = credits)]
pub max_per_page: Option<f64>,
#[arg(long, global = true, value_name = "N")]
pub attempts: Option<u8>,
#[arg(long, global = true, value_name = "SECONDS")]
pub wall: Option<u64>,
#[arg(long, global = true, value_name = "N")]
pub max_tokens: Option<usize>,
#[arg(long, global = true, value_name = "MODE", value_parser = fetch_modes())]
pub mode: Option<Mode>,
#[arg(long, global = true)]
pub with_links: bool,
#[arg(long, global = true, value_enum)]
pub proxy: Option<Pool>,
#[arg(long, global = true, value_name = "CODE")]
pub country: Option<String>,
#[arg(long, global = true, value_name = "SECONDS")]
pub timeout: Option<u64>,
#[arg(long, global = true)]
pub session: bool,
#[arg(long, global = true)]
pub no_update: bool,
#[arg(long, global = true)]
pub no_router: bool,
#[arg(long, global = true, value_name = "MODE", value_parser = optimizer_modes(), conflicts_with = "no_optimize")]
pub optimize: Option<Mode>,
#[arg(
long,
global = true,
value_name = "FILE",
conflicts_with = "no_optimize"
)]
pub optimize_model: Option<String>,
#[arg(long, global = true)]
pub no_optimize: bool,
#[arg(short, long, global = true, conflicts_with = "verbose")]
pub quiet: bool,
#[arg(short, long, global = true)]
pub verbose: bool,
}
pub(crate) fn credits(text: &str) -> Result<f64, String> {
let value: f64 = text
.parse()
.map_err(|_| format!("{text} is not a number of credits"))?;
if !value.is_finite() || value < 0.0 {
return Err(format!(
"{text} is not a number of credits. Give a number that is zero or more."
));
}
Ok(value)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Format {
Text,
Json,
Ndjson,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Mode {
Http,
Smart,
Browser,
Fallback,
First,
Off,
Shadow,
Apply,
}
fn fetch_modes() -> impl clap::builder::TypedValueParser<Value = Mode> {
use clap::builder::{PossibleValue, PossibleValuesParser, TypedValueParser};
PossibleValuesParser::new([
PossibleValue::new("http").help("Plain HTTP. Cheapest, and enough for most pages"),
PossibleValue::new("smart").help("The service decides per page"),
PossibleValue::new("browser").help("A full browser"),
])
.try_map(|name| match name.as_str() {
"http" => Ok(Mode::Http),
"smart" => Ok(Mode::Smart),
"browser" => Ok(Mode::Browser),
other => Err(format!("{other} is not a fetch mode")),
})
}
fn router_modes() -> impl clap::builder::TypedValueParser<Value = Mode> {
use clap::builder::{PossibleValue, PossibleValuesParser, TypedValueParser};
PossibleValuesParser::new([
PossibleValue::new("fallback")
.help("Behind Spider's own fetch, tried only when that fails"),
PossibleValue::new("first")
.help("In front of Spider's own fetch. Needs a provider and its key"),
PossibleValue::new("off").help("No outside provider"),
])
.try_map(|name| match name.as_str() {
"fallback" => Ok(Mode::Fallback),
"first" => Ok(Mode::First),
"off" => Ok(Mode::Off),
other => Err(format!("{other} is not a router mode")),
})
}
fn optimizer_modes() -> impl clap::builder::TypedValueParser<Value = Mode> {
use clap::builder::{PossibleValue, PossibleValuesParser, TypedValueParser};
PossibleValuesParser::new([
PossibleValue::new("off").help("Send the request as the router left it"),
PossibleValue::new("shadow").help("Score and record, and change nothing"),
PossibleValue::new("apply").help("Write the chosen edits onto the fields you left unset"),
])
.try_map(|name| match name.as_str() {
"off" => Ok(Mode::Off),
"shadow" => Ok(Mode::Shadow),
"apply" => Ok(Mode::Apply),
other => Err(format!("{other} is not an optimizer mode")),
})
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Pool {
Isp,
Residential,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Goal {
Text,
Markdown,
Html,
Links,
Metadata,
Fields,
Screenshot,
Raw,
}
#[derive(Debug, Args)]
pub struct Targets {
#[arg(value_name = "URL")]
pub urls: Vec<String>,
#[arg(long, value_name = "FILE")]
pub urls_from: Option<String>,
}
#[derive(Debug, Args)]
pub struct ScrapeArgs {
#[command(flatten)]
pub targets: Targets,
#[arg(long, value_enum, default_value = "markdown")]
pub goal: Goal,
#[arg(long, value_name = "FILE")]
pub selectors: Option<String>,
}
#[derive(Debug, Args)]
pub struct FetchArgs {
#[arg(value_name = "DOMAIN")]
pub domain: String,
#[arg(value_name = "PATH", default_value = "/")]
pub path: String,
#[arg(long, value_enum, default_value = "raw")]
pub goal: Goal,
}
#[derive(Debug, Args)]
pub struct CrawlArgs {
#[command(flatten)]
pub targets: Targets,
#[arg(long, value_enum, default_value = "markdown")]
pub goal: Goal,
#[arg(long, value_name = "FILE")]
pub selectors: Option<String>,
#[arg(long, value_name = "N")]
pub limit: Option<u32>,
#[arg(long, value_name = "N")]
pub depth: Option<u32>,
}
#[derive(Debug, Args)]
pub struct ExtractArgs {
#[command(flatten)]
pub targets: Targets,
#[arg(long, value_name = "FILE", required = true)]
pub selectors: String,
#[arg(long, value_name = "PATH")]
pub at: Option<String>,
}
#[derive(Debug, Args)]
pub struct LinksArgs {
#[command(flatten)]
pub targets: Targets,
}
#[derive(Debug, Args)]
pub struct SearchArgs {
#[arg(value_name = "QUERY", required = true)]
pub query: Vec<String>,
#[arg(long, value_name = "N")]
pub limit: Option<u32>,
#[arg(long)]
pub fetch_pages: bool,
}
#[derive(Debug, Args)]
pub struct ScreenshotArgs {
#[command(flatten)]
pub targets: Targets,
}
#[derive(Debug, Args)]
pub struct TransformArgs {
#[arg(long, value_name = "FILE", required = true)]
pub input: String,
#[arg(long, value_name = "URL")]
pub url: Option<String>,
#[arg(long = "to", value_enum, default_value = "markdown")]
pub to: Goal,
#[arg(long)]
pub readability: bool,
}
#[derive(Debug, Args)]
pub struct RunArgs {
#[command(flatten)]
pub targets: Targets,
#[arg(long, value_enum)]
pub goal: Option<Goal>,
#[arg(long, value_name = "FILE")]
pub selectors: Option<String>,
#[arg(long, value_name = "N")]
pub expand: Option<usize>,
#[arg(long, value_name = "N")]
pub max_pages: Option<usize>,
#[arg(long, value_name = "FILE")]
pub plan: Option<String>,
}
#[derive(Debug, Args)]
pub struct RowsArgs {
#[arg(long, value_name = "N")]
pub limit: Option<u32>,
#[arg(long, value_name = "N")]
pub page: Option<u32>,
}
impl RowsArgs {
pub const fn none() -> RowsArgs {
RowsArgs {
limit: None,
page: None,
}
}
}
#[derive(Debug, Args)]
pub struct SetupArgs {
#[arg(long)]
pub dry_run: bool,
#[arg(long, value_enum, value_name = "NAME")]
pub client: Vec<Client>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Client {
ClaudeCode,
Codex,
Cursor,
Windsurf,
ClaudeDesktop,
Vscode,
}
impl Client {
pub const fn label(self) -> &'static str {
match self {
Client::ClaudeCode => "claude-code",
Client::Codex => "codex",
Client::Cursor => "cursor",
Client::Windsurf => "windsurf",
Client::ClaudeDesktop => "claude-desktop",
Client::Vscode => "vscode",
}
}
}
#[derive(Debug, Args)]
pub struct LoginArgs {
#[arg(long)]
pub print: bool,
#[arg(long)]
pub device: bool,
#[arg(long, requires = "device", conflicts_with_all = ["resume", "print"])]
pub no_wait: bool,
#[arg(long, requires = "device")]
pub resume: bool,
#[arg(long, requires = "device", value_name = "SECONDS")]
pub wait: Option<u64>,
}
#[derive(Debug, Args)]
pub struct RouterArgs {
#[command(subcommand)]
pub command: RouterCommand,
}
#[derive(Debug, Subcommand)]
pub enum RouterCommand {
Set(RouterSetArgs),
Show,
Clear,
}
#[derive(Debug, Args)]
pub struct RouterSetArgs {
#[arg(long, value_name = "NAME")]
pub provider: Option<String>,
#[arg(long, value_name = "MODE", value_parser = router_modes())]
pub mode: Option<Mode>,
#[arg(long, value_enum)]
pub funding: Option<Funding>,
#[arg(long, conflicts_with = "no_token")]
pub token_stdin: bool,
#[arg(long, value_name = "VALUE", num_args = 0..=1, hide = true)]
pub token: Option<Option<String>>,
#[arg(long)]
pub no_token: bool,
#[arg(long, value_name = "NAME=VALUE|NAME-stdin")]
pub credential: Vec<String>,
#[arg(long, value_name = "NAME")]
pub no_credential: Vec<String>,
#[arg(long, value_name = "PROVIDER.KEY=VALUE")]
pub option: Vec<String>,
#[arg(long, value_name = "PROVIDER.KEY")]
pub no_option: Vec<String>,
}
#[derive(Debug, Args)]
pub struct OptimizeArgs {
#[command(subcommand)]
pub command: OptimizeCommand,
}
#[derive(Debug, Subcommand)]
pub enum OptimizeCommand {
Set(OptimizeSetArgs),
Show,
Clear,
}
#[derive(Debug, Args)]
pub struct OptimizeSetArgs {
#[arg(long, value_name = "MODE", value_parser = optimizer_modes())]
pub mode: Option<Mode>,
#[arg(long, value_name = "FILE", conflicts_with = "no_model")]
pub model: Option<String>,
#[arg(long)]
pub no_model: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Funding {
Any,
Own,
}
#[derive(Debug, Args)]
pub struct RouteArgs {
#[command(flatten)]
pub targets: Targets,
#[arg(long, value_enum, default_value = "markdown")]
pub goal: Goal,
}