use clap::{Args, Parser, Subcommand, ValueEnum};
pub const KEY_RESOLUTION: &str = "The key comes from SPIDER_API_KEY, then SPIDER_CLOUD_API_KEY, then the keychain, when the binary was built with the keyring feature, then ~/.spider/credentials.";
#[derive(Debug, Parser)]
#[command(
name = "spider-agent",
version,
about = "Fetch, crawl, search and extract with Spider Cloud.",
long_about = "Fetch, crawl, search and extract with Spider Cloud.
Transport is chosen locally before a call goes out, escalation answers the \
status the site returned, and a budget you set stops the run. No large model \
is called and no model key is read.
Results go to stdout and diagnostics go to stderr, so a pipe carries the \
payload and nothing else. --ndjson writes one object per line, flushed as each \
arrives, and that schema is a contract: `spider-agent schema` prints it.
Exit codes: 0 done, 1 failed, 2 usage, 3 auth, 4 budget, 5 the site refused, \
6 transport, 7 output refused.
There is no flag for the key, because an argument is \
readable in the process list.",
after_long_help = KEY_RESOLUTION,
args_conflicts_with_subcommands = true,
subcommand_negates_reqs = true,
disable_help_subcommand = true
)]
pub struct Cli {
#[command(flatten)]
pub global: Global,
#[command(flatten)]
pub default: ScrapeArgs,
#[command(subcommand)]
pub command: Option<Command>,
}
#[derive(Debug, Subcommand)]
pub enum Command {
Scrape(ScrapeArgs),
#[command(long_about = "Read one path under the config the service holds for it.
This is not scrape. Scrape sends an address and the settings you chose. Fetch \
names a host and a path, and the service answers with a config it already \
worked out for that path: which fields to pull, whether the page needs a \
browser, what to wait for.
The first call on a path nobody has asked for yet goes away and works one out, \
which takes far longer than a scrape and can fail while it tries. The query \
string is not part of the target, so a path carrying one reaches the page \
without it. Reach for scrape unless you want the stored config.")]
Fetch(FetchArgs),
Crawl(CrawlArgs),
Extract(ExtractArgs),
Links(LinksArgs),
Search(SearchArgs),
Screenshot(ScreenshotArgs),
Transform(TransformArgs),
Run(RunArgs),
Credits,
Logs(RowsArgs),
#[command(long_about = "The sites configured on the account.
One row per site, with the settings the service holds for it. Columns: \
anti_bot, blacklist, cache, crawl_budget, created_at, domain, fts, \
full_resources, gpt_config, headless, id, last_checked_at, metadata, \
proxy_enabled, return_format, scheme, smart_mode, subdomains, tld, \
updated_at, url, user_id, whitelist.
The columns belong to the service and change without a release of this tool, \
so a row is passed through as it arrived.")]
Sites(RowsArgs),
#[command(long_about = "The API keys on the account, as metadata.
No key is returned. There is no token, secret or hash column in this reply, \
and token_name is the label you gave a key rather than any part of it. \
Columns: created_at, id, last_used, team_id, team_member_id, token_name, \
updated_at, user_id.
Use it to find a key that has not been used in months, or to count what is \
issued against the account. Rotating or revoking one is done on the \
dashboard, not here.")]
Keys(RowsArgs),
#[command(
long_about = "The account itself: plan limits, totals and billing caps.
One row. Columns include ai_function_calls_limit, ai_requests_limit, \
ai_tokens_limit, approved_usage, auto_recharge, auto_recharge_threshold, \
billing_allowed, billing_limit, billing_limit_soft, disabled_at, \
disabled_reason, email, email_alert_percent, has_stripe, has_subscription, \
id, provider, total_crawl_count, total_credits_spent, usage and user_role.
For what is left to spend right now, `credits` is the shorter answer."
)]
Profile,
Login(LoginArgs),
#[command(
long_about = "Store a provider fallback that every page request carries.
The request parameter router names an outside scraping provider and how it \
takes part: fallback puts it behind Spider's own fetch, tried only when that \
fails; first puts it in front, and needs a provider and its key; off keeps the \
request away from outside providers. funding own allows only routes your own \
keys pay for. Providers that take one token: firecrawl, zenrows, scrapingbee, \
scraperapi, brightdata, zyte, apify, crawlbase, diffbot. oxylabs and \
dataforseo take a username and password as credentials instead.
The config lives in ~/.spider/router.json, readable by its owner only. Every \
command that fetches pages sends it unless the request already names a router. \
Pass --no-router, or set SPIDER_AGENT_NO_ROUTER to any value, to skip it for \
one run."
)]
Router(RouterArgs),
Route(RouteArgs),
Schema,
#[command(
long_about = "Install the newest release now instead of on a later run.
Once a day, a run checks for a newer release in the background, downloads it, \
checks it against the release's SHA256SUMS.txt and the minisign signature \
on that file, and leaves it beside the \
binary. The next run moves it into place and carries on under the new \
version. This command does all of that at once and says what it did on \
stderr.
Exit codes: 0 installed or already the newest, 1 the release was refused or \
the install failed, 2 self update is turned off, 6 the release host could not \
be reached, 7 this binary belongs to cargo, a package manager, or a directory \
this user cannot write, so it is left alone.
Set SPIDER_AGENT_NO_UPDATE to any value, or pass --no-update, to turn self \
update off, including the background check and an update already waiting. \
With CI set, runs skip the background check and the waiting update, and this \
command still works."
)]
Update,
}
#[derive(Debug, Args)]
pub struct Global {
#[arg(long, global = true, conflicts_with_all = ["ndjson", "format"])]
pub json: bool,
#[arg(long, global = true, conflicts_with = "format")]
pub ndjson: bool,
#[arg(long, global = true, value_enum)]
pub format: Option<Format>,
#[arg(short = 'o', long, global = true, value_name = "FILE")]
pub output: Option<String>,
#[arg(
short = 'd',
long,
global = true,
value_name = "DIR",
conflicts_with = "output"
)]
pub output_dir: Option<String>,
#[arg(long, global = true, requires = "output")]
pub append: bool,
#[arg(long, global = true)]
pub mkdir: bool,
#[arg(long, global = true)]
pub force: bool,
#[arg(long, global = true, value_name = "CREDITS", value_parser = credits)]
pub budget: Option<f64>,
#[arg(long, global = true, value_name = "CREDITS", value_parser = credits)]
pub max_per_page: Option<f64>,
#[arg(long, global = true, value_name = "N")]
pub attempts: Option<u8>,
#[arg(long, global = true, value_name = "SECONDS")]
pub wall: Option<u64>,
#[arg(long, global = true, value_name = "N")]
pub max_tokens: Option<usize>,
#[arg(long, global = true, value_name = "MODE", value_parser = fetch_modes())]
pub mode: Option<Mode>,
#[arg(long, global = true)]
pub with_links: bool,
#[arg(long, global = true, value_enum)]
pub proxy: Option<Pool>,
#[arg(long, global = true, value_name = "CODE")]
pub country: Option<String>,
#[arg(long, global = true, value_name = "SECONDS")]
pub timeout: Option<u64>,
#[arg(long, global = true)]
pub session: bool,
#[arg(long, global = true)]
pub no_update: bool,
#[arg(long, global = true)]
pub no_router: bool,
#[arg(short, long, global = true, conflicts_with = "verbose")]
pub quiet: bool,
#[arg(short, long, global = true)]
pub verbose: bool,
}
pub(crate) fn credits(text: &str) -> Result<f64, String> {
let value: f64 = text
.parse()
.map_err(|_| format!("{text} is not a number of credits"))?;
if !value.is_finite() || value < 0.0 {
return Err(format!(
"{text} is not a number of credits. Give a number that is zero or more."
));
}
Ok(value)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Format {
Text,
Json,
Ndjson,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Mode {
Http,
Smart,
Browser,
Fallback,
First,
Off,
}
fn fetch_modes() -> impl clap::builder::TypedValueParser<Value = Mode> {
use clap::builder::{PossibleValue, PossibleValuesParser, TypedValueParser};
PossibleValuesParser::new([
PossibleValue::new("http").help("Plain HTTP. Cheapest, and enough for most pages"),
PossibleValue::new("smart").help("The service decides per page"),
PossibleValue::new("browser").help("A full browser"),
])
.try_map(|name| match name.as_str() {
"http" => Ok(Mode::Http),
"smart" => Ok(Mode::Smart),
"browser" => Ok(Mode::Browser),
other => Err(format!("{other} is not a fetch mode")),
})
}
fn router_modes() -> impl clap::builder::TypedValueParser<Value = Mode> {
use clap::builder::{PossibleValue, PossibleValuesParser, TypedValueParser};
PossibleValuesParser::new([
PossibleValue::new("fallback")
.help("Behind Spider's own fetch, tried only when that fails"),
PossibleValue::new("first")
.help("In front of Spider's own fetch. Needs a provider and its key"),
PossibleValue::new("off").help("No outside provider"),
])
.try_map(|name| match name.as_str() {
"fallback" => Ok(Mode::Fallback),
"first" => Ok(Mode::First),
"off" => Ok(Mode::Off),
other => Err(format!("{other} is not a router mode")),
})
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Pool {
Isp,
Residential,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Goal {
Text,
Markdown,
Html,
Links,
Metadata,
Fields,
Screenshot,
Raw,
}
#[derive(Debug, Args)]
pub struct Targets {
#[arg(value_name = "URL")]
pub urls: Vec<String>,
#[arg(long, value_name = "FILE")]
pub urls_from: Option<String>,
}
#[derive(Debug, Args)]
pub struct ScrapeArgs {
#[command(flatten)]
pub targets: Targets,
#[arg(long, value_enum, default_value = "markdown")]
pub goal: Goal,
#[arg(long, value_name = "FILE")]
pub selectors: Option<String>,
}
#[derive(Debug, Args)]
pub struct FetchArgs {
#[arg(value_name = "DOMAIN")]
pub domain: String,
#[arg(value_name = "PATH", default_value = "/")]
pub path: String,
#[arg(long, value_enum, default_value = "raw")]
pub goal: Goal,
}
#[derive(Debug, Args)]
pub struct CrawlArgs {
#[command(flatten)]
pub targets: Targets,
#[arg(long, value_enum, default_value = "markdown")]
pub goal: Goal,
#[arg(long, value_name = "FILE")]
pub selectors: Option<String>,
#[arg(long, value_name = "N")]
pub limit: Option<u32>,
#[arg(long, value_name = "N")]
pub depth: Option<u32>,
}
#[derive(Debug, Args)]
pub struct ExtractArgs {
#[command(flatten)]
pub targets: Targets,
#[arg(long, value_name = "FILE", required = true)]
pub selectors: String,
#[arg(long, value_name = "PATH")]
pub at: Option<String>,
}
#[derive(Debug, Args)]
pub struct LinksArgs {
#[command(flatten)]
pub targets: Targets,
}
#[derive(Debug, Args)]
pub struct SearchArgs {
#[arg(value_name = "QUERY", required = true)]
pub query: Vec<String>,
#[arg(long, value_name = "N")]
pub limit: Option<u32>,
#[arg(long)]
pub fetch_pages: bool,
}
#[derive(Debug, Args)]
pub struct ScreenshotArgs {
#[command(flatten)]
pub targets: Targets,
}
#[derive(Debug, Args)]
pub struct TransformArgs {
#[arg(long, value_name = "FILE", required = true)]
pub input: String,
#[arg(long, value_name = "URL")]
pub url: Option<String>,
#[arg(long = "to", value_enum, default_value = "markdown")]
pub to: Goal,
#[arg(long)]
pub readability: bool,
}
#[derive(Debug, Args)]
pub struct RunArgs {
#[command(flatten)]
pub targets: Targets,
#[arg(long, value_enum)]
pub goal: Option<Goal>,
#[arg(long, value_name = "FILE")]
pub selectors: Option<String>,
#[arg(long, value_name = "N")]
pub expand: Option<usize>,
#[arg(long, value_name = "N")]
pub max_pages: Option<usize>,
#[arg(long, value_name = "FILE")]
pub plan: Option<String>,
}
#[derive(Debug, Args)]
pub struct RowsArgs {
#[arg(long, value_name = "N")]
pub limit: Option<u32>,
#[arg(long, value_name = "N")]
pub page: Option<u32>,
}
impl RowsArgs {
pub const fn none() -> RowsArgs {
RowsArgs {
limit: None,
page: None,
}
}
}
#[derive(Debug, Args)]
pub struct LoginArgs {
#[arg(long)]
pub print: bool,
}
#[derive(Debug, Args)]
pub struct RouterArgs {
#[command(subcommand)]
pub command: RouterCommand,
}
#[derive(Debug, Subcommand)]
pub enum RouterCommand {
Set(RouterSetArgs),
Show,
Clear,
}
#[derive(Debug, Args)]
pub struct RouterSetArgs {
#[arg(long, value_name = "NAME")]
pub provider: Option<String>,
#[arg(long, value_name = "MODE", value_parser = router_modes())]
pub mode: Option<Mode>,
#[arg(long, value_enum)]
pub funding: Option<Funding>,
#[arg(long, conflicts_with = "no_token")]
pub token_stdin: bool,
#[arg(long, value_name = "VALUE", num_args = 0..=1, hide = true)]
pub token: Option<Option<String>>,
#[arg(long)]
pub no_token: bool,
#[arg(long, value_name = "NAME=VALUE|NAME-stdin")]
pub credential: Vec<String>,
#[arg(long, value_name = "NAME")]
pub no_credential: Vec<String>,
#[arg(long, value_name = "PROVIDER.KEY=VALUE")]
pub option: Vec<String>,
#[arg(long, value_name = "PROVIDER.KEY")]
pub no_option: Vec<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Funding {
Any,
Own,
}
#[derive(Debug, Args)]
pub struct RouteArgs {
#[command(flatten)]
pub targets: Targets,
#[arg(long, value_enum, default_value = "markdown")]
pub goal: Goal,
}