use clap::{Args, Parser, Subcommand, ValueEnum};
#[derive(Debug, Parser)]
#[command(
name = "spider-agent",
version,
about = "Fetch, crawl, search and extract with Spider Cloud.",
long_about = "Fetch, crawl, search and extract with Spider Cloud.
Transport is chosen locally before a call goes out, escalation answers the \
status the site returned, and a budget you set stops the run. No large model \
is called and no model key is read.
Results go to stdout and diagnostics go to stderr, so a pipe carries the \
payload and nothing else. --ndjson writes one object per line, flushed as each \
arrives, and that schema is a contract: `spider-agent schema` prints it.
Exit codes: 0 done, 1 failed, 2 usage, 3 auth, 4 budget, 5 the site refused, \
6 transport, 7 output refused.
The key comes from SPIDER_API_KEY, then SPIDER_CLOUD_API_KEY, then \
~/.spider/credentials. There is no flag for it, because an argument is \
readable in the process list.",
args_conflicts_with_subcommands = true,
subcommand_negates_reqs = true,
disable_help_subcommand = true
)]
pub struct Cli {
#[command(flatten)]
pub global: Global,
#[command(flatten)]
pub default: ScrapeArgs,
#[command(subcommand)]
pub command: Option<Command>,
}
#[derive(Debug, Subcommand)]
pub enum Command {
Scrape(ScrapeArgs),
#[command(long_about = "Read one path under the config the service holds for it.
This is not scrape. Scrape sends an address and the settings you chose. Fetch \
names a host and a path, and the service answers with a config it already \
worked out for that path: which fields to pull, whether the page needs a \
browser, what to wait for.
The first call on a path nobody has asked for yet goes away and works one out, \
which takes far longer than a scrape and can fail while it tries. The query \
string is not part of the target, so a path carrying one reaches the page \
without it. Reach for scrape unless you want the stored config.")]
Fetch(FetchArgs),
Crawl(CrawlArgs),
Extract(ExtractArgs),
Links(LinksArgs),
Search(SearchArgs),
Screenshot(ScreenshotArgs),
Transform(TransformArgs),
Run(RunArgs),
Credits,
Logs(RowsArgs),
#[command(long_about = "The sites configured on the account.
One row per site, with the settings the service holds for it. Columns: \
anti_bot, blacklist, cache, crawl_budget, created_at, domain, fts, \
full_resources, gpt_config, headless, id, last_checked_at, metadata, \
proxy_enabled, return_format, scheme, smart_mode, subdomains, tld, \
updated_at, url, user_id, whitelist.
The columns belong to the service and change without a release of this tool, \
so a row is passed through as it arrived.")]
Sites(RowsArgs),
#[command(long_about = "The API keys on the account, as metadata.
No key is returned. There is no token, secret or hash column in this reply, \
and token_name is the label you gave a key rather than any part of it. \
Columns: created_at, id, last_used, team_id, team_member_id, token_name, \
updated_at, user_id.
Use it to find a key that has not been used in months, or to count what is \
issued against the account. Rotating or revoking one is done on the \
dashboard, not here.")]
Keys(RowsArgs),
#[command(
long_about = "The account itself: plan limits, totals and billing caps.
One row. Columns include ai_function_calls_limit, ai_requests_limit, \
ai_tokens_limit, approved_usage, auto_recharge, auto_recharge_threshold, \
billing_allowed, billing_limit, billing_limit_soft, disabled_at, \
disabled_reason, email, email_alert_percent, has_stripe, has_subscription, \
id, provider, total_crawl_count, total_credits_spent, usage and user_role.
For what is left to spend right now, `credits` is the shorter answer."
)]
Profile,
Login(LoginArgs),
Route(RouteArgs),
Schema,
}
#[derive(Debug, Args)]
pub struct Global {
#[arg(long, global = true, conflicts_with_all = ["ndjson", "format"])]
pub json: bool,
#[arg(long, global = true, conflicts_with = "format")]
pub ndjson: bool,
#[arg(long, global = true, value_enum)]
pub format: Option<Format>,
#[arg(short = 'o', long, global = true, value_name = "FILE")]
pub output: Option<String>,
#[arg(
short = 'd',
long,
global = true,
value_name = "DIR",
conflicts_with = "output"
)]
pub output_dir: Option<String>,
#[arg(long, global = true, requires = "output")]
pub append: bool,
#[arg(long, global = true)]
pub mkdir: bool,
#[arg(long, global = true)]
pub force: bool,
#[arg(long, global = true, value_name = "CREDITS")]
pub budget: Option<f64>,
#[arg(long, global = true, value_name = "CREDITS")]
pub max_per_page: Option<f64>,
#[arg(long, global = true, value_name = "N")]
pub attempts: Option<u8>,
#[arg(long, global = true, value_name = "SECONDS")]
pub wall: Option<u64>,
#[arg(long, global = true, value_name = "N")]
pub max_tokens: Option<usize>,
#[arg(long, global = true, value_enum)]
pub mode: Option<Mode>,
#[arg(long, global = true)]
pub with_links: bool,
#[arg(long, global = true, value_enum)]
pub proxy: Option<Pool>,
#[arg(long, global = true, value_name = "CODE")]
pub country: Option<String>,
#[arg(long, global = true, value_name = "SECONDS")]
pub timeout: Option<u64>,
#[arg(long, global = true)]
pub session: bool,
#[arg(short, long, global = true, conflicts_with = "verbose")]
pub quiet: bool,
#[arg(short, long, global = true)]
pub verbose: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Format {
Text,
Json,
Ndjson,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Mode {
Http,
Smart,
Browser,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Pool {
Isp,
Residential,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Goal {
Text,
Markdown,
Html,
Links,
Metadata,
Fields,
Screenshot,
Raw,
}
#[derive(Debug, Args)]
pub struct Targets {
#[arg(value_name = "URL")]
pub urls: Vec<String>,
#[arg(long, value_name = "FILE")]
pub urls_from: Option<String>,
}
#[derive(Debug, Args)]
pub struct ScrapeArgs {
#[command(flatten)]
pub targets: Targets,
#[arg(long, value_enum, default_value = "markdown")]
pub goal: Goal,
#[arg(long, value_name = "FILE")]
pub selectors: Option<String>,
}
#[derive(Debug, Args)]
pub struct FetchArgs {
#[arg(value_name = "DOMAIN")]
pub domain: String,
#[arg(value_name = "PATH", default_value = "/")]
pub path: String,
#[arg(long, value_enum, default_value = "raw")]
pub goal: Goal,
}
#[derive(Debug, Args)]
pub struct CrawlArgs {
#[command(flatten)]
pub targets: Targets,
#[arg(long, value_enum, default_value = "markdown")]
pub goal: Goal,
#[arg(long, value_name = "FILE")]
pub selectors: Option<String>,
#[arg(long, value_name = "N")]
pub limit: Option<u32>,
#[arg(long, value_name = "N")]
pub depth: Option<u32>,
}
#[derive(Debug, Args)]
pub struct ExtractArgs {
#[command(flatten)]
pub targets: Targets,
#[arg(long, value_name = "FILE", required = true)]
pub selectors: String,
#[arg(long, value_name = "PATH")]
pub at: Option<String>,
}
#[derive(Debug, Args)]
pub struct LinksArgs {
#[command(flatten)]
pub targets: Targets,
}
#[derive(Debug, Args)]
pub struct SearchArgs {
#[arg(value_name = "QUERY", required = true)]
pub query: Vec<String>,
#[arg(long, value_name = "N")]
pub limit: Option<u32>,
#[arg(long)]
pub fetch_pages: bool,
}
#[derive(Debug, Args)]
pub struct ScreenshotArgs {
#[command(flatten)]
pub targets: Targets,
}
#[derive(Debug, Args)]
pub struct TransformArgs {
#[arg(long, value_name = "FILE", required = true)]
pub input: String,
#[arg(long, value_name = "URL")]
pub url: Option<String>,
#[arg(long = "to", value_enum, default_value = "markdown")]
pub to: Goal,
#[arg(long)]
pub readability: bool,
}
#[derive(Debug, Args)]
pub struct RunArgs {
#[command(flatten)]
pub targets: Targets,
#[arg(long, value_enum, default_value = "markdown")]
pub goal: Goal,
#[arg(long, value_name = "FILE")]
pub selectors: Option<String>,
#[arg(long, default_value_t = 0, value_name = "N")]
pub expand: usize,
#[arg(long, value_name = "N")]
pub max_pages: Option<usize>,
#[arg(long, value_name = "FILE")]
pub plan: Option<String>,
}
#[derive(Debug, Args)]
pub struct RowsArgs {
#[arg(long, value_name = "N")]
pub limit: Option<u32>,
#[arg(long, value_name = "N")]
pub page: Option<u32>,
}
impl RowsArgs {
pub const fn none() -> RowsArgs {
RowsArgs {
limit: None,
page: None,
}
}
}
#[derive(Debug, Args)]
pub struct LoginArgs {
#[arg(long)]
pub print: bool,
}
#[derive(Debug, Args)]
pub struct RouteArgs {
#[command(flatten)]
pub targets: Targets,
#[arg(long, value_enum, default_value = "markdown")]
pub goal: Goal,
}