use clap::{Args, Parser, Subcommand, ValueEnum};
#[derive(Debug, Parser)]
#[command(
name = "spider-agent",
version,
about = "Fetch, crawl, search and extract with Spider Cloud.",
long_about = "Fetch, crawl, search and extract with Spider Cloud.
Transport is chosen locally before a call goes out, escalation answers the \
status the site returned, and a budget you set stops the run. No large model \
is called and no model key is read.
Results go to stdout and diagnostics go to stderr, so a pipe carries the \
payload and nothing else. --ndjson writes one object per line, flushed as each \
arrives, and that schema is a contract: `spider-agent schema` prints it.
Exit codes: 0 done, 1 failed, 2 usage, 3 auth, 4 budget, 5 the site refused, \
6 transport, 7 output refused.
The key comes from SPIDER_API_KEY, then SPIDER_CLOUD_API_KEY, then \
~/.spider/credentials. There is no flag for it, because an argument is \
readable in the process list.",
args_conflicts_with_subcommands = true,
subcommand_negates_reqs = true,
disable_help_subcommand = true
)]
pub struct Cli {
#[command(flatten)]
pub global: Global,
#[command(flatten)]
pub default: ScrapeArgs,
#[command(subcommand)]
pub command: Option<Command>,
}
#[derive(Debug, Subcommand)]
pub enum Command {
Scrape(ScrapeArgs),
#[command(long_about = "Read one path under the config the service holds for it.
This is not scrape. Scrape sends an address and the settings you chose. Fetch \
names a host and a path, and the service answers with a config it already \
worked out for that path: which fields to pull, whether the page needs a \
browser, what to wait for.
The first call on a path nobody has asked for yet goes away and works one out, \
which takes far longer than a scrape and can fail while it tries. The query \
string is not part of the target, so a path carrying one reaches the page \
without it. Reach for scrape unless you want the stored config.")]
Fetch(FetchArgs),
Crawl(CrawlArgs),
Extract(ExtractArgs),
Links(LinksArgs),
Search(SearchArgs),
Screenshot(ScreenshotArgs),
Transform(TransformArgs),
Run(RunArgs),
Credits,
Logs(RowsArgs),
Table(TableArgs),
Login(LoginArgs),
Route(RouteArgs),
Schema,
}
#[derive(Debug, Args)]
pub struct Global {
#[arg(long, global = true, conflicts_with_all = ["ndjson", "format"])]
pub json: bool,
#[arg(long, global = true, conflicts_with = "format")]
pub ndjson: bool,
#[arg(long, global = true, value_enum)]
pub format: Option<Format>,
#[arg(short = 'o', long, global = true, value_name = "FILE")]
pub output: Option<String>,
#[arg(
short = 'd',
long,
global = true,
value_name = "DIR",
conflicts_with = "output"
)]
pub output_dir: Option<String>,
#[arg(long, global = true, requires = "output")]
pub append: bool,
#[arg(long, global = true)]
pub mkdir: bool,
#[arg(long, global = true)]
pub force: bool,
#[arg(long, global = true, value_name = "CREDITS")]
pub budget: Option<f64>,
#[arg(long, global = true, value_name = "CREDITS")]
pub max_per_page: Option<f64>,
#[arg(long, global = true, value_name = "N")]
pub attempts: Option<u8>,
#[arg(long, global = true, value_name = "SECONDS")]
pub wall: Option<u64>,
#[arg(long, global = true, value_name = "N")]
pub max_tokens: Option<usize>,
#[arg(long, global = true, value_enum)]
pub mode: Option<Mode>,
#[arg(long, global = true, value_enum)]
pub proxy: Option<Pool>,
#[arg(long, global = true, value_name = "CODE")]
pub country: Option<String>,
#[arg(long, global = true, value_name = "SECONDS")]
pub timeout: Option<u64>,
#[arg(long, global = true)]
pub session: bool,
#[arg(short, long, global = true, conflicts_with = "verbose")]
pub quiet: bool,
#[arg(short, long, global = true)]
pub verbose: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Format {
Text,
Json,
Ndjson,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Mode {
Http,
Smart,
Browser,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Pool {
Isp,
Residential,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Goal {
Text,
Markdown,
Html,
Links,
Metadata,
Fields,
Screenshot,
Raw,
}
#[derive(Debug, Args)]
pub struct Targets {
#[arg(value_name = "URL")]
pub urls: Vec<String>,
#[arg(long, value_name = "FILE")]
pub urls_from: Option<String>,
}
#[derive(Debug, Args)]
pub struct ScrapeArgs {
#[command(flatten)]
pub targets: Targets,
#[arg(long, value_enum, default_value = "markdown")]
pub goal: Goal,
#[arg(long, value_name = "FILE")]
pub selectors: Option<String>,
}
#[derive(Debug, Args)]
pub struct FetchArgs {
#[arg(value_name = "DOMAIN")]
pub domain: String,
#[arg(value_name = "PATH", default_value = "/")]
pub path: String,
#[arg(long, value_enum, default_value = "raw")]
pub goal: Goal,
}
#[derive(Debug, Args)]
pub struct CrawlArgs {
#[command(flatten)]
pub targets: Targets,
#[arg(long, value_enum, default_value = "markdown")]
pub goal: Goal,
#[arg(long, value_name = "FILE")]
pub selectors: Option<String>,
#[arg(long, value_name = "N")]
pub limit: Option<u32>,
#[arg(long, value_name = "N")]
pub depth: Option<u32>,
}
#[derive(Debug, Args)]
pub struct ExtractArgs {
#[command(flatten)]
pub targets: Targets,
#[arg(long, value_name = "FILE", required = true)]
pub selectors: String,
#[arg(long, value_name = "PATH")]
pub at: Option<String>,
}
#[derive(Debug, Args)]
pub struct LinksArgs {
#[command(flatten)]
pub targets: Targets,
}
#[derive(Debug, Args)]
pub struct SearchArgs {
#[arg(value_name = "QUERY", required = true)]
pub query: Vec<String>,
#[arg(long, value_name = "N")]
pub limit: Option<u32>,
#[arg(long)]
pub fetch_pages: bool,
}
#[derive(Debug, Args)]
pub struct ScreenshotArgs {
#[command(flatten)]
pub targets: Targets,
}
#[derive(Debug, Args)]
pub struct TransformArgs {
#[arg(long, value_name = "FILE", required = true)]
pub input: String,
#[arg(long, value_name = "URL")]
pub url: Option<String>,
#[arg(long = "to", value_enum, default_value = "markdown")]
pub to: Goal,
#[arg(long)]
pub readability: bool,
}
#[derive(Debug, Args)]
pub struct RunArgs {
#[command(flatten)]
pub targets: Targets,
#[arg(long, value_enum, default_value = "markdown")]
pub goal: Goal,
#[arg(long, value_name = "FILE")]
pub selectors: Option<String>,
#[arg(long, default_value_t = 0, value_name = "N")]
pub expand: usize,
#[arg(long, value_name = "N")]
pub max_pages: Option<usize>,
#[arg(long, value_name = "FILE")]
pub plan: Option<String>,
}
#[derive(Debug, Args)]
pub struct RowsArgs {
#[arg(long, value_name = "N")]
pub limit: Option<u32>,
#[arg(long, value_name = "N")]
pub page: Option<u32>,
}
#[derive(Debug, Args)]
pub struct TableArgs {
#[arg(value_name = "NAME")]
pub name: String,
#[command(flatten)]
pub rows: RowsArgs,
}
#[derive(Debug, Args)]
pub struct LoginArgs {
#[arg(long)]
pub print: bool,
}
#[derive(Debug, Args)]
pub struct RouteArgs {
#[command(flatten)]
pub targets: Targets,
#[arg(long, value_enum, default_value = "markdown")]
pub goal: Goal,
}