spider-agent-cli 0.3.0

The spider-agent command line tool: fetch, crawl, search and extract, with local routing, status-aware escalation and a budget.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
//! The command line, as clap reads it.
//!
//! Help text is copy. It says what a flag does and what it costs, and it names
//! the output contract where a caller would otherwise have to guess.

use clap::{Args, Parser, Subcommand, ValueEnum};

/// Fetch, crawl, search and extract, choosing transport locally and stopping on
/// a budget.
#[derive(Debug, Parser)]
#[command(
    name = "spider-agent",
    version,
    about = "Fetch, crawl, search and extract with Spider Cloud.",
    long_about = "Fetch, crawl, search and extract with Spider Cloud.

Transport is chosen locally before a call goes out, escalation answers the \
status the site returned, and a budget you set stops the run. No large model \
is called and no model key is read.

Results go to stdout and diagnostics go to stderr, so a pipe carries the \
payload and nothing else. --ndjson writes one object per line, flushed as each \
arrives, and that schema is a contract: `spider-agent schema` prints it.

Exit codes: 0 done, 1 failed, 2 usage, 3 auth, 4 budget, 5 the site refused, \
6 transport, 7 output refused.

The key comes from SPIDER_API_KEY, then SPIDER_CLOUD_API_KEY, then \
~/.spider/credentials. There is no flag for it, because an argument is \
readable in the process list.",
    args_conflicts_with_subcommands = true,
    subcommand_negates_reqs = true,
    disable_help_subcommand = true
)]
pub struct Cli {
    /// Settings every command reads.
    #[command(flatten)]
    pub global: Global,

    /// Addresses to read when no command is named.
    #[command(flatten)]
    pub default: ScrapeArgs,

    /// What to do.
    #[command(subcommand)]
    pub command: Option<Command>,
}

/// The commands.
#[derive(Debug, Subcommand)]
pub enum Command {
    /// Read one page or a list of them. What the bare form runs.
    Scrape(ScrapeArgs),
    /// Read one path under the config the service holds for it.
    #[command(long_about = "Read one path under the config the service holds for it.

This is not scrape. Scrape sends an address and the settings you chose. Fetch \
names a host and a path, and the service answers with a config it already \
worked out for that path: which fields to pull, whether the page needs a \
browser, what to wait for.

The first call on a path nobody has asked for yet goes away and works one out, \
which takes far longer than a scrape and can fail while it tries. The query \
string is not part of the target, so a path carrying one reaches the page \
without it. Reach for scrape unless you want the stored config.")]
    Fetch(FetchArgs),
    /// Read a site, following its links.
    Crawl(CrawlArgs),
    /// Pull named fields out of pages. The page body never crosses the wire.
    Extract(ExtractArgs),
    /// Collect the links on a page without paying for its content.
    Links(LinksArgs),
    /// Run a query.
    Search(SearchArgs),
    /// Take a picture of a page. Needs --output or --output-dir.
    Screenshot(ScreenshotArgs),
    /// Convert markup you already hold, without fetching anything.
    Transform(TransformArgs),
    /// Work a goal over one or more addresses until it is met or the budget
    /// stops it.
    Run(RunArgs),
    /// What is left on the account.
    Credits,
    /// The record of past crawls.
    Logs(RowsArgs),
    /// The sites configured on the account.
    #[command(long_about = "The sites configured on the account.

One row per site, with the settings the service holds for it. Columns: \
anti_bot, blacklist, cache, crawl_budget, created_at, domain, fts, \
full_resources, gpt_config, headless, id, last_checked_at, metadata, \
proxy_enabled, return_format, scheme, smart_mode, subdomains, tld, \
updated_at, url, user_id, whitelist.

The columns belong to the service and change without a release of this tool, \
so a row is passed through as it arrived.")]
    Sites(RowsArgs),
    /// The API keys on the account, as metadata. No key is returned.
    #[command(long_about = "The API keys on the account, as metadata.

No key is returned. There is no token, secret or hash column in this reply, \
and token_name is the label you gave a key rather than any part of it. \
Columns: created_at, id, last_used, team_id, team_member_id, token_name, \
updated_at, user_id.

Use it to find a key that has not been used in months, or to count what is \
issued against the account. Rotating or revoking one is done on the \
dashboard, not here.")]
    Keys(RowsArgs),
    /// The account itself: plan limits, totals and billing caps.
    #[command(
        long_about = "The account itself: plan limits, totals and billing caps.

One row. Columns include ai_function_calls_limit, ai_requests_limit, \
ai_tokens_limit, approved_usage, auto_recharge, auto_recharge_threshold, \
billing_allowed, billing_limit, billing_limit_soft, disabled_at, \
disabled_reason, email, email_alert_percent, has_stripe, has_subscription, \
id, provider, total_crawl_count, total_credits_spent, usage and user_role.

For what is left to spend right now, `credits` is the shorter answer."
    )]
    Profile,
    /// Sign in through a browser and store the key.
    Login(LoginArgs),
    /// What transport would be chosen for an address. Local, no call, no spend.
    Route(RouteArgs),
    /// The command tree and the record schema, as JSON.
    Schema,
}

/// Settings every command reads.
#[derive(Debug, Args)]
pub struct Global {
    /// Write one JSON document. Buffers, so nothing is written until the run
    /// ends.
    #[arg(long, global = true, conflicts_with_all = ["ndjson", "format"])]
    pub json: bool,

    /// Write one JSON object per line, flushed as each arrives.
    #[arg(long, global = true, conflicts_with = "format")]
    pub ndjson: bool,

    /// How results are written. Each command has its own default.
    #[arg(long, global = true, value_enum)]
    pub format: Option<Format>,

    /// Write results to this file instead of stdout.
    #[arg(short = 'o', long, global = true, value_name = "FILE")]
    pub output: Option<String>,

    /// Write one file per page into this directory, named from the address.
    #[arg(
        short = 'd',
        long,
        global = true,
        value_name = "DIR",
        conflicts_with = "output"
    )]
    pub output_dir: Option<String>,

    /// Append to the output file rather than replacing it.
    #[arg(long, global = true, requires = "output")]
    pub append: bool,

    /// Create the parent directories of the output path.
    #[arg(long, global = true)]
    pub mkdir: bool,

    /// Overwrite a file that is already there.
    #[arg(long, global = true)]
    pub force: bool,

    /// The most the whole run may spend, in credits. The first call on a page
    /// always goes out, because a price is not known until it is asked for.
    /// The cap stops the escalations after it and the pages after that.
    #[arg(long, global = true, value_name = "CREDITS")]
    pub budget: Option<f64>,

    /// The most any one page may spend, in credits. Fractions survive here.
    #[arg(long, global = true, value_name = "CREDITS")]
    pub max_per_page: Option<f64>,

    /// The most calls one page may take, first attempt and escalations
    /// together.
    #[arg(long, global = true, value_name = "N")]
    pub attempts: Option<u8>,

    /// The longest the run may take, in seconds, waits included.
    #[arg(long, global = true, value_name = "SECONDS")]
    pub wall: Option<u64>,

    /// The most to hand back, in approximate tokens, shared across the pages.
    #[arg(long, global = true, value_name = "N")]
    pub max_tokens: Option<usize>,

    /// Fix how the page is fetched. Left alone, the router picks it.
    #[arg(long, global = true, value_enum)]
    pub mode: Option<Mode>,

    /// Ask for the links on every page as well as its content. They come back
    /// in the same call rather than costing another.
    #[arg(long, global = true)]
    pub with_links: bool,

    /// Fix which pool the request leaves from.
    #[arg(long, global = true, value_enum)]
    pub proxy: Option<Pool>,

    /// The country to appear to be in, as a two letter code.
    #[arg(long, global = true, value_name = "CODE")]
    pub country: Option<String>,

    /// How long one page has to come back, in seconds.
    #[arg(long, global = true, value_name = "SECONDS")]
    pub timeout: Option<u64>,

    /// Keep cookies and headers across the requests made to one site.
    #[arg(long, global = true)]
    pub session: bool,

    /// Print nothing on stderr but failures.
    #[arg(short, long, global = true, conflicts_with = "verbose")]
    pub quiet: bool,

    /// Print a line on stderr for every page and every escalation.
    #[arg(short, long, global = true)]
    pub verbose: bool,
}

/// How results are written.
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Format {
    /// The content alone, with no wrapper.
    Text,
    /// One JSON document for the whole run.
    Json,
    /// One JSON object per line.
    Ndjson,
}

/// How a page is fetched.
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Mode {
    /// Plain HTTP. Cheapest, and enough for most pages.
    Http,
    /// The service decides per page.
    Smart,
    /// A full browser.
    Browser,
}

/// Which pool a request leaves from.
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Pool {
    /// The cheaper pool.
    Isp,
    /// Residential addresses, dearer per page.
    Residential,
}

/// What the caller wants back.
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Goal {
    /// The page as plain text, furniture stripped.
    Text,
    /// The page as markdown.
    Markdown,
    /// The markup, cleaned of scripts, styles and comments.
    Html,
    /// The links, and none of the content.
    Links,
    /// Title, description and the rest of the declared metadata.
    Metadata,
    /// Named fields only. Needs --selectors, and no page bytes cross the wire.
    Fields,
    /// A picture of the page.
    Screenshot,
    /// Whatever the service returns on its own, with nothing switched off.
    Raw,
}

/// Addresses, however they arrive.
#[derive(Debug, Args)]
pub struct Targets {
    /// Addresses to work on.
    #[arg(value_name = "URL")]
    pub urls: Vec<String>,

    /// Read addresses from a file, one per line. Use - for stdin.
    #[arg(long, value_name = "FILE")]
    pub urls_from: Option<String>,
}

/// Read one page or a list of them.
#[derive(Debug, Args)]
pub struct ScrapeArgs {
    /// Where to read from.
    #[command(flatten)]
    pub targets: Targets,

    /// What to ask for. Asking for less is where the saving is.
    #[arg(long, value_enum, default_value = "markdown")]
    pub goal: Goal,

    /// Named fields to pull out, as a JSON object of name to selector. Use -
    /// for stdin. Implies --goal fields.
    #[arg(long, value_name = "FILE")]
    pub selectors: Option<String>,
}

/// Read one path under the config the service holds for it.
#[derive(Debug, Args)]
pub struct FetchArgs {
    /// The host, with no scheme. Example: example.com
    #[arg(value_name = "DOMAIN")]
    pub domain: String,

    /// The path on that host. Defaults to the root.
    #[arg(value_name = "PATH", default_value = "/")]
    pub path: String,

    /// What to ask for. Defaults to raw, which leaves the stored config to
    /// decide. Naming anything else argues with it and usually wins.
    #[arg(long, value_enum, default_value = "raw")]
    pub goal: Goal,
}

/// Fetch a site, following its links.
#[derive(Debug, Args)]
pub struct CrawlArgs {
    /// Where to start.
    #[command(flatten)]
    pub targets: Targets,

    /// What to ask for on every page.
    #[arg(long, value_enum, default_value = "markdown")]
    pub goal: Goal,

    /// Named fields to pull off every page. Use - for stdin.
    #[arg(long, value_name = "FILE")]
    pub selectors: Option<String>,

    /// How many pages the crawl may visit. A crawl with no limit reads the
    /// whole site.
    #[arg(long, value_name = "N")]
    pub limit: Option<u32>,

    /// How many links deep from the starting page the crawl may go.
    #[arg(long, value_name = "N")]
    pub depth: Option<u32>,
}

/// Pull named fields out of pages.
#[derive(Debug, Args)]
pub struct ExtractArgs {
    /// Which pages to read.
    #[command(flatten)]
    pub targets: Targets,

    /// A JSON object of field name to selector, or to a list of selectors
    /// tried in order. A selector starting with a slash is read as XPath. Use
    /// - for stdin.
    #[arg(long, value_name = "FILE", required = true)]
    pub selectors: String,

    /// The path the fields apply to. Worth setting on a crawl, where a listing
    /// page and a product page hold different fields.
    #[arg(long, value_name = "PATH")]
    pub at: Option<String>,
}

/// Collect the links on a page.
#[derive(Debug, Args)]
pub struct LinksArgs {
    /// Which pages to read.
    #[command(flatten)]
    pub targets: Targets,
}

/// Run a query.
#[derive(Debug, Args)]
pub struct SearchArgs {
    /// What to search for.
    #[arg(value_name = "QUERY", required = true)]
    pub query: Vec<String>,

    /// How many results to work through.
    #[arg(long, value_name = "N")]
    pub limit: Option<u32>,

    /// Read the page behind every result. One fetch per result, so a query
    /// returning fifty of them is fifty fetches.
    #[arg(long)]
    pub fetch_pages: bool,
}

/// Take a picture of a page.
#[derive(Debug, Args)]
pub struct ScreenshotArgs {
    /// Which pages to shoot.
    #[command(flatten)]
    pub targets: Targets,
}

/// Convert markup you already hold.
#[derive(Debug, Args)]
pub struct TransformArgs {
    /// The markup to convert. Use - for stdin.
    #[arg(long, value_name = "FILE", required = true)]
    pub input: String,

    /// Where the markup came from, so links resolve.
    #[arg(long, value_name = "URL")]
    pub url: Option<String>,

    /// The shape to convert into.
    #[arg(long = "to", value_enum, default_value = "markdown")]
    pub to: Goal,

    /// Strip navigation, adverts and the rest of the furniture first.
    #[arg(long)]
    pub readability: bool,
}

/// Work a goal over addresses until it is met or the budget stops it.
#[derive(Debug, Args)]
pub struct RunArgs {
    /// Where to start.
    #[command(flatten)]
    pub targets: Targets,

    /// What counts as done for every address.
    #[arg(long, value_enum, default_value = "markdown")]
    pub goal: Goal,

    /// Named fields, as a JSON object. Use - for stdin. Implies --goal fields.
    #[arg(long, value_name = "FILE")]
    pub selectors: Option<String>,

    /// Follow up to this many same-host links found on the pages already read.
    #[arg(long, default_value_t = 0, value_name = "N")]
    pub expand: usize,

    /// Stop after this many pages, however they were reached.
    #[arg(long, value_name = "N")]
    pub max_pages: Option<usize>,

    /// Read goal, addresses and caps from a JSON file. Use - for stdin.
    /// Anything named on the command line wins.
    #[arg(long, value_name = "FILE")]
    pub plan: Option<String>,
}

/// Read rows off the account.
#[derive(Debug, Args)]
pub struct RowsArgs {
    /// How many rows to read.
    #[arg(long, value_name = "N")]
    pub limit: Option<u32>,

    /// Which page of rows, counting from one.
    #[arg(long, value_name = "N")]
    pub page: Option<u32>,
}

impl RowsArgs {
    /// Paging for a read that answers with one row, where a limit and a page
    /// would mean nothing.
    pub const fn none() -> RowsArgs {
        RowsArgs {
            limit: None,
            page: None,
        }
    }
}

/// Sign in through a browser.
#[derive(Debug, Args)]
pub struct LoginArgs {
    /// Print the key instead of storing it. It goes to stdout and nowhere
    /// else, so redirect it or do not use it.
    #[arg(long)]
    pub print: bool,
}

/// Ask what transport would be chosen.
#[derive(Debug, Args)]
pub struct RouteArgs {
    /// The addresses to decide about.
    #[command(flatten)]
    pub targets: Targets,

    /// What the request would ask for, which changes the answer.
    #[arg(long, value_enum, default_value = "markdown")]
    pub goal: Goal,
}