spider-agent-cli 0.8.0

The spider-agent command line tool, built to be called by an agent: fetch, crawl, search and extract as JSON or NDJSON, with local routing, status-aware escalation and a budget.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
//! The command line, as clap reads it.
//!
//! Help text is copy. It says what a flag does and what it costs, and it names
//! the output contract where a caller would otherwise have to guess.

use clap::{Args, Parser, Subcommand, ValueEnum};

/// Shared by help and schema so callers see the same resolution order.
pub const KEY_RESOLUTION: &str = "The key comes from SPIDER_API_KEY, then SPIDER_CLOUD_API_KEY, then the keychain, when the binary was built with the keyring feature, then ~/.spider/credentials.";

/// Fetch, crawl, search and extract, choosing transport locally and stopping on
/// a budget.
#[derive(Debug, Parser)]
#[command(
    name = "spider-agent",
    version,
    about = "Fetch, crawl, search and extract with Spider Cloud.",
    long_about = "Fetch, crawl, search and extract with Spider Cloud.

Transport is chosen locally before a call goes out, escalation answers the \
status the site returned, and a budget you set stops the run. No large model \
is called and no model key is read.

Results go to stdout and diagnostics go to stderr, so a pipe carries the \
payload and nothing else. --ndjson writes one object per line, flushed as each \
arrives, and that schema is a contract: `spider-agent schema` prints it.

Exit codes: 0 done, 1 failed, 2 usage, 3 auth, 4 budget, 5 the site refused, \
6 transport, 7 output refused.

There is no flag for the key, because an argument is \
readable in the process list.",
    after_long_help = KEY_RESOLUTION,
    args_conflicts_with_subcommands = true,
    subcommand_negates_reqs = true,
    disable_help_subcommand = true
)]
pub struct Cli {
    /// Settings every command reads.
    #[command(flatten)]
    pub global: Global,

    /// Addresses to read when no command is named.
    #[command(flatten)]
    pub default: ScrapeArgs,

    /// What to do.
    #[command(subcommand)]
    pub command: Option<Command>,
}

/// The commands.
#[derive(Debug, Subcommand)]
pub enum Command {
    /// Read one page or a list of them. What the bare form runs.
    Scrape(ScrapeArgs),
    /// Read one path under the config the service holds for it.
    #[command(long_about = "Read one path under the config the service holds for it.

This is not scrape. Scrape sends an address and the settings you chose. Fetch \
names a host and a path, and the service answers with a config it already \
worked out for that path: which fields to pull, whether the page needs a \
browser, what to wait for.

The first call on a path nobody has asked for yet goes away and works one out, \
which takes far longer than a scrape and can fail while it tries. The query \
string is not part of the target, so a path carrying one reaches the page \
without it. Reach for scrape unless you want the stored config.")]
    Fetch(FetchArgs),
    /// Read a site, following its links.
    Crawl(CrawlArgs),
    /// Pull named fields out of pages. The page body never crosses the wire.
    Extract(ExtractArgs),
    /// Collect the links on a page without paying for its content.
    Links(LinksArgs),
    /// Run a query.
    Search(SearchArgs),
    /// Take a picture of a page. Needs --output or --output-dir.
    Screenshot(ScreenshotArgs),
    /// Convert markup you already hold, without fetching anything.
    Transform(TransformArgs),
    /// Work a goal over one or more addresses until it is met or the budget
    /// stops it.
    Run(RunArgs),
    /// What is left on the account.
    Credits,
    /// The record of past crawls.
    Logs(RowsArgs),
    /// The sites configured on the account.
    #[command(long_about = "The sites configured on the account.

One row per site, with the settings the service holds for it. Columns: \
anti_bot, blacklist, cache, crawl_budget, created_at, domain, fts, \
full_resources, gpt_config, headless, id, last_checked_at, metadata, \
proxy_enabled, return_format, scheme, smart_mode, subdomains, tld, \
updated_at, url, user_id, whitelist.

The columns belong to the service and change without a release of this tool, \
so a row is passed through as it arrived.")]
    Sites(RowsArgs),
    /// The API keys on the account, as metadata. No key is returned.
    #[command(long_about = "The API keys on the account, as metadata.

No key is returned. There is no token, secret or hash column in this reply, \
and token_name is the label you gave a key rather than any part of it. \
Columns: created_at, id, last_used, team_id, team_member_id, token_name, \
updated_at, user_id.

Use it to find a key that has not been used in months, or to count what is \
issued against the account. Rotating or revoking one is done on the \
dashboard, not here.")]
    Keys(RowsArgs),
    /// The account itself: plan limits, totals and billing caps.
    #[command(
        long_about = "The account itself: plan limits, totals and billing caps.

One row. Columns include ai_function_calls_limit, ai_requests_limit, \
ai_tokens_limit, approved_usage, auto_recharge, auto_recharge_threshold, \
billing_allowed, billing_limit, billing_limit_soft, disabled_at, \
disabled_reason, email, email_alert_percent, has_stripe, has_subscription, \
id, provider, total_crawl_count, total_credits_spent, usage and user_role.

For what is left to spend right now, `credits` is the shorter answer."
    )]
    Profile,
    /// Sign in and store the key.
    #[command(long_about = "Sign in and store the key.

With no flag a browser opens on this machine. On a machine with no browser, \
such as one reached over SSH, a container or an agent's sandbox, pass --device: \
the command prints a short code and a web address, a person opens the address \
on any device and types the code, and the key arrives here.

A caller that cannot wait while a person finds a browser splits it in two. \
--device --no-wait prints the code and leaves. --device --resume collects the \
key afterwards, and leaves with code 3 while the code is still unapproved.")]
    Login(LoginArgs),
    /// Store a provider fallback that every page request carries.
    #[command(
        long_about = "Store a provider fallback that every page request carries.

The request parameter router names an outside scraping provider and how it \
takes part: fallback puts it behind Spider's own fetch, tried only when that \
fails; first puts it in front, and needs a provider and its key; off keeps the \
request away from outside providers. funding own allows only routes your own \
keys pay for. Providers that take one token: firecrawl, zenrows, scrapingbee, \
scraperapi, brightdata, zyte, apify, crawlbase, diffbot. oxylabs and \
dataforseo take a username and password as credentials instead.

The config lives in ~/.spider/router.json, readable by its owner only. Every \
command that fetches pages sends it unless the request already names a router. \
Pass --no-router, or set SPIDER_AGENT_NO_ROUTER to any value, to skip it for \
one run."
    )]
    Router(RouterArgs),
    /// Store how the parameter optimizer runs.
    #[command(
        long_about = "Store how the parameter optimizer runs, in ~/.spider/optimize.json.

The optimizer is a second decision layer. After the router picks the mode, the \
pool and the wait, it lists the valid edits to the request, scores each one \
against leaving the request alone, and writes at most one set that clears the \
gate. It never edits a field you set, it only touches the first attempt, and a \
request that fixes the mode, the pool or the country is not passed to it.

shadow scores and changes nothing. apply writes the pick onto the fields you \
left unset. Without weights the scorer abstains, so apply edits nothing: name \
a weights file with --model, or with --optimize-model for one run.

Every command that fetches pages reads this file. --optimize and \
--optimize-model override it for one run, and --no-optimize, or \
SPIDER_AGENT_NO_OPTIMIZE set to any value, leaves the optimizer out. Local, no \
call."
    )]
    Optimize(OptimizeArgs),
    /// What transport would be chosen for an address. Local, no call, no spend.
    Route(RouteArgs),
    /// The command tree and the record schema, as JSON.
    Schema,
    /// Add the MCP server to the agent clients on this machine.
    #[command(long_about = "Add the MCP server to the agent clients on this machine.

Looks for Claude Code, Codex, Cursor, Windsurf, Claude Desktop and VS Code, and \
registers spider-agent mcp with each one it finds, under the name spider. \
Claude Code, Codex and VS Code are asked through their own commands. The \
others keep a JSON file, and one entry is added to it. Every other server a \
client has is left alone, and a second run changes nothing.

--dry-run says what would change and writes nothing. --client names the \
clients to set up, instead of every one found.")]
    Setup(SetupArgs),
    /// Serve these commands as tools over the Model Context Protocol, on
    /// stdin and stdout.
    #[command(
        long_about = "Serve these commands as tools over the Model Context Protocol, on stdin and stdout.

An agent client starts this once and calls it many times. The process stays \
up, so every call after the first reuses the connection the first one opened \
and pays for no process start. Calls that arrive together run together.

Tools: spider_scrape, spider_crawl, spider_search, spider_links, \
spider_transform and spider_get_credits. Each runs the command of the same \
name and answers with the JSON document that command writes under --json, \
closing report included. A call that fails answers with isError set and the \
reason in words.

spider_crawl reads 10 pages unless the call names a limit, because a crawl \
with no limit reads the whole site. Every tool that fetches takes a budget in \
credits, and it caps that one call.

Nothing but protocol messages is written to stdout. Register it with a client \
as the command `spider-agent mcp`."
    )]
    Mcp,
    /// Install the newest release now instead of on a later run.
    #[command(
        long_about = "Install the newest release now instead of on a later run.

Once a day, a run checks for a newer release in the background, downloads it, \
checks it against the release's SHA256SUMS.txt and the minisign signature \
on that file, and leaves it beside the \
binary. The next run moves it into place and carries on under the new \
version. This command does all of that at once and says what it did on \
stderr.

Exit codes: 0 installed or already the newest, 1 the release was refused or \
the install failed, 2 self update is turned off, 6 the release host could not \
be reached, 7 this binary belongs to cargo, a package manager, or a directory \
this user cannot write, so it is left alone.

Set SPIDER_AGENT_NO_UPDATE to any value, or pass --no-update, to turn self \
update off, including the background check and an update already waiting. \
With CI set, runs skip the background check and the waiting update, and this \
command still works."
    )]
    Update,
}

/// Settings every command reads.
#[derive(Debug, Args)]
pub struct Global {
    /// Write one JSON document. Buffers, so nothing is written until the run
    /// ends.
    #[arg(long, global = true, conflicts_with_all = ["ndjson", "format"])]
    pub json: bool,

    /// Write one JSON object per line, flushed as each arrives.
    #[arg(long, global = true, conflicts_with = "format")]
    pub ndjson: bool,

    /// How results are written. Each command has its own default.
    #[arg(long, global = true, value_enum)]
    pub format: Option<Format>,

    /// Write results to this file instead of stdout. A dash is stdout.
    #[arg(short = 'o', long, global = true, value_name = "FILE")]
    pub output: Option<String>,

    /// Write one file per page into this directory, named from the address.
    #[arg(
        short = 'd',
        long,
        global = true,
        value_name = "DIR",
        conflicts_with = "output"
    )]
    pub output_dir: Option<String>,

    /// Append to the output file rather than replacing it.
    #[arg(long, global = true, requires = "output")]
    pub append: bool,

    /// Create the parent directories of the output path.
    #[arg(long, global = true)]
    pub mkdir: bool,

    /// Overwrite a file that is already there.
    #[arg(long, global = true)]
    pub force: bool,

    /// The most the whole run may spend, in credits. The first call on a page
    /// always goes out, because a price is not known until it is asked for.
    /// The cap stops the escalations after it and the pages after that.
    #[arg(long, global = true, value_name = "CREDITS", value_parser = credits)]
    pub budget: Option<f64>,

    /// The most any one page may spend, in credits. Fractions survive here.
    #[arg(long, global = true, value_name = "CREDITS", value_parser = credits)]
    pub max_per_page: Option<f64>,

    /// The most calls one page may take, first attempt and escalations
    /// together.
    #[arg(long, global = true, value_name = "N")]
    pub attempts: Option<u8>,

    /// The longest the run may take, in seconds, waits included.
    #[arg(long, global = true, value_name = "SECONDS")]
    pub wall: Option<u64>,

    /// The most to hand back, in approximate tokens, shared across the pages.
    #[arg(long, global = true, value_name = "N")]
    pub max_tokens: Option<usize>,

    /// Fix how the page is fetched. Left alone, the router picks it.
    #[arg(long, global = true, value_name = "MODE", value_parser = fetch_modes())]
    pub mode: Option<Mode>,

    /// Ask for the links on every page as well as its content. They come back
    /// in the same call rather than costing another.
    #[arg(long, global = true)]
    pub with_links: bool,

    /// Fix which pool the request leaves from.
    #[arg(long, global = true, value_enum)]
    pub proxy: Option<Pool>,

    /// The country to appear to be in, as a two letter code.
    #[arg(long, global = true, value_name = "CODE")]
    pub country: Option<String>,

    /// How long one page has to come back, in seconds.
    #[arg(long, global = true, value_name = "SECONDS")]
    pub timeout: Option<u64>,

    /// Keep cookies and headers across the requests made to one site.
    #[arg(long, global = true)]
    pub session: bool,

    /// Do not check for, download or install a new release on this run. The
    /// same as setting SPIDER_AGENT_NO_UPDATE.
    #[arg(long, global = true)]
    pub no_update: bool,

    /// Send no stored provider fallback on this run. The same as setting
    /// SPIDER_AGENT_NO_ROUTER.
    #[arg(long, global = true)]
    pub no_router: bool,

    /// Run the parameter optimizer on the first attempt of each page. It never
    /// edits a field you set. Left alone, the stored settings decide.
    #[arg(long, global = true, value_name = "MODE", value_parser = optimizer_modes(), conflicts_with = "no_optimize")]
    pub optimize: Option<Mode>,

    /// The weights the optimizer scores with. Without them it abstains, and
    /// apply edits nothing.
    #[arg(
        long,
        global = true,
        value_name = "FILE",
        conflicts_with = "no_optimize"
    )]
    pub optimize_model: Option<String>,

    /// Leave the optimizer out of this run, the stored settings included. The
    /// same as setting SPIDER_AGENT_NO_OPTIMIZE.
    #[arg(long, global = true)]
    pub no_optimize: bool,

    /// Print nothing on stderr but failures.
    #[arg(short, long, global = true, conflicts_with = "verbose")]
    pub quiet: bool,

    /// Print a line on stderr for every page and every escalation.
    #[arg(short, long, global = true)]
    pub verbose: bool,
}

/// An amount of credits a caller typed.
///
/// `f64` reads `nan`, `inf` and a negative number as numbers. As a cap, a
/// NaN compares false against everything and a negative is spent before the
/// first page, so each is a cap that does not do what the caller who typed
/// one meant. Refused here, where the answer is a usage failure.
pub(crate) fn credits(text: &str) -> Result<f64, String> {
    let value: f64 = text
        .parse()
        .map_err(|_| format!("{text} is not a number of credits"))?;
    if !value.is_finite() || value < 0.0 {
        return Err(format!(
            "{text} is not a number of credits. Give a number that is zero or more."
        ));
    }
    Ok(value)
}

/// How results are written.
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Format {
    /// The content alone, with no wrapper.
    Text,
    /// One JSON document for the whole run.
    Json,
    /// One JSON object per line.
    Ndjson,
}

/// A `--mode` value.
///
/// Three flags are spelled `--mode`: the global one, for how a page is
/// fetched, the one on `router set`, for how a provider takes part, and the one
/// on `optimize set`, for what the optimizer does with its pick. clap keeps one
/// slot per name and hands a value on a subcommand up to the global slot, so
/// all three parse into this one type, and each accepts only its own names.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Mode {
    /// Plain HTTP. Cheapest, and enough for most pages.
    Http,
    /// The service decides per page.
    Smart,
    /// A full browser.
    Browser,
    /// A provider behind Spider's own fetch, tried only when that fails.
    Fallback,
    /// A provider in front of Spider's own fetch.
    First,
    /// No outside provider, and the optimizer left out of the request.
    Off,
    /// The optimizer scores and records, and changes nothing.
    Shadow,
    /// The optimizer writes its pick onto the fields the caller left unset.
    Apply,
}

/// The global `--mode`: how a page is fetched.
fn fetch_modes() -> impl clap::builder::TypedValueParser<Value = Mode> {
    use clap::builder::{PossibleValue, PossibleValuesParser, TypedValueParser};
    PossibleValuesParser::new([
        PossibleValue::new("http").help("Plain HTTP. Cheapest, and enough for most pages"),
        PossibleValue::new("smart").help("The service decides per page"),
        PossibleValue::new("browser").help("A full browser"),
    ])
    .try_map(|name| match name.as_str() {
        "http" => Ok(Mode::Http),
        "smart" => Ok(Mode::Smart),
        "browser" => Ok(Mode::Browser),
        other => Err(format!("{other} is not a fetch mode")),
    })
}

/// `router set --mode`: how a provider takes part.
fn router_modes() -> impl clap::builder::TypedValueParser<Value = Mode> {
    use clap::builder::{PossibleValue, PossibleValuesParser, TypedValueParser};
    PossibleValuesParser::new([
        PossibleValue::new("fallback")
            .help("Behind Spider's own fetch, tried only when that fails"),
        PossibleValue::new("first")
            .help("In front of Spider's own fetch. Needs a provider and its key"),
        PossibleValue::new("off").help("No outside provider"),
    ])
    .try_map(|name| match name.as_str() {
        "fallback" => Ok(Mode::Fallback),
        "first" => Ok(Mode::First),
        "off" => Ok(Mode::Off),
        other => Err(format!("{other} is not a router mode")),
    })
}

/// `--optimize` and `optimize set --mode`: what the optimizer does with its
/// pick.
fn optimizer_modes() -> impl clap::builder::TypedValueParser<Value = Mode> {
    use clap::builder::{PossibleValue, PossibleValuesParser, TypedValueParser};
    PossibleValuesParser::new([
        PossibleValue::new("off").help("Send the request as the router left it"),
        PossibleValue::new("shadow").help("Score and record, and change nothing"),
        PossibleValue::new("apply").help("Write the chosen edits onto the fields you left unset"),
    ])
    .try_map(|name| match name.as_str() {
        "off" => Ok(Mode::Off),
        "shadow" => Ok(Mode::Shadow),
        "apply" => Ok(Mode::Apply),
        other => Err(format!("{other} is not an optimizer mode")),
    })
}

/// Which pool a request leaves from.
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Pool {
    /// The cheaper pool.
    Isp,
    /// Residential addresses, dearer per page.
    Residential,
}

/// What the caller wants back.
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Goal {
    /// The page as plain text, furniture stripped.
    Text,
    /// The page as markdown.
    Markdown,
    /// The markup, cleaned of scripts, styles and comments.
    Html,
    /// The links, and none of the content.
    Links,
    /// Title, description and the rest of the declared metadata.
    Metadata,
    /// Named fields only. Needs --selectors, and no page bytes cross the wire.
    Fields,
    /// A picture of the page.
    Screenshot,
    /// Whatever the service returns on its own, with nothing switched off.
    Raw,
}

/// Addresses, however they arrive.
#[derive(Debug, Args)]
pub struct Targets {
    /// Addresses to work on.
    #[arg(value_name = "URL")]
    pub urls: Vec<String>,

    /// Read addresses from a file, one per line. Use - for stdin.
    #[arg(long, value_name = "FILE")]
    pub urls_from: Option<String>,
}

/// Read one page or a list of them.
#[derive(Debug, Args)]
pub struct ScrapeArgs {
    /// Where to read from.
    #[command(flatten)]
    pub targets: Targets,

    /// What to ask for. Asking for less is where the saving is.
    #[arg(long, value_enum, default_value = "markdown")]
    pub goal: Goal,

    /// Named fields to pull out, as a JSON object of name to selector. Use -
    /// for stdin. Implies --goal fields.
    #[arg(long, value_name = "FILE")]
    pub selectors: Option<String>,
}

/// Read one path under the config the service holds for it.
#[derive(Debug, Args)]
pub struct FetchArgs {
    /// The host, with no scheme. Example: example.com
    #[arg(value_name = "DOMAIN")]
    pub domain: String,

    /// The path on that host. Defaults to the root.
    #[arg(value_name = "PATH", default_value = "/")]
    pub path: String,

    /// What to ask for. Defaults to raw, which leaves the stored config to
    /// decide. Naming anything else argues with it and usually wins.
    #[arg(long, value_enum, default_value = "raw")]
    pub goal: Goal,
}

/// Fetch a site, following its links.
#[derive(Debug, Args)]
pub struct CrawlArgs {
    /// Where to start.
    #[command(flatten)]
    pub targets: Targets,

    /// What to ask for on every page.
    #[arg(long, value_enum, default_value = "markdown")]
    pub goal: Goal,

    /// Named fields to pull off every page. Use - for stdin.
    #[arg(long, value_name = "FILE")]
    pub selectors: Option<String>,

    /// How many pages the crawl may visit. A crawl with no limit reads the
    /// whole site.
    #[arg(long, value_name = "N")]
    pub limit: Option<u32>,

    /// How many links deep from the starting page the crawl may go.
    #[arg(long, value_name = "N")]
    pub depth: Option<u32>,
}

/// Pull named fields out of pages.
#[derive(Debug, Args)]
pub struct ExtractArgs {
    /// Which pages to read.
    #[command(flatten)]
    pub targets: Targets,

    /// A JSON object of field name to selector, or to a list of selectors
    /// tried in order. A selector starting with a slash is read as XPath. Use
    /// - for stdin.
    #[arg(long, value_name = "FILE", required = true)]
    pub selectors: String,

    /// The path the fields apply to. Worth setting on a crawl, where a listing
    /// page and a product page hold different fields.
    #[arg(long, value_name = "PATH")]
    pub at: Option<String>,
}

/// Collect the links on a page.
#[derive(Debug, Args)]
pub struct LinksArgs {
    /// Which pages to read.
    #[command(flatten)]
    pub targets: Targets,
}

/// Run a query.
#[derive(Debug, Args)]
pub struct SearchArgs {
    /// What to search for.
    #[arg(value_name = "QUERY", required = true)]
    pub query: Vec<String>,

    /// How many results to work through.
    #[arg(long, value_name = "N")]
    pub limit: Option<u32>,

    /// Read the page behind every result. One fetch per result, so a query
    /// returning fifty of them is fifty fetches.
    #[arg(long)]
    pub fetch_pages: bool,
}

/// Take a picture of a page.
#[derive(Debug, Args)]
pub struct ScreenshotArgs {
    /// Which pages to shoot.
    #[command(flatten)]
    pub targets: Targets,
}

/// Convert markup you already hold.
#[derive(Debug, Args)]
pub struct TransformArgs {
    /// The markup to convert. Use - for stdin.
    #[arg(long, value_name = "FILE", required = true)]
    pub input: String,

    /// Where the markup came from, so links resolve.
    #[arg(long, value_name = "URL")]
    pub url: Option<String>,

    /// The shape to convert into.
    #[arg(long = "to", value_enum, default_value = "markdown")]
    pub to: Goal,

    /// Strip navigation, adverts and the rest of the furniture first.
    #[arg(long)]
    pub readability: bool,
}

/// Work a goal over addresses until it is met or the budget stops it.
#[derive(Debug, Args)]
pub struct RunArgs {
    /// Where to start.
    #[command(flatten)]
    pub targets: Targets,

    /// What counts as done for every address.
    #[arg(long, value_enum)]
    pub goal: Option<Goal>,

    /// Named fields, as a JSON object. Use - for stdin. Implies --goal fields.
    #[arg(long, value_name = "FILE")]
    pub selectors: Option<String>,

    /// Follow up to this many same-host links found on the pages already read.
    #[arg(long, value_name = "N")]
    pub expand: Option<usize>,

    /// Stop after this many pages, however they were reached.
    #[arg(long, value_name = "N")]
    pub max_pages: Option<usize>,

    /// Read goal, addresses and caps from a JSON file. Use - for stdin.
    /// Anything named on the command line wins.
    #[arg(long, value_name = "FILE")]
    pub plan: Option<String>,
}

/// Read rows off the account.
#[derive(Debug, Args)]
pub struct RowsArgs {
    /// How many rows to read.
    #[arg(long, value_name = "N")]
    pub limit: Option<u32>,

    /// Which page of rows, counting from one.
    #[arg(long, value_name = "N")]
    pub page: Option<u32>,
}

impl RowsArgs {
    /// Paging for a read that answers with one row, where a limit and a page
    /// would mean nothing.
    pub const fn none() -> RowsArgs {
        RowsArgs {
            limit: None,
            page: None,
        }
    }
}

/// Which agent clients to set up.
#[derive(Debug, Args)]
pub struct SetupArgs {
    /// Say what would change and write nothing.
    #[arg(long)]
    pub dry_run: bool,
    /// Set up only these clients. Without it, every client found.
    #[arg(long, value_enum, value_name = "NAME")]
    pub client: Vec<Client>,
}

/// An agent client `setup` knows how to register the server with.
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Client {
    /// Claude Code, through `claude mcp add`.
    ClaudeCode,
    /// Codex, through `codex mcp add`.
    Codex,
    /// Cursor, in `~/.cursor/mcp.json`.
    Cursor,
    /// Windsurf, in `~/.codeium/windsurf/mcp_config.json`.
    Windsurf,
    /// Claude Desktop, in its `claude_desktop_config.json`.
    ClaudeDesktop,
    /// VS Code, through `code --add-mcp`.
    Vscode,
}

impl Client {
    /// The name on the command line and in a record.
    pub const fn label(self) -> &'static str {
        match self {
            Client::ClaudeCode => "claude-code",
            Client::Codex => "codex",
            Client::Cursor => "cursor",
            Client::Windsurf => "windsurf",
            Client::ClaudeDesktop => "claude-desktop",
            Client::Vscode => "vscode",
        }
    }
}

/// Sign in.
#[derive(Debug, Args)]
pub struct LoginArgs {
    /// Print the key instead of storing it. It goes to stdout and nowhere
    /// else, so redirect it or do not use it.
    #[arg(long)]
    pub print: bool,
    /// Sign in with a code typed on another device, for a machine with no
    /// browser.
    #[arg(long)]
    pub device: bool,
    /// With --device: print the code and leave without waiting for approval.
    #[arg(long, requires = "device", conflicts_with_all = ["resume", "print"])]
    pub no_wait: bool,
    /// With --device: collect the key for the code --no-wait printed.
    #[arg(long, requires = "device")]
    pub resume: bool,
    /// With --device: the most seconds to wait for approval. Without it the
    /// wait lasts as long as the code does, or 30 seconds under --resume.
    #[arg(long, requires = "device", value_name = "SECONDS")]
    pub wait: Option<u64>,
}

/// The stored provider fallback.
#[derive(Debug, Args)]
pub struct RouterArgs {
    /// What to do with it.
    #[command(subcommand)]
    pub command: RouterCommand,
}

/// What `router` does.
#[derive(Debug, Subcommand)]
pub enum RouterCommand {
    /// Store or change the fallback. A flag you do not pass keeps its stored
    /// value.
    Set(RouterSetArgs),
    /// Print the stored fallback, with every key and setting value redacted.
    Show,
    /// Delete the stored fallback.
    Clear,
}

/// Store or change the provider fallback.
#[derive(Debug, Args)]
pub struct RouterSetArgs {
    /// The provider, by name. A name this version does not know is stored with
    /// a note, because providers are added without a release.
    #[arg(long, value_name = "NAME")]
    pub provider: Option<String>,

    /// How the provider takes part.
    #[arg(long, value_name = "MODE", value_parser = router_modes())]
    pub mode: Option<Mode>,

    /// Which routes may be used: any, or only those your own keys pay for.
    #[arg(long, value_enum)]
    pub funding: Option<Funding>,

    /// Read the provider token from stdin. SPIDER_ROUTER_TOKEN works too.
    #[arg(long, conflicts_with = "no_token")]
    pub token_stdin: bool,

    /// Refused: a key on the command line lands in shell history. Use
    /// --token-stdin or SPIDER_ROUTER_TOKEN.
    #[arg(long, value_name = "VALUE", num_args = 0..=1, hide = true)]
    pub token: Option<Option<String>>,

    /// Remove the stored token.
    #[arg(long)]
    pub no_token: bool,

    /// A credential, as NAME=VALUE for a value that is not secret, or
    /// NAME-stdin to read the value from stdin. Repeat for more.
    #[arg(long, value_name = "NAME=VALUE|NAME-stdin")]
    pub credential: Vec<String>,

    /// Remove a stored credential. Repeat for more.
    #[arg(long, value_name = "NAME")]
    pub no_credential: Vec<String>,

    /// A provider setting, as PROVIDER.KEY=VALUE. A value that reads as JSON is
    /// stored as JSON, anything else as text. Applied only on routes your own
    /// key pays for. Repeat for more.
    #[arg(long, value_name = "PROVIDER.KEY=VALUE")]
    pub option: Vec<String>,

    /// Remove a stored provider setting. Repeat for more.
    #[arg(long, value_name = "PROVIDER.KEY")]
    pub no_option: Vec<String>,
}

/// The stored optimizer settings.
#[derive(Debug, Args)]
pub struct OptimizeArgs {
    /// What to do with them.
    #[command(subcommand)]
    pub command: OptimizeCommand,
}

/// What `optimize` does.
#[derive(Debug, Subcommand)]
pub enum OptimizeCommand {
    /// Store or change the settings. A flag you do not pass keeps its stored
    /// value.
    Set(OptimizeSetArgs),
    /// Print the stored settings.
    Show,
    /// Delete the stored settings.
    Clear,
}

/// Store or change how the optimizer runs.
#[derive(Debug, Args)]
pub struct OptimizeSetArgs {
    /// What to do with the pick.
    #[arg(long, value_name = "MODE", value_parser = optimizer_modes())]
    pub mode: Option<Mode>,

    /// The weights file to score with. It is read on every run that fetches a
    /// page, so keep it where it is.
    #[arg(long, value_name = "FILE", conflicts_with = "no_model")]
    pub model: Option<String>,

    /// Forget the stored weights file. The mode is left as it is.
    #[arg(long)]
    pub no_model: bool,
}

/// Which routes a request may be paid through.
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Funding {
    /// Any route.
    Any,
    /// Only routes your own keys pay for.
    Own,
}

/// Ask what transport would be chosen.
#[derive(Debug, Args)]
pub struct RouteArgs {
    /// The addresses to decide about.
    #[command(flatten)]
    pub targets: Targets,

    /// What the request would ask for, which changes the answer.
    #[arg(long, value_enum, default_value = "markdown")]
    pub goal: Goal,
}