spider_cli 1.31.7

The fastest web crawler CLI written in Rust.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

use clap::Subcommand;

#[derive(Subcommand)]
pub enum Commands {
    /// crawl the website extracting links.
    CRAWL {
        /// sequentially one by one crawl pages
        #[clap(short, long)]
        sync: bool,
        /// stdout all links crawled
        #[clap(short, long)]
        output_links: bool,
    },
    /// scrape the website extracting html and links.
    SCRAPE {
        /// stdout all pages links crawled
        #[clap(short, long)]
        output_links: bool,
        /// stdout all pages html crawled
        #[clap(long)]
        output_html: bool,
    },
}