spider_cli 1.7.14

Multithreaded web crawler written in Rust.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

use crate::options::sub_command::Commands;
use clap::Parser;

/// program to crawl a website and gather valid web urls.
#[derive(Parser)]
#[clap(author, version, about, long_about = None)]
pub struct Cli {
    /// Build main sub commands
    #[clap(subcommand)]
    pub command: Option<Commands>,
    /// Domain to crawl
    #[clap(short, long)]
    pub domain: String,
    /// Respect robots.txt file
    #[clap(short, long)]
    pub respect_robots_txt: bool,
    /// Print page visited on standard output
    #[clap(short, long)]
    pub verbose: bool,
    /// Polite crawling delay in milli seconds
    #[clap(short, long)]
    pub delay: Option<u64>,
    /// How many request can be run simultaneously
    #[clap(short, long)]
    pub concurrency: Option<usize>,
    /// Comma seperated string list of pages to not crawl or regex with feature enabled
    #[clap(short, long)]
    pub blacklist_url: Option<String>,
    /// User-Agent
    #[clap(short, long)]
    pub user_agent: Option<String>,
}