Struct CrawlOpts 

pub struct CrawlOpts {
    pub seed_url: Option<String>,
    pub global_concurrency: usize,
    pub per_host_concurrency: usize,
    pub per_host_delay_ms: u64,
    pub max_depth: u16,
    pub redis_url: Option<String>,
    pub redis_rate_limit: bool,
    pub storage_dir: Option<PathBuf>,
}

Fields

seed_url: Option<String>

Starting URL. Required unless using –redis-url as consumer-only.

global_concurrency: usize

per_host_concurrency: usize

per_host_delay_ms: u64

max_depth: u16

redis_url: Option<String>

Redis URL for distributed mode. If set with no value, uses redis://127.0.0.1:6379/ (matches docker-compose).

redis_rate_limit: bool

When using Redis, use Redis for per-host rate limiting so all processes share the same limit.

storage_dir: Option<PathBuf>

Directory to persist fetch results (metadata JSON + body files). If unset, nothing is written to disk.

Trait Implementations

impl Args for CrawlOpts

fn group_id() -> Option<Id>

Report the ArgGroup::id for this set of arguments

fn augment_args<'b>(__clap_app: Command) -> Command

Append to Command so it can instantiate Self via FromArgMatches::from_arg_matches_mut

fn augment_args_for_update<'b>(__clap_app: Command) -> Command

Append to Command so it can instantiate self via FromArgMatches::update_from_arg_matches_mut

impl CommandFactory for CrawlOpts

fn command<'b>() -> Command

Build a Command that can instantiate Self.

fn command_for_update<'b>() -> Command

Build a Command that can update self.

impl Debug for CrawlOpts

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl FromArgMatches for CrawlOpts

fn from_arg_matches(__clap_arg_matches: &ArgMatches) -> Result<Self, Error>

Instantiate Self from ArgMatches, parsing the arguments as needed.

fn from_arg_matches_mut( __clap_arg_matches: &mut ArgMatches, ) -> Result<Self, Error>

Instantiate Self from ArgMatches, parsing the arguments as needed.

fn update_from_arg_matches( &mut self, __clap_arg_matches: &ArgMatches, ) -> Result<(), Error>

Assign values from ArgMatches to self.

fn update_from_arg_matches_mut( &mut self, __clap_arg_matches: &mut ArgMatches, ) -> Result<(), Error>

Assign values from ArgMatches to self.

impl Parser for CrawlOpts

fn parse() -> Self

Parse from std::env::args_os(), exit on error.

fn try_parse() -> Result<Self, Error>

Parse from std::env::args_os(), return Err on error.

fn parse_from<I, T>(itr: I) -> Self

where I: IntoIterator<Item = T>, T: Into<OsString> + Clone,

Parse from iterator, exit on error.

fn try_parse_from<I, T>(itr: I) -> Result<Self, Error>

where I: IntoIterator<Item = T>, T: Into<OsString> + Clone,

Parse from iterator, return Err on error.

fn update_from<I, T>(&mut self, itr: I)

where I: IntoIterator<Item = T>, T: Into<OsString> + Clone,

Update from iterator, exit on error.

fn try_update_from<I, T>(&mut self, itr: I) -> Result<(), Error>

where I: IntoIterator<Item = T>, T: Into<OsString> + Clone,

Update from iterator, return Err on error.