robotparser 0.10.0

robots.txt parser for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

[package]
authors = ["messense <messense@icloud.com>"]
description = "robots.txt parser for Rust"
documentation = "https://messense.github.io/robotparser-rs"
homepage = "https://github.com/messense/robotparser-rs"
keywords = ["robot", "robots", "parser", "spider", "crawler"]
license = "MIT"
name = "robotparser"
readme = "README.md"
repository = "https://github.com/messense/robotparser-rs"
version = "0.10.0"

[dependencies]
url = "1.2"

[dependencies.reqwest]
version = "0.8"
optional = true

[dependencies.clippy]
optional = true
version = "^0.*"

[features]
default = ["http"]
http = ["reqwest"]
unstable = []