[package]
name = "halldyll-robots"
version.workspace = true
edition.workspace = true
authors.workspace = true
license.workspace = true
repository.workspace = true
documentation = "https://docs.rs/halldyll-robots"
description = "robots.txt parser and compliance for halldyll scraper"
keywords = ["robots", "txt", "crawler", "compliance", "legal"]
categories = ["web-programming", "parser-implementations"]
[dependencies]
texting_robots = { workspace = true }
reqwest.workspace = true
tokio.workspace = true
url.workspace = true
thiserror.workspace = true
dashmap.workspace = true
chrono.workspace = true
tracing.workspace = true
serde.workspace = true
serde_json = { workspace = true }
[dev-dependencies]
tokio-test = "0.4"