[package]
name = "crusty-core"
version = "0.82.0"
authors = ["Sergey F. <let4be@gmail.com>"]
edition = "2021"
description = "Library for creating blazing fast and configurable web crawlers"
readme = "../README.md"
license = "GPL-3.0"
repository = "https://github.com/let4be/crusty-core"
keywords = ["crawler", "spider", "rust", "async", "multi-threaded"]
[package.metadata.release]
[features]
select_rs = ["select"]
[[example]]
name="find_duplicate_titles"
required-features = ["select_rs"]
[[example]]
name="simple"
required-features = ["select_rs"]
[[example]]
name="simple-short"
required-features = ["select_rs"]
[dependencies]
tokio = { version = "^1.14", features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros"] }
strum = { version = "~0.23", features = ["derive"] }
tracing-tools = "~0.5.0"
flume = "~0.10.5"
futures-lite = "~1.12.0"
serde = {version = "~1.0.125", features = ["derive"] }
hyper = {version = "~0.14.12", features = ["http1", "http2", "client"] }
robotstxt-with-cache = {version = "~0.4.0"}
trust-dns-resolver = {version = "~0.20.2", features = ["system-config", "tokio-runtime", "serde-config"]}
hyper-tls = "~0.5.0"
url = "~2.2.1"
http = "~0.2.3"
bytes = "~1.1.0"
flate2 = "~1.0.20"
select = {version = "~0.5.0", optional = true}
ipnet = "~2.3.0"
humansize = "~1.1.0"
humanize-rs = "~0.1.5"
tracing = "~0.1.25"
thiserror = "~1.0.24"
anyhow = "~1.0.40"
pin-project = "~1.0.7"
lazy_static = "~1.4.0"
num_cpus = "~1.13.0"
rand = "~0.8.0"
derivative = "~2.2.0"
core_affinity = "~0.5.10"
[dev-dependencies]
html5ever = "~0.25.1"
markup5ever = "~0.10.1"
tracing-subscriber = "~0.3.2"
tracing = "~0.1.25"
[package.metadata.cargo-udeps.ignore]
development = ["tracing-subscriber", "tracing"]