scrape-le 0.3.1

Check whether a page is scrapeable before the scraper is written, and say when it cannot tell
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78

[package]
name = "scrape-le"
version = "0.3.1"
edition = "2024"
rust-version = "1.88"
license = "MIT"
readme = "README.md"
description = "Check whether a page is scrapeable before the scraper is written, and say when it cannot tell"
authors = ["Nolin D Naidoo"]
homepage = "https://letools.dev/tools/scrape-le"
repository = "https://github.com/nolindnaidoo/scrape-le"
keywords = ["robots-txt", "web-scraping", "crawler", "anti-bot", "scraping"]
categories = ["command-line-utilities", "web-programming", "development-tools", "text-processing", "parsing"]
# What a consumer gets. SPEC.md ships because it is the behavioural
# contract — verdicts, exit codes, batching — and someone scripting
# against the exit codes needs it. AGENTS.md and CLAUDE.md are how this
# repo is worked on, not how the tool is used, and examples/spike.rs is
# a network-touching development spike. signatures/ and fixtures/ are
# not optional: the build embeds the first and the tests read both.
exclude = ["AGENTS.md", "CLAUDE.md", "rust-toolchain.toml", "examples/"]
# Excluding a directory does not stop cargo auto-discovering a target in
# it, so every `cargo package` printed "ignoring example `spike`". A
# warning on every run of a command is one people stop reading.
autoexamples = false

[[bin]]
name = "scrape-le"
path = "src/main.rs"

[dependencies]
# The corpus is TOML by design (reviewable by non-Rust readers); serde +
# toml parse it at startup from the embedded files. regex ports the
# extension's URL and robots patterns verbatim — hand-rolling them would
# be parity drift waiting to happen. url is the native-URL half of
# validateUrl and is already in the tree via ureq.
serde = { version = "1", features = ["derive"] }
serde_json = "1"
toml = "0.9"
regex = "1"
url = "2"
# The decided stack (AGENTS.md): sync, no runtime. ureq is the HTTP
# half, headless_chrome the sync-CDP render half.
ureq = "3"
headless_chrome = "1"

# anyhow is error glue for examples/spike.rs alone, which `exclude`
# keeps out of the published package — so this is a dependency of the
# repository, never of the crate a consumer builds.
[dev-dependencies]
anyhow = "1"

[profile.release]
strip = true
lto = "thin"
codegen-units = 1
# Every output of this tool is a claim someone scripts against — a verdict,
# a timing, a byte count. A wrapped number is silently wrong data in a
# report whose whole value is honesty, which is worse than a crash — so
# overflow keeps panicking in release, as it does in debug. (Same rule as
# pixelcoords and pixelactions.)
overflow-checks = true

[lints.rust]
unsafe_code = "forbid"

[lints.clippy]
all = { level = "warn", priority = -1 }
pedantic = { level = "warn", priority = -1 }
# Deliberate relaxations, same policy as pixelcoords/pixelactions: no
# inline #[allow] anywhere (CI enforces); anything relaxed must be listed
# here, visibly.
cast_possible_truncation = "allow"
cast_possible_wrap = "allow"
cast_sign_loss = "allow"
cast_precision_loss = "allow"
must_use_candidate = "allow"
missing_errors_doc = "allow"
missing_panics_doc = "allow"