spider 2.51.69

A web crawler and scraper, building blocks for data curation workloads.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

//! Comprehensive integration test suite based on <https://www.crawler-test.com/>.
//!
//! Gated behind `RUN_LIVE_TESTS` env var. Requires `basic` feature at minimum.
//! Smart-mode tests require `smart` feature. Chrome-mode tests require `chrome` feature.
//!
//! Run:
//!   RUN_LIVE_TESTS=1 cargo test --test crawler_test_com --features "basic"
//!   RUN_LIVE_TESTS=1 cargo test --test crawler_test_com --features "basic,smart"
//!   RUN_LIVE_TESTS=1 cargo test --test crawler_test_com --features "basic,chrome"

mod helpers;

mod canonical_tags;
mod content;
mod description_tags;
mod encoding;
mod javascript;
mod links;
mod mobile;
mod other;
mod redirects;
mod robots_protocol;
mod social_tags;
mod status_codes;
mod titles;
mod urls;