rust_scrapling 0.2.3

A Rust port of Scrapling - modern web scraping framework with CSS selectors, async HTTP, and spider-based crawling
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92

//! [`SpiderResponse`]: the response type handed to
//! [`Spider::parse`](crate::spiders::spider::Spider::parse), wrapping a
//! fetcher [`Response`](crate::fetchers::response::Response) with direct
//! extraction shortcuts.

use crate::fetchers::response::Response as FetcherResponse;
use crate::parser::{Selector, Selectors};

/// A fetched (or cache-replayed) response as seen by a spider's `parse`
/// callback, with parsing shortcuts. Note each `css*` call re-parses the
/// body; when making many queries, call [`SpiderResponse::selector`] once
/// and query that.
pub struct SpiderResponse {
    inner: FetcherResponse,
}

impl SpiderResponse {
    /// Wrap a fetcher response.
    pub fn new(inner: FetcherResponse) -> Self {
        Self { inner }
    }

    /// The underlying fetcher response (full headers, content type, body).
    #[must_use]
    pub fn response(&self) -> &FetcherResponse {
        &self.inner
    }

    /// The HTTP status code.
    #[must_use]
    pub fn status(&self) -> u16 {
        self.inner.status()
    }

    /// The decoded response body.
    #[must_use]
    pub fn text(&self) -> &str {
        self.inner.text()
    }

    /// The final URL of the response, after any redirects.
    #[must_use]
    pub fn url(&self) -> &str {
        self.inner.url()
    }

    /// Parse the body as HTML into a [`Selector`] rooted at the document
    /// (base URL set for `urljoin`).
    #[must_use]
    pub fn selector(&self) -> Selector {
        self.inner.selector()
    }

    /// Parse the body and run a plain CSS query against it (see
    /// [`Selector::css`]).
    #[must_use]
    pub fn css(&self, selector: &str) -> Selectors {
        self.selector().css(selector)
    }

    /// Parsel-inspired CSS query with `::text` / `::attr(name)` support; all
    /// extracted values. See [`Selector::css_getall`].
    #[must_use]
    pub fn css_getall(&self, query: &str) -> Vec<crate::core::text_handler::TextHandler> {
        self.selector().css_getall(query)
    }

    /// Parsel-inspired CSS query with `::text` / `::attr(name)` support; first
    /// extracted value. See [`Selector::css_get`].
    #[must_use]
    pub fn css_get(&self, query: &str) -> Option<crate::core::text_handler::TextHandler> {
        self.selector().css_get(query)
    }

    /// Parse the body as JSON.
    pub fn json(&self) -> Result<serde_json::Value, serde_json::Error> {
        self.inner.json()
    }

    /// Whether the status code is one of the bot-blocking codes (see
    /// [`Response::is_blocked`](crate::fetchers::response::Response::is_blocked)).
    #[must_use]
    pub fn is_blocked(&self) -> bool {
        self.inner.is_blocked()
    }

    /// Size of the decoded body in bytes.
    #[must_use]
    pub fn content_length(&self) -> usize {
        self.inner.content_length()
    }
}