Struct spider::page::Page

pub struct Page {
    pub status_code: StatusCode,
    pub error_status: Option<String>,
    pub external_domains_caseless: Box<HashSet<CaseInsensitiveString>>,
    pub final_redirect_destination: Option<String>,
    /* private fields */
}

Represent a page visited. This page contains HTML scraped with scraper.

Fields

status_code: StatusCode

The status code of the page request.

error_status: Option<String>

The error of the request if any.

external_domains_caseless: Box<HashSet<CaseInsensitiveString>>

The external urls to group with the domain

final_redirect_destination: Option<String>

The final destination of the page if redirects were performed [Not implemented in the chrome feature].

Implementations

impl Page

pub async fn new_page(url: &str, client: &Client) -> Self

Instantiate a new page and gather the html repro of standard fetch_page_html.

pub async fn new(url: &str, client: &Client) -> Self

Instantiate a new page and gather the html.

pub async fn screenshot( &self, _full_page: bool, _omit_background: bool, _format: CaptureScreenshotFormat, _quality: Option<i64>, _output_path: Option<impl AsRef<Path>>, _clip: Option<ClipViewport> ) -> Vec<u8>

Take a screenshot of the page. If the output path is set to None the screenshot will not be saved. The feature flag chrome_store_page is required.

pub fn is_empty(&self) -> bool

Page request fulfilled.

pub fn get_url(&self) -> &str

Url getter for page.

pub fn get_url_final(&self) -> &str

Url getter for page after redirects.

pub fn set_external( &mut self, external_domains_caseless: Box<HashSet<CaseInsensitiveString>> )

Set the external domains to treat as one

pub fn set_html_bytes(&mut self, html: Option<Bytes>)

Set the html directly of the page

pub fn get_url_parsed(&self) -> &Url

Parsed URL getter for page.

pub fn get_bytes(&self) -> Option<&Bytes>

Html getter for bytes on the page.

pub fn get_html(&self) -> String

Html getter for bytes on the page as string.

pub fn get_html_bytes_u8(&self) -> &[u8]

Html getter for page to u8.

pub fn get_html_encoded(&self, _label: &str) -> String

Html getter for getting the content with proper encoding. Pass in a proper encoding label like SHIFT_JIS. This fallsback to get_html without the encoding flag enabled.

pub fn push_link<A: PartialEq + Eq + Hash + From<String>>( &self, href: &str, map: &mut HashSet<A>, base_domain: &CompactString, parent_host: &CompactString, parent_host_scheme: &CompactString )

Validate link and push into the map

pub async fn links_stream_xml_links_stream_base<A: PartialEq + Eq + Hash + From<String>>( &self, selectors: &(&CompactString, &SmallVec<[CompactString; 2]>), xml: &str, map: &mut HashSet<A> )

Find the links as a stream using string resource validation for XML files

pub async fn links_stream_base<A: PartialEq + Eq + Hash + From<String>>( &self, selectors: &(&CompactString, &SmallVec<[CompactString; 2]>), html: &str ) -> HashSet<A>

Find the links as a stream using string resource validation

pub async fn links_stream<A: PartialEq + Eq + Hash + From<String>>( &self, selectors: &(&CompactString, &SmallVec<[CompactString; 2]>) ) -> HashSet<A>

Find the links as a stream using string resource validation

pub async fn links_stream_full_resource<A: PartialEq + Eq + Hash + From<String>>( &self, selectors: &(&CompactString, &SmallVec<[CompactString; 2]>) ) -> HashSet<A>

Find the links as a stream using string resource validation

pub async fn links( &self, selectors: &(CompactString, SmallVec<[CompactString; 2]>) ) -> HashSet<CaseInsensitiveString>

Find all href links and return them using CSS selectors.

pub async fn links_full( &self, selectors: &(CompactString, SmallVec<[CompactString; 2]>) ) -> HashSet<CaseInsensitiveString>

Find all href links and return them using CSS selectors gathering all resources.

Trait Implementations

impl Clone for Page

fn clone(&self) -> Page

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Page

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.