Struct ParseOutput 

pub struct ParseOutput<I> { /* private fields */ }

The output of a spider’s parse method.

Implementations

impl<I> ParseOutput<I>

pub fn new() -> Self

Creates a new, empty ParseOutput.

Examples found in repository

examples/quotes_scraper.rs (line 39)

    async fn parse(&mut self, response: Response) -> Result<ParseOutput<Self::Item>, SpiderError> {
        let html = response.to_html()?;
        let mut output = ParseOutput::new();

        for quote_element in html.select(&".quote".to_selector()?) {
            let text = quote_element
                .select(&".text".to_selector()?)
                .next()
                .map(|e| e.text().collect::<String>().trim().to_string())
                .unwrap_or_default();

            let author = quote_element
                .select(&".author".to_selector()?)
                .next()
                .map(|e| e.text().collect::<String>().trim().to_string())
                .unwrap_or_default();

            let tags: Vec<String> = quote_element
                .select(&".tags .tag".to_selector()?)
                .map(|e| e.text().collect::<String>().trim().to_string())
                .collect();
            let tags_str = tags.join(", ");

            let item = QuoteItem {
                text,
                author,
                tags: tags_str,
            };
            output.add_item(item);
        }

        if let Some(next_href) = html
            .select(&".next > a".to_selector()?)
            .next()
            .and_then(|a| a.attr("href"))
        {
            let next_url = response.url.join(next_href)?;
            let next_request = Request::new(next_url);
            output.add_request(next_request);
        }

        Ok(output)
    }

pub fn into_parts(self) -> (Vec<I>, Vec<Request>)

Consumes the ParseOutput and returns its inner items and requests.

pub fn add_item(&mut self, item: I)

Adds a scraped item to the output.

Examples found in repository

examples/quotes_scraper.rs (line 65)

    async fn parse(&mut self, response: Response) -> Result<ParseOutput<Self::Item>, SpiderError> {
        let html = response.to_html()?;
        let mut output = ParseOutput::new();

        for quote_element in html.select(&".quote".to_selector()?) {
            let text = quote_element
                .select(&".text".to_selector()?)
                .next()
                .map(|e| e.text().collect::<String>().trim().to_string())
                .unwrap_or_default();

            let author = quote_element
                .select(&".author".to_selector()?)
                .next()
                .map(|e| e.text().collect::<String>().trim().to_string())
                .unwrap_or_default();

            let tags: Vec<String> = quote_element
                .select(&".tags .tag".to_selector()?)
                .map(|e| e.text().collect::<String>().trim().to_string())
                .collect();
            let tags_str = tags.join(", ");

            let item = QuoteItem {
                text,
                author,
                tags: tags_str,
            };
            output.add_item(item);
        }

        if let Some(next_href) = html
            .select(&".next > a".to_selector()?)
            .next()
            .and_then(|a| a.attr("href"))
        {
            let next_url = response.url.join(next_href)?;
            let next_request = Request::new(next_url);
            output.add_request(next_request);
        }

        Ok(output)
    }

pub fn add_request(&mut self, request: Request)

Adds a new request to be crawled.

Examples found in repository

examples/quotes_scraper.rs (line 75)

    async fn parse(&mut self, response: Response) -> Result<ParseOutput<Self::Item>, SpiderError> {
        let html = response.to_html()?;
        let mut output = ParseOutput::new();

        for quote_element in html.select(&".quote".to_selector()?) {
            let text = quote_element
                .select(&".text".to_selector()?)
                .next()
                .map(|e| e.text().collect::<String>().trim().to_string())
                .unwrap_or_default();

            let author = quote_element
                .select(&".author".to_selector()?)
                .next()
                .map(|e| e.text().collect::<String>().trim().to_string())
                .unwrap_or_default();

            let tags: Vec<String> = quote_element
                .select(&".tags .tag".to_selector()?)
                .map(|e| e.text().collect::<String>().trim().to_string())
                .collect();
            let tags_str = tags.join(", ");

            let item = QuoteItem {
                text,
                author,
                tags: tags_str,
            };
            output.add_item(item);
        }

        if let Some(next_href) = html
            .select(&".next > a".to_selector()?)
            .next()
            .and_then(|a| a.attr("href"))
        {
            let next_url = response.url.join(next_href)?;
            let next_request = Request::new(next_url);
            output.add_request(next_request);
        }

        Ok(output)
    }

pub fn add_items(&mut self, items: impl IntoIterator<Item = I>)

Adds multiple scraped items to the output.

pub fn add_requests(&mut self, requests: impl IntoIterator<Item = Request>)

Adds multiple new requests to be crawled.

Trait Implementations

impl<I: Clone> Clone for ParseOutput<I>

fn clone(&self) -> ParseOutput<I>

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<I: Debug> Debug for ParseOutput<I>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<I> Default for ParseOutput<I>

fn default() -> Self

Returns the “default value” for a type.