scrapman 0.1.1

A high-level declarative web scraping framework
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

use crate::{
    client::ScrapeClient,
    pipeline::{ScrapeContext, ScrapeError, ScrapePipeline},
    value::JsonValue,
};
use fantoccini::ClientBuilder;
use log::info;

pub type ScrapeResult = Result<ScrapeContext, ScrapeError>;

pub struct Scrapman {
    webdriver_url: String,
}

impl Scrapman {
    pub fn new<T: Into<String>>(webdriver_url: T) -> Self {
        Scrapman {
            webdriver_url: webdriver_url.into(),
        }
    }

    pub async fn launch<T>(&self, pipeline: ScrapePipeline, values: T) -> ScrapeResult
    where
        T: Into<Option<JsonValue>>,
    {
        let client = ClientBuilder::native()
            .connect(&self.webdriver_url)
            .await
            .map_err(ScrapeError::WebdriverConnectionError)?;

        self.launch_with_client(pipeline, values, client).await
    }

    pub async fn launch_with_client<Values, Client>(
        &self,
        pipeline: ScrapePipeline,
        values: Values,
        client: Client,
    ) -> ScrapeResult
    where
        Values: Into<Option<JsonValue>>,
        Client: ScrapeClient + 'static,
    {
        let mut context = ScrapeContext::new(client, values);
        info!("Launching pipeline execution");
        pipeline.execute(&mut context).await?;
        context.client.disconnect().await?;
        Ok(context)
    }
}