scout-interpreter 0.7.1

A web crawling programming language
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88

use get_port::Ops;

use crate::{env::EnvPointer, eval::ScrapeResultsPtr, EnvVars, Interpreter};

#[derive(Debug)]
pub enum BuilderError {
    BrowserStartup(String),
    EnvError(String),
}

#[derive(Default)]
pub struct InterpreterBuilder {
    env: Option<EnvPointer>,
    crawler: Option<fantoccini::Client>,
    results: Option<ScrapeResultsPtr>,
}

impl InterpreterBuilder {
    pub fn with_env(mut self, env: EnvPointer) -> Self {
        self.env = Some(env);
        self
    }

    pub fn with_crawler(mut self, crawler: fantoccini::Client) -> Self {
        self.crawler = Some(crawler);
        self
    }

    pub fn with_results(mut self, results: ScrapeResultsPtr) -> Self {
        self.results = Some(results);
        self
    }

    pub async fn build(self) -> Result<Interpreter, BuilderError> {
        let env_vars =
            envy::from_env::<EnvVars>().map_err(|e| BuilderError::EnvError(e.to_string()))?;
        let port = env_vars
            .port()
            .unwrap_or_else(|| get_port::tcp::TcpPort::any("127.0.0.1").unwrap() as usize);
        let child = crate::GeckDriverProc::new(port);
        let crawler = match self.crawler {
            Some(c) => Ok(c),
            None => new_crawler(&env_vars, port).await,
        }?;

        let interpreter = Interpreter::new(
            self.env.unwrap_or_default(),
            self.results.unwrap_or_default(),
            crawler,
            child,
        );

        Ok(interpreter)
    }
}

async fn new_crawler(env_vars: &EnvVars, port: usize) -> Result<fantoccini::Client, BuilderError> {
    let mut caps = serde_json::map::Map::new();
    if !env_vars.scout_debug {
        let opts = serde_json::json!({ "args": ["--headless"] });
        caps.insert("moz:firefoxOptions".into(), opts);
    }
    if let Some(proxy) = env_vars.scout_proxy.clone() {
        let opt = serde_json::json!({
            "proxyType": "manual",
            "httpProxy": proxy,
        });
        caps.insert("proxy".into(), opt);
    }
    let conn_url = format!("http://localhost:{}", port);
    let crawler = fantoccini::ClientBuilder::native()
        .capabilities(caps)
        .connect(&conn_url)
        .await
        .map_err(|e| BuilderError::BrowserStartup(e.to_string()))?;
    Ok(crawler)
}

impl std::fmt::Display for BuilderError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            BuilderError::BrowserStartup(e) => write!(f, "{}", e),
            BuilderError::EnvError(e) => write!(f, "{}", e),
        }
    }
}

impl std::error::Error for BuilderError {}