website_crawler 0.9.9

gRPC tokio based web crawler built with spider
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

pub mod website {
    tonic::include_proto!("website");
}

use std::env::var;

use tonic::{transport::Channel, Streaming};

use spider::utils::log;

// gRPC client
pub use website::{
    website_service_client::WebsiteServiceClient, ScanInitParams, ScanParams, ScanStreamResponse,
};

/// create gRPC client from the API server.
pub async fn create_client() -> Result<WebsiteServiceClient<Channel>, tonic::transport::Error> {
    lazy_static! {
        static ref CLIENT: String = format!(
            "http://{}",
            var("GRPC_HOST_API").unwrap_or_else(|_| "[::1]:50051".to_string())
        );
    };

    let client = WebsiteServiceClient::connect(CLIENT.as_ref()).await?;

    Ok(client)
}

/// request to the API server that scan has started.
pub async fn monitor_page_start(
    client: &mut WebsiteServiceClient<Channel>,
    page: ScanInitParams,
) -> Result<(), tonic::Status> {
    let request = tonic::Request::new(page);

    client.scan_start(request).await?;

    Ok(())
}

/// request to the API server that scan has finished.
pub async fn monitor_page_complete(
    client: &mut WebsiteServiceClient<Channel>,
    page: ScanInitParams,
) -> Result<(), tonic::Status> {
    let request = tonic::Request::new(page);

    client.scan_end(request).await?;

    Ok(())
}

/// request to the API server to perform scan action to gather results.
pub async fn monitor_page_async(page: ScanParams) -> Result<(), tonic::Status> {
    let mut client = create_client().await.unwrap();
    let request = tonic::Request::new(page);

    client.scan(request).await?;

    Ok(())
}

/// run a accessibility scan waiting for results.
pub async fn monitor(
    client: &mut WebsiteServiceClient<Channel>,
    link: String,
    user_id: u32,
    html: String,
) -> bool {
    let request = tonic::Request::new(ScanParams {
        pages: vec![link],
        user_id,
        html,
        ..Default::default()
    });
    let stream: Option<Streaming<ScanStreamResponse>> = match client.scan_stream(request).await {
        Ok(val) => Some(val.into_inner()),
        Err(e) => {
            log("error status-code :", &e.code().to_string());

            None
        }
    };

    let mut perform_shutdown = false; // shutdown was performed on website - terminate entire crawl

    match stream {
        Some(mut res) => {
            while let Some(r) = res.message().await.unwrap_or_default() {
                if r.message == "shutdown" {
                    perform_shutdown = true;
                }
            }
        }
        None => {
            // perform shut down true
            perform_shutdown = true;
        }
    };

    !perform_shutdown
}