website_crawler 0.9.9

gRPC tokio based web crawler built with spider
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

syntax = "proto3";

package website;

// Central API that manages your website between starting single and multi page scans.
service WebsiteService {
   rpc ScanStart (ScanInitParams) returns (Empty) {} // track when scan starts.
   rpc ScanEnd (ScanInitParams) returns (Empty) {} // tracks when scan completes.
   rpc Scan (ScanParams) returns (Empty) {} // non stream scanning allowing for full track up time for keep alive cost.
   rpc ScanStream (ScanParams) returns (stream ScanStreamResponse) {} // stream the scan request and return if scan should continue.
}

// params to send when scanning pages.
message ScanParams {
   repeated string pages = 1; // list of pages returned.
   string domain = 2; // the url base of the crawl.
   uint32 user_id = 3; // user id performing scan.
   bool full = 4; // full crawl performed with all links.
   string html = 5; // raw HTML to verify.
}

// params to declare scan stream initiation [Should use bidirectional streams instead].
message ScanInitParams {
   string domain = 1; // the url base of the crawl.
   uint32 user_id = 2; // user id performing scan.
}

// send nothing mainly for triggering events.
message Empty {}

// send streamed response
message ScanStreamResponse {
   string message = 1; // message of the scan success or if should terminate.
}