syntax = "proto3";
package crawler;
// The web indexer to find links async.
service Crawler {
rpc Scan (ScanRequest) returns (ScanReply) {} // sends feedback async as links found
rpc Crawl (ScanRequest) returns (ScanReply) {} // waits for all links to be found before returning
}
// basic reply message.
message ScanReply {
string message = 1; // message of the scan success.
}
// Request params for crawling generically.
message ScanRequest {
string url = 1; // the base request to start crawling/indexing pages.
uint32 id = 2; // the user id or identifier to track crawl subs.
bool norobots = 3; // ignore respect robots txt file rules.
string agent = 4; // User agent to use when crawling.
bool subdomains = 5; // allow subdomain crawling.
bool tld = 6; // allow tld crawling all . ext.
string proxy = 7; // connect to proxy.
bool sitemap = 8; // extend crawl with sitemap links.
}