stork_http

This is a stork implementation for the HTTP protocol and specifically HTML-based web scraping. Given an initial page to scrape, stork_http will find all indexable links on the page and yield them back to you - ready to scrape again in an instant or store for later to come back to at another time, all using futures to allow for parallel processing.

At this time rel="nofollow" is strictly enforced and not possible to change although this will come in time as more filters are added.

Example usage:

# use stork::FilterSet;
# use failure::err_msg;
# use stork_http::{HttpStorkable, filters::*};
# use futures::StreamExt;
#
# #[tokio::main]
# async fn main() -> failure::Fallible<()> {
// start scanning https://example.com/ for links with the given filters
let stream = HttpStorkable::new("https://example.com/".parse()?)
.with_filters(
FilterSet::default()
.add_filter(DomainFilter::new("www.iana.org"))
.add_filter(SchemeFilter::new("https"))
)
.exec();
# futures::pin_mut!(stream); // needed for iteration
// get the first link from example.com and ensure its the one we expected
// it to be
let first_link_on_example: HttpStorkable = match stream.next().await {
Some(Ok(link)) => {
assert_eq!(link.val().text(), Some("More information...".to_string()));
assert_eq!(link.val().url().as_str(), "https://www.iana.org/domains/example");
assert_eq!(link.parent().unwrap().val().url().as_str(), "https://example.com/");
link
},
_ => panic!("failed to get links from page")
};

// add another filter looking for the root path and start scanning for links
let filters = first_link_on_example.filters().clone()
.add_filter(PathFilter::new(FilterType::Equals, "/"));
let stream = first_link_on_example
.with_filters(filters)
.exec();
# futures::pin_mut!(stream); // needed for iteration
// get the first link from the stream and ensure its a link to the homepage
match stream.next().await {
Some(Ok(link)) => {
assert_eq!(link.val().url().as_str(), "https://www.iana.org/");
assert_eq!(link.parent().unwrap().val().url().as_str(), "https://www.iana.org/domains/example");
assert_eq!(link.parent().unwrap().parent().unwrap().val().url().as_str(), "https://example.com/")
},
_ => panic!("failed to get links from page")
}
// ensure theres no other unexpected links on the homepage
assert!(stream.next().await.is_none(), "should've been only one homepage link on the page!");
# Ok(())
# }