scrapfly-sdk 0.2.4

Async Rust client for the Scrapfly web scraping, screenshot, extraction and crawler APIs
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

//! Concurrent scrape example via `buffer_unordered`.

use futures_util::StreamExt;
use scrapfly_sdk::{Client, ScrapeConfig};

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let key = std::env::var("SCRAPFLY_KEY").expect("SCRAPFLY_KEY env var required");
    let client = Client::builder().api_key(key).build()?;
    let configs: Vec<ScrapeConfig> = (1..=5)
        .map(|i| {
            ScrapeConfig::builder(format!("https://httpbin.dev/anything?i={}", i))
                .build()
                .expect("build")
        })
        .collect();
    let mut stream = client.concurrent_scrape(configs, 3);
    while let Some(result) = stream.next().await {
        match result {
            Ok(r) => println!("ok: {}", r.result.status_code),
            Err(e) => eprintln!("err: {}", e),
        }
    }
    Ok(())
}