webscrape-ai 0.1.0

Official Rust SDK for the webscrape.ai web scraping and LLM-extraction API
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59

//! Structured extraction with an `output_schema`, decoded into a typed struct.
//!
//! Run with: `WEBSCRAPE_API_KEY=wsg_live_... cargo run --example smartscraper`

use serde::Deserialize;
use serde_json::json;
use webscrape_ai::{Client, SmartScraperRequest};

#[derive(Debug, Deserialize)]
struct Page {
    stories: Vec<Story>,
}

#[derive(Debug, Deserialize)]
struct Story {
    title: Option<String>,
    url: Option<String>,
    score: Option<i64>,
}

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let client = Client::from_env()?;

    let resp = client
        .smartscraper(
            SmartScraperRequest::new(
                "https://news.ycombinator.com",
                "Extract the front-page stories with title, url, and score.",
            )
            .output_schema(json!({
                "type": "object",
                "properties": {
                    "stories": {
                        "type": "array",
                        "items": {
                            "type": "object",
                            "properties": {
                                "title": { "type": "string" },
                                "url": { "type": "string" },
                                "score": { "type": "integer" }
                            }
                        }
                    }
                }
            })),
        )
        .await?;

    println!("credits_used={:?}", resp.credits_used);

    // The raw result is a serde_json::Value; decode it into your own type.
    let page: Page = resp.data.result_as()?;
    for story in page.stories.iter().take(10) {
        println!("[{:?}] {:?} ({:?})", story.score, story.title, story.url);
    }

    Ok(())
}