hydrafetch 0.4.0

Official Rust client for the Hydrafetch web data API. Turn any URL into clean Markdown and structured data.
Documentation
use hydrafetch::{
    BatchOptions, Client, ExtractOptions, Format, Location, RateResultInput, Rating, ScrapeOptions,
    WaitOptions,
};
use serde_json::{json, Value};
use std::time::Duration;
use wiremock::matchers::{header, method, path, query_param};
use wiremock::{Mock, MockServer, Request, ResponseTemplate};

async fn client_for(server: &MockServer) -> Client {
    Client::builder()
        .api_key("hf_test")
        .base_url(server.uri())
        .max_retries(0)
        .build()
        .unwrap()
}

fn ok(body: Value) -> ResponseTemplate {
    ResponseTemplate::new(200).set_body_json(body)
}

#[tokio::test]
async fn sends_api_key_header_and_never_bearer() {
    let server = MockServer::start().await;
    Mock::given(method("POST"))
        .and(path("/v1/web/scrape"))
        .and(header("x-api-key", "hf_test"))
        .respond_with(ok(
            json!({ "success": true, "data": { "url": "u", "markdown": "# hi" } }),
        ))
        .mount(&server)
        .await;

    let hf = client_for(&server).await;
    let page = hf.scrape("https://example.com", None).await.unwrap();
    assert_eq!(page.markdown.as_deref(), Some("# hi"));

    let requests = server.received_requests().await.unwrap();
    assert!(requests[0].headers.get("authorization").is_none());
}

#[tokio::test]
async fn missing_key_is_rejected_at_construction() {
    std::env::remove_var("HYDRAFETCH_API_KEY");
    let err = Client::builder().build().unwrap_err();
    assert_eq!(err.code(), "MISSING_API_KEY");
}

#[tokio::test]
async fn options_ride_alongside_the_url() {
    let server = MockServer::start().await;
    Mock::given(method("POST"))
        .and(path("/v1/web/scrape"))
        .respond_with(ok(json!({ "success": true, "data": { "url": "u" } })))
        .mount(&server)
        .await;

    let hf = client_for(&server).await;
    hf.scrape(
        "https://example.com",
        Some(ScrapeOptions {
            formats: Some(vec![Format::Markdown, Format::Links]),
            prefer_structure: Some(true),
            ..Default::default()
        }),
    )
    .await
    .unwrap();

    let requests = server.received_requests().await.unwrap();
    let body: Value = requests[0].body_json().unwrap();
    assert_eq!(body["url"], "https://example.com");
    assert_eq!(body["preferStructure"], true);
    assert_eq!(body["formats"], json!(["markdown", "links"]));
    assert!(body.get("timeout").is_none());
}

#[tokio::test]
async fn extract_sends_location_and_headers() {
    let server = MockServer::start().await;
    Mock::given(method("POST"))
        .and(path("/v1/web/extract"))
        .respond_with(ok(json!({ "success": true, "data": { "results": [] } })))
        .mount(&server)
        .await;

    let hf = client_for(&server).await;
    hf.extract(
        &["https://shop.example/s?q=ghee"],
        Some(ExtractOptions {
            prompt: Some("products and prices".into()),
            location: Some(Location {
                country: Some("in".into()),
                languages: Some(vec!["en-IN".into()]),
            }),
            headers: Some([("Cookie".to_string(), "store_id=42".to_string())].into()),
            ..Default::default()
        }),
    )
    .await
    .unwrap();

    let requests = server.received_requests().await.unwrap();
    let body: Value = requests[0].body_json().unwrap();
    assert_eq!(
        body["location"],
        json!({ "country": "in", "languages": ["en-IN"] })
    );
    assert_eq!(body["headers"], json!({ "Cookie": "store_id=42" }));
}

#[tokio::test]
async fn get_endpoints_use_query_parameters() {
    let server = MockServer::start().await;
    Mock::given(method("GET"))
        .and(path("/v1/web/brand/logo"))
        .and(query_param("domain", "stripe.com"))
        .and(query_param("theme", "dark"))
        .respond_with(ok(
            json!({ "success": true, "data": { "url": "https://img" } }),
        ))
        .mount(&server)
        .await;

    let hf = client_for(&server).await;
    hf.logo(
        "stripe.com",
        Some(hydrafetch::LogoOptions {
            theme: Some("dark".into()),
            kind: None,
        }),
    )
    .await
    .unwrap();
}

#[tokio::test]
async fn error_carries_code_status_and_request_id() {
    let server = MockServer::start().await;
    Mock::given(method("POST"))
        .respond_with(ResponseTemplate::new(401).set_body_json(json!({
            "success": false,
            "error": { "code": "UNAUTHORIZED", "message": "Invalid API key" },
            "meta": { "requestId": "abc" }
        })))
        .mount(&server)
        .await;

    let hf = client_for(&server).await;
    let err = hf.scrape("https://example.com", None).await.unwrap_err();
    assert_eq!(err.code(), "UNAUTHORIZED");
    assert_eq!(err.status(), Some(401));
    assert_eq!(err.request_id(), Some("abc"));
    assert!(err.is_auth());
}

#[tokio::test]
async fn classifies_statuses() {
    for (status, check) in [
        (401u16, "auth"),
        (402, "credits"),
        (422, "invalid"),
        (429, "retryable"),
        (503, "retryable"),
    ] {
        let server = MockServer::start().await;
        Mock::given(method("POST"))
            .respond_with(
                ResponseTemplate::new(status)
                    .set_body_json(json!({ "success": false, "error": { "message": "x" } })),
            )
            .mount(&server)
            .await;

        let hf = client_for(&server).await;
        let err = hf.scrape("https://example.com", None).await.unwrap_err();
        let matched = match check {
            "auth" => err.is_auth(),
            "credits" => err.is_out_of_credits(),
            "invalid" => err.is_invalid_request(),
            _ => err.is_retryable(),
        };
        assert!(matched, "status {status} should classify as {check}");
    }
}

#[tokio::test]
async fn does_not_retry_validation_failures() {
    let server = MockServer::start().await;
    Mock::given(method("POST"))
        .respond_with(
            ResponseTemplate::new(422)
                .set_body_json(json!({ "success": false, "error": { "message": "bad url" } })),
        )
        .expect(1)
        .mount(&server)
        .await;

    let hf = Client::builder()
        .api_key("hf_test")
        .base_url(server.uri())
        .max_retries(3)
        .build()
        .unwrap();

    assert!(hf.scrape("nope", None).await.is_err());
}

#[tokio::test]
async fn batch_id_comes_from_top_level() {
    let server = MockServer::start().await;
    Mock::given(method("POST"))
        .and(path("/v1/web/batch"))
        .respond_with(
            ResponseTemplate::new(201)
                .set_body_json(json!({ "batchId": "b_1", "status": "pending" })),
        )
        .mount(&server)
        .await;

    let hf = client_for(&server).await;
    assert_eq!(
        hf.start_batch(&["https://a.example"], None).await.unwrap(),
        "b_1"
    );
}

#[tokio::test]
async fn per_page_options_nest_under_scrape_options() {
    let server = MockServer::start().await;
    Mock::given(method("POST"))
        .and(path("/v1/web/batch"))
        .respond_with(ResponseTemplate::new(201).set_body_json(json!({ "batchId": "b_1" })))
        .mount(&server)
        .await;

    let hf = client_for(&server).await;
    hf.start_batch(
        &["https://a.example"],
        Some(BatchOptions {
            scrape_options: Some(ScrapeOptions {
                formats: Some(vec![Format::Markdown]),
                ..Default::default()
            }),
            ..Default::default()
        }),
    )
    .await
    .unwrap();

    let requests = server.received_requests().await.unwrap();
    let body: Value = requests[0].body_json().unwrap();
    assert_eq!(body["scrapeOptions"]["formats"], json!(["markdown"]));
    assert!(body.get("formats").is_none());
}

#[tokio::test]
async fn polls_until_terminal() {
    let server = MockServer::start().await;
    Mock::given(method("POST"))
        .and(path("/v1/web/batch"))
        .respond_with(ResponseTemplate::new(201).set_body_json(json!({ "batchId": "b_1" })))
        .mount(&server)
        .await;

    let statuses = std::sync::Mutex::new(vec!["completed", "running", "pending"]);
    Mock::given(method("GET"))
        .and(path("/v1/web/batch/b_1"))
        .respond_with(move |_: &Request| {
            let next = statuses.lock().unwrap().pop().unwrap_or("completed");
            ok(json!({ "success": true, "data": { "id": "b_1", "status": next } }))
        })
        .mount(&server)
        .await;

    let hf = client_for(&server).await;
    let job = hf
        .batch(
            &["https://a.example"],
            None,
            Some(WaitOptions {
                poll_interval: Duration::from_millis(1),
                timeout: Duration::from_secs(5),
            }),
        )
        .await
        .unwrap();

    assert_eq!(job.status, "completed");
}

#[tokio::test]
async fn job_polling_times_out() {
    let server = MockServer::start().await;
    Mock::given(method("POST"))
        .and(path("/v1/web/batch"))
        .respond_with(ResponseTemplate::new(201).set_body_json(json!({ "batchId": "b_1" })))
        .mount(&server)
        .await;
    Mock::given(method("GET"))
        .and(path("/v1/web/batch/b_1"))
        .respond_with(ok(
            json!({ "success": true, "data": { "id": "b_1", "status": "running" } }),
        ))
        .mount(&server)
        .await;

    let hf = client_for(&server).await;
    let err = hf
        .batch(
            &["https://a.example"],
            None,
            Some(WaitOptions {
                poll_interval: Duration::from_millis(1),
                timeout: Duration::from_millis(5),
            }),
        )
        .await
        .unwrap_err();

    assert!(err.is_timeout());
}

#[tokio::test]
async fn styleguide_reads_one_page_fresh() {
    let server = MockServer::start().await;
    Mock::given(method("GET"))
        .and(path("/v1/web/styleguide"))
        .and(query_param("url", "https://stripe.com/pricing"))
        .and(query_param("maxAge", "0"))
        .respond_with(ok(json!({ "success": true, "data": {} })))
        .expect(1)
        .mount(&server)
        .await;

    let hf = client_for(&server).await;
    hf.styleguide_with(
        "https://stripe.com/pricing",
        hydrafetch::StyleguideOptions { max_age: Some(0) },
    )
    .await
    .unwrap();
}

#[tokio::test]
async fn rate_result_posts_the_rating_for_the_last_request() {
    let server = MockServer::start().await;
    Mock::given(method("POST"))
        .and(path("/v1/web/scrape"))
        .respond_with(
            ok(json!({ "success": true, "data": { "url": "u", "markdown": "# hi" } }))
                .insert_header("x-request-id", "r1"),
        )
        .mount(&server)
        .await;
    Mock::given(method("POST"))
        .and(path("/v1/web/feedback"))
        .respond_with(ok(
            json!({ "success": true, "data": { "id": "f1", "requestId": "r1", "rating": "bad" } }),
        ))
        .mount(&server)
        .await;

    let hf = client_for(&server).await;
    hf.scrape("https://example.com", None).await.unwrap();
    assert_eq!(hf.last_request_id().as_deref(), Some("r1"));

    let mut input = RateResultInput::for_request(hf.last_request_id().unwrap(), Rating::Bad);
    input.reasons = vec!["missing_content".to_string()];
    input.note = Some("no prices".to_string());
    let out = hf.rate_result(input).await.unwrap();
    assert_eq!(out.rating, Rating::Bad);

    let requests = server.received_requests().await.unwrap();
    let body: Value = serde_json::from_slice(&requests[1].body).unwrap();
    assert_eq!(
        body,
        json!({ "requestId": "r1", "rating": "bad", "reasons": ["missing_content"], "note": "no prices" })
    );
}

#[tokio::test]
async fn rate_result_needs_a_target() {
    let server = MockServer::start().await;
    let hf = client_for(&server).await;
    let input = RateResultInput {
        request_id: None,
        url: None,
        rating: Rating::Good,
        reasons: Vec::new(),
        note: None,
    };
    assert!(hf.rate_result(input).await.is_err());
}