faucet-stream 1.4.0

A declarative, config-driven data pipeline with pluggable source and sink connectors
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

//! XML/SOAP → MongoDB — full builder showcase for both connectors.
//!
//! XML source uses Bearer auth and offset pagination. MongoDB sink shows
//! batch sizing.
//!
//! Run:
//! ```bash
//! cargo run -p faucet-stream --example xml_to_mongodb \
//!     --features "source-xml sink-mongodb"
//! ```

use faucet_stream::Pipeline;
use faucet_stream::sink::mongodb::{MongoSink, MongoSinkConfig};
use faucet_stream::source::xml::{XmlAuth, XmlPagination, XmlStream, XmlStreamConfig};
use reqwest::header::{HeaderMap, HeaderValue};

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut headers = HeaderMap::new();
    headers.insert("Accept", HeaderValue::from_static("application/xml"));

    let source = XmlStream::new(
        XmlStreamConfig::new("https://feeds.example.com", "/catalog")
            .method(reqwest::Method::GET)
            .auth(XmlAuth::Bearer {
                token: std::env::var("FEED_TOKEN")?,
            })
            .headers(headers)
            .query_param("format", "xml")
            .records_element_path("catalog.products.product")
            .pagination(XmlPagination::Offset {
                offset_param: "offset".into(),
                limit_param: "limit".into(),
                limit: 250,
            })
            .max_pages(usize::MAX),
    );

    let sink = MongoSink::new(
        MongoSinkConfig::new("mongodb://localhost:27017", "warehouse", "catalog_items")
            .with_batch_size(1000),
    )
    .await?;

    let result = Pipeline::new(&source, &sink).run().await?;
    println!(
        "inserted {} catalog items into MongoDB",
        result.records_written
    );
    Ok(())
}