chewdata 3.6.1

Extract Transform and Load data
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

pub mod cursor;
pub mod offset;

use cursor::Cursor;
use futures::Stream;
use offset::Offset;
use serde::{Deserialize, Serialize};
use std::io::Result;
use std::pin::Pin;

use crate::connector::{curl::Curl, Connector};

#[derive(Debug, Deserialize, Serialize, Clone)]
#[serde(tag = "type")]
pub enum PaginatorType {
    #[serde(alias = "offset")]
    Offset(Offset),
    #[serde(rename = "cursor")]
    Cursor(Cursor),
}

impl Default for PaginatorType {
    fn default() -> Self {
        PaginatorType::Offset(Offset::default())
    }
}

impl PaginatorType {
    pub async fn paginate(
        &self,
        connector: &Curl,
    ) -> Result<Pin<Box<dyn Stream<Item = Result<Box<dyn Connector>>> + Send>>> {
        match self {
            PaginatorType::Offset(paginator) => {
                let mut paginator = paginator.clone();
                if paginator.count.is_none() && connector.counter_type.is_some() {
                    paginator.count = Some(connector.len().await?);
                }

                paginator.paginate(connector).await
            }
            PaginatorType::Cursor(paginator) => paginator.paginate(connector).await,
        }
    }
}