Trait DataSource 

pub trait DataSource: Send + Sync {
    // Required methods
    fn source_id(&self) -> &str;
    fn fetch_batch<'life0, 'async_trait>(
        &'life0 self,
        cursor: Option<String>,
        batch_size: usize,
    ) -> Pin<Box<dyn Future<Output = Result<(Vec<DataRecord>, Option<String>)>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
    fn total_count<'life0, 'async_trait>(
        &'life0 self,
    ) -> Pin<Box<dyn Future<Output = Result<Option<u64>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
    fn health_check<'life0, 'async_trait>(
        &'life0 self,
    ) -> Pin<Box<dyn Future<Output = Result<bool>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
}

Trait for data sources that can be ingested

Required Methods

fn source_id(&self) -> &str

Source identifier

fn fetch_batch<'life0, 'async_trait>( &'life0 self, cursor: Option<String>, batch_size: usize, ) -> Pin<Box<dyn Future<Output = Result<(Vec<DataRecord>, Option<String>)>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Fetch a batch of records starting from cursor

fn total_count<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<Option<u64>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Get total record count (if known)

fn health_check<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<bool>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Check if source is available

Implementors

impl DataSource for CoreClient

impl DataSource for EricClient

impl DataSource for ruvector_data_framework::academic_clients::OpenAlexClient

impl DataSource for UnpaywallClient

impl DataSource for EdgarClient

impl DataSource for NoaaClient

impl DataSource for ruvector_data_framework::api_clients::OpenAlexClient

impl DataSource for GuardianClient

impl DataSource for HackerNewsClient

impl DataSource for NewsDataClient

impl DataSource for RedditClient

impl DataSource for WikidataClient

impl DataSource for WikipediaClient