Struct HudiDataSource

pub struct HudiDataSource { /* private fields */ }

Create a HudiDataSource. Used for Datafusion to query Hudi tables

Examples

use std::sync::Arc;

use datafusion::error::Result;
use datafusion::prelude::{DataFrame, SessionContext};
use hudi_datafusion::HudiDataSource;

// Initialize a new DataFusion session context
let ctx = SessionContext::new();

// Create a new HudiDataSource with specific read options
let hudi = HudiDataSource::new_with_options(
    "/tmp/trips_table",
    [("hoodie.read.input.partitions", 5)]).await?;

// Register the Hudi table with the session context
ctx.register_table("trips_table", Arc::new(hudi))?;
let df: DataFrame = ctx.sql("SELECT * from trips_table where city = 'san_francisco'").await?;
df.show().await?;

Implementations

impl HudiDataSource

pub async fn new(base_uri: &str) -> Result<Self>

pub async fn new_with_options<I, K, V>( base_uri: &str, options: I, ) -> Result<Self>

where I: IntoIterator<Item = (K, V)>, K: AsRef<str>, V: Into<String>,

Trait Implementations

impl Clone for HudiDataSource

fn clone(&self) -> HudiDataSource

Returns a duplicate of the value.

const fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for HudiDataSource

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl TableProvider for HudiDataSource

fn as_any(&self) -> &dyn Any

Returns the table provider as Any so that it can be downcast to a specific implementation.

fn schema(&self) -> SchemaRef

Get a reference to the schema for this table

fn table_type(&self) -> TableType

Get the type of this table for metadata/catalog purposes.

fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, state: &'life1 dyn Session, projection: Option<&'life2 Vec<usize>>, filters: &'life3 [Expr], limit: Option<usize>, ) -> Pin<Box<dyn Future<Output = Result<Arc<dyn ExecutionPlan>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait,

Create an ExecutionPlan for scanning the table with optionally specified projection, filter and limit, described below.

fn supports_filters_pushdown( &self, filters: &[&Expr], ) -> Result<Vec<TableProviderFilterPushDown>>

Specify if DataFusion should provide filter expressions to the TableProvider to apply during the scan.

fn constraints(&self) -> Option<&Constraints>

Get a reference to the constraints of the table. Returns:

fn get_table_definition(&self) -> Option<&str>

Get the create statement used to create this table, if available.

fn get_logical_plan(&self) -> Option<Cow<'_, LogicalPlan>>

Get the LogicalPlan of this table, if available.

fn get_column_default(&self, _column: &str) -> Option<&Expr>

Get the default value for a column, if available.

fn statistics(&self) -> Option<Statistics>

Get statistics for this table, if available Although not presently used in mainline DataFusion, this allows implementation specific behavior for downstream repositories, in conjunction with specialized optimizer rules to perform operations such as re-ordering of joins.

fn insert_into<'life0, 'life1, 'async_trait>( &'life0 self, _state: &'life1 dyn Session, _input: Arc<dyn ExecutionPlan>, _insert_op: InsertOp, ) -> Pin<Box<dyn Future<Output = Result<Arc<dyn ExecutionPlan>, DataFusionError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self: 'async_trait,

Return an ExecutionPlan to insert data into this table, if supported.