Skip to main content

deltalake_core/datafile/
datafusion_ext.rs

1//! DataFusion-backed extensions to the basic data-file traits: write an
2//! `ExecutionPlan`'s output, and read through `DeltaScanNext` (pushdown,
3//! deletion vectors, transforms). The writer extension late-materializes a plan
4//! into the basic record-batch stream and delegates to [`DeltaDataWriter`].
5
6use std::sync::Arc;
7
8use datafusion::catalog::{Session, TableProvider};
9use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
10use datafusion::physical_plan::{
11    ExecutionPlan, SendableRecordBatchStream, execute_stream_partitioned,
12};
13use futures::stream::{StreamExt as _, select_all};
14
15use super::writer::DeltaWriter;
16use super::{BatchStream, DeltaDataReader, DeltaDataWriter, ReadOptions, results_to_stream};
17use crate::DeltaTable;
18use crate::errors::{DeltaResult, DeltaTableError};
19use crate::kernel::Add;
20
21/// Adapt several DataFusion partition streams into one basic [`BatchStream`],
22/// polling all of them concurrently.
23fn sendable_streams_to_stream(streams: Vec<SendableRecordBatchStream>) -> BatchStream {
24    // `select_all` panics on an empty iterator; an empty input is just an empty stream.
25    if streams.is_empty() {
26        return futures::stream::empty().boxed();
27    }
28    results_to_stream(select_all(streams))
29}
30
31/// Run `provider.scan(..)` and coalesce its output partitions into one stream.
32/// Shared by [`DataFusionDataReader::scan`] and the `load` operation.
33pub(crate) async fn coalesce_provider_scan(
34    provider: &Arc<dyn TableProvider>,
35    session: &dyn Session,
36    projection: Option<&Vec<usize>>,
37    limit: Option<usize>,
38) -> DeltaResult<SendableRecordBatchStream> {
39    let scan_plan = provider.scan(session, projection, &[], limit).await?;
40    Ok(CoalescePartitionsExec::new(scan_plan).execute(0, session.task_ctx())?)
41}
42
43/// Options controlling a DataFusion-backed scan.
44#[derive(Debug, Default, Clone)]
45pub struct ScanOptions {
46    /// Project to this subset of (logical) column names. `None` reads all columns.
47    pub projection: Option<Vec<String>>,
48    /// Stop after returning at least this many rows. `None` reads the whole table.
49    pub limit: Option<usize>,
50}
51
52impl From<ReadOptions> for ScanOptions {
53    fn from(value: ReadOptions) -> Self {
54        Self {
55            projection: value.projection,
56            limit: value.limit,
57        }
58    }
59}
60
61/// DataFusion extension to [`DeltaDataWriter`]: write the output of an execution plan.
62#[async_trait::async_trait]
63pub trait DeltaDataWriterExt {
64    /// Execute `plan` (already containing any validation/repartition/CDC nodes)
65    /// against `session` and write its output through the basic writer.
66    async fn write_plan(
67        self: Box<Self>,
68        session: &dyn Session,
69        plan: Arc<dyn ExecutionPlan>,
70    ) -> DeltaResult<Vec<Add>>;
71}
72
73#[async_trait::async_trait]
74impl DeltaDataWriterExt for DeltaWriter {
75    async fn write_plan(
76        self: Box<Self>,
77        session: &dyn Session,
78        plan: Arc<dyn ExecutionPlan>,
79    ) -> DeltaResult<Vec<Add>> {
80        let streams = execute_stream_partitioned(plan, session.task_ctx())?;
81        self.write_all(sendable_streams_to_stream(streams)).await
82    }
83}
84
85/// DataFusion extension to [`DeltaDataReader`]: a full scan with pushdown.
86#[async_trait::async_trait]
87pub trait DeltaDataReaderExt: DeltaDataReader {
88    /// Scan the table through the DataFusion `DeltaScanNext` provider, returning
89    /// a coalesced single-partition stream.
90    ///
91    /// `session` must have the table's object store registered — for
92    /// [`DataFusionDataReader`], the session passed to
93    /// [`try_new`](DataFusionDataReader::try_new) (which registers it) or one
94    /// sharing that session's `RuntimeEnv`. A reader built via
95    /// [`new`](DataFusionDataReader::new) registers nothing, and scanning with
96    /// an unrelated session fails at execution with an
97    /// "object store not registered" error.
98    async fn scan(
99        &self,
100        session: &dyn Session,
101        options: ScanOptions,
102    ) -> DeltaResult<SendableRecordBatchStream>;
103}
104
105/// A DataFusion-backed reader wrapping the existing `DeltaScanNext` provider.
106/// It carries its own session so it can also satisfy [`DeltaDataReader`].
107pub struct DataFusionDataReader {
108    provider: Arc<dyn TableProvider>,
109    session: Arc<dyn Session>,
110}
111
112impl DataFusionDataReader {
113    /// Create a reader from an already-built table provider and session.
114    pub fn new(provider: Arc<dyn TableProvider>, session: Arc<dyn Session>) -> Self {
115        Self { provider, session }
116    }
117
118    /// Build a reader for `table`, registering the table's object store with
119    /// `session` (idempotent) and resolving the `DeltaScanNext` provider.
120    pub async fn try_new(table: &DeltaTable, session: Arc<dyn Session>) -> DeltaResult<Self> {
121        table.update_datafusion_session(session.as_ref())?;
122        let provider = table.table_provider().await?;
123        Ok(Self::new(provider, session))
124    }
125}
126
127/// Resolve logical projection column names to their indices in `schema`.
128/// Shared by [`DataFusionDataReader::scan`] and the `load` operation.
129pub(crate) fn projection_indices(
130    schema: &arrow_schema::Schema,
131    columns: Option<&[String]>,
132) -> DeltaResult<Option<Vec<usize>>> {
133    columns
134        .map(|cols| {
135            cols.iter()
136                .map(|col| {
137                    schema
138                        .column_with_name(col)
139                        .map(|(idx, _)| idx)
140                        .ok_or_else(|| DeltaTableError::SchemaMismatch {
141                            msg: format!("Column '{col}' does not exist in table schema."),
142                        })
143                })
144                .collect::<Result<Vec<_>, _>>()
145        })
146        .transpose()
147}
148
149#[async_trait::async_trait]
150impl DeltaDataReaderExt for DataFusionDataReader {
151    async fn scan(
152        &self,
153        session: &dyn Session,
154        options: ScanOptions,
155    ) -> DeltaResult<SendableRecordBatchStream> {
156        let projection =
157            projection_indices(&self.provider.schema(), options.projection.as_deref())?;
158        coalesce_provider_scan(&self.provider, session, projection.as_ref(), options.limit).await
159    }
160}
161
162#[async_trait::async_trait]
163impl DeltaDataReader for DataFusionDataReader {
164    async fn read(&self, options: ReadOptions) -> DeltaResult<BatchStream> {
165        let stream = self.scan(self.session.as_ref(), options.into()).await?;
166        Ok(results_to_stream(stream))
167    }
168}
169
170#[cfg(test)]
171mod tests {
172    use super::*;
173    use std::sync::Arc;
174
175    use datafusion::prelude::SessionContext;
176    use futures::stream::TryStreamExt as _;
177
178    use crate::operations::create::CreateBuilder;
179    use crate::writer::DeltaWriter as _;
180    use crate::writer::RecordBatchWriter;
181    use crate::writer::test_utils::{get_delta_schema, get_record_batch};
182
183    #[tokio::test]
184    async fn test_sendable_streams_to_stream_empty_is_empty() {
185        // Empty input must not panic in `select_all`; it yields an empty stream.
186        let mut stream = sendable_streams_to_stream(vec![]);
187        assert!(stream.next().await.is_none());
188    }
189
190    /// Write a small unpartitioned table with one batch; returns the temp dir
191    /// (kept alive for the on-disk files), the loaded table, and the row count.
192    async fn table_with_one_batch() -> (tempfile::TempDir, DeltaTable, usize) {
193        let schema = get_delta_schema();
194        let tmp = tempfile::tempdir().unwrap();
195        let mut table = CreateBuilder::new()
196            .with_location(tmp.path().to_str().unwrap())
197            .with_columns(schema.fields().cloned())
198            .await
199            .unwrap();
200
201        let batch = get_record_batch(None, false);
202        let rows = batch.num_rows();
203        let mut writer = RecordBatchWriter::for_table(&table).unwrap();
204        writer.write(batch).await.unwrap();
205        writer.flush_and_commit(&mut table).await.unwrap();
206        (tmp, table, rows)
207    }
208
209    fn session() -> Arc<dyn Session> {
210        Arc::new(SessionContext::new().state())
211    }
212
213    #[tokio::test]
214    async fn test_datafusion_data_reader_reads_all_rows() {
215        // `try_new` + `read(default)` round-trips every written row through the
216        // advanced (DataFusion-backed) reader.
217        let (_tmp, table, rows) = table_with_one_batch().await;
218        let reader = DataFusionDataReader::try_new(&table, session())
219            .await
220            .unwrap();
221
222        let stream = reader.read(ReadOptions::default()).await.unwrap();
223        let batches: Vec<_> = stream.try_collect().await.unwrap();
224
225        let total: usize = batches.iter().map(|b| b.num_rows()).sum();
226        assert_eq!(total, rows);
227    }
228
229    #[tokio::test]
230    async fn test_datafusion_data_reader_projection() {
231        // A column projection is resolved against the schema and applied to the scan.
232        let (_tmp, table, _rows) = table_with_one_batch().await;
233        let session = session();
234        let reader = DataFusionDataReader::try_new(&table, session.clone())
235            .await
236            .unwrap();
237
238        let options = ScanOptions {
239            projection: Some(vec!["id".to_string()]),
240            limit: None,
241        };
242        let stream = reader.scan(session.as_ref(), options).await.unwrap();
243
244        assert_eq!(stream.schema().fields().len(), 1);
245        assert_eq!(stream.schema().field(0).name(), "id");
246    }
247
248    #[tokio::test]
249    async fn test_datafusion_data_reader_projection_unknown_column_errors() {
250        // Projecting a column that isn't in the schema is a clear error, not a panic.
251        let (_tmp, table, _rows) = table_with_one_batch().await;
252        let session = session();
253        let reader = DataFusionDataReader::try_new(&table, session.clone())
254            .await
255            .unwrap();
256
257        let options = ScanOptions {
258            projection: Some(vec!["does_not_exist".to_string()]),
259            limit: None,
260        };
261        let err = reader
262            .scan(session.as_ref(), options)
263            .await
264            .err()
265            .expect("projecting an unknown column should error");
266        assert!(
267            matches!(err, DeltaTableError::SchemaMismatch { .. }),
268            "got: {err:?}"
269        );
270    }
271}