deltalake_core/datafile/
datafusion_ext.rs1use std::sync::Arc;
7
8use datafusion::catalog::{Session, TableProvider};
9use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
10use datafusion::physical_plan::{
11 ExecutionPlan, SendableRecordBatchStream, execute_stream_partitioned,
12};
13use futures::stream::{StreamExt as _, select_all};
14
15use super::writer::DeltaWriter;
16use super::{BatchStream, DeltaDataReader, DeltaDataWriter, ReadOptions, results_to_stream};
17use crate::DeltaTable;
18use crate::errors::{DeltaResult, DeltaTableError};
19use crate::kernel::Add;
20
21fn sendable_streams_to_stream(streams: Vec<SendableRecordBatchStream>) -> BatchStream {
24 if streams.is_empty() {
26 return futures::stream::empty().boxed();
27 }
28 results_to_stream(select_all(streams))
29}
30
31pub(crate) async fn coalesce_provider_scan(
34 provider: &Arc<dyn TableProvider>,
35 session: &dyn Session,
36 projection: Option<&Vec<usize>>,
37 limit: Option<usize>,
38) -> DeltaResult<SendableRecordBatchStream> {
39 let scan_plan = provider.scan(session, projection, &[], limit).await?;
40 Ok(CoalescePartitionsExec::new(scan_plan).execute(0, session.task_ctx())?)
41}
42
43#[derive(Debug, Default, Clone)]
45pub struct ScanOptions {
46 pub projection: Option<Vec<String>>,
48 pub limit: Option<usize>,
50}
51
52impl From<ReadOptions> for ScanOptions {
53 fn from(value: ReadOptions) -> Self {
54 Self {
55 projection: value.projection,
56 limit: value.limit,
57 }
58 }
59}
60
61#[async_trait::async_trait]
63pub trait DeltaDataWriterExt {
64 async fn write_plan(
67 self: Box<Self>,
68 session: &dyn Session,
69 plan: Arc<dyn ExecutionPlan>,
70 ) -> DeltaResult<Vec<Add>>;
71}
72
73#[async_trait::async_trait]
74impl DeltaDataWriterExt for DeltaWriter {
75 async fn write_plan(
76 self: Box<Self>,
77 session: &dyn Session,
78 plan: Arc<dyn ExecutionPlan>,
79 ) -> DeltaResult<Vec<Add>> {
80 let streams = execute_stream_partitioned(plan, session.task_ctx())?;
81 self.write_all(sendable_streams_to_stream(streams)).await
82 }
83}
84
85#[async_trait::async_trait]
87pub trait DeltaDataReaderExt: DeltaDataReader {
88 async fn scan(
99 &self,
100 session: &dyn Session,
101 options: ScanOptions,
102 ) -> DeltaResult<SendableRecordBatchStream>;
103}
104
105pub struct DataFusionDataReader {
108 provider: Arc<dyn TableProvider>,
109 session: Arc<dyn Session>,
110}
111
112impl DataFusionDataReader {
113 pub fn new(provider: Arc<dyn TableProvider>, session: Arc<dyn Session>) -> Self {
115 Self { provider, session }
116 }
117
118 pub async fn try_new(table: &DeltaTable, session: Arc<dyn Session>) -> DeltaResult<Self> {
121 table.update_datafusion_session(session.as_ref())?;
122 let provider = table.table_provider().await?;
123 Ok(Self::new(provider, session))
124 }
125}
126
127pub(crate) fn projection_indices(
130 schema: &arrow_schema::Schema,
131 columns: Option<&[String]>,
132) -> DeltaResult<Option<Vec<usize>>> {
133 columns
134 .map(|cols| {
135 cols.iter()
136 .map(|col| {
137 schema
138 .column_with_name(col)
139 .map(|(idx, _)| idx)
140 .ok_or_else(|| DeltaTableError::SchemaMismatch {
141 msg: format!("Column '{col}' does not exist in table schema."),
142 })
143 })
144 .collect::<Result<Vec<_>, _>>()
145 })
146 .transpose()
147}
148
149#[async_trait::async_trait]
150impl DeltaDataReaderExt for DataFusionDataReader {
151 async fn scan(
152 &self,
153 session: &dyn Session,
154 options: ScanOptions,
155 ) -> DeltaResult<SendableRecordBatchStream> {
156 let projection =
157 projection_indices(&self.provider.schema(), options.projection.as_deref())?;
158 coalesce_provider_scan(&self.provider, session, projection.as_ref(), options.limit).await
159 }
160}
161
162#[async_trait::async_trait]
163impl DeltaDataReader for DataFusionDataReader {
164 async fn read(&self, options: ReadOptions) -> DeltaResult<BatchStream> {
165 let stream = self.scan(self.session.as_ref(), options.into()).await?;
166 Ok(results_to_stream(stream))
167 }
168}
169
170#[cfg(test)]
171mod tests {
172 use super::*;
173 use std::sync::Arc;
174
175 use datafusion::prelude::SessionContext;
176 use futures::stream::TryStreamExt as _;
177
178 use crate::operations::create::CreateBuilder;
179 use crate::writer::DeltaWriter as _;
180 use crate::writer::RecordBatchWriter;
181 use crate::writer::test_utils::{get_delta_schema, get_record_batch};
182
183 #[tokio::test]
184 async fn test_sendable_streams_to_stream_empty_is_empty() {
185 let mut stream = sendable_streams_to_stream(vec![]);
187 assert!(stream.next().await.is_none());
188 }
189
190 async fn table_with_one_batch() -> (tempfile::TempDir, DeltaTable, usize) {
193 let schema = get_delta_schema();
194 let tmp = tempfile::tempdir().unwrap();
195 let mut table = CreateBuilder::new()
196 .with_location(tmp.path().to_str().unwrap())
197 .with_columns(schema.fields().cloned())
198 .await
199 .unwrap();
200
201 let batch = get_record_batch(None, false);
202 let rows = batch.num_rows();
203 let mut writer = RecordBatchWriter::for_table(&table).unwrap();
204 writer.write(batch).await.unwrap();
205 writer.flush_and_commit(&mut table).await.unwrap();
206 (tmp, table, rows)
207 }
208
209 fn session() -> Arc<dyn Session> {
210 Arc::new(SessionContext::new().state())
211 }
212
213 #[tokio::test]
214 async fn test_datafusion_data_reader_reads_all_rows() {
215 let (_tmp, table, rows) = table_with_one_batch().await;
218 let reader = DataFusionDataReader::try_new(&table, session())
219 .await
220 .unwrap();
221
222 let stream = reader.read(ReadOptions::default()).await.unwrap();
223 let batches: Vec<_> = stream.try_collect().await.unwrap();
224
225 let total: usize = batches.iter().map(|b| b.num_rows()).sum();
226 assert_eq!(total, rows);
227 }
228
229 #[tokio::test]
230 async fn test_datafusion_data_reader_projection() {
231 let (_tmp, table, _rows) = table_with_one_batch().await;
233 let session = session();
234 let reader = DataFusionDataReader::try_new(&table, session.clone())
235 .await
236 .unwrap();
237
238 let options = ScanOptions {
239 projection: Some(vec!["id".to_string()]),
240 limit: None,
241 };
242 let stream = reader.scan(session.as_ref(), options).await.unwrap();
243
244 assert_eq!(stream.schema().fields().len(), 1);
245 assert_eq!(stream.schema().field(0).name(), "id");
246 }
247
248 #[tokio::test]
249 async fn test_datafusion_data_reader_projection_unknown_column_errors() {
250 let (_tmp, table, _rows) = table_with_one_batch().await;
252 let session = session();
253 let reader = DataFusionDataReader::try_new(&table, session.clone())
254 .await
255 .unwrap();
256
257 let options = ScanOptions {
258 projection: Some(vec!["does_not_exist".to_string()]),
259 limit: None,
260 };
261 let err = reader
262 .scan(session.as_ref(), options)
263 .await
264 .err()
265 .expect("projecting an unknown column should error");
266 assert!(
267 matches!(err, DeltaTableError::SchemaMismatch { .. }),
268 "got: {err:?}"
269 );
270 }
271}