lance-datafusion 4.0.0

Internal utilities used by other lance modules to simplify working with datafusion
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The Lance Authors

use std::sync::Arc;

use arrow_array::RecordBatchReader;
use datafusion::{
    execution::SendableRecordBatchStream,
    physical_plan::{ExecutionPlan, stream::RecordBatchStreamAdapter},
};
use datafusion_common::DataFusionError;
use futures::TryStreamExt;
use lance_core::Error;
use lance_datagen::{BatchCount, BatchGeneratorBuilder, ByteCount, RoundingBehavior, RowCount};

use crate::exec::OneShotExec;

pub trait DatafusionDatagenExt {
    fn into_df_stream(
        self,
        batch_size: RowCount,
        num_batches: BatchCount,
    ) -> SendableRecordBatchStream;

    fn into_df_stream_bytes(
        self,
        batch_size: ByteCount,
        num_batches: BatchCount,
        rounding_behavior: RoundingBehavior,
    ) -> Result<SendableRecordBatchStream, Error>;

    fn into_df_exec(self, batch_size: RowCount, num_batches: BatchCount) -> Arc<dyn ExecutionPlan>;
}

impl DatafusionDatagenExt for BatchGeneratorBuilder {
    fn into_df_stream(
        self,
        batch_size: RowCount,
        num_batches: BatchCount,
    ) -> SendableRecordBatchStream {
        let (stream, schema) = self.into_reader_stream(batch_size, num_batches);
        let stream = stream.map_err(DataFusionError::from);
        Box::pin(RecordBatchStreamAdapter::new(schema, stream))
    }

    fn into_df_stream_bytes(
        self,
        batch_size: ByteCount,
        num_batches: BatchCount,
        rounding_behavior: RoundingBehavior,
    ) -> Result<SendableRecordBatchStream, Error> {
        let stream = self.into_reader_bytes(batch_size, num_batches, rounding_behavior)?;
        let schema = stream.schema();
        let stream = futures::stream::iter(stream).map_err(DataFusionError::from);
        Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream)))
    }

    fn into_df_exec(self, batch_size: RowCount, num_batches: BatchCount) -> Arc<dyn ExecutionPlan> {
        let stream = self.into_df_stream(batch_size, num_batches);
        Arc::new(OneShotExec::new(stream))
    }
}