algeon-datafusion 0.1.2

DataFusion physical-plan adapter for native GPU execution.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

use std::sync::Arc;

use async_trait::async_trait;
use datafusion::{
    catalog::Session,
    common::Result,
    logical_expr::{LogicalPlan, UserDefinedLogicalNode, physical_planning_context::PhysicalPlanningContext},
    physical_expr::{LexOrdering, create_physical_sort_exprs},
    physical_plan::ExecutionPlan,
    physical_planner::{ExtensionPlanner, PhysicalPlanner},
};

use super::{CumlAlgoExec, CumlLogicalNode};
use crate::session::planning_native_execution_config;

#[derive(Debug)]
/// DataFusion physical extension planner for validated cuML logical nodes.
///
/// Calls outside this family remain available to other extension planners.
pub struct CumlExtensionPlanner;

#[async_trait]
impl ExtensionPlanner for CumlExtensionPlanner {
    async fn plan_extension(
        &self,
        _planner: &dyn PhysicalPlanner,
        node: &dyn UserDefinedLogicalNode,
        logical_inputs: &[&LogicalPlan],
        physical_inputs: &[Arc<dyn ExecutionPlan>],
        session: &dyn Session,
        planning_ctx: &PhysicalPlanningContext,
    ) -> Result<Option<Arc<dyn ExecutionPlan>>> {
        let Some(node) = node.as_any().downcast_ref::<CumlLogicalNode>() else {
            return Ok(None);
        };
        let native_execution_config = planning_native_execution_config(session);
        let input_orderings = node
            .input_orderings()
            .iter()
            .zip(logical_inputs)
            .map(|(ordering, input)| {
                create_physical_sort_exprs(ordering, input.schema(), session.execution_props(), planning_ctx)
                    .map(LexOrdering::new)
            })
            .collect::<Result<Vec<_>>>()?;
        CumlAlgoExec::try_new(
            node.operation(),
            node.validated_options().clone(),
            node.model().cloned(),
            physical_inputs.to_vec(),
            input_orderings,
            node.output_schema(),
            native_execution_config,
        )
        .map(|exec| Some(Arc::new(exec) as Arc<dyn ExecutionPlan>))
    }
}