polars-stream 0.53.0

Private crate for the streaming execution engine for the Polars DataFrame library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96

use super::compute_node_prelude::*;

/// A node that will pass-through up to length rows, starting at start_offset.
/// Since start_offset must be non-negative this can be done in a streaming
/// manner.
pub struct StreamingSliceNode {
    start_offset: usize,
    length: usize,
    stream_offset: usize,
}

impl StreamingSliceNode {
    pub fn new(start_offset: usize, length: usize) -> Self {
        Self {
            start_offset,
            length,
            stream_offset: 0,
        }
    }
}

impl ComputeNode for StreamingSliceNode {
    fn name(&self) -> &str {
        "streaming-slice"
    }

    fn update_state(
        &mut self,
        recv: &mut [PortState],
        send: &mut [PortState],
        _state: &StreamingExecutionState,
    ) -> PolarsResult<()> {
        if self.stream_offset >= self.start_offset.saturating_add(self.length) || self.length == 0 {
            recv[0] = PortState::Done;
            send[0] = PortState::Done;
        } else {
            recv.swap_with_slice(send);
        }
        Ok(())
    }

    fn spawn<'env, 's>(
        &'env mut self,
        scope: &'s TaskScope<'s, 'env>,
        recv_ports: &mut [Option<RecvPort<'_>>],
        send_ports: &mut [Option<SendPort<'_>>],
        _state: &'s StreamingExecutionState,
        join_handles: &mut Vec<JoinHandle<PolarsResult<()>>>,
    ) {
        assert!(recv_ports.len() == 1 && send_ports.len() == 1);
        let mut recv = recv_ports[0].take().unwrap().serial();
        let mut send = send_ports[0].take().unwrap().serial();
        join_handles.push(scope.spawn_task(TaskPriority::High, async move {
            let stop_offset = self.start_offset.saturating_add(self.length);

            while let Ok(morsel) = recv.recv().await {
                let morsel = morsel.map(|df| {
                    let height = df.height();

                    // Calculate start/stop offsets within df and update global offset.
                    let relative_start_offset = self
                        .start_offset
                        .saturating_sub(self.stream_offset)
                        .min(height);
                    let relative_stop_offset =
                        stop_offset.saturating_sub(self.stream_offset).min(height);
                    self.stream_offset += height;

                    let new_height = relative_stop_offset.saturating_sub(relative_start_offset);
                    if new_height != height {
                        df.slice(relative_start_offset as i64, new_height)
                    } else {
                        df
                    }
                });

                // Technically not necessary, but it's nice to already tell the
                // source to stop producing more morsels as we won't be
                // interested in the results anyway.
                if self.stream_offset >= stop_offset {
                    morsel.source_token().stop();
                }

                if morsel.df().height() > 0 && send.send(morsel).await.is_err() {
                    break;
                }

                if self.stream_offset >= stop_offset {
                    break;
                }
            }

            Ok(())
        }))
    }
}