vortex-layout 0.89.0

Vortex layouts provide a way to perform lazy push-down scans over abstract storage
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

//! Looking codes up in dictionary values.

use vortex_array::ArrayRef;
use vortex_array::IntoArray;
use vortex_array::arrays::DictArray;
use vortex_error::VortexResult;

use super::UNBOUNDED;
use super::join_inlet;
use crate::arrays::shared::Shared;
use crate::plan::physical::TakePlan;
use crate::plan::physical::operator::Cx;
use crate::plan::physical::operator::Input;
use crate::plan::physical::operator::Operator;
use crate::plan::physical::operator::Source;
use crate::plan::physical::operator::SourceBlocked;
use crate::plan::physical::operator::SourceReady;
use crate::plan::physical::operator::Step;
use crate::plan::physical::pipeline::DEFAULT_CAPACITY;
use crate::plan::physical::pipeline::ValueCache;

/// Wraps each batch of codes as a dictionary over the values, once the values inlet has ended.
pub(crate) struct TakeSource {
    plan: TakePlan,
    /// The values, once their inlet has ended.
    values: Option<ArrayRef>,
    /// Where values joined from the inlet are kept for the split's later scans.
    cache: Option<ValueCache>,
}

const CODES: usize = 0;
const VALUES: usize = 1;

impl TakeSource {
    /// A take over `values`, or over what its values inlet produces when `None`, keeping those
    /// in `cache`.
    pub(crate) fn new(plan: TakePlan, values: Option<ArrayRef>, cache: Option<ValueCache>) -> Self {
        Self {
            plan,
            values,
            cache,
        }
    }

    /// The batches inlet `inlet` holds: the values are taken whole, so they never block.
    pub(crate) fn capacity(inlet: usize) -> usize {
        if inlet == VALUES {
            UNBOUNDED
        } else {
            DEFAULT_CAPACITY
        }
    }
}

impl Source for TakeSource {
    fn ready(&mut self, cx: &mut Cx<'_>) -> VortexResult<SourceReady> {
        if self.values.is_none() && !cx.inlet(VALUES).closed() {
            return Ok(SourceReady::Blocked(SourceBlocked::Inlet(VALUES)));
        }
        let codes = cx.inlet(CODES);
        Ok(if codes.is_empty() && !codes.closed() {
            SourceReady::Blocked(SourceBlocked::Inlet(CODES))
        } else {
            SourceReady::Ready
        })
    }
}

impl Operator for TakeSource {
    fn compute(&mut self, _input: Input, cx: &mut Cx<'_>) -> VortexResult<Step> {
        let values = match &self.values {
            Some(values) => values.clone(),
            None => {
                let values_plan = self.plan.values()?;
                let values = shared(join_inlet(cx, VALUES, values_plan.dtype())?);
                let values = match &self.cache {
                    Some(cache) => cache.insert(&values_plan, values),
                    None => values,
                };
                self.values = Some(values.clone());
                values
            }
        };
        Ok(match cx.inlet(CODES).take() {
            Some(codes) => Step::Final(DictArray::try_new(codes, values)?.into_array()),
            None => Step::End,
        })
    }
}

/// `array` as a [`SharedArray`](crate::arrays::shared::SharedArray), so however many dictionaries use it, it is canonicalized once.
fn shared(array: ArrayRef) -> ArrayRef {
    if array.is::<Shared>() {
        array
    } else {
        Shared::new(array).into_array()
    }
}