arrow-data 30.0.1

Array data abstractions for Apache Arrow
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

use crate::data::{contains_nulls, ArrayData};
use arrow_buffer::bit_chunk_iterator::BitChunks;
use arrow_schema::DataType;

// whether bits along the positions are equal
// `lhs_start`, `rhs_start` and `len` are _measured in bits_.
#[inline]
pub(super) fn equal_bits(
    lhs_values: &[u8],
    rhs_values: &[u8],
    lhs_start: usize,
    rhs_start: usize,
    len: usize,
) -> bool {
    let lhs = BitChunks::new(lhs_values, lhs_start, len);
    let rhs = BitChunks::new(rhs_values, rhs_start, len);

    for (a, b) in lhs.iter().zip(rhs.iter()) {
        if a != b {
            return false;
        }
    }

    lhs.remainder_bits() == rhs.remainder_bits()
}

#[inline]
pub(super) fn equal_nulls(
    lhs: &ArrayData,
    rhs: &ArrayData,
    lhs_start: usize,
    rhs_start: usize,
    len: usize,
) -> bool {
    let lhs_offset = lhs_start + lhs.offset();
    let rhs_offset = rhs_start + rhs.offset();

    match (lhs.null_buffer(), rhs.null_buffer()) {
        (Some(lhs), Some(rhs)) => {
            equal_bits(lhs.as_slice(), rhs.as_slice(), lhs_offset, rhs_offset, len)
        }
        (Some(lhs), None) => !contains_nulls(Some(lhs), lhs_offset, len),
        (None, Some(rhs)) => !contains_nulls(Some(rhs), rhs_offset, len),
        (None, None) => true,
    }
}

#[inline]
pub(super) fn base_equal(lhs: &ArrayData, rhs: &ArrayData) -> bool {
    let equal_type = match (lhs.data_type(), rhs.data_type()) {
        (DataType::Union(l_fields, _, l_mode), DataType::Union(r_fields, _, r_mode)) => {
            l_fields == r_fields && l_mode == r_mode
        }
        (DataType::Map(l_field, l_sorted), DataType::Map(r_field, r_sorted)) => {
            let field_equal = match (l_field.data_type(), r_field.data_type()) {
                (DataType::Struct(l_fields), DataType::Struct(r_fields))
                    if l_fields.len() == 2 && r_fields.len() == 2 =>
                {
                    let l_key_field = l_fields.get(0).unwrap();
                    let r_key_field = r_fields.get(0).unwrap();
                    let l_value_field = l_fields.get(1).unwrap();
                    let r_value_field = r_fields.get(1).unwrap();

                    // We don't enforce the equality of field names
                    let data_type_equal = l_key_field.data_type()
                        == r_key_field.data_type()
                        && l_value_field.data_type() == r_value_field.data_type();
                    let nullability_equal = l_key_field.is_nullable()
                        == r_key_field.is_nullable()
                        && l_value_field.is_nullable() == r_value_field.is_nullable();
                    let metadata_equal = l_key_field.metadata() == r_key_field.metadata()
                        && l_value_field.metadata() == r_value_field.metadata();
                    data_type_equal && nullability_equal && metadata_equal
                }
                _ => panic!("Map type should have 2 fields Struct in its field"),
            };
            field_equal && l_sorted == r_sorted
        }
        (l_data_type, r_data_type) => l_data_type == r_data_type,
    };
    equal_type && lhs.len() == rhs.len()
}

// whether the two memory regions are equal
#[inline]
pub(super) fn equal_len(
    lhs_values: &[u8],
    rhs_values: &[u8],
    lhs_start: usize,
    rhs_start: usize,
    len: usize,
) -> bool {
    lhs_values[lhs_start..(lhs_start + len)] == rhs_values[rhs_start..(rhs_start + len)]
}