questdb/ingress/column_sender/validity.rs
1/*******************************************************************************
2 * ___ _ ____ ____
3 * / _ \ _ _ ___ ___| |_| _ \| __ )
4 * | | | | | | |/ _ \/ __| __| | | | _ \
5 * | |_| | |_| | __/\__ \ |_| |_| | |_) |
6 * \__\_\\__,_|\___||___/\__|____/|____/
7 *
8 * Copyright (c) 2014-2019 Appsicle
9 * Copyright (c) 2019-2025 QuestDB
10 *
11 * Licensed under the Apache License, Version 2.0 (the "License");
12 * you may not use this file except in compliance with the License.
13 * You may obtain a copy of the License at
14 *
15 * http://www.apache.org/licenses/LICENSE-2.0
16 *
17 * Unless required by applicable law or agreed to in writing, software
18 * distributed under the License is distributed on an "AS IS" BASIS,
19 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20 * See the License for the specific language governing permissions and
21 * limitations under the License.
22 *
23 ******************************************************************************/
24
25//! Validity bitmap helpers for the column-major sender.
26//!
27//! Users pass validity in **Arrow shape**: bit = 1 means valid, LSB-first
28//! inside each byte. The QWP wire shape is the inverse: bit = 1 means
29//! NULL. The conversion happens here; helpers below also count non-null
30//! rows and stream Arrow-bit-set positions for the gather path.
31
32use crate::{Result, error};
33
34/// Public Arrow-shaped validity bitmap: one LSB-first bit per row, where one
35/// means valid and zero means null.
36#[derive(Debug)]
37pub struct Validity<'a> {
38 pub(crate) bits: &'a [u8],
39 pub(crate) bit_len: usize,
40}
41
42impl<'a> Validity<'a> {
43 /// Borrow `bits` as a validity bitmap of length `bit_len` rows.
44 ///
45 /// `bits.len()` must be at least `ceil(bit_len / 8)`. Bits past
46 /// `bit_len` are ignored by the encoder, so callers do not need to
47 /// zero them. `bit_len` is rejected above
48 /// [`super::MAX_CHUNK_ROWS`] so the inferred slice length cannot
49 /// approach `isize::MAX` on the FFI fabrication path.
50 pub fn from_bitmap(bits: &'a [u8], bit_len: usize) -> Result<Self> {
51 if bit_len > super::MAX_CHUNK_ROWS {
52 return Err(error::fmt!(
53 InvalidApiCall,
54 "validity bit_len {} exceeds MAX_CHUNK_ROWS ({})",
55 bit_len,
56 super::MAX_CHUNK_ROWS
57 ));
58 }
59 let required_bytes = bit_len.div_ceil(8);
60 if bits.len() < required_bytes {
61 return Err(error::fmt!(
62 InvalidApiCall,
63 "validity bitmap too short: {} bytes for {} bits (need at least {})",
64 bits.len(),
65 bit_len,
66 required_bytes
67 ));
68 }
69 Ok(Self { bits, bit_len })
70 }
71
72 /// Logical length in bits / rows.
73 pub fn bit_len(&self) -> usize {
74 self.bit_len
75 }
76
77 /// `true` iff bit `idx` is set (row `idx` is **valid**, Arrow shape).
78 #[inline]
79 pub(crate) fn is_valid(&self, idx: usize) -> bool {
80 debug_assert!(idx < self.bit_len);
81 let byte = self.bits[idx / 8];
82 (byte >> (idx % 8)) & 1 == 1
83 }
84
85 /// Count non-null (i.e. valid) rows.
86 pub(crate) fn non_null_count(&self) -> usize {
87 let full_bytes = self.bit_len / 8;
88 let trailing_bits = self.bit_len % 8;
89 let mut count: usize = 0;
90 for &byte in &self.bits[..full_bytes] {
91 count += byte.count_ones() as usize;
92 }
93 if trailing_bits != 0 {
94 let mask = (1u8 << trailing_bits) - 1;
95 count += (self.bits[full_bytes] & mask).count_ones() as usize;
96 }
97 count
98 }
99}
100
101/// Validate that a caller-supplied `data` length matches a chunk's locked
102/// row count and any validity bitmap. Returns the row count to use.
103pub(crate) fn check_row_count(
104 locked: Option<usize>,
105 data_len: usize,
106 validity: Option<&Validity<'_>>,
107) -> Result<usize> {
108 let row_count = data_len;
109 if let Some(existing) = locked
110 && existing != row_count
111 {
112 return Err(error::fmt!(
113 InvalidApiCall,
114 "Column length mismatch: chunk row_count is {} but this column has {} rows",
115 existing,
116 row_count
117 ));
118 }
119 if let Some(v) = validity
120 && v.bit_len != row_count
121 {
122 return Err(error::fmt!(
123 InvalidApiCall,
124 "Validity bitmap length ({} bits) does not match column data length ({} rows)",
125 v.bit_len,
126 row_count
127 ));
128 }
129 Ok(row_count)
130}
131
132#[cfg(test)]
133mod tests {
134 use super::*;
135
136 #[test]
137 fn non_null_count_handles_trailing_bits() {
138 // 9 bits: 0b1010_1010, 0b0000_0001 — bits 1,3,5,7 valid in byte 0;
139 // bit 8 (== row 8) valid in byte 1. Trailing bits past row 8 must
140 // be masked.
141 let bits = [0b1010_1010, 0xFFu8]; // second byte has every bit set
142 let v = Validity::from_bitmap(&bits, 9).unwrap();
143 assert_eq!(v.non_null_count(), 4 + 1);
144 }
145
146 #[test]
147 fn from_bitmap_rejects_short_buffer() {
148 let err = Validity::from_bitmap(&[0u8], 9).unwrap_err();
149 assert_eq!(err.code(), crate::ErrorCode::InvalidApiCall);
150 }
151
152 #[test]
153 fn from_bitmap_rejects_bit_len_above_max() {
154 // bit_len is checked before the buffer-size check, so a tiny slice
155 // with an oversized bit_len returns the cap error rather than
156 // requiring a multi-megabyte allocation.
157 let err = Validity::from_bitmap(&[0u8], super::super::MAX_CHUNK_ROWS + 1).unwrap_err();
158 assert_eq!(err.code(), crate::ErrorCode::InvalidApiCall);
159 assert!(err.msg().contains("MAX_CHUNK_ROWS"), "{}", err.msg());
160 }
161}