Skip to main content

tantivy_common/
lib.rs

1#![allow(clippy::len_without_is_empty)]
2
3use std::ops::Deref;
4
5pub use byteorder::LittleEndian as Endianness;
6
7mod bitset;
8pub mod bounds;
9mod byte_count;
10mod datetime;
11pub mod file_slice;
12mod group_by;
13pub mod json_path_writer;
14mod key_tracking;
15mod serialize;
16mod versatile_buffer;
17mod vint;
18mod writer;
19pub use bitset::*;
20pub use byte_count::ByteCount;
21pub use datetime::{DateTime, DateTimePrecision};
22pub use group_by::GroupByIteratorExtended;
23pub use json_path_writer::JsonPathWriter;
24pub use key_tracking::{KeyTracking, WithoutKeys};
25pub use ownedbytes::{OwnedBytes, StableDeref};
26pub use serialize::{BinarySerializable, DeserializeFrom, FixedSize};
27pub use versatile_buffer::VersatileBuffer;
28pub use vint::{
29    VInt, VIntU128, read_u32_vint, read_u32_vint_no_advance, serialize_vint_u32, write_u32_vint,
30};
31pub use writer::{AntiCallToken, CountingWriter, TerminatingWrite};
32
33/// Has length trait
34pub trait HasLen {
35    /// Return length
36    fn len(&self) -> usize;
37
38    /// Returns true iff empty.
39    fn is_empty(&self) -> bool {
40        self.len() == 0
41    }
42}
43
44impl<T: Deref<Target = [u8]>> HasLen for T {
45    fn len(&self) -> usize {
46        self.deref().len()
47    }
48}
49
50const HIGHEST_BIT: u64 = 1 << 63;
51
52/// Maps a `i64` to `u64`
53///
54/// For simplicity, tantivy internally handles `i64` as `u64`.
55/// The mapping is defined by this function.
56///
57/// Maps `i64` to `u64` so that
58/// `-2^63 .. 2^63-1` is mapped
59///     to
60/// `0 .. 2^64-1`
61/// in that order.
62///
63/// This is more suited than simply casting (`val as u64`)
64/// because of bitpacking.
65///
66/// Imagine a list of `i64` ranging from -10 to 10.
67/// When casting negative values, the negative values are projected
68/// to values over 2^63, and all values end up requiring 64 bits.
69///
70/// # See also
71/// The reverse mapping is [`u64_to_i64()`].
72#[inline]
73pub fn i64_to_u64(val: i64) -> u64 {
74    (val as u64) ^ HIGHEST_BIT
75}
76
77/// Reverse the mapping given by [`i64_to_u64()`].
78#[inline]
79pub fn u64_to_i64(val: u64) -> i64 {
80    (val ^ HIGHEST_BIT) as i64
81}
82
83/// Maps a `f64` to `u64`
84///
85/// For simplicity, tantivy internally handles `f64` as `u64`.
86/// The mapping is defined by this function.
87///
88/// Maps `f64` to `u64` in a monotonic manner, so that bytes lexical order is preserved.
89///
90/// This is more suited than simply casting (`val as u64`)
91/// which would truncate the result
92///
93/// # Reference
94///
95/// Daniel Lemire's [blog post](https://lemire.me/blog/2020/12/14/converting-floating-point-numbers-to-integers-while-preserving-order/)
96/// explains the mapping in a clear manner.
97///
98/// # See also
99/// The reverse mapping is [`u64_to_f64()`].
100#[inline]
101pub fn f64_to_u64(val: f64) -> u64 {
102    let bits = val.to_bits();
103    if val.is_sign_positive() {
104        bits ^ HIGHEST_BIT
105    } else {
106        !bits
107    }
108}
109
110/// Reverse the mapping given by [`f64_to_u64()`].
111#[inline]
112pub fn u64_to_f64(val: u64) -> f64 {
113    f64::from_bits(if val & HIGHEST_BIT != 0 {
114        val ^ HIGHEST_BIT
115    } else {
116        !val
117    })
118}
119
120/// Replaces a given byte in the `bytes` slice of bytes.
121///
122/// This function assumes that the needle is rarely contained in the bytes string
123/// and offers a fast path if the needle is not present.
124#[inline]
125pub fn replace_in_place(needle: u8, replacement: u8, bytes: &mut [u8]) {
126    if !bytes.contains(&needle) {
127        return;
128    }
129    for b in bytes {
130        if *b == needle {
131            *b = replacement;
132        }
133    }
134}
135
136#[cfg(test)]
137pub(crate) mod test {
138
139    use proptest::prelude::*;
140
141    use super::{f64_to_u64, i64_to_u64, u64_to_f64, u64_to_i64};
142
143    fn test_i64_converter_helper(val: i64) {
144        assert_eq!(u64_to_i64(i64_to_u64(val)), val);
145    }
146
147    fn test_f64_converter_helper(val: f64) {
148        assert_eq!(u64_to_f64(f64_to_u64(val)), val);
149    }
150
151    proptest! {
152        #[test]
153        fn test_f64_converter_monotonicity_proptest((left, right) in (proptest::num::f64::NORMAL, proptest::num::f64::NORMAL)) {
154            let left_u64 = f64_to_u64(left);
155            let right_u64 = f64_to_u64(right);
156            assert_eq!(left_u64 < right_u64,  left < right);
157        }
158    }
159
160    #[test]
161    fn test_i64_converter() {
162        assert_eq!(i64_to_u64(i64::MIN), u64::MIN);
163        assert_eq!(i64_to_u64(i64::MAX), u64::MAX);
164        test_i64_converter_helper(0i64);
165        test_i64_converter_helper(i64::MIN);
166        test_i64_converter_helper(i64::MAX);
167        for i in -1000i64..1000i64 {
168            test_i64_converter_helper(i);
169        }
170    }
171
172    #[test]
173    fn test_f64_converter() {
174        test_f64_converter_helper(f64::INFINITY);
175        test_f64_converter_helper(f64::NEG_INFINITY);
176        test_f64_converter_helper(0.0);
177        test_f64_converter_helper(-0.0);
178        test_f64_converter_helper(1.0);
179        test_f64_converter_helper(-1.0);
180    }
181
182    #[test]
183    fn test_f64_order() {
184        assert!(
185            !(f64_to_u64(f64::NEG_INFINITY)..f64_to_u64(f64::INFINITY))
186                .contains(&f64_to_u64(f64::NAN))
187        ); // nan is not a number
188        assert!(f64_to_u64(1.5) > f64_to_u64(1.0)); // same exponent, different mantissa
189        assert!(f64_to_u64(2.0) > f64_to_u64(1.0)); // same mantissa, different exponent
190        assert!(f64_to_u64(2.0) > f64_to_u64(1.5)); // different exponent and mantissa
191        assert!(f64_to_u64(1.0) > f64_to_u64(-1.0)); // pos > neg
192        assert!(f64_to_u64(-1.5) < f64_to_u64(-1.0));
193        assert!(f64_to_u64(-2.0) < f64_to_u64(1.0));
194        assert!(f64_to_u64(-2.0) < f64_to_u64(-1.5));
195    }
196
197    #[test]
198    fn test_replace_in_place() {
199        let test_aux = |before_replacement: &[u8], expected: &[u8]| {
200            let mut bytes: Vec<u8> = before_replacement.to_vec();
201            super::replace_in_place(b'b', b'c', &mut bytes);
202            assert_eq!(&bytes[..], expected);
203        };
204        test_aux(b"", b"");
205        test_aux(b"b", b"c");
206        test_aux(b"baaa", b"caaa");
207        test_aux(b"aaab", b"aaac");
208        test_aux(b"aaabaa", b"aaacaa");
209        test_aux(b"aaaaaa", b"aaaaaa");
210        test_aux(b"bbbb", b"cccc");
211    }
212}