lb_tantivy/schema/document/
de.rs

1//! Document binary deserialization API
2//!
3//! The deserialization API is strongly inspired by serde's API but with
4//! some tweaks, mostly around some of the types being concrete (errors)
5//! and some more specific types being visited (Ips, datetime, etc...)
6//!
7//! The motivation behind this API is to provide a easy to implement and
8//! efficient way of deserializing a potentially arbitrarily nested object.
9
10use std::collections::{BTreeMap, HashMap};
11use std::fmt::Display;
12use std::io;
13use std::io::Read;
14use std::marker::PhantomData;
15use std::net::Ipv6Addr;
16use std::sync::Arc;
17
18use columnar::MonotonicallyMappableToU128;
19use common::{u64_to_f64, BinarySerializable, DateTime, VInt};
20
21use super::se::BinaryObjectSerializer;
22use super::{OwnedValue, Value};
23use crate::schema::document::type_codes;
24use crate::schema::{Facet, Field};
25use crate::store::DocStoreVersion;
26use crate::tokenizer::PreTokenizedString;
27
28#[derive(Debug, thiserror::Error, Clone)]
29/// An error which occurs while attempting to deserialize a given value
30/// by using the provided value visitor.
31pub enum DeserializeError {
32    #[error("Unsupported Type: {0:?} cannot be deserialized from the given visitor")]
33    /// The value cannot be deserialized from the given type.
34    UnsupportedType(ValueType),
35    #[error("Type Mismatch: Expected {expected:?} but found {actual:?}")]
36    /// The value cannot be deserialized from the given type.
37    TypeMismatch {
38        /// The expected value type.
39        expected: ValueType,
40        /// The actual value type read.
41        actual: ValueType,
42    },
43    #[error("The value could not be read: {0}")]
44    /// The value was unable to be read due to the error.
45    CorruptedValue(Arc<io::Error>),
46    #[error("{0}")]
47    /// A custom error message.
48    Custom(String),
49    #[error("Version {0}, Max version supported: {1}")]
50    /// Unsupported version error.
51    UnsupportedVersion(u32, u32),
52}
53
54impl DeserializeError {
55    /// Creates a new custom deserialize error.
56    pub fn custom(msg: impl Display) -> Self {
57        Self::Custom(msg.to_string())
58    }
59}
60
61impl From<io::Error> for DeserializeError {
62    fn from(error: io::Error) -> Self {
63        Self::CorruptedValue(Arc::new(error))
64    }
65}
66
67/// The core trait for deserializing a document.
68///
69/// TODO: Improve docs
70pub trait DocumentDeserialize: Sized {
71    /// Attempts to deserialize Self from a given document deserializer.
72    fn deserialize<'de, D>(deserializer: D) -> Result<Self, DeserializeError>
73    where D: DocumentDeserializer<'de>;
74}
75
76/// A deserializer that can walk through each entry in the document.
77pub trait DocumentDeserializer<'de> {
78    /// A indicator as to how many values are in the document.
79    ///
80    /// This can be used to pre-allocate entries but should not
81    /// be depended on as a fixed size.
82    fn size_hint(&self) -> usize;
83
84    /// Attempts to deserialize the next field in the document.
85    fn next_field<V: ValueDeserialize>(&mut self) -> Result<Option<(Field, V)>, DeserializeError>;
86}
87
88/// The core trait for deserializing values.
89///
90/// TODO: Improve docs
91pub trait ValueDeserialize: Sized {
92    /// Attempts to deserialize Self from a given value deserializer.
93    fn deserialize<'de, D>(deserializer: D) -> Result<Self, DeserializeError>
94    where D: ValueDeserializer<'de>;
95}
96
97/// A value deserializer.
98pub trait ValueDeserializer<'de> {
99    /// Attempts to deserialize a null value from the deserializer.
100    fn deserialize_null(self) -> Result<(), DeserializeError>;
101
102    /// Attempts to deserialize a string value from the deserializer.
103    fn deserialize_string(self) -> Result<String, DeserializeError>;
104
105    /// Attempts to deserialize a u64 value from the deserializer.
106    fn deserialize_u64(self) -> Result<u64, DeserializeError>;
107
108    /// Attempts to deserialize an i64 value from the deserializer.
109    fn deserialize_i64(self) -> Result<i64, DeserializeError>;
110
111    /// Attempts to deserialize a f64 value from the deserializer.
112    fn deserialize_f64(self) -> Result<f64, DeserializeError>;
113
114    /// Attempts to deserialize a datetime value from the deserializer.
115    fn deserialize_datetime(self) -> Result<DateTime, DeserializeError>;
116
117    /// Attempts to deserialize a facet value from the deserializer.
118    fn deserialize_facet(self) -> Result<Facet, DeserializeError>;
119
120    /// Attempts to deserialize a bytes value from the deserializer.
121    fn deserialize_bytes(self) -> Result<Vec<u8>, DeserializeError>;
122
123    /// Attempts to deserialize an IP address value from the deserializer.
124    fn deserialize_ip_address(self) -> Result<Ipv6Addr, DeserializeError>;
125
126    /// Attempts to deserialize a bool value from the deserializer.
127    fn deserialize_bool(self) -> Result<bool, DeserializeError>;
128
129    /// Attempts to deserialize a pre-tokenized string value from the deserializer.
130    fn deserialize_pre_tokenized_string(self) -> Result<PreTokenizedString, DeserializeError>;
131
132    /// Attempts to deserialize the value using a given visitor.
133    fn deserialize_any<V>(self, visitor: V) -> Result<V::Value, DeserializeError>
134    where V: ValueVisitor;
135}
136
137#[derive(Debug, Copy, Clone, Eq, PartialEq)]
138/// The type of the value attempting to be deserialized.
139pub enum ValueType {
140    /// A null value.
141    Null,
142    /// A string value.
143    String,
144    /// A u64 value.
145    U64,
146    /// A i64 value.
147    I64,
148    /// A f64 value.
149    F64,
150    /// A datetime value.
151    DateTime,
152    /// A facet value.
153    Facet,
154    /// A bytes value.
155    Bytes,
156    /// A IP address value.
157    IpAddr,
158    /// A boolean value.
159    Bool,
160    /// A pre-tokenized string value.
161    PreTokStr,
162    /// An array of value.
163    Array,
164    /// A dynamic object value.
165    Object,
166    /// A JSON object value. Deprecated.
167    #[deprecated(note = "We keep this for backwards compatibility, use Object instead")]
168    JSONObject,
169}
170
171/// A value visitor for deserializing a document value.
172///
173/// This is strongly inspired by serde but has a few extra types.
174///
175/// TODO: Improve docs
176pub trait ValueVisitor {
177    /// The value produced by the visitor.
178    type Value;
179
180    #[inline]
181    /// Called when the deserializer visits a string value.
182    fn visit_null(&self) -> Result<Self::Value, DeserializeError> {
183        Err(DeserializeError::UnsupportedType(ValueType::Null))
184    }
185
186    #[inline]
187    /// Called when the deserializer visits a string value.
188    fn visit_string(&self, _val: String) -> Result<Self::Value, DeserializeError> {
189        Err(DeserializeError::UnsupportedType(ValueType::String))
190    }
191
192    #[inline]
193    /// Called when the deserializer visits a u64 value.
194    fn visit_u64(&self, _val: u64) -> Result<Self::Value, DeserializeError> {
195        Err(DeserializeError::UnsupportedType(ValueType::U64))
196    }
197
198    #[inline]
199    /// Called when the deserializer visits a i64 value.
200    fn visit_i64(&self, _val: i64) -> Result<Self::Value, DeserializeError> {
201        Err(DeserializeError::UnsupportedType(ValueType::I64))
202    }
203
204    #[inline]
205    /// Called when the deserializer visits a f64 value.
206    fn visit_f64(&self, _val: f64) -> Result<Self::Value, DeserializeError> {
207        Err(DeserializeError::UnsupportedType(ValueType::F64))
208    }
209
210    #[inline]
211    /// Called when the deserializer visits a bool value.
212    fn visit_bool(&self, _val: bool) -> Result<Self::Value, DeserializeError> {
213        Err(DeserializeError::UnsupportedType(ValueType::Bool))
214    }
215
216    #[inline]
217    /// Called when the deserializer visits a datetime value.
218    fn visit_datetime(&self, _val: DateTime) -> Result<Self::Value, DeserializeError> {
219        Err(DeserializeError::UnsupportedType(ValueType::DateTime))
220    }
221
222    #[inline]
223    /// Called when the deserializer visits an IP address value.
224    fn visit_ip_address(&self, _val: Ipv6Addr) -> Result<Self::Value, DeserializeError> {
225        Err(DeserializeError::UnsupportedType(ValueType::IpAddr))
226    }
227
228    #[inline]
229    /// Called when the deserializer visits a facet value.
230    fn visit_facet(&self, _val: Facet) -> Result<Self::Value, DeserializeError> {
231        Err(DeserializeError::UnsupportedType(ValueType::Facet))
232    }
233
234    #[inline]
235    /// Called when the deserializer visits a bytes value.
236    fn visit_bytes(&self, _val: Vec<u8>) -> Result<Self::Value, DeserializeError> {
237        Err(DeserializeError::UnsupportedType(ValueType::Bytes))
238    }
239
240    #[inline]
241    /// Called when the deserializer visits a pre-tokenized string value.
242    fn visit_pre_tokenized_string(
243        &self,
244        _val: PreTokenizedString,
245    ) -> Result<Self::Value, DeserializeError> {
246        Err(DeserializeError::UnsupportedType(ValueType::PreTokStr))
247    }
248
249    #[inline]
250    /// Called when the deserializer visits an array.
251    fn visit_array<'de, A>(&self, _access: A) -> Result<Self::Value, DeserializeError>
252    where A: ArrayAccess<'de> {
253        Err(DeserializeError::UnsupportedType(ValueType::Array))
254    }
255
256    #[inline]
257    /// Called when the deserializer visits a object value.
258    fn visit_object<'de, A>(&self, _access: A) -> Result<Self::Value, DeserializeError>
259    where A: ObjectAccess<'de> {
260        Err(DeserializeError::UnsupportedType(ValueType::Object))
261    }
262}
263
264/// Access to a sequence of values which can be deserialized.
265pub trait ArrayAccess<'de> {
266    /// A indicator as to how many values are in the object.
267    ///
268    /// This can be used to pre-allocate entries but should not
269    /// be depended on as a fixed size.
270    fn size_hint(&self) -> usize;
271
272    /// Attempts to deserialize the next element in the sequence.
273    fn next_element<V: ValueDeserialize>(&mut self) -> Result<Option<V>, DeserializeError>;
274}
275
276/// TODO: Improve docs
277pub trait ObjectAccess<'de> {
278    /// A indicator as to how many values are in the object.
279    ///
280    /// This can be used to pre-allocate entries but should not
281    /// be depended on as a fixed size.
282    fn size_hint(&self) -> usize;
283
284    /// Attempts to deserialize the next key-value pair in the object.
285    fn next_entry<V: ValueDeserialize>(&mut self) -> Result<Option<(String, V)>, DeserializeError>;
286}
287
288/// The document deserializer used to read the tantivy documents serialized with
289/// `BinarySerializable`.
290///
291/// This acts very similarly to serde's deserialize types and can incrementally
292/// deserialize each field of the document from the provided reader (`R`).
293///
294/// TODO: Switch to slice instead?
295pub struct BinaryDocumentDeserializer<'de, R> {
296    length: usize,
297    position: usize,
298    doc_store_version: DocStoreVersion,
299    reader: &'de mut R,
300}
301
302impl<'de, R> BinaryDocumentDeserializer<'de, R>
303where R: Read
304{
305    /// Attempts to create a new document deserializer from a given reader.
306    pub(crate) fn from_reader(
307        reader: &'de mut R,
308        doc_store_version: DocStoreVersion,
309    ) -> Result<Self, DeserializeError> {
310        let length = VInt::deserialize(reader)?;
311
312        Ok(Self {
313            length: length.val() as usize,
314            position: 0,
315            doc_store_version,
316            reader,
317        })
318    }
319
320    /// Returns true if the deserializer has deserialized all the entries
321    /// within the document.
322    fn is_complete(&self) -> bool {
323        self.position >= self.length
324    }
325}
326
327impl<'de, R> DocumentDeserializer<'de> for BinaryDocumentDeserializer<'de, R>
328where R: Read
329{
330    #[inline]
331    fn size_hint(&self) -> usize {
332        self.length
333    }
334
335    fn next_field<V: ValueDeserialize>(&mut self) -> Result<Option<(Field, V)>, DeserializeError> {
336        if self.is_complete() {
337            return Ok(None);
338        }
339
340        let field = Field::deserialize(self.reader).map_err(DeserializeError::from)?;
341        let deserializer =
342            BinaryValueDeserializer::from_reader(self.reader, self.doc_store_version)?;
343        let value = V::deserialize(deserializer)?;
344
345        self.position += 1;
346
347        Ok(Some((field, value)))
348    }
349}
350
351/// A single value deserializer that deserializes a value serialized with `BinarySerializable`.
352/// TODO: Improve docs
353pub struct BinaryValueDeserializer<'de, R> {
354    value_type: ValueType,
355    reader: &'de mut R,
356    doc_store_version: DocStoreVersion,
357}
358
359impl<'de, R> BinaryValueDeserializer<'de, R>
360where R: Read
361{
362    /// Attempts to create a new value deserializer from a given reader.
363    fn from_reader(
364        reader: &'de mut R,
365        doc_store_version: DocStoreVersion,
366    ) -> Result<Self, DeserializeError> {
367        let type_code = <u8 as BinarySerializable>::deserialize(reader)?;
368
369        let value_type = match type_code {
370            type_codes::TEXT_CODE => ValueType::String,
371            type_codes::U64_CODE => ValueType::U64,
372            type_codes::I64_CODE => ValueType::I64,
373            type_codes::F64_CODE => ValueType::F64,
374            type_codes::BOOL_CODE => ValueType::Bool,
375            type_codes::DATE_CODE => ValueType::DateTime,
376            type_codes::HIERARCHICAL_FACET_CODE => ValueType::Facet,
377            type_codes::BYTES_CODE => ValueType::Bytes,
378            type_codes::EXT_CODE => {
379                let ext_type_code = <u8 as BinarySerializable>::deserialize(reader)?;
380
381                match ext_type_code {
382                    type_codes::TOK_STR_EXT_CODE => ValueType::PreTokStr,
383                    _ => {
384                        return Err(DeserializeError::from(io::Error::new(
385                            io::ErrorKind::InvalidData,
386                            format!(
387                                "No extended field type is associated with code {ext_type_code:?}"
388                            ),
389                        )))
390                    }
391                }
392            }
393            type_codes::IP_CODE => ValueType::IpAddr,
394            type_codes::NULL_CODE => ValueType::Null,
395            type_codes::ARRAY_CODE => ValueType::Array,
396            type_codes::OBJECT_CODE => ValueType::Object,
397            #[expect(deprecated)]
398            type_codes::JSON_OBJ_CODE => ValueType::JSONObject,
399            _ => {
400                return Err(DeserializeError::from(io::Error::new(
401                    io::ErrorKind::InvalidData,
402                    format!("No field type is associated with code {type_code:?}"),
403                )))
404            }
405        };
406
407        Ok(Self {
408            value_type,
409            reader,
410            doc_store_version,
411        })
412    }
413
414    fn validate_type(&self, expected_type: ValueType) -> Result<(), DeserializeError> {
415        if self.value_type == expected_type {
416            Ok(())
417        } else {
418            Err(DeserializeError::TypeMismatch {
419                expected: expected_type,
420                actual: self.value_type,
421            })
422        }
423    }
424}
425
426impl<'de, R> ValueDeserializer<'de> for BinaryValueDeserializer<'de, R>
427where R: Read
428{
429    fn deserialize_null(self) -> Result<(), DeserializeError> {
430        self.validate_type(ValueType::Null)?;
431        Ok(())
432    }
433
434    fn deserialize_string(self) -> Result<String, DeserializeError> {
435        self.validate_type(ValueType::String)?;
436        <String as BinarySerializable>::deserialize(self.reader).map_err(DeserializeError::from)
437    }
438
439    fn deserialize_u64(self) -> Result<u64, DeserializeError> {
440        self.validate_type(ValueType::U64)?;
441        <u64 as BinarySerializable>::deserialize(self.reader).map_err(DeserializeError::from)
442    }
443
444    fn deserialize_i64(self) -> Result<i64, DeserializeError> {
445        self.validate_type(ValueType::I64)?;
446        <i64 as BinarySerializable>::deserialize(self.reader).map_err(DeserializeError::from)
447    }
448
449    fn deserialize_f64(self) -> Result<f64, DeserializeError> {
450        self.validate_type(ValueType::F64)?;
451        <u64 as BinarySerializable>::deserialize(self.reader)
452            .map(u64_to_f64)
453            .map_err(DeserializeError::from)
454    }
455
456    fn deserialize_datetime(self) -> Result<DateTime, DeserializeError> {
457        self.validate_type(ValueType::DateTime)?;
458        match self.doc_store_version {
459            DocStoreVersion::V1 => {
460                let timestamp_micros = <i64 as BinarySerializable>::deserialize(self.reader)?;
461                Ok(DateTime::from_timestamp_micros(timestamp_micros))
462            }
463            DocStoreVersion::V2 => {
464                let timestamp_nanos = <i64 as BinarySerializable>::deserialize(self.reader)?;
465                Ok(DateTime::from_timestamp_nanos(timestamp_nanos))
466            }
467        }
468    }
469
470    fn deserialize_facet(self) -> Result<Facet, DeserializeError> {
471        self.validate_type(ValueType::Facet)?;
472        <Facet as BinarySerializable>::deserialize(self.reader).map_err(DeserializeError::from)
473    }
474
475    fn deserialize_bytes(self) -> Result<Vec<u8>, DeserializeError> {
476        self.validate_type(ValueType::Bytes)?;
477        <Vec<u8> as BinarySerializable>::deserialize(self.reader).map_err(DeserializeError::from)
478    }
479
480    fn deserialize_ip_address(self) -> Result<Ipv6Addr, DeserializeError> {
481        self.validate_type(ValueType::IpAddr)?;
482        <u128 as BinarySerializable>::deserialize(self.reader)
483            .map(Ipv6Addr::from_u128)
484            .map_err(DeserializeError::from)
485    }
486
487    fn deserialize_bool(self) -> Result<bool, DeserializeError> {
488        self.validate_type(ValueType::Bool)?;
489        <bool as BinarySerializable>::deserialize(self.reader).map_err(DeserializeError::from)
490    }
491
492    fn deserialize_pre_tokenized_string(self) -> Result<PreTokenizedString, DeserializeError> {
493        self.validate_type(ValueType::PreTokStr)?;
494        <PreTokenizedString as BinarySerializable>::deserialize(self.reader)
495            .map_err(DeserializeError::from)
496    }
497
498    fn deserialize_any<V>(self, visitor: V) -> Result<V::Value, DeserializeError>
499    where V: ValueVisitor {
500        match self.value_type {
501            ValueType::Null => visitor.visit_null(),
502            ValueType::String => {
503                let val = self.deserialize_string()?;
504                visitor.visit_string(val)
505            }
506            ValueType::U64 => {
507                let val = self.deserialize_u64()?;
508                visitor.visit_u64(val)
509            }
510            ValueType::I64 => {
511                let val = self.deserialize_i64()?;
512                visitor.visit_i64(val)
513            }
514            ValueType::F64 => {
515                let val = self.deserialize_f64()?;
516                visitor.visit_f64(val)
517            }
518            ValueType::DateTime => {
519                let val = self.deserialize_datetime()?;
520                visitor.visit_datetime(val)
521            }
522            ValueType::Facet => {
523                let val = self.deserialize_facet()?;
524                visitor.visit_facet(val)
525            }
526            ValueType::Bytes => {
527                let val = self.deserialize_bytes()?;
528                visitor.visit_bytes(val)
529            }
530            ValueType::IpAddr => {
531                let val = self.deserialize_ip_address()?;
532                visitor.visit_ip_address(val)
533            }
534            ValueType::Bool => {
535                let val = self.deserialize_bool()?;
536                visitor.visit_bool(val)
537            }
538            ValueType::PreTokStr => {
539                let val = self.deserialize_pre_tokenized_string()?;
540                visitor.visit_pre_tokenized_string(val)
541            }
542            ValueType::Array => {
543                let access =
544                    BinaryArrayDeserializer::from_reader(self.reader, self.doc_store_version)?;
545                visitor.visit_array(access)
546            }
547            ValueType::Object => {
548                let access =
549                    BinaryObjectDeserializer::from_reader(self.reader, self.doc_store_version)?;
550                visitor.visit_object(access)
551            }
552            #[allow(deprecated)]
553            ValueType::JSONObject => {
554                // This is a compatibility layer
555                // The implementation is slow, but is temporary anyways
556                let mut de = serde_json::Deserializer::from_reader(self.reader);
557                let json_map = <serde_json::Map::<String, serde_json::Value> as serde::Deserialize>::deserialize(&mut de).map_err(|err| DeserializeError::Custom(err.to_string()))?;
558                let mut out = Vec::new();
559                let mut serializer = BinaryObjectSerializer::begin(json_map.len(), &mut out)?;
560                for (key, val) in json_map {
561                    let val: OwnedValue = val.into();
562                    serializer.serialize_entry(&key, (&val).as_value())?;
563                }
564                serializer.end()?;
565
566                let out_rc = std::rc::Rc::new(out);
567                let mut slice: &[u8] = &out_rc;
568                let access =
569                    BinaryObjectDeserializer::from_reader(&mut slice, self.doc_store_version)?;
570
571                visitor.visit_object(access)
572            }
573        }
574    }
575}
576
577/// A deserializer for an array of values serialized with `BinarySerializable`.
578/// TODO: Improve docs
579pub struct BinaryArrayDeserializer<'de, R> {
580    length: usize,
581    position: usize,
582    reader: &'de mut R,
583    doc_store_version: DocStoreVersion,
584}
585
586impl<'de, R> BinaryArrayDeserializer<'de, R>
587where R: Read
588{
589    /// Attempts to create a new array deserializer from a given reader.
590    fn from_reader(
591        reader: &'de mut R,
592        doc_store_version: DocStoreVersion,
593    ) -> Result<Self, DeserializeError> {
594        let length = <VInt as BinarySerializable>::deserialize(reader)?;
595
596        Ok(Self {
597            length: length.val() as usize,
598            position: 0,
599            reader,
600            doc_store_version,
601        })
602    }
603
604    /// Returns true if the deserializer has deserialized all the elements
605    /// within the array.
606    fn is_complete(&self) -> bool {
607        self.position >= self.length
608    }
609}
610
611impl<'de, R> ArrayAccess<'de> for BinaryArrayDeserializer<'de, R>
612where R: Read
613{
614    #[inline]
615    fn size_hint(&self) -> usize {
616        self.length
617    }
618
619    fn next_element<V: ValueDeserialize>(&mut self) -> Result<Option<V>, DeserializeError> {
620        if self.is_complete() {
621            return Ok(None);
622        }
623
624        let deserializer =
625            BinaryValueDeserializer::from_reader(self.reader, self.doc_store_version)?;
626        let value = V::deserialize(deserializer)?;
627
628        // Advance the position cursor.
629        self.position += 1;
630
631        Ok(Some(value))
632    }
633}
634
635/// A deserializer for a object consisting of key-value pairs.
636pub struct BinaryObjectDeserializer<'de, R> {
637    /// The inner deserializer.
638    ///
639    /// Internally an object is just represented by an array
640    /// in the format of `[key, value, key, value, key, value]`.
641    inner: BinaryArrayDeserializer<'de, R>,
642}
643
644impl<'de, R> BinaryObjectDeserializer<'de, R>
645where R: Read
646{
647    /// Attempts to create a new object deserializer from a given reader.
648    fn from_reader(
649        reader: &'de mut R,
650        doc_store_version: DocStoreVersion,
651    ) -> Result<Self, DeserializeError> {
652        let inner = BinaryArrayDeserializer::from_reader(reader, doc_store_version)?;
653        Ok(Self { inner })
654    }
655}
656
657impl<'de, R> ObjectAccess<'de> for BinaryObjectDeserializer<'de, R>
658where R: Read
659{
660    #[inline]
661    /// A indicator as to how many values are in the object.
662    ///
663    /// This can be used to pre-allocate entries but should not
664    /// be depended on as a fixed size.
665    fn size_hint(&self) -> usize {
666        // We divide by 2 here as we know our elements are going to be
667        // in the format of `[key, value, key, value, key, value]`.
668        self.inner.size_hint() / 2
669    }
670
671    /// Attempts to deserialize the next key-value pair in the object.
672    fn next_entry<V: ValueDeserialize>(&mut self) -> Result<Option<(String, V)>, DeserializeError> {
673        if self.inner.is_complete() {
674            return Ok(None);
675        }
676
677        let key = self.inner.next_element::<String>()?.expect(
678            "Deserializer should not be empty as it is not marked as complete, this is a bug",
679        );
680        let value = self.inner.next_element::<V>()?.expect(
681            "Deserializer should not be empty as it is not marked as complete, this is a bug",
682        );
683
684        Ok(Some((key, value)))
685    }
686}
687
688// Core type implementations
689
690impl ValueDeserialize for String {
691    #[inline]
692    fn deserialize<'de, D>(deserializer: D) -> Result<Self, DeserializeError>
693    where D: ValueDeserializer<'de> {
694        deserializer.deserialize_string()
695    }
696}
697
698impl ValueDeserialize for u64 {
699    #[inline]
700    fn deserialize<'de, D>(deserializer: D) -> Result<Self, DeserializeError>
701    where D: ValueDeserializer<'de> {
702        deserializer.deserialize_u64()
703    }
704}
705
706impl ValueDeserialize for i64 {
707    #[inline]
708    fn deserialize<'de, D>(deserializer: D) -> Result<Self, DeserializeError>
709    where D: ValueDeserializer<'de> {
710        deserializer.deserialize_i64()
711    }
712}
713
714impl ValueDeserialize for f64 {
715    #[inline]
716    fn deserialize<'de, D>(deserializer: D) -> Result<Self, DeserializeError>
717    where D: ValueDeserializer<'de> {
718        deserializer.deserialize_f64()
719    }
720}
721
722impl ValueDeserialize for DateTime {
723    #[inline]
724    fn deserialize<'de, D>(deserializer: D) -> Result<Self, DeserializeError>
725    where D: ValueDeserializer<'de> {
726        deserializer.deserialize_datetime()
727    }
728}
729
730impl ValueDeserialize for Ipv6Addr {
731    #[inline]
732    fn deserialize<'de, D>(deserializer: D) -> Result<Self, DeserializeError>
733    where D: ValueDeserializer<'de> {
734        deserializer.deserialize_ip_address()
735    }
736}
737
738impl ValueDeserialize for Facet {
739    #[inline]
740    fn deserialize<'de, D>(deserializer: D) -> Result<Self, DeserializeError>
741    where D: ValueDeserializer<'de> {
742        deserializer.deserialize_facet()
743    }
744}
745
746impl ValueDeserialize for Vec<u8> {
747    #[inline]
748    fn deserialize<'de, D>(deserializer: D) -> Result<Self, DeserializeError>
749    where D: ValueDeserializer<'de> {
750        deserializer.deserialize_bytes()
751    }
752}
753
754impl ValueDeserialize for PreTokenizedString {
755    #[inline]
756    fn deserialize<'de, D>(deserializer: D) -> Result<Self, DeserializeError>
757    where D: ValueDeserializer<'de> {
758        deserializer.deserialize_pre_tokenized_string()
759    }
760}
761
762// Collections kind of suck, but can't think of a nicer way of doing this generically
763// without quite literally cloning serde entirely...
764
765struct VecVisitor<T: ValueDeserialize>(PhantomData<T>);
766impl<T: ValueDeserialize> ValueVisitor for VecVisitor<T> {
767    type Value = Vec<T>;
768
769    fn visit_array<'de, A>(&self, mut access: A) -> Result<Self::Value, DeserializeError>
770    where A: ArrayAccess<'de> {
771        let mut entries = Vec::with_capacity(access.size_hint());
772        while let Some(value) = access.next_element()? {
773            entries.push(value);
774        }
775        Ok(entries)
776    }
777}
778impl<T: ValueDeserialize> ValueDeserialize for Vec<T> {
779    #[inline]
780    fn deserialize<'de, D>(deserializer: D) -> Result<Self, DeserializeError>
781    where D: ValueDeserializer<'de> {
782        deserializer.deserialize_any(VecVisitor(PhantomData))
783    }
784}
785
786struct BTreeMapVisitor<T: ValueDeserialize>(PhantomData<T>);
787impl<T: ValueDeserialize> ValueVisitor for BTreeMapVisitor<T> {
788    type Value = BTreeMap<String, T>;
789
790    fn visit_object<'de, A>(&self, mut access: A) -> Result<Self::Value, DeserializeError>
791    where A: ObjectAccess<'de> {
792        let mut entries = BTreeMap::new();
793        while let Some((key, value)) = access.next_entry()? {
794            entries.insert(key, value);
795        }
796        Ok(entries)
797    }
798}
799impl<T: ValueDeserialize> ValueDeserialize for BTreeMap<String, T> {
800    #[inline]
801    fn deserialize<'de, D>(deserializer: D) -> Result<Self, DeserializeError>
802    where D: ValueDeserializer<'de> {
803        deserializer.deserialize_any(BTreeMapVisitor(PhantomData))
804    }
805}
806
807struct HashMapVisitor<T: ValueDeserialize>(PhantomData<T>);
808impl<T: ValueDeserialize> ValueVisitor for HashMapVisitor<T> {
809    type Value = HashMap<String, T>;
810
811    fn visit_object<'de, A>(&self, mut access: A) -> Result<Self::Value, DeserializeError>
812    where A: ObjectAccess<'de> {
813        let mut entries = HashMap::with_capacity(access.size_hint());
814        while let Some((key, value)) = access.next_entry()? {
815            entries.insert(key, value);
816        }
817        Ok(entries)
818    }
819}
820impl<T: ValueDeserialize> ValueDeserialize for HashMap<String, T> {
821    #[inline]
822    fn deserialize<'de, D>(deserializer: D) -> Result<Self, DeserializeError>
823    where D: ValueDeserializer<'de> {
824        deserializer.deserialize_any(HashMapVisitor(PhantomData))
825    }
826}
827
828struct KeyValuesVecVisitor<T: ValueDeserialize>(PhantomData<T>);
829impl<T: ValueDeserialize> ValueVisitor for KeyValuesVecVisitor<T> {
830    type Value = Vec<(String, T)>;
831
832    fn visit_object<'de, A>(&self, mut access: A) -> Result<Self::Value, DeserializeError>
833    where A: ObjectAccess<'de> {
834        let mut entries = Vec::with_capacity(access.size_hint());
835        while let Some(entry) = access.next_entry()? {
836            entries.push(entry);
837        }
838        Ok(entries)
839    }
840}
841impl<T: ValueDeserialize> ValueDeserialize for Vec<(String, T)> {
842    #[inline]
843    fn deserialize<'de, D>(deserializer: D) -> Result<Self, DeserializeError>
844    where D: ValueDeserializer<'de> {
845        deserializer.deserialize_any(KeyValuesVecVisitor(PhantomData))
846    }
847}
848
849#[cfg(test)]
850mod tests {
851    use std::io::Cursor;
852
853    use serde_json::Number;
854    use tokenizer_api::Token;
855
856    use super::*;
857    use crate::schema::document::existing_type_impls::JsonObjectIter;
858    use crate::schema::document::se::BinaryValueSerializer;
859    use crate::schema::document::{ReferenceValue, ReferenceValueLeaf};
860    use crate::store::DOC_STORE_VERSION;
861
862    fn serialize_value<'a>(value: ReferenceValue<'a, &'a serde_json::Value>) -> Vec<u8> {
863        let mut writer = Vec::new();
864
865        let mut serializer = BinaryValueSerializer::new(&mut writer);
866        serializer.serialize_value(value).expect("Serialize value");
867
868        writer
869    }
870
871    fn serialize_owned_value<'a>(value: ReferenceValue<'a, &'a OwnedValue>) -> Vec<u8> {
872        let mut writer = Vec::new();
873
874        let mut serializer = BinaryValueSerializer::new(&mut writer);
875        serializer.serialize_value(value).expect("Serialize value");
876
877        writer
878    }
879
880    fn deserialize_value(buffer: Vec<u8>) -> crate::schema::OwnedValue {
881        let mut cursor = Cursor::new(buffer);
882        let deserializer =
883            BinaryValueDeserializer::from_reader(&mut cursor, DOC_STORE_VERSION).unwrap();
884        crate::schema::OwnedValue::deserialize(deserializer).expect("Deserialize value")
885    }
886
887    #[test]
888    fn test_simple_value_serialize() {
889        let result = serialize_value(ReferenceValueLeaf::Null.into());
890        let value = deserialize_value(result);
891        assert_eq!(value, crate::schema::OwnedValue::Null);
892
893        let result = serialize_value(ReferenceValueLeaf::Str("hello, world").into());
894        let value = deserialize_value(result);
895        assert_eq!(
896            value,
897            crate::schema::OwnedValue::Str(String::from("hello, world"))
898        );
899
900        let result = serialize_value(ReferenceValueLeaf::U64(123).into());
901        let value = deserialize_value(result);
902        assert_eq!(value, crate::schema::OwnedValue::U64(123));
903
904        let result = serialize_value(ReferenceValueLeaf::I64(-123).into());
905        let value = deserialize_value(result);
906        assert_eq!(value, crate::schema::OwnedValue::I64(-123));
907
908        let result = serialize_value(ReferenceValueLeaf::F64(123.3845).into());
909        let value = deserialize_value(result);
910        assert_eq!(value, crate::schema::OwnedValue::F64(123.3845));
911
912        let result = serialize_value(ReferenceValueLeaf::Bool(false).into());
913        let value = deserialize_value(result);
914        assert_eq!(value, crate::schema::OwnedValue::Bool(false));
915
916        let result =
917            serialize_value(ReferenceValueLeaf::Date(DateTime::from_timestamp_micros(100)).into());
918        let value = deserialize_value(result);
919        assert_eq!(
920            value,
921            crate::schema::OwnedValue::Date(DateTime::from_timestamp_micros(100))
922        );
923
924        let facet = Facet::from_text("/hello/world").unwrap();
925        let result = serialize_value(ReferenceValueLeaf::Facet(facet.encoded_str()).into());
926        let value = deserialize_value(result);
927        assert_eq!(value, crate::schema::OwnedValue::Facet(facet));
928
929        let pre_tok_str = PreTokenizedString {
930            text: "hello, world".to_string(),
931            tokens: vec![Token::default(), Token::default()],
932        };
933        let result =
934            serialize_value(ReferenceValueLeaf::PreTokStr(pre_tok_str.clone().into()).into());
935        let value = deserialize_value(result);
936        assert_eq!(value, crate::schema::OwnedValue::PreTokStr(pre_tok_str));
937    }
938
939    #[test]
940    fn test_array_serialize() {
941        let elements = [serde_json::Value::Null, serde_json::Value::Null];
942        let result = serialize_value(ReferenceValue::Array(elements.iter()));
943        let value = deserialize_value(result);
944        assert_eq!(
945            value,
946            crate::schema::OwnedValue::Array(vec![
947                crate::schema::OwnedValue::Null,
948                crate::schema::OwnedValue::Null,
949            ]),
950        );
951
952        let elements = [
953            serde_json::Value::String("Hello, world".into()),
954            serde_json::Value::String("Some demo".into()),
955        ];
956        let result = serialize_value(ReferenceValue::Array(elements.iter()));
957        let value = deserialize_value(result);
958        assert_eq!(
959            value,
960            crate::schema::OwnedValue::Array(vec![
961                crate::schema::OwnedValue::Str(String::from("Hello, world")),
962                crate::schema::OwnedValue::Str(String::from("Some demo")),
963            ]),
964        );
965
966        let elements = [];
967        let result = serialize_value(ReferenceValue::Array(elements.iter()));
968        let value = deserialize_value(result);
969        assert_eq!(value, crate::schema::OwnedValue::Array(vec![]));
970
971        let elements = [
972            serde_json::Value::Null,
973            serde_json::Value::String("Hello, world".into()),
974            serde_json::Value::Number(12345.into()),
975        ];
976        let result = serialize_value(ReferenceValue::Array(elements.iter()));
977        let value = deserialize_value(result);
978        assert_eq!(
979            value,
980            crate::schema::OwnedValue::Array(vec![
981                crate::schema::OwnedValue::Null,
982                crate::schema::OwnedValue::Str(String::from("Hello, world")),
983                crate::schema::OwnedValue::I64(12345),
984            ]),
985        );
986    }
987
988    #[test]
989    fn test_object_serialize() {
990        let mut object = serde_json::Map::new();
991        object.insert(
992            "my-first-key".into(),
993            serde_json::Value::String("Hello".into()),
994        );
995        object.insert("my-second-key".into(), serde_json::Value::Null);
996        object.insert(
997            "my-third-key".into(),
998            serde_json::Value::Number(Number::from_f64(123.0).unwrap()),
999        );
1000        let result = serialize_value(ReferenceValue::Object(JsonObjectIter(object.iter())));
1001        let value = deserialize_value(result);
1002
1003        let mut expected_object = BTreeMap::new();
1004        expected_object.insert(
1005            "my-first-key".to_string(),
1006            crate::schema::OwnedValue::Str(String::from("Hello")),
1007        );
1008        expected_object.insert("my-second-key".to_string(), crate::schema::OwnedValue::Null);
1009        expected_object.insert(
1010            "my-third-key".to_string(),
1011            crate::schema::OwnedValue::F64(123.0),
1012        );
1013        assert_eq!(
1014            value,
1015            crate::schema::OwnedValue::Object(expected_object.into_iter().collect())
1016        );
1017
1018        let object = serde_json::Map::new();
1019        let result = serialize_value(ReferenceValue::Object(JsonObjectIter(object.iter())));
1020        let value = deserialize_value(result);
1021        let expected_object = BTreeMap::new();
1022        assert_eq!(
1023            value,
1024            crate::schema::OwnedValue::Object(expected_object.into_iter().collect())
1025        );
1026
1027        let mut object = serde_json::Map::new();
1028        object.insert("my-first-key".into(), serde_json::Value::Null);
1029        object.insert("my-second-key".into(), serde_json::Value::Null);
1030        object.insert("my-third-key".into(), serde_json::Value::Null);
1031        let result = serialize_value(ReferenceValue::Object(JsonObjectIter(object.iter())));
1032        let value = deserialize_value(result);
1033        let mut expected_object = BTreeMap::new();
1034        expected_object.insert("my-first-key".to_string(), crate::schema::OwnedValue::Null);
1035        expected_object.insert("my-second-key".to_string(), crate::schema::OwnedValue::Null);
1036        expected_object.insert("my-third-key".to_string(), crate::schema::OwnedValue::Null);
1037        assert_eq!(
1038            value,
1039            crate::schema::OwnedValue::Object(expected_object.into_iter().collect())
1040        );
1041    }
1042
1043    #[test]
1044    fn test_json_compat() {
1045        let data = [
1046            8, 123, 34, 107, 101, 121, 97, 58, 34, 58, 34, 98, 108, 117, 98, 34, 44, 34, 118, 97,
1047            108, 115, 34, 58, 123, 34, 104, 101, 121, 34, 58, 34, 104, 111, 34, 125, 125,
1048        ]
1049        .to_vec();
1050        let expected = json!({
1051            "keya:": "blub",
1052            "vals": {
1053                "hey": "ho"
1054            }
1055        });
1056        let expected_val: OwnedValue = expected.clone().into();
1057
1058        let value = deserialize_value(data);
1059        assert_eq!(value, expected_val);
1060    }
1061
1062    #[test]
1063    fn test_nested_date_precision() {
1064        let object = OwnedValue::Object(vec![(
1065            "my-date".into(),
1066            OwnedValue::Date(DateTime::from_timestamp_nanos(323456)),
1067        )]);
1068        let result = serialize_owned_value((&object).as_value());
1069        let value = deserialize_value(result);
1070        assert_eq!(value, object);
1071    }
1072
1073    #[test]
1074    fn test_nested_serialize() {
1075        let mut object = serde_json::Map::new();
1076        object.insert(
1077            "my-array".into(),
1078            serde_json::Value::Array(vec![
1079                serde_json::Value::Null,
1080                serde_json::Value::String(String::from("bobby of the sea")),
1081            ]),
1082        );
1083        object.insert(
1084            "my-object".into(),
1085            serde_json::Value::Object(
1086                vec![
1087                    (
1088                        "inner-1".to_string(),
1089                        serde_json::Value::Number((-123i64).into()),
1090                    ),
1091                    (
1092                        "inner-2".to_string(),
1093                        serde_json::Value::String(String::from("bobby of the sea 2")),
1094                    ),
1095                ]
1096                .into_iter()
1097                .collect(),
1098            ),
1099        );
1100        let result = serialize_value(ReferenceValue::Object(JsonObjectIter(object.iter())));
1101        let value = deserialize_value(result);
1102
1103        let mut expected_object = BTreeMap::new();
1104        expected_object.insert(
1105            "my-array".to_string(),
1106            crate::schema::OwnedValue::Array(vec![
1107                crate::schema::OwnedValue::Null,
1108                crate::schema::OwnedValue::Str(String::from("bobby of the sea")),
1109            ]),
1110        );
1111        expected_object.insert(
1112            "my-object".to_string(),
1113            crate::schema::OwnedValue::Object(
1114                vec![
1115                    (
1116                        "inner-1".to_string(),
1117                        crate::schema::OwnedValue::I64(-123i64),
1118                    ),
1119                    (
1120                        "inner-2".to_string(),
1121                        crate::schema::OwnedValue::Str(String::from("bobby of the sea 2")),
1122                    ),
1123                ]
1124                .into_iter()
1125                .collect(),
1126            ),
1127        );
1128        assert_eq!(
1129            value,
1130            crate::schema::OwnedValue::Object(expected_object.into_iter().collect())
1131        );
1132
1133        // Some more extreme nesting that might behave weirdly
1134        let mut object = serde_json::Map::new();
1135        object.insert(
1136            "my-array".into(),
1137            serde_json::Value::Array(vec![serde_json::Value::Array(vec![
1138                serde_json::Value::Array(vec![]),
1139                serde_json::Value::Array(vec![serde_json::Value::Null]),
1140            ])]),
1141        );
1142        let result = serialize_value(ReferenceValue::Object(JsonObjectIter(object.iter())));
1143        let value = deserialize_value(result);
1144
1145        let mut expected_object = BTreeMap::new();
1146        expected_object.insert(
1147            "my-array".to_string(),
1148            OwnedValue::Array(vec![OwnedValue::Array(vec![
1149                OwnedValue::Array(vec![]),
1150                OwnedValue::Array(vec![OwnedValue::Null]),
1151            ])]),
1152        );
1153        assert_eq!(
1154            value,
1155            OwnedValue::Object(expected_object.into_iter().collect())
1156        );
1157    }
1158}
lb_tantivy/schema/document/de.rs

lb_tantivy/schema/document/
de.rs
