lb_tantivy/schema/document/
owned_value.rs

1use std::collections::BTreeMap;
2use std::fmt;
3use std::net::Ipv6Addr;
4
5use base64::engine::general_purpose::STANDARD as BASE64;
6use base64::Engine;
7use serde::de::{MapAccess, SeqAccess};
8use time::format_description::well_known::Rfc3339;
9use time::OffsetDateTime;
10
11use super::existing_type_impls::can_be_rfc3339_date_time;
12use super::ReferenceValueLeaf;
13use crate::schema::document::{
14    ArrayAccess, DeserializeError, ObjectAccess, ReferenceValue, Value, ValueDeserialize,
15    ValueDeserializer, ValueVisitor,
16};
17use crate::schema::Facet;
18use crate::tokenizer::PreTokenizedString;
19use crate::DateTime;
20
21/// This is a owned variant of `Value`, that can be passed around without lifetimes.
22/// Represents the value of a any field.
23/// It is an enum over all over all of the possible field type.
24#[derive(Debug, Clone, PartialEq)]
25pub enum OwnedValue {
26    /// A null value.
27    Null,
28    /// The str type is used for any text information.
29    Str(String),
30    /// Pre-tokenized str type,
31    PreTokStr(PreTokenizedString),
32    /// Unsigned 64-bits Integer `u64`
33    U64(u64),
34    /// Signed 64-bits Integer `i64`
35    I64(i64),
36    /// 64-bits Float `f64`
37    F64(f64),
38    /// Bool value
39    Bool(bool),
40    /// Date/time with nanoseconds precision
41    Date(DateTime),
42    /// Facet
43    Facet(Facet),
44    /// Arbitrarily sized byte array
45    Bytes(Vec<u8>),
46    /// A set of values.
47    Array(Vec<Self>),
48    /// Dynamic object value.
49    Object(Vec<(String, Self)>),
50    /// IpV6 Address. Internally there is no IpV4, it needs to be converted to `Ipv6Addr`.
51    IpAddr(Ipv6Addr),
52}
53
54impl AsRef<OwnedValue> for OwnedValue {
55    #[inline]
56    fn as_ref(&self) -> &OwnedValue {
57        self
58    }
59}
60
61impl<'a> Value<'a> for &'a OwnedValue {
62    type ArrayIter = std::slice::Iter<'a, OwnedValue>;
63    type ObjectIter = ObjectMapIter<'a>;
64
65    fn as_value(&self) -> ReferenceValue<'a, Self> {
66        match self {
67            OwnedValue::Null => ReferenceValueLeaf::Null.into(),
68            OwnedValue::Str(val) => ReferenceValueLeaf::Str(val).into(),
69            OwnedValue::PreTokStr(val) => ReferenceValueLeaf::PreTokStr(val.clone().into()).into(),
70            OwnedValue::U64(val) => ReferenceValueLeaf::U64(*val).into(),
71            OwnedValue::I64(val) => ReferenceValueLeaf::I64(*val).into(),
72            OwnedValue::F64(val) => ReferenceValueLeaf::F64(*val).into(),
73            OwnedValue::Bool(val) => ReferenceValueLeaf::Bool(*val).into(),
74            OwnedValue::Date(val) => ReferenceValueLeaf::Date(*val).into(),
75            OwnedValue::Facet(val) => ReferenceValueLeaf::Facet(val.encoded_str()).into(),
76            OwnedValue::Bytes(val) => ReferenceValueLeaf::Bytes(val).into(),
77            OwnedValue::IpAddr(val) => ReferenceValueLeaf::IpAddr(*val).into(),
78            OwnedValue::Array(array) => ReferenceValue::Array(array.iter()),
79            OwnedValue::Object(object) => ReferenceValue::Object(ObjectMapIter(object.iter())),
80        }
81    }
82}
83
84impl ValueDeserialize for OwnedValue {
85    fn deserialize<'de, D>(deserializer: D) -> Result<Self, DeserializeError>
86    where D: ValueDeserializer<'de> {
87        struct Visitor;
88
89        impl ValueVisitor for Visitor {
90            type Value = OwnedValue;
91
92            fn visit_null(&self) -> Result<Self::Value, DeserializeError> {
93                Ok(OwnedValue::Null)
94            }
95
96            fn visit_string(&self, val: String) -> Result<Self::Value, DeserializeError> {
97                Ok(OwnedValue::Str(val))
98            }
99
100            fn visit_u64(&self, val: u64) -> Result<Self::Value, DeserializeError> {
101                Ok(OwnedValue::U64(val))
102            }
103
104            fn visit_i64(&self, val: i64) -> Result<Self::Value, DeserializeError> {
105                Ok(OwnedValue::I64(val))
106            }
107
108            fn visit_f64(&self, val: f64) -> Result<Self::Value, DeserializeError> {
109                Ok(OwnedValue::F64(val))
110            }
111
112            fn visit_bool(&self, val: bool) -> Result<Self::Value, DeserializeError> {
113                Ok(OwnedValue::Bool(val))
114            }
115
116            fn visit_datetime(&self, val: DateTime) -> Result<Self::Value, DeserializeError> {
117                Ok(OwnedValue::Date(val))
118            }
119
120            fn visit_ip_address(&self, val: Ipv6Addr) -> Result<Self::Value, DeserializeError> {
121                Ok(OwnedValue::IpAddr(val))
122            }
123
124            fn visit_facet(&self, val: Facet) -> Result<Self::Value, DeserializeError> {
125                Ok(OwnedValue::Facet(val))
126            }
127
128            fn visit_bytes(&self, val: Vec<u8>) -> Result<Self::Value, DeserializeError> {
129                Ok(OwnedValue::Bytes(val))
130            }
131
132            fn visit_pre_tokenized_string(
133                &self,
134                val: PreTokenizedString,
135            ) -> Result<Self::Value, DeserializeError> {
136                Ok(OwnedValue::PreTokStr(val))
137            }
138
139            fn visit_array<'de, A>(&self, mut access: A) -> Result<Self::Value, DeserializeError>
140            where A: ArrayAccess<'de> {
141                let mut elements = Vec::with_capacity(access.size_hint());
142
143                while let Some(value) = access.next_element()? {
144                    elements.push(value);
145                }
146
147                Ok(OwnedValue::Array(elements))
148            }
149
150            fn visit_object<'de, A>(&self, mut access: A) -> Result<Self::Value, DeserializeError>
151            where A: ObjectAccess<'de> {
152                let mut elements = Vec::with_capacity(access.size_hint());
153
154                while let Some((key, value)) = access.next_entry()? {
155                    elements.push((key, value));
156                }
157
158                Ok(OwnedValue::Object(elements))
159            }
160        }
161
162        deserializer.deserialize_any(Visitor)
163    }
164}
165
166impl Eq for OwnedValue {}
167
168impl serde::Serialize for OwnedValue {
169    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
170    where S: serde::Serializer {
171        use serde::ser::SerializeMap;
172        match *self {
173            OwnedValue::Null => serializer.serialize_unit(),
174            OwnedValue::Str(ref v) => serializer.serialize_str(v),
175            OwnedValue::PreTokStr(ref v) => v.serialize(serializer),
176            OwnedValue::U64(u) => serializer.serialize_u64(u),
177            OwnedValue::I64(u) => serializer.serialize_i64(u),
178            OwnedValue::F64(u) => serializer.serialize_f64(u),
179            OwnedValue::Bool(b) => serializer.serialize_bool(b),
180            OwnedValue::Date(ref date) => {
181                time::serde::rfc3339::serialize(&date.into_utc(), serializer)
182            }
183            OwnedValue::Facet(ref facet) => facet.serialize(serializer),
184            OwnedValue::Bytes(ref bytes) => serializer.serialize_str(&BASE64.encode(bytes)),
185            OwnedValue::Object(ref obj) => {
186                let mut map = serializer.serialize_map(Some(obj.len()))?;
187                for (k, v) in obj {
188                    map.serialize_entry(k, v)?;
189                }
190                map.end()
191            }
192            OwnedValue::IpAddr(ref ip_v6) => {
193                // Ensure IpV4 addresses get serialized as IpV4, but excluding IpV6 loopback.
194                if let Some(ip_v4) = ip_v6.to_ipv4_mapped() {
195                    ip_v4.serialize(serializer)
196                } else {
197                    ip_v6.serialize(serializer)
198                }
199            }
200            OwnedValue::Array(ref array) => array.serialize(serializer),
201        }
202    }
203}
204
205impl<'de> serde::Deserialize<'de> for OwnedValue {
206    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
207    where D: serde::Deserializer<'de> {
208        struct ValueVisitor;
209
210        impl<'de> serde::de::Visitor<'de> for ValueVisitor {
211            type Value = OwnedValue;
212
213            fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
214                formatter.write_str("a string or u32")
215            }
216
217            fn visit_bool<E>(self, v: bool) -> Result<Self::Value, E> {
218                Ok(OwnedValue::Bool(v))
219            }
220
221            fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E> {
222                Ok(OwnedValue::I64(v))
223            }
224
225            fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E> {
226                Ok(OwnedValue::U64(v))
227            }
228
229            fn visit_f64<E>(self, v: f64) -> Result<Self::Value, E> {
230                Ok(OwnedValue::F64(v))
231            }
232
233            fn visit_str<E>(self, v: &str) -> Result<Self::Value, E> {
234                Ok(OwnedValue::Str(v.to_owned()))
235            }
236
237            fn visit_string<E>(self, v: String) -> Result<Self::Value, E> {
238                Ok(OwnedValue::Str(v))
239            }
240
241            fn visit_unit<E>(self) -> Result<Self::Value, E>
242            where E: serde::de::Error {
243                Ok(OwnedValue::Null)
244            }
245
246            fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
247            where A: SeqAccess<'de> {
248                let mut elements = Vec::with_capacity(seq.size_hint().unwrap_or_default());
249
250                while let Some(value) = seq.next_element()? {
251                    elements.push(value);
252                }
253
254                Ok(OwnedValue::Array(elements))
255            }
256
257            fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
258            where A: MapAccess<'de> {
259                let mut object = map.size_hint().map(Vec::with_capacity).unwrap_or_default();
260                while let Some((key, value)) = map.next_entry()? {
261                    object.push((key, value));
262                }
263                Ok(OwnedValue::Object(object))
264            }
265        }
266
267        deserializer.deserialize_any(ValueVisitor)
268    }
269}
270
271impl<'a, V: Value<'a>> From<ReferenceValue<'a, V>> for OwnedValue {
272    fn from(val: ReferenceValue<'a, V>) -> OwnedValue {
273        match val {
274            ReferenceValue::Leaf(leaf) => match leaf {
275                ReferenceValueLeaf::Null => OwnedValue::Null,
276                ReferenceValueLeaf::Str(val) => OwnedValue::Str(val.to_string()),
277                ReferenceValueLeaf::U64(val) => OwnedValue::U64(val),
278                ReferenceValueLeaf::I64(val) => OwnedValue::I64(val),
279                ReferenceValueLeaf::F64(val) => OwnedValue::F64(val),
280                ReferenceValueLeaf::Date(val) => OwnedValue::Date(val),
281                ReferenceValueLeaf::Facet(val) => {
282                    OwnedValue::Facet(Facet::from_encoded_string(val.to_string()))
283                }
284                ReferenceValueLeaf::Bytes(val) => OwnedValue::Bytes(val.to_vec()),
285                ReferenceValueLeaf::IpAddr(val) => OwnedValue::IpAddr(val),
286                ReferenceValueLeaf::Bool(val) => OwnedValue::Bool(val),
287                ReferenceValueLeaf::PreTokStr(val) => OwnedValue::PreTokStr(*val.clone()),
288            },
289            ReferenceValue::Array(val) => {
290                OwnedValue::Array(val.map(|v| v.as_value().into()).collect())
291            }
292            ReferenceValue::Object(val) => OwnedValue::Object(
293                val.map(|(k, v)| (k.to_string(), v.as_value().into()))
294                    .collect(),
295            ),
296        }
297    }
298}
299
300impl From<String> for OwnedValue {
301    fn from(s: String) -> OwnedValue {
302        OwnedValue::Str(s)
303    }
304}
305
306impl From<Ipv6Addr> for OwnedValue {
307    fn from(v: Ipv6Addr) -> OwnedValue {
308        OwnedValue::IpAddr(v)
309    }
310}
311
312impl From<u64> for OwnedValue {
313    fn from(v: u64) -> OwnedValue {
314        OwnedValue::U64(v)
315    }
316}
317
318impl From<i64> for OwnedValue {
319    fn from(v: i64) -> OwnedValue {
320        OwnedValue::I64(v)
321    }
322}
323
324impl From<f64> for OwnedValue {
325    fn from(v: f64) -> OwnedValue {
326        OwnedValue::F64(v)
327    }
328}
329
330impl From<bool> for OwnedValue {
331    fn from(b: bool) -> Self {
332        OwnedValue::Bool(b)
333    }
334}
335
336impl From<DateTime> for OwnedValue {
337    fn from(dt: DateTime) -> OwnedValue {
338        OwnedValue::Date(dt)
339    }
340}
341
342impl<'a> From<&'a str> for OwnedValue {
343    fn from(s: &'a str) -> OwnedValue {
344        OwnedValue::Str(s.to_string())
345    }
346}
347
348impl<'a> From<&'a [u8]> for OwnedValue {
349    fn from(bytes: &'a [u8]) -> OwnedValue {
350        OwnedValue::Bytes(bytes.to_vec())
351    }
352}
353
354impl From<Facet> for OwnedValue {
355    fn from(facet: Facet) -> OwnedValue {
356        OwnedValue::Facet(facet)
357    }
358}
359
360impl From<Vec<u8>> for OwnedValue {
361    fn from(bytes: Vec<u8>) -> OwnedValue {
362        OwnedValue::Bytes(bytes)
363    }
364}
365
366impl From<PreTokenizedString> for OwnedValue {
367    fn from(pretokenized_string: PreTokenizedString) -> OwnedValue {
368        OwnedValue::PreTokStr(pretokenized_string)
369    }
370}
371
372impl From<BTreeMap<String, OwnedValue>> for OwnedValue {
373    fn from(object: BTreeMap<String, OwnedValue>) -> OwnedValue {
374        let key_values = object.into_iter().collect();
375        OwnedValue::Object(key_values)
376    }
377}
378
379impl From<serde_json::Value> for OwnedValue {
380    fn from(value: serde_json::Value) -> Self {
381        match value {
382            serde_json::Value::Null => Self::Null,
383            serde_json::Value::Bool(val) => Self::Bool(val),
384            serde_json::Value::Number(number) => {
385                if let Some(val) = number.as_i64() {
386                    Self::I64(val)
387                } else if let Some(val) = number.as_u64() {
388                    Self::U64(val)
389                } else if let Some(val) = number.as_f64() {
390                    Self::F64(val)
391                } else {
392                    panic!("Unsupported serde_json number {number}");
393                }
394            }
395            serde_json::Value::String(text) => {
396                if can_be_rfc3339_date_time(&text) {
397                    match OffsetDateTime::parse(&text, &Rfc3339) {
398                        Ok(dt) => {
399                            let dt_utc = dt.to_offset(time::UtcOffset::UTC);
400                            Self::Date(DateTime::from_utc(dt_utc))
401                        }
402                        Err(_) => Self::Str(text),
403                    }
404                } else {
405                    Self::Str(text)
406                }
407            }
408            serde_json::Value::Array(elements) => {
409                let converted_elements = elements.into_iter().map(Self::from).collect();
410                Self::Array(converted_elements)
411            }
412            serde_json::Value::Object(object) => Self::from(object),
413        }
414    }
415}
416
417impl From<serde_json::Map<String, serde_json::Value>> for OwnedValue {
418    fn from(map: serde_json::Map<String, serde_json::Value>) -> Self {
419        let object: Vec<(String, OwnedValue)> = map
420            .into_iter()
421            .map(|(key, value)| (key, OwnedValue::from(value)))
422            .collect();
423        OwnedValue::Object(object)
424    }
425}
426
427/// A wrapper type for iterating over a serde_json object producing reference values.
428pub struct ObjectMapIter<'a>(std::slice::Iter<'a, (String, OwnedValue)>);
429
430impl<'a> Iterator for ObjectMapIter<'a> {
431    type Item = (&'a str, &'a OwnedValue);
432
433    fn next(&mut self) -> Option<Self::Item> {
434        let (key, value) = self.0.next()?;
435        Some((key.as_str(), value))
436    }
437}
438
439#[cfg(test)]
440mod tests {
441    use super::*;
442    use crate::schema::{BytesOptions, Schema};
443    use crate::{Document, TantivyDocument};
444
445    #[test]
446    fn test_parse_bytes_doc() {
447        let mut schema_builder = Schema::builder();
448        let bytes_options = BytesOptions::default();
449        let bytes_field = schema_builder.add_bytes_field("my_bytes", bytes_options);
450        let schema = schema_builder.build();
451        let mut doc = TantivyDocument::default();
452        doc.add_bytes(bytes_field, "this is a test".as_bytes());
453        let json_string = doc.to_json(&schema);
454        assert_eq!(json_string, r#"{"my_bytes":["dGhpcyBpcyBhIHRlc3Q="]}"#);
455    }
456
457    #[test]
458    fn test_parse_empty_bytes_doc() {
459        let mut schema_builder = Schema::builder();
460        let bytes_options = BytesOptions::default();
461        let bytes_field = schema_builder.add_bytes_field("my_bytes", bytes_options);
462        let schema = schema_builder.build();
463        let mut doc = TantivyDocument::default();
464        doc.add_bytes(bytes_field, "".as_bytes());
465        let json_string = doc.to_json(&schema);
466
467        assert_eq!(json_string, r#"{"my_bytes":[""]}"#);
468    }
469
470    #[test]
471    fn test_parse_many_bytes_doc() {
472        let mut schema_builder = Schema::builder();
473        let bytes_options = BytesOptions::default();
474        let bytes_field = schema_builder.add_bytes_field("my_bytes", bytes_options);
475        let schema = schema_builder.build();
476        let mut doc = TantivyDocument::default();
477        doc.add_bytes(
478            bytes_field,
479            "A bigger test I guess\nspanning on multiple lines\nhoping this will work".as_bytes(),
480        );
481        let json_string = doc.to_json(&schema);
482        assert_eq!(
483            json_string,
484            r#"{"my_bytes":["QSBiaWdnZXIgdGVzdCBJIGd1ZXNzCnNwYW5uaW5nIG9uIG11bHRpcGxlIGxpbmVzCmhvcGluZyB0aGlzIHdpbGwgd29yaw=="]}"#
485        );
486    }
487
488    #[test]
489    fn test_serialize_date() {
490        let value = OwnedValue::from(DateTime::from_utc(
491            OffsetDateTime::parse("1996-12-20T00:39:57+00:00", &Rfc3339).unwrap(),
492        ));
493        let serialized_value_json = serde_json::to_string_pretty(&value).unwrap();
494        assert_eq!(serialized_value_json, r#""1996-12-20T00:39:57Z""#);
495        let value = OwnedValue::from(DateTime::from_utc(
496            OffsetDateTime::parse("1996-12-20T00:39:57-01:00", &Rfc3339).unwrap(),
497        ));
498        let serialized_value_json = serde_json::to_string_pretty(&value).unwrap();
499        // The time zone information gets lost by conversion into `Value::Date` and
500        // implicitly becomes UTC.
501        assert_eq!(serialized_value_json, r#""1996-12-20T01:39:57Z""#);
502    }
503}
lb_tantivy/schema/document/owned_value.rs

lb_tantivy/schema/document/
owned_value.rs
