qdrant_edge/segment/json_path/
mod.rs

1use std::fmt::{Display, Formatter};
2use std::hash::Hash;
3
4use data_encoding::BASE32_DNSSEC;
5use itertools::Itertools as _;
6use schemars::JsonSchema;
7use schemars::r#gen::SchemaGenerator;
8use schemars::schema::Schema;
9use serde::{Deserialize, Deserializer, Serialize, Serializer};
10use serde_json::Value;
11use sha2::{Digest as _, Sha256};
12
13use crate::segment::common::anonymize::Anonymize;
14use crate::segment::common::utils::{MultiValue, merge_map};
15
16mod parse;
17
18#[derive(Debug, Clone, PartialEq, Eq,  Ord, Hash, PartialOrd)]
19pub struct JsonPath {
20    pub first_key: String,
21    pub rest: Vec<JsonPathItem>,
22}
23
24#[derive(Debug, PartialEq, Clone, Eq,  Ord, Hash, PartialOrd)]
25pub enum JsonPathItem {
26    /// A key in a JSON object, e.g. `.foo`
27    Key(String),
28    /// An index in a JSON array, e.g. `[3]`
29    
30    Index(usize),
31    /// All indices in a JSON array, i.e. `[]`
32    WildcardIndex,
33}
34
35impl JsonPath {
36    /// Create a new `JsonPath` from a string. For production code, use `FromStr::parse` instead.
37    ///
38    /// # Panics
39    ///
40    /// Panics if the string is not a valid path. Thus, this function should only be used in tests.
41    #[cfg(feature = "testing")]
42    pub fn new(p: &str) -> Self {
43        p.parse().unwrap()
44    }
45
46    /// Get values at a given JSON path from a JSON map.
47    pub fn value_get<'a>(
48        &self,
49        json_map: &'a serde_json::Map<String, Value>,
50    ) -> MultiValue<&'a Value> {
51        let mut result = MultiValue::new();
52        if let Some(value) = json_map.get(&self.first_key) {
53            value_get(&self.rest, Some(value), &mut result);
54        }
55        result
56    }
57
58    /// Set values at a given JSON path in a JSON map.
59    pub fn value_set<'a>(
60        path: Option<&Self>,
61        dest: &'a mut serde_json::Map<String, Value>,
62        src: &'a serde_json::Map<String, Value>,
63    ) {
64        if let Some(path) = path {
65            value_set_map(&path.first_key, &path.rest, dest, src);
66        } else {
67            merge_map(dest, src);
68        }
69    }
70
71    /// Remove values at a given JSON path from a JSON map. Returns values that were removed.
72    pub fn value_remove(&self, json_map: &mut serde_json::Map<String, Value>) -> MultiValue<Value> {
73        let mut result = MultiValue::new();
74        if let Some((rest1, restn)) = self.rest.split_first() {
75            if let Some(value) = json_map.get_mut(&self.first_key) {
76                value_remove(rest1, restn, value, &mut result);
77            }
78        } else if let Some(value) = json_map.remove(&self.first_key) {
79            result.push(value);
80        }
81        result
82    }
83
84    /// Filter values in a JSON map based on a predicate.
85    pub fn value_filter(
86        json_map: &serde_json::Map<String, Value>,
87        filter: impl Fn(&Self, &Value) -> bool,
88    ) -> serde_json::Map<String, Value> {
89        let mut new_map = serde_json::Map::new();
90        let mut path = JsonPath {
91            first_key: "".to_string(),
92            rest: Vec::new(),
93        };
94        for (key, value) in json_map.iter() {
95            path.first_key.clone_from(key);
96            if filter(&path, value) {
97                let value = run_filter(&mut path, value, &filter);
98                new_map.insert(key.clone(), value);
99            }
100        }
101        new_map
102    }
103
104    /// Remove the wildcard suffix from the path, if it exists.
105    /// E.g. `a.b[]` -> `a.b`.
106    pub fn strip_wildcard_suffix(&self) -> Self {
107        match self.rest.split_last() {
108            Some((JsonPathItem::WildcardIndex, rest)) => JsonPath {
109                first_key: self.first_key.clone(),
110                rest: rest.to_vec(),
111            },
112            _ => self.clone(),
113        }
114    }
115
116    /// If `self` starts with `prefix`, returns a new path with the prefix removed.
117    pub fn strip_prefix(&self, prefix: &Self) -> Option<Self> {
118        if self.first_key != prefix.first_key {
119            return None;
120        }
121        let mut self_it = self.rest.iter().peekable();
122        let mut prefix_it = prefix.rest.iter().peekable();
123        loop {
124            match (self_it.peek(), prefix_it.peek()) {
125                (Some(self_item), Some(prefix_item)) if self_item == prefix_item => {
126                    self_it.next();
127                    prefix_it.next();
128                }
129                (Some(_), Some(_)) => return None,
130                (Some(JsonPathItem::Key(k)), None) => {
131                    return Some(JsonPath {
132                        first_key: k.clone(),
133                        rest: self_it.skip(1).cloned().collect(),
134                    });
135                }
136                (Some(_), None) => {
137                    // We don't support json paths starting with `[`. So
138                    // `strip_prefix("foo[]", "foo")` is not possible.
139                    return None;
140                }
141                (None, Some(_)) => return None,
142                (None, None) => {
143                    // Paths are equal. We don't support empty json paths.
144                    return None;
145                }
146            }
147        }
148    }
149
150    /// Extend the path with another path.
151    pub fn extend(&self, other: &Self) -> Self {
152        let mut rest = Vec::with_capacity(self.rest.len() + 1 + other.rest.len());
153        rest.extend_from_slice(&self.rest);
154        rest.push(JsonPathItem::Key(other.first_key.clone()));
155        rest.extend_from_slice(&other.rest);
156        JsonPath {
157            first_key: self.first_key.clone(),
158            rest,
159        }
160    }
161
162    /// Returns a new path with an array key appended to the end.
163    /// E.g. `a.b` -> `a.b[]`.
164    pub fn array_key(&self) -> Self {
165        let mut result = JsonPath {
166            first_key: self.first_key.clone(),
167            rest: Vec::with_capacity(self.rest.len() + 1),
168        };
169        result.rest.extend_from_slice(&self.rest);
170        if result.rest.last() != Some(&JsonPathItem::WildcardIndex) {
171            result.rest.push(JsonPathItem::WildcardIndex);
172        }
173        result
174    }
175
176    pub fn has_wildcard_suffix(&self) -> bool {
177        self.rest.last() == Some(&JsonPathItem::WildcardIndex)
178    }
179
180    /// Check if a path is included in a list of patterns.
181    ///
182    /// Basically, it checks if either the pattern or path is a prefix of the other.
183    pub fn check_include_pattern(&self, pattern: &Self) -> bool {
184        self.first_key == pattern.first_key
185            && self.rest.iter().zip(&pattern.rest).all(|(a, b)| a == b)
186    }
187
188    /// Check if a path should be excluded by a pattern.
189    ///
190    /// Basically, it checks if pattern is a prefix of path, but not the other way around.
191    pub fn check_exclude_pattern(&self, pattern: &Self) -> bool {
192        self.first_key == pattern.first_key && pattern.rest.starts_with(&self.rest)
193    }
194
195    pub fn extend_or_new(base: Option<&Self>, other: &Self) -> Self {
196        base.map_or_else(|| other.clone(), |base| base.extend(other))
197    }
198
199    /// Check if a path is a compatible prefix of another path or vice versa.
200    pub fn compatible(&self, other: &Self) -> bool {
201        if self.first_key != other.first_key {
202            return false;
203        }
204        self.rest
205            .iter()
206            .zip(&other.rest)
207            .all(|(a, b)| match (a, b) {
208                (JsonPathItem::Key(a), JsonPathItem::Key(b)) => a == b,
209                (JsonPathItem::Index(a), JsonPathItem::Index(b)) => a == b,
210                (JsonPathItem::WildcardIndex, JsonPathItem::WildcardIndex) => true,
211                (JsonPathItem::Index(_), JsonPathItem::WildcardIndex) => true,
212                (JsonPathItem::WildcardIndex, JsonPathItem::Index(_)) => true,
213                _ => false,
214            })
215    }
216
217    /// Check if the path will be affected by a call to `path_to_remove.value_remove(_)`.
218    pub fn is_affected_by_value_remove(&self, path_to_remove: &JsonPath) -> bool {
219        // If we have, e.g., indexed field "a.b", then it is not safe to delete any of of "a",
220        // "a.b", or "a.b.c".
221        path_to_remove.compatible(self)
222    }
223
224    /// Check if the path will be affected by a call to `path_to_set.value_set(_, payload)`.
225    pub fn is_affected_by_value_set(
226        &self,
227        payload: &serde_json::Map<String, Value>,
228        path_to_set: Option<&JsonPath>,
229    ) -> bool {
230        // Suppose we have a `path_to_set=a.b.c` and a `payload={"x": 1, "y": 2, "z": {"q": 0}}`.
231        // It's safe to set the payload if the indexed fields doesn't intersect[^1] with the
232        // following paths:
233        // - `a.b.c.x`
234        // - `a.b.c.y`
235        // - `a.b.c.z` // Note that only top-level keys of the payload are considered.
236        //
237        // [^1]: In simple cases, we consider two paths to intersect if one of them is a prefix of
238        // the other.  For example, `a.b` and `a.b.c` intersect, but `a.b` and `a.c` don't. More
239        // nuanced cases include wildcard indexes, e.g., `a[0].b` and `a[].b` intersect.
240        // Additionally, we consider path with incompatible types (e.g. `a[0]` and `a.b`) to
241        // intersect because `valuse_set` could override the subtree by replacing an array with an
242        // object (or vice versa), deleting indexed fields.
243
244        let Some(path_to_set) = path_to_set else {
245            return payload.contains_key(&self.first_key);
246        };
247        if self.first_key != path_to_set.first_key {
248            return false;
249        }
250        let mut it_a = self.rest.iter();
251        let mut it_b = path_to_set.rest.iter();
252        loop {
253            let (a, b) = match (it_a.next(), it_b.next()) {
254                (Some(a), Some(b)) => (a, b),
255                (None, _) => return true, // indexed_path is a compatible prefix of path_to_set
256
257                (Some(JsonPathItem::Key(a)), None) => return payload.contains_key(a),
258                (Some(JsonPathItem::Index(_)), None) => return true,
259                (Some(JsonPathItem::WildcardIndex), None) => return true,
260            };
261
262            match (a, b) {
263                // Paths items match each other => continue.
264                (JsonPathItem::Key(a), JsonPathItem::Key(b)) if a == b => (),
265                (JsonPathItem::Index(a), JsonPathItem::Index(b)) if a == b => (),
266                (JsonPathItem::WildcardIndex, JsonPathItem::WildcardIndex) => (),
267                (JsonPathItem::Index(_), JsonPathItem::WildcardIndex) => (),
268                (JsonPathItem::WildcardIndex, JsonPathItem::Index(_)) => (),
269
270                // Paths diverge, but their types are compatible, e.g. `a.b` and `a.c`, or `a[0]`
271                // and `a[1]`.  This means that payload and indexed fields point to different
272                // subtrees, so it's safe to set the payload.
273                (JsonPathItem::Key(_), JsonPathItem::Key(_)) => return false,
274                (JsonPathItem::Index(_), JsonPathItem::Index(_)) => return false,
275
276                // Types are not compatible. This means that `value_set` could override the
277                // subtree, deleting indexed fields.
278                (JsonPathItem::Key(_), JsonPathItem::Index(_) | JsonPathItem::WildcardIndex) => {
279                    return true;
280                }
281                (JsonPathItem::Index(_) | JsonPathItem::WildcardIndex, JsonPathItem::Key(_)) => {
282                    return true;
283                }
284            }
285        }
286    }
287
288    /// Convert the path into a string suitable for use as a filename by adhering to the following
289    /// restrictions: max length, limited character set, but still being relatively unique.
290    pub fn filename(&self) -> String {
291        const MAX_LENGTH: usize = 33;
292        const HASH_LENGTH: usize = 24; // In base32 characters, i.e. 5 bits per character.
293
294        let text = self.to_string();
295        let mut result = String::with_capacity(MAX_LENGTH);
296
297        BASE32_DNSSEC.encode_append(
298            &Sha256::digest(text.as_bytes())[0..(HASH_LENGTH * 5).div_ceil(8)],
299            &mut result,
300        );
301        debug_assert_eq!(result.len(), HASH_LENGTH);
302
303        result.push('-');
304
305        text.chars()
306            .map(|c| match c {
307                'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' => c.to_ascii_lowercase(),
308                _ => '_',
309            })
310            .dedup_by(|&a, &b| a == '_' && b == '_')
311            .take(MAX_LENGTH - result.len())
312            .for_each(|c| result.push(c));
313
314        debug_assert!(result.len() <= MAX_LENGTH);
315        result
316    }
317}
318
319fn value_get<'a>(
320    path: &[JsonPathItem],
321    value: Option<&'a Value>,
322    result: &mut MultiValue<&'a Value>,
323) {
324    if let Some((head, tail)) = path.split_first() {
325        match (head, value) {
326            (JsonPathItem::Key(key), Some(Value::Object(map))) => {
327                value_get(tail, map.get(key), result)
328            }
329            (JsonPathItem::Index(index), Some(Value::Array(array))) => {
330                if let Some(value) = array.get(*index) {
331                    value_get(tail, Some(value), result);
332                }
333            }
334            (JsonPathItem::WildcardIndex, Some(Value::Array(array))) => array
335                .iter()
336                .for_each(|value| value_get(tail, Some(value), result)),
337            _ => (),
338        }
339    } else if let Some(value) = value {
340        result.push(value);
341    }
342}
343
344fn value_set(path: &[JsonPathItem], dest: &mut Value, src: &serde_json::Map<String, Value>) {
345    if let Some((head, rest)) = path.split_first() {
346        match head {
347            JsonPathItem::Key(key) => {
348                if !dest.is_object() {
349                    *dest = Value::Object(serde_json::Map::new());
350                }
351                let map = dest.as_object_mut().unwrap();
352                value_set_map(key, rest, map, src);
353            }
354            &JsonPathItem::Index(i) => {
355                if !dest.is_array() {
356                    *dest = Value::Array(Vec::new());
357                }
358                let array = dest.as_array_mut().unwrap();
359                if let Some(v) = array.get_mut(i) {
360                    value_set(rest, v, src);
361                }
362            }
363            JsonPathItem::WildcardIndex => {
364                if dest.is_array() {
365                    for value in dest.as_array_mut().unwrap() {
366                        value_set(rest, value, src);
367                    }
368                } else {
369                    *dest = Value::Array(Vec::new());
370                }
371            }
372        }
373    } else {
374        if !dest.is_object() {
375            *dest = Value::Object(serde_json::Map::new());
376        }
377        let map = dest.as_object_mut().unwrap();
378        merge_map(map, src);
379    }
380}
381
382fn value_set_map(
383    key: &str,
384    path: &[JsonPathItem],
385    dest_map: &mut serde_json::Map<String, Value>,
386    src: &serde_json::Map<String, Value>,
387) {
388    if let Some(value) = dest_map.get_mut(key) {
389        value_set(path, value, src);
390    } else {
391        let mut value = Value::Null;
392        value_set(path, &mut value, src);
393        dest_map.insert(key.to_string(), value);
394    }
395}
396
397fn value_remove(
398    head: &JsonPathItem,
399    rest: &[JsonPathItem],
400    value: &mut Value,
401    result: &mut MultiValue<Value>,
402) {
403    if let Some((rest1, restn)) = rest.split_first() {
404        match (head, value) {
405            (JsonPathItem::Key(k), Value::Object(map)) => {
406                if let Some(value) = map.get_mut(k) {
407                    value_remove(rest1, restn, value, result);
408                }
409            }
410            (JsonPathItem::Index(i), Value::Array(array)) => {
411                if let Some(value) = array.get_mut(*i) {
412                    value_remove(rest1, restn, value, result);
413                }
414            }
415            (JsonPathItem::WildcardIndex, Value::Array(array)) => {
416                for value in array {
417                    value_remove(rest1, restn, value, result);
418                }
419            }
420            _ => (),
421        }
422    } else {
423        match (head, value) {
424            (JsonPathItem::Key(k), Value::Object(map)) => {
425                if let Some(v) = map.remove(k) {
426                    result.push(v);
427                }
428            }
429            (JsonPathItem::Index(_), Value::Array(_)) => {
430                // Deleting array indices is not idempotent, so we don't support it.
431            }
432            (JsonPathItem::WildcardIndex, Value::Array(array)) => {
433                result.push(Value::Array(std::mem::take(array)));
434            }
435            _ => (),
436        }
437    }
438}
439
440fn run_filter<'a>(
441    path: &mut JsonPath,
442    value: &'a Value,
443    filter: &dyn Fn(&JsonPath, &Value) -> bool,
444) -> Value {
445    match &value {
446        Value::Null => value.clone(),
447        Value::Bool(_) => value.clone(),
448        Value::Number(_) => value.clone(),
449        Value::String(_) => value.clone(),
450        Value::Array(array) => {
451            let mut new_array = Vec::new();
452            path.rest.push(JsonPathItem::WildcardIndex);
453            for value in array.iter() {
454                if filter(path, value) {
455                    let value = run_filter(path, value, filter);
456                    new_array.push(value);
457                }
458            }
459            path.rest.pop();
460            Value::Array(new_array)
461        }
462        Value::Object(object) => {
463            let mut new_object = serde_json::Map::new();
464            for (key, value) in object.iter() {
465                path.rest.push(JsonPathItem::Key(key.clone()));
466                if filter(path, value) {
467                    let value = run_filter(path, value, filter);
468                    new_object.insert(key.clone(), value);
469                }
470                path.rest.pop();
471            }
472            Value::Object(new_object)
473        }
474    }
475}
476
477impl Display for JsonPath {
478    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
479        let write_key = |f: &mut Formatter<'_>, key: &str| {
480            if parse::key_needs_quoting(key) {
481                write!(f, "\"{key}\"")
482            } else {
483                f.write_str(key)
484            }
485        };
486
487        write_key(f, &self.first_key)?;
488        for item in &self.rest {
489            match item {
490                JsonPathItem::Key(key) => {
491                    f.write_str(".")?;
492                    write_key(f, key)?;
493                }
494                JsonPathItem::Index(index) => write!(f, "[{index}]")?,
495                JsonPathItem::WildcardIndex => f.write_str("[]")?,
496            }
497        }
498        Ok(())
499    }
500}
501
502impl TryFrom<&str> for JsonPath {
503    type Error = ();
504
505    fn try_from(value: &str) -> Result<Self, Self::Error> {
506        value.parse()
507    }
508}
509
510impl Serialize for JsonPath {
511    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
512        serializer.serialize_str(&self.to_string())
513    }
514}
515
516impl<'de> Deserialize<'de> for JsonPath {
517    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
518        let string = String::deserialize(deserializer)?;
519        string
520            .parse()
521            .map_err(|_| serde::de::Error::custom(format!("Invalid json path: \'{string}\'")))
522    }
523}
524
525impl JsonSchema for JsonPath {
526    fn is_referenceable() -> bool {
527        false
528    }
529
530    fn schema_name() -> String {
531        "JsonPath".to_string()
532    }
533
534    fn json_schema(generator: &mut SchemaGenerator) -> Schema {
535        String::json_schema(generator)
536    }
537}
538
539#[cfg(test)]
540mod tests {
541    use super::*;
542    use crate::segment::common::utils::check_is_empty;
543
544    fn json(str: &str) -> serde_json::Map<String, Value> {
545        serde_json::from_str(str).unwrap()
546    }
547
548    #[test]
549    fn test_is_affected_by_value_set() {
550        assert!(!JsonPath::new("a").is_affected_by_value_set(&json(r#"{"b": 1, "c": 1}"#), None));
551        assert!(JsonPath::new("a").is_affected_by_value_set(&json(r#"{"a": 1, "b": 1}"#), None));
552        assert!(JsonPath::new("a.x").is_affected_by_value_set(&json(r#"{"a": {"y": 1}}"#), None));
553        assert!(!JsonPath::new("a.x").is_affected_by_value_set(&json(r#"{"b": {"x": 1}}"#), None));
554    }
555
556    #[test]
557    fn test_is_affected_by_value_remove() {
558        assert!(JsonPath::new("a").is_affected_by_value_remove(&JsonPath::new("a")));
559        assert!(!JsonPath::new("a").is_affected_by_value_remove(&JsonPath::new("b")));
560        assert!(JsonPath::new("a.b").is_affected_by_value_remove(&JsonPath::new("a")));
561        assert!(JsonPath::new("a.b").is_affected_by_value_remove(&JsonPath::new("a.b")));
562        assert!(JsonPath::new("a.b").is_affected_by_value_remove(&JsonPath::new("a.b.c")));
563    }
564
565    /// This test checks that `is_affected_by_value_set` and `is_affected_by_value_remove` don't
566    /// produce false negatives.
567    /// The penalty for a false positive is just degraded performance, but the penalty for a false
568    /// negative is inconsistency in the indexed fields.
569    #[test]
570    fn test_no_false_negatives() {
571        let paths: Vec<JsonPath> = ["a", "a.a", "a[]", "a[0]", "a[0].a", "a[0].a[]"]
572            .iter()
573            .map(|s| s.parse().unwrap())
574            .collect();
575        let payloads = vec![
576            json(r#"{"b": 1}"#),
577            json(r#"{"a": 1, "b": 2}"#),
578            json(r#"{"a": [], "b": 1}"#),
579            json(r#"{"a": [1], "b": 2}"#),
580            json(r#"{"a": {}, "b": 1}"#),
581            json(r#"{"a": {"a": 1, "b": 2}, "b": 3}"#),
582            json(r#"{"a": [{"a": 1, "b": 2}, {"a": 3, "b": 4}], "b": 5}"#),
583            json(r#"{"a": [{"a": [1], "b": 2}, {"a": [3], "b": 4}], "b": 5}"#),
584        ];
585
586        for init_payload in &payloads {
587            for indexed_path in &paths {
588                for value_key in &["a", "b"] {
589                    check_set(init_payload, indexed_path, None, value_key);
590                    for path_to_set in &paths {
591                        check_set(init_payload, indexed_path, Some(path_to_set), value_key);
592                    }
593                }
594                for path_to_remove in &paths {
595                    check_remove(init_payload, indexed_path, path_to_remove);
596                }
597            }
598        }
599    }
600
601    fn check_set(
602        init_payload: &serde_json::Map<String, serde_json::Value>,
603        indexed_path: &JsonPath,
604        path_to_set: Option<&JsonPath>,
605        value_key: &str,
606    ) {
607        let mut new_payload = init_payload.clone();
608        let init_values = indexed_path.value_get(init_payload);
609
610        JsonPath::value_set(
611            path_to_set,
612            &mut new_payload,
613            &json(r#"{"value_key": 100}"#),
614        );
615        let new_values = indexed_path.value_get(&new_payload);
616
617        // Ground truth
618        let indexed_value_changed = init_values != new_values;
619
620        // Our prediction
621        let is_affected =
622            indexed_path.is_affected_by_value_set(&json(r#"{"value_key": 100}"#), path_to_set);
623
624        assert!(
625            is_affected || !indexed_value_changed,
626            "init_payload: {:?}\nnew_payload: {:?}\nindex_path: {:?}\npath_to_set: {:?}\nvalue_key: {:?}",
627            init_payload,
628            new_payload,
629            indexed_path.to_string(),
630            path_to_set.map(|p| p.to_string()),
631            value_key,
632        );
633    }
634
635    fn check_remove(
636        init_payload: &serde_json::Map<String, serde_json::Value>,
637        indexed_path: &JsonPath,
638        path_to_remove: &JsonPath,
639    ) {
640        let mut new_payload = init_payload.clone();
641        let init_values = indexed_path.value_get(init_payload);
642        path_to_remove.value_remove(&mut new_payload);
643        let new_values = indexed_path.value_get(&new_payload);
644
645        // Ground truth
646        let indexed_value_changed = init_values != new_values;
647
648        // Our prediction
649        let is_affected = indexed_path.is_affected_by_value_remove(path_to_remove);
650
651        assert!(
652            is_affected || !indexed_value_changed,
653            "init_payload: {:?}\nnew_payload: {:?}\nindex_path: {:?}\npath_to_remove: {:?}",
654            init_payload,
655            new_payload,
656            indexed_path.to_string(),
657            path_to_remove.to_string(),
658        );
659    }
660
661    #[test]
662    fn test_get_nested_value_from_json_map() {
663        let map = json(
664            r#"
665            {
666                "a": {"b": {"c": 1}},
667                "d": 2
668            }
669            "#,
670        );
671
672        assert_eq!(
673            JsonPath::new("a.b").value_get(&map).into_vec(),
674            vec![&Value::Object(serde_json::Map::from_iter(vec![(
675                "c".to_string(),
676                Value::Number(1.into())
677            )]))]
678        );
679
680        // going deeper
681        assert_eq!(
682            JsonPath::new("a.b.c").value_get(&map).into_vec(),
683            vec![&Value::Number(1.into())]
684        );
685
686        // missing path
687        assert!(check_is_empty(
688            JsonPath::new("a.b.c.d").value_get(&map).iter().copied()
689        ));
690    }
691
692    #[test]
693    fn test_is_empty() {
694        let map = json(
695            r#"
696                {
697                   "a": [
698                     { "b": 1 },
699                     { "b": 2 },
700                     { "b": null },
701                     { "d": [] },
702                     { "d": [] },
703                     { "f": null }
704                   ]
705                }
706            "#,
707        );
708        let multivalue = JsonPath::new("a[].b").value_get(&map);
709        let is_empty = check_is_empty(multivalue.iter().copied());
710
711        assert!(!is_empty, "a[].b is not empty");
712
713        let multivalue = JsonPath::new("a[].c").value_get(&map);
714        let is_empty = check_is_empty(multivalue.iter().copied());
715
716        assert!(is_empty, "a[].c is empty");
717
718        let multivalue = JsonPath::new("a[].d").value_get(&map);
719        let is_empty = check_is_empty(multivalue.iter().copied());
720        assert!(is_empty, "a[].d is empty");
721
722        let multivalue = JsonPath::new("a[].f").value_get(&map);
723        let is_empty = check_is_empty(multivalue.iter().copied());
724        assert!(is_empty, "a[].f is empty");
725    }
726
727    #[test]
728    fn test_get_nested_array_value_from_json_map() {
729        let map = json(
730            r#"
731            {
732                "a": {
733                    "b": [
734                        { "c": 1 },
735                        { "c": 2 },
736                        { "d": { "e": 3 } }
737                    ]
738                },
739                "f": 3,
740                "g": ["g0", "g1", "g2"]
741            }
742            "#,
743        );
744
745        // get JSON array
746        assert_eq!(
747            JsonPath::new("a.b").value_get(&map).into_vec(),
748            vec![&Value::Array(vec![
749                Value::Object(serde_json::Map::from_iter(vec![(
750                    "c".to_string(),
751                    Value::Number(1.into())
752                )])),
753                Value::Object(serde_json::Map::from_iter(vec![(
754                    "c".to_string(),
755                    Value::Number(2.into())
756                )])),
757                Value::Object(serde_json::Map::from_iter(vec![(
758                    "d".to_string(),
759                    Value::Object(serde_json::Map::from_iter(vec![(
760                        "e".to_string(),
761                        Value::Number(3.into())
762                    )]))
763                )]))
764            ])]
765        );
766
767        // a.b[] extract all elements from array
768        assert_eq!(
769            JsonPath::new("a.b[]").value_get(&map).into_vec(),
770            vec![
771                &Value::Object(serde_json::Map::from_iter(vec![(
772                    "c".to_string(),
773                    Value::Number(1.into())
774                )])),
775                &Value::Object(serde_json::Map::from_iter(vec![(
776                    "c".to_string(),
777                    Value::Number(2.into())
778                )])),
779                &Value::Object(serde_json::Map::from_iter(vec![(
780                    "d".to_string(),
781                    Value::Object(serde_json::Map::from_iter(vec![(
782                        "e".to_string(),
783                        Value::Number(3.into())
784                    )]))
785                )]))
786            ]
787        );
788
789        // project scalar field through array
790        assert_eq!(
791            JsonPath::new("a.b[].c").value_get(&map).into_vec(),
792            vec![&Value::Number(1.into()), &Value::Number(2.into())]
793        );
794
795        // project object field through array
796        assert_eq!(
797            JsonPath::new("a.b[].d").value_get(&map).into_vec(),
798            vec![&Value::Object(serde_json::Map::from_iter(vec![(
799                "e".to_string(),
800                Value::Number(3.into())
801            )]))]
802        );
803
804        // select scalar element from array
805        assert_eq!(
806            JsonPath::new("a.b[0]").value_get(&map).into_vec(),
807            vec![&Value::Object(serde_json::Map::from_iter(vec![(
808                "c".to_string(),
809                Value::Number(1.into())
810            )]))]
811        );
812
813        // select scalar object from array different index
814        assert_eq!(
815            JsonPath::new("a.b[1]").value_get(&map).into_vec(),
816            vec![&Value::Object(serde_json::Map::from_iter(vec![(
817                "c".to_string(),
818                Value::Number(2.into())
819            )]))]
820        );
821
822        // select field element from array different index
823        assert_eq!(
824            JsonPath::new("a.b[1].c").value_get(&map).into_vec(),
825            vec![&Value::Number(2.into())]
826        );
827
828        // select scalar element from array different index
829        assert_eq!(
830            JsonPath::new("g[2]").value_get(&map).into_vec(),
831            vec![&Value::String("g2".to_string())]
832        );
833
834        // select object element from array
835        assert_eq!(
836            JsonPath::new("a.b[2]").value_get(&map).into_vec(),
837            vec![&Value::Object(serde_json::Map::from_iter(vec![(
838                "d".to_string(),
839                Value::Object(serde_json::Map::from_iter(vec![(
840                    "e".to_string(),
841                    Value::Number(3.into())
842                )]))
843            )]))]
844        );
845
846        // select out of bound index from array
847        assert!(check_is_empty(
848            JsonPath::new("a.b[3]").value_get(&map).iter().copied()
849        ));
850    }
851
852    #[test]
853    fn test_get_deeply_nested_array_value_from_json_map() {
854        let map = json(
855            r#"
856            {
857                "arr1": [
858                    {
859                        "arr2": [
860                            {"a": 1, "b": 2}
861                        ]
862                    },
863                    {
864                        "arr2": [
865                            {"a": 3, "b": 4},
866                            {"a": 5, "b": 6}
867                        ]
868                    }
869                ]
870            }
871            "#,
872        );
873
874        // extract and flatten all elements from arrays
875        assert_eq!(
876            JsonPath::new("arr1[].arr2[].a").value_get(&map).into_vec(),
877            vec![
878                &Value::Number(1.into()),
879                &Value::Number(3.into()),
880                &Value::Number(5.into()),
881            ]
882        );
883    }
884
885    #[test]
886    fn test_no_flatten_array_value_from_json_map() {
887        let map = json(
888            r#"
889            {
890                "arr": [
891                    { "a": [1, 2, 3] },
892                    { "a": 4 },
893                    { "b": 5 }
894                ]
895            }
896            "#,
897        );
898
899        // extract and retain structure for arrays arrays
900        assert_eq!(
901            JsonPath::new("arr[].a").value_get(&map).into_vec(),
902            vec![
903                &Value::Array(vec![
904                    Value::Number(1.into()),
905                    Value::Number(2.into()),
906                    Value::Number(3.into()),
907                ]),
908                &Value::Number(4.into()),
909            ]
910        );
911
912        // expect an array as leaf, ignore non arrays
913        assert_eq!(
914            JsonPath::new("arr[].a[]").value_get(&map).into_vec(),
915            vec![
916                &Value::Number(1.into()),
917                &Value::Number(2.into()),
918                &Value::Number(3.into()),
919            ]
920        );
921    }
922
923    #[test]
924    fn test_get_null_and_absent_values() {
925        let map = json(
926            r#"
927            {
928                "a": null,
929                "b": [null, null],
930                "c": []
931            }
932            "#,
933        );
934
935        assert_eq!(
936            JsonPath::new("a").value_get(&map).as_slice(),
937            &[&Value::Null],
938        );
939
940        assert!(JsonPath::new("a[]").value_get(&map).is_empty());
941
942        assert_eq!(
943            JsonPath::new("b").value_get(&map).as_slice(),
944            &[&Value::Array(vec![Value::Null, Value::Null])],
945        );
946
947        assert_eq!(
948            JsonPath::new("b[]").value_get(&map).as_slice(),
949            &[&Value::Null, &Value::Null],
950        );
951
952        assert_eq!(
953            JsonPath::new("c").value_get(&map).as_slice(),
954            &[&Value::Array(vec![])],
955        );
956
957        assert!(JsonPath::new("c[]").value_get(&map).is_empty());
958
959        assert!(JsonPath::new("d").value_get(&map).is_empty());
960
961        assert!(JsonPath::new("d[]").value_get(&map).is_empty());
962    }
963
964    #[test]
965    fn test_filter_json() {
966        let map = json(
967            r#"
968            {
969                "a": {
970                    "b": [
971                        { "c": 1 },
972                        { "c": 2 },
973                        { "d": { "e": 3 } }
974                    ]
975                },
976                "f": 3,
977                "g": ["g0", "g1", "g2"]
978            }
979            "#,
980        );
981
982        let res = JsonPath::value_filter(&map, |path, _value| {
983            let path = path.to_string();
984            path.starts_with("a.b[].c") || "a.b[].c".starts_with(&path)
985        });
986
987        assert_eq!(
988            res,
989            json(
990                r#"
991                {
992                    "a": {
993                        "b": [
994                            { "c": 1 },
995                            { "c": 2 },
996                            {}
997                        ]
998                    }
999                }
1000                "#,
1001            ),
1002        );
1003    }
1004
1005    #[test]
1006    fn test_check_include_pattern() {
1007        assert!(JsonPath::new("a.b.c").check_include_pattern(&JsonPath::new("a.b.c")));
1008        assert!(JsonPath::new("a.b.c").check_include_pattern(&JsonPath::new("a.b")));
1009        assert!(!JsonPath::new("a.b.c").check_include_pattern(&JsonPath::new("a.b.d")));
1010        assert!(JsonPath::new("a.b.c").check_include_pattern(&JsonPath::new("a")));
1011        assert!(JsonPath::new("a").check_include_pattern(&JsonPath::new("a.d")));
1012    }
1013
1014    #[test]
1015    fn test_check_exclude_pattern() {
1016        assert!(JsonPath::new("a.b.c").check_exclude_pattern(&JsonPath::new("a.b.c")));
1017        assert!(!JsonPath::new("a.b.c").check_exclude_pattern(&JsonPath::new("a.b")));
1018        assert!(!JsonPath::new("a.b.c").check_exclude_pattern(&JsonPath::new("a.b.d")));
1019        assert!(!JsonPath::new("a.b.c").check_exclude_pattern(&JsonPath::new("a")));
1020        assert!(JsonPath::new("a").check_exclude_pattern(&JsonPath::new("a.d")));
1021    }
1022
1023    #[test]
1024    fn test_set_value_to_json_with_empty_key() {
1025        let mut map = json(
1026            r#"
1027            {
1028                "a": {
1029                    "b": [
1030                        { "c": 1 },
1031                        { "c": 2 },
1032                        { "d": { "e": 3 } }
1033                    ]
1034                },
1035                "f": 3,
1036                "g": ["g0", "g1", "g2"]
1037            }
1038            "#,
1039        );
1040
1041        JsonPath::value_set(None, &mut map, &json(r#"{"c": 5}"#));
1042
1043        assert_eq!(
1044            map,
1045            json(
1046                r#"
1047                {
1048                    "a": {
1049                        "b": [
1050                            { "c": 1 },
1051                            { "c": 2 },
1052                            { "d": { "e": 3 } }
1053                        ]
1054                    },
1055                    "f": 3,
1056                    "g": ["g0", "g1", "g2"],
1057                    "c": 5
1058                }
1059                "#,
1060            ),
1061        );
1062    }
1063
1064    #[test]
1065    fn test_set_value_to_json_with_one_level_key() {
1066        let mut map = json(
1067            r#"
1068            {
1069                "a": {
1070                    "b": [
1071                        { "c": 1 },
1072                        { "c": 2 },
1073                        { "d": { "e": 3 } }
1074                    ]
1075                },
1076                "f": 3,
1077                "g": ["g0", "g1", "g2"]
1078            }
1079            "#,
1080        );
1081
1082        JsonPath::value_set(Some(&JsonPath::new("a")), &mut map, &json(r#"{"b": 5}"#));
1083
1084        assert_eq!(
1085            map,
1086            json(
1087                r#"
1088                {
1089                    "a": {
1090                        "b": 5
1091                    },
1092                    "f": 3,
1093                    "g": ["g0", "g1", "g2"]
1094                }
1095                "#,
1096            ),
1097        );
1098    }
1099
1100    #[test]
1101    fn test_set_value_to_json_with_array_index() {
1102        let mut map = json(
1103            r#"
1104            {
1105                "a": {
1106                    "b": [
1107                        { "c": 1 },
1108                        { "c": 2 },
1109                        { "d": { "e": 3 } }
1110                    ]
1111                },
1112                "f": 3,
1113                "g": ["g0", "g1", "g2"]
1114            }
1115            "#,
1116        );
1117
1118        JsonPath::value_set(
1119            Some(&JsonPath::new("a.b[1]")),
1120            &mut map,
1121            &json(r#"{"c": 5}"#),
1122        );
1123
1124        assert_eq!(
1125            map,
1126            json(
1127                r#"
1128                {
1129                    "a": {
1130                        "b": [
1131                            { "c": 1 },
1132                            { "c": 5 },
1133                            { "d": { "e": 3 } }
1134                        ]
1135                    },
1136                    "f": 3,
1137                    "g": ["g0", "g1", "g2"]
1138                }
1139                "#,
1140            ),
1141        );
1142    }
1143
1144    #[test]
1145    fn test_set_value_to_json_with_empty_src() {
1146        let mut map = json(
1147            r#"
1148            {
1149                "a": {
1150                    "b": [
1151                        { "c": 1 },
1152                        { "c": 2 },
1153                        { "d": { "e": 3 } }
1154                    ]
1155                },
1156                "f": 3,
1157                "g": ["g0", "g1", "g2"]
1158            }
1159            "#,
1160        );
1161
1162        JsonPath::value_set(Some(&JsonPath::new("a.b[1]")), &mut map, &json("{}"));
1163
1164        assert_eq!(
1165            map,
1166            json(
1167                r#"
1168                {
1169                    "a": {
1170                        "b": [
1171                            { "c": 1 },
1172                            { "c": 2 },
1173                            { "d": { "e": 3 } }
1174                        ]
1175                    },
1176                    "f": 3,
1177                    "g": ["g0", "g1", "g2"]
1178                }
1179                "#,
1180            ),
1181        );
1182    }
1183
1184    #[test]
1185    fn test_set_value_to_json_with_empty_dest() {
1186        let mut map = json("{}");
1187
1188        JsonPath::value_set(None, &mut map, &json(r#"{"c": 1}"#));
1189
1190        assert_eq!(map, json(r#"{"c": 1}"#));
1191    }
1192
1193    #[test]
1194    fn test_set_value_to_json_with_empty_dest_nested_key() {
1195        let mut map = json("{}");
1196
1197        JsonPath::value_set(
1198            Some(&JsonPath::new("key1.key2")),
1199            &mut map,
1200            &json(r#"{"c": 1}"#),
1201        );
1202
1203        assert_eq!(map, json(r#"{"key1": {"key2": {"c": 1}}}"#));
1204    }
1205
1206    #[test]
1207    fn test_set_value_to_json_with_empty_dest_nested_array_index_key() {
1208        let mut map = json("{}");
1209        let src = json(r#"{"c": 1}"#);
1210        JsonPath::value_set(Some(&JsonPath::new("key1.key2[3]")), &mut map, &src);
1211        assert_eq!(map, json(r#" {"key1": {"key2": []}} "#));
1212
1213        let mut map = json("{}");
1214        let src = json(r#"{"c": 1}"#);
1215        JsonPath::value_set(Some(&JsonPath::new("key1.key2[0]")), &mut map, &src);
1216        assert_eq!(map, json(r#" {"key1": {"key2": []}} "#));
1217    }
1218
1219    #[test]
1220    fn test_expand_payload_with_non_existing_array() {
1221        let mut map = json("{}");
1222
1223        JsonPath::value_set(
1224            Some(&JsonPath::new("key1.key2[].key3")),
1225            &mut map,
1226            &json(r#"{"c": 1}"#),
1227        );
1228
1229        assert_eq!(map, json(r#"{"key1": {"key2": [] }}"#));
1230    }
1231
1232    #[test]
1233    fn test_replace_scalar_key_with_object() {
1234        let mut map = json(r#"{"a": 10}"#);
1235
1236        JsonPath::value_set(
1237            Some(&JsonPath::new("a.b.c")),
1238            &mut map,
1239            &json(r#"{"x": 1}"#),
1240        );
1241
1242        assert_eq!(map, json(r#"{"a": {"b": {"c": {"x": 1}}}}"#));
1243    }
1244
1245    #[test]
1246    fn test_remove_key() {
1247        let mut payload = json(
1248            r#"
1249            {
1250                "a": 1,
1251                "b": {
1252                    "c": 123,
1253                    "e": {
1254                        "f": [1,2,3],
1255                        "g": 7,
1256                        "h": "text",
1257                        "i": [
1258                            {
1259                                "j": 1,
1260                                "k": 2
1261                            },
1262                            {
1263                                "j": 3,
1264                                "k": 4
1265                            }
1266                        ]
1267                    }
1268                }
1269            }
1270            "#,
1271        );
1272        let removed = JsonPath::new("b.c").value_remove(&mut payload).into_vec();
1273        assert_eq!(removed, vec![Value::Number(123.into())]);
1274        assert_ne!(payload, Default::default());
1275
1276        let removed = JsonPath::new("b.e.i[0].j")
1277            .value_remove(&mut payload)
1278            .into_vec();
1279        assert_eq!(removed, vec![Value::Number(1.into())]);
1280        assert_ne!(payload, Default::default());
1281
1282        let removed = JsonPath::new("b.e.i[].k")
1283            .value_remove(&mut payload)
1284            .into_vec();
1285        assert_eq!(
1286            removed,
1287            vec![Value::Number(2.into()), Value::Number(4.into())]
1288        );
1289        assert_ne!(payload, Default::default());
1290
1291        let removed = JsonPath::new("b.e.i[]")
1292            .value_remove(&mut payload)
1293            .into_vec();
1294        assert_eq!(
1295            removed,
1296            vec![Value::Array(vec![
1297                Value::Object(serde_json::Map::from_iter(vec![])),
1298                Value::Object(serde_json::Map::from_iter(vec![(
1299                    "j".to_string(),
1300                    Value::Number(3.into())
1301                ),])),
1302            ])]
1303        );
1304        assert_ne!(payload, Default::default());
1305
1306        let removed = JsonPath::new("b.e.i").value_remove(&mut payload).into_vec();
1307        assert_eq!(removed, vec![Value::Array(vec![])]);
1308        assert_ne!(payload, Default::default());
1309
1310        let removed = JsonPath::new("b.e.f").value_remove(&mut payload).into_vec();
1311        assert_eq!(
1312            removed,
1313            vec![Value::Array(vec![1.into(), 2.into(), 3.into()])]
1314        );
1315        assert_ne!(payload, Default::default());
1316
1317        let removed = JsonPath::new("k").value_remove(&mut payload);
1318        assert!(check_is_empty(&removed));
1319        assert_ne!(payload, Default::default());
1320
1321        let removed = JsonPath::new("b.e.l").value_remove(&mut payload);
1322        assert!(check_is_empty(&removed));
1323        assert_ne!(payload, Default::default());
1324
1325        let removed = JsonPath::new("a").value_remove(&mut payload).into_vec();
1326        assert_eq!(removed, vec![Value::Number(1.into())]);
1327        assert_ne!(payload, Default::default());
1328
1329        let removed = JsonPath::new("b.e").value_remove(&mut payload).into_vec();
1330        assert_eq!(
1331            removed,
1332            vec![Value::Object(serde_json::Map::from_iter(vec![
1333                // ("f".to_string(), Value::Array(vec![1.into(), 2.into(), 3.into()])), has been removed
1334                ("g".to_string(), Value::Number(7.into())),
1335                ("h".to_string(), Value::String("text".to_owned())),
1336            ]))]
1337        );
1338        assert_ne!(payload, Default::default());
1339
1340        let removed = JsonPath::new("b").value_remove(&mut payload).into_vec();
1341        assert_eq!(
1342            removed,
1343            vec![Value::Object(serde_json::Map::from_iter(vec![]))]
1344        ); // empty object left
1345        assert_eq!(payload, Default::default());
1346    }
1347
1348    #[test]
1349    fn test_filename() {
1350        assert_eq!(
1351            JsonPath::new("foo").filename(),
1352            "5gjb8qr8vv38vucr8ku1qc21-foo",
1353        );
1354        assert_eq!(
1355            JsonPath::new("a.\"b c\".d").filename(),
1356            "59if87e118rvurkl7j0q10mc-a_b_c_d",
1357        );
1358        assert_eq!(
1359            JsonPath::new("a.b[0][]").filename(),
1360            "vk82udqfa8drecufa7j5mo6v-a_b_0_",
1361        );
1362        assert_eq!(
1363            JsonPath::new("really.loooooooooooooooooooooooooooooooooooooooooooong.path").filename(),
1364            "sh47i3hjfgn44gch5jvm3bum-really_l",
1365        );
1366        assert_eq!(
1367            JsonPath::new("Müesli").filename(),
1368            "4huj4rn1fflrtriqo0tieqhh-m_esli",
1369        );
1370    }
1371}
qdrant_edge/segment/json_path/mod.rs

qdrant_edge/segment/json_path/
mod.rs
