lb_tantivy/query/
exist_query.rs

1use core::fmt::Debug;
2
3use columnar::{ColumnIndex, DynamicColumn};
4
5use super::{ConstScorer, EmptyScorer};
6use crate::docset::{DocSet, TERMINATED};
7use crate::index::SegmentReader;
8use crate::query::explanation::does_not_match;
9use crate::query::{EnableScoring, Explanation, Query, Scorer, Weight};
10use crate::schema::Type;
11use crate::{DocId, Score, TantivyError};
12
13/// Query that matches all documents with a non-null value in the specified
14/// field.
15///
16/// When querying inside a JSON field, "exists" queries can be executed strictly
17/// on the field name or check all the subpaths. In that second case a document
18/// will be matched if a non-null value exists in any subpath. For example,
19/// assuming the following document where `myfield` is a JSON fast field:
20/// ```json
21/// {
22///   "myfield": {
23///     "mysubfield": "hello"
24///   }
25/// }
26/// ```
27/// With `json_subpaths` enabled queries on either `myfield` or
28/// `myfield.mysubfield` will match the document. If it is set to false, only
29/// `myfield.mysubfield` will match it.
30///
31/// All of the matched documents get the score 1.0.
32#[derive(Clone, Debug)]
33pub struct ExistsQuery {
34    field_name: String,
35    json_subpaths: bool,
36}
37
38impl ExistsQuery {
39    /// Creates a new `ExistQuery` from the given field.
40    ///
41    /// This query matches all documents with at least one non-null value in the specified field.
42    /// This constructor never fails, but executing the search with this query will return an
43    /// error if the specified field doesn't exists or is not a fast field.
44    #[deprecated]
45    pub fn new_exists_query(field: String) -> ExistsQuery {
46        ExistsQuery {
47            field_name: field,
48            json_subpaths: false,
49        }
50    }
51
52    /// Creates a new `ExistQuery` from the given field.
53    ///
54    /// This query matches all documents with at least one non-null value in the
55    /// specified field. If `json_subpaths` is set to true, documents with
56    /// non-null values in any JSON subpath will also be matched.
57    ///
58    /// This constructor never fails, but executing the search with this query will
59    /// return an error if the specified field doesn't exists or is not a fast
60    /// field.
61    pub fn new(field: String, json_subpaths: bool) -> Self {
62        Self {
63            field_name: field,
64            json_subpaths,
65        }
66    }
67}
68
69impl Query for ExistsQuery {
70    fn weight(&self, enable_scoring: EnableScoring) -> crate::Result<Box<dyn Weight>> {
71        let schema = enable_scoring.schema();
72        let Some((field, _path)) = schema.find_field(&self.field_name) else {
73            return Err(TantivyError::FieldNotFound(self.field_name.clone()));
74        };
75        let field_type = schema.get_field_entry(field).field_type();
76        if !field_type.is_fast() {
77            return Err(TantivyError::SchemaError(format!(
78                "Field {} is not a fast field.",
79                self.field_name
80            )));
81        }
82        Ok(Box::new(ExistsWeight {
83            field_name: self.field_name.clone(),
84            field_type: field_type.value_type(),
85            json_subpaths: self.json_subpaths,
86        }))
87    }
88}
89
90/// Weight associated with the `ExistsQuery` query.
91pub struct ExistsWeight {
92    field_name: String,
93    field_type: Type,
94    json_subpaths: bool,
95}
96
97impl Weight for ExistsWeight {
98    fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
99        let fast_field_reader = reader.fast_fields();
100        let mut column_handles = fast_field_reader.dynamic_column_handles(&self.field_name)?;
101        if self.field_type == Type::Json && self.json_subpaths {
102            let mut sub_columns =
103                fast_field_reader.dynamic_subpath_column_handles(&self.field_name)?;
104            column_handles.append(&mut sub_columns);
105        }
106        let dynamic_columns: crate::Result<Vec<DynamicColumn>> = column_handles
107            .into_iter()
108            .map(|handle| handle.open().map_err(|io_error| io_error.into()))
109            .collect();
110        let mut non_empty_columns = Vec::new();
111        for column in dynamic_columns? {
112            if !matches!(column.column_index(), ColumnIndex::Empty { .. }) {
113                non_empty_columns.push(column)
114            }
115        }
116        // TODO: we can optimizer more here since in most cases we will have only one index
117        if !non_empty_columns.is_empty() {
118            let docset = ExistsDocSet::new(non_empty_columns, reader.max_doc());
119            Ok(Box::new(ConstScorer::new(docset, boost)))
120        } else {
121            Ok(Box::new(EmptyScorer))
122        }
123    }
124
125    fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
126        let mut scorer = self.scorer(reader, 1.0)?;
127        if scorer.seek(doc) != doc {
128            return Err(does_not_match(doc));
129        }
130        Ok(Explanation::new("ExistsQuery", 1.0))
131    }
132}
133
134pub(crate) struct ExistsDocSet {
135    columns: Vec<DynamicColumn>,
136    doc: DocId,
137    max_doc: DocId,
138}
139
140impl ExistsDocSet {
141    pub(crate) fn new(columns: Vec<DynamicColumn>, max_doc: DocId) -> Self {
142        let mut set = Self {
143            columns,
144            doc: 0u32,
145            max_doc,
146        };
147        set.find_next();
148        set
149    }
150
151    fn find_next(&mut self) -> DocId {
152        while self.doc < self.max_doc {
153            if self
154                .columns
155                .iter()
156                .any(|col| col.column_index().has_value(self.doc))
157            {
158                return self.doc;
159            }
160            self.doc += 1;
161        }
162        self.doc = TERMINATED;
163        TERMINATED
164    }
165}
166
167impl DocSet for ExistsDocSet {
168    fn advance(&mut self) -> DocId {
169        self.seek(self.doc + 1)
170    }
171
172    fn size_hint(&self) -> u32 {
173        0
174    }
175
176    fn doc(&self) -> DocId {
177        self.doc
178    }
179
180    #[inline(always)]
181    fn seek(&mut self, target: DocId) -> DocId {
182        self.doc = target;
183        self.find_next()
184    }
185}
186
187#[cfg(test)]
188mod tests {
189    use std::net::Ipv6Addr;
190    use std::ops::Bound;
191
192    use common::DateTime;
193    use time::OffsetDateTime;
194
195    use crate::collector::Count;
196    use crate::query::exist_query::ExistsQuery;
197    use crate::query::{BooleanQuery, RangeQuery};
198    use crate::schema::{Facet, FacetOptions, Schema, FAST, INDEXED, STRING, TEXT};
199    use crate::{Index, Searcher, Term};
200
201    #[test]
202    fn test_exists_query_simple() -> crate::Result<()> {
203        let mut schema_builder = Schema::builder();
204        let all_field = schema_builder.add_u64_field("all", INDEXED | FAST);
205        let even_field = schema_builder.add_u64_field("even", INDEXED | FAST);
206        let odd_field = schema_builder.add_text_field("odd", STRING | FAST);
207        let multi_field = schema_builder.add_text_field("multi", FAST);
208        let _never_field = schema_builder.add_u64_field("never", INDEXED | FAST);
209        let schema = schema_builder.build();
210
211        let index = Index::create_in_ram(schema);
212        {
213            let mut index_writer = index.writer_for_tests()?;
214            for i in 0u64..100u64 {
215                if i % 2 == 0 {
216                    if i % 10 == 0 {
217                        index_writer.add_document(doc!(all_field => i, even_field => i, multi_field => i.to_string(), multi_field => (i + 1).to_string()))?;
218                    } else {
219                        index_writer.add_document(doc!(all_field => i, even_field => i))?;
220                    }
221                } else {
222                    index_writer.add_document(doc!(all_field => i, odd_field => i.to_string()))?;
223                }
224            }
225            index_writer.commit()?;
226        }
227        let reader = index.reader()?;
228        let searcher = reader.searcher();
229
230        assert_eq!(count_existing_fields(&searcher, "all", false)?, 100);
231        assert_eq!(count_existing_fields(&searcher, "odd", false)?, 50);
232        assert_eq!(count_existing_fields(&searcher, "even", false)?, 50);
233        assert_eq!(count_existing_fields(&searcher, "multi", false)?, 10);
234        assert_eq!(count_existing_fields(&searcher, "multi", true)?, 10);
235        assert_eq!(count_existing_fields(&searcher, "never", false)?, 0);
236
237        // exercise seek
238        let query = BooleanQuery::intersection(vec![
239            Box::new(RangeQuery::new(
240                Bound::Included(Term::from_field_u64(all_field, 50)),
241                Bound::Unbounded,
242            )),
243            Box::new(ExistsQuery::new("even".to_string(), false)),
244        ]);
245        assert_eq!(searcher.search(&query, &Count)?, 25);
246
247        let query = BooleanQuery::intersection(vec![
248            Box::new(RangeQuery::new(
249                Bound::Included(Term::from_field_u64(all_field, 0)),
250                Bound::Included(Term::from_field_u64(all_field, 50)),
251            )),
252            Box::new(ExistsQuery::new("odd".to_string(), false)),
253        ]);
254        assert_eq!(searcher.search(&query, &Count)?, 25);
255
256        Ok(())
257    }
258
259    #[test]
260    fn test_exists_query_json() -> crate::Result<()> {
261        let mut schema_builder = Schema::builder();
262        let json = schema_builder.add_json_field("json", TEXT | FAST);
263        let schema = schema_builder.build();
264
265        let index = Index::create_in_ram(schema);
266        {
267            let mut index_writer = index.writer_for_tests()?;
268            for i in 0u64..100u64 {
269                if i % 2 == 0 {
270                    index_writer.add_document(doc!(json => json!({"all": i, "even": true})))?;
271                } else {
272                    index_writer
273                        .add_document(doc!(json => json!({"all": i.to_string(), "odd": true})))?;
274                }
275            }
276            index_writer.commit()?;
277        }
278        let reader = index.reader()?;
279        let searcher = reader.searcher();
280
281        assert_eq!(count_existing_fields(&searcher, "json.all", false)?, 100);
282        assert_eq!(count_existing_fields(&searcher, "json.even", false)?, 50);
283        assert_eq!(count_existing_fields(&searcher, "json.even", true)?, 50);
284        assert_eq!(count_existing_fields(&searcher, "json.odd", false)?, 50);
285        assert_eq!(count_existing_fields(&searcher, "json", false)?, 0);
286        assert_eq!(count_existing_fields(&searcher, "json", true)?, 100);
287
288        // Handling of non-existing fields:
289        assert_eq!(count_existing_fields(&searcher, "json.absent", false)?, 0);
290        assert_eq!(count_existing_fields(&searcher, "json.absent", true)?, 0);
291        assert_does_not_exist(&searcher, "does_not_exists.absent", true);
292        assert_does_not_exist(&searcher, "does_not_exists.absent", false);
293
294        Ok(())
295    }
296
297    #[test]
298    fn test_exists_query_misc_supported_types() -> crate::Result<()> {
299        let mut schema_builder = Schema::builder();
300        let bool = schema_builder.add_bool_field("bool", FAST);
301        let bytes = schema_builder.add_bytes_field("bytes", FAST);
302        let date = schema_builder.add_date_field("date", FAST);
303        let f64 = schema_builder.add_f64_field("f64", FAST);
304        let ip_addr = schema_builder.add_ip_addr_field("ip_addr", FAST);
305        let facet = schema_builder.add_facet_field("facet", FacetOptions::default());
306        let schema = schema_builder.build();
307
308        let index = Index::create_in_ram(schema);
309        {
310            let mut index_writer = index.writer_for_tests()?;
311            let now = OffsetDateTime::now_utc().unix_timestamp();
312            for i in 0u8..100u8 {
313                if i % 2 == 0 {
314                    let date_val = DateTime::from_utc(OffsetDateTime::from_unix_timestamp(
315                        now + i as i64 * 100,
316                    )?);
317                    index_writer.add_document(
318                        doc!(bool => i % 3 == 0, bytes => vec![i, i + 1,  i + 2], date => date_val),
319                    )?;
320                } else {
321                    let ip_addr_v6 = Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, i.into());
322                    index_writer
323                        .add_document(doc!(f64 => i as f64 * 0.5, ip_addr => ip_addr_v6, facet => Facet::from("/facet/foo"), facet => Facet::from("/facet/bar")))?;
324                }
325            }
326            index_writer.commit()?;
327        }
328        let reader = index.reader()?;
329        let searcher = reader.searcher();
330
331        assert_eq!(count_existing_fields(&searcher, "bool", false)?, 50);
332        assert_eq!(count_existing_fields(&searcher, "bool", true)?, 50);
333        assert_eq!(count_existing_fields(&searcher, "bytes", false)?, 50);
334        assert_eq!(count_existing_fields(&searcher, "date", false)?, 50);
335        assert_eq!(count_existing_fields(&searcher, "f64", false)?, 50);
336        assert_eq!(count_existing_fields(&searcher, "ip_addr", false)?, 50);
337        assert_eq!(count_existing_fields(&searcher, "facet", false)?, 50);
338
339        Ok(())
340    }
341
342    #[test]
343    fn test_exists_query_unsupported_types() -> crate::Result<()> {
344        let mut schema_builder = Schema::builder();
345        let not_fast = schema_builder.add_text_field("not_fast", TEXT);
346        let schema = schema_builder.build();
347
348        let index = Index::create_in_ram(schema);
349        {
350            let mut index_writer = index.writer_for_tests()?;
351            index_writer.add_document(doc!(
352                not_fast => "slow",
353            ))?;
354            index_writer.commit()?;
355        }
356        let reader = index.reader()?;
357        let searcher = reader.searcher();
358
359        assert_eq!(
360            searcher
361                .search(&ExistsQuery::new("not_fast".to_string(), false), &Count)
362                .unwrap_err()
363                .to_string(),
364            "Schema error: 'Field not_fast is not a fast field.'"
365        );
366
367        assert_does_not_exist(&searcher, "does_not_exists", false);
368
369        Ok(())
370    }
371
372    fn count_existing_fields(
373        searcher: &Searcher,
374        field: &str,
375        json_subpaths: bool,
376    ) -> crate::Result<usize> {
377        let query = ExistsQuery::new(field.to_string(), json_subpaths);
378        searcher.search(&query, &Count)
379    }
380
381    fn assert_does_not_exist(searcher: &Searcher, field: &str, json_subpaths: bool) {
382        assert_eq!(
383            searcher
384                .search(&ExistsQuery::new(field.to_string(), json_subpaths), &Count)
385                .unwrap_err()
386                .to_string(),
387            format!("The field does not exist: '{field}'")
388        );
389    }
390}
lb_tantivy/query/exist_query.rs

lb_tantivy/query/
exist_query.rs
