1use std::io;
2use std::ops::Bound;
3
4use common::bounds::{map_bound, BoundsRange};
5use common::BitSet;
6
7use super::range_query_fastfield::FastFieldRangeWeight;
8use crate::index::SegmentReader;
9use crate::query::explanation::does_not_match;
10use crate::query::range_query::is_type_valid_for_fastfield_range_query;
11use crate::query::{BitSetDocSet, ConstScorer, EnableScoring, Explanation, Query, Scorer, Weight};
12use crate::schema::{Field, IndexRecordOption, Term, Type};
13use crate::termdict::{TermDictionary, TermStreamer};
14use crate::{DocId, Score};
15
16#[derive(Clone, Debug)]
71pub struct RangeQuery {
72 bounds: BoundsRange<Term>,
73}
74
75impl RangeQuery {
76 pub fn new(lower_bound: Bound<Term>, upper_bound: Bound<Term>) -> RangeQuery {
81 RangeQuery {
82 bounds: BoundsRange::new(lower_bound, upper_bound),
83 }
84 }
85
86 pub fn field(&self) -> Field {
88 self.get_term().field()
89 }
90
91 pub fn value_type(&self) -> Type {
93 self.get_term().typ()
94 }
95
96 pub(crate) fn get_term(&self) -> &Term {
97 self.bounds
98 .get_inner()
99 .expect("At least one bound must be set")
100 }
101}
102
103impl Query for RangeQuery {
104 fn weight(&self, enable_scoring: EnableScoring<'_>) -> crate::Result<Box<dyn Weight>> {
105 let schema = enable_scoring.schema();
106 let field_type = schema.get_field_entry(self.field()).field_type();
107
108 if field_type.is_fast() && is_type_valid_for_fastfield_range_query(self.value_type()) {
109 Ok(Box::new(FastFieldRangeWeight::new(self.bounds.clone())))
110 } else {
111 if field_type.is_json() {
112 return Err(crate::TantivyError::InvalidArgument(
113 "RangeQuery on JSON is only supported for fast fields currently".to_string(),
114 ));
115 }
116 Ok(Box::new(InvertedIndexRangeWeight::new(
117 self.field(),
118 &self.bounds.lower_bound,
119 &self.bounds.upper_bound,
120 None,
121 )))
122 }
123 }
124}
125
126#[derive(Clone, Debug)]
127pub struct InvertedIndexRangeQuery {
129 bounds: BoundsRange<Term>,
130 limit: Option<u64>,
131}
132impl InvertedIndexRangeQuery {
133 pub fn new(lower_bound: Bound<Term>, upper_bound: Bound<Term>) -> InvertedIndexRangeQuery {
135 InvertedIndexRangeQuery {
136 bounds: BoundsRange::new(lower_bound, upper_bound),
137 limit: None,
138 }
139 }
140 pub fn limit(&mut self, limit: u64) {
145 self.limit = Some(limit);
146 }
147}
148
149impl Query for InvertedIndexRangeQuery {
150 fn weight(&self, _enable_scoring: EnableScoring<'_>) -> crate::Result<Box<dyn Weight>> {
151 let field = self
152 .bounds
153 .get_inner()
154 .expect("At least one bound must be set")
155 .field();
156
157 Ok(Box::new(InvertedIndexRangeWeight::new(
158 field,
159 &self.bounds.lower_bound,
160 &self.bounds.upper_bound,
161 self.limit,
162 )))
163 }
164}
165
166pub struct InvertedIndexRangeWeight {
168 field: Field,
169 lower_bound: Bound<Vec<u8>>,
170 upper_bound: Bound<Vec<u8>>,
171 limit: Option<u64>,
172}
173
174impl InvertedIndexRangeWeight {
175 pub fn new(
179 field: Field,
180 lower_bound: &Bound<Term>,
181 upper_bound: &Bound<Term>,
182 limit: Option<u64>,
183 ) -> Self {
184 let verify_and_unwrap_term = |val: &Term| val.serialized_value_bytes().to_owned();
185 Self {
186 field,
187 lower_bound: map_bound(lower_bound, verify_and_unwrap_term),
188 upper_bound: map_bound(upper_bound, verify_and_unwrap_term),
189 limit,
190 }
191 }
192
193 fn term_range<'a>(&self, term_dict: &'a TermDictionary) -> io::Result<TermStreamer<'a>> {
194 use std::ops::Bound::*;
195 let mut term_stream_builder = term_dict.range();
196 term_stream_builder = match self.lower_bound {
197 Included(ref term_val) => term_stream_builder.ge(term_val),
198 Excluded(ref term_val) => term_stream_builder.gt(term_val),
199 Unbounded => term_stream_builder,
200 };
201 term_stream_builder = match self.upper_bound {
202 Included(ref term_val) => term_stream_builder.le(term_val),
203 Excluded(ref term_val) => term_stream_builder.lt(term_val),
204 Unbounded => term_stream_builder,
205 };
206 #[cfg(feature = "quickwit")]
207 if let Some(limit) = self.limit {
208 term_stream_builder = term_stream_builder.limit(limit);
209 }
210 term_stream_builder.into_stream()
211 }
212}
213
214impl Weight for InvertedIndexRangeWeight {
215 fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
216 let max_doc = reader.max_doc();
217 let mut doc_bitset = BitSet::with_max_value(max_doc);
218
219 let inverted_index = reader.inverted_index(self.field)?;
220 let term_dict = inverted_index.terms();
221 let mut term_range = self.term_range(term_dict)?;
222 let mut processed_count = 0;
223 while term_range.advance() {
224 if let Some(limit) = self.limit {
225 if limit <= processed_count {
226 break;
227 }
228 }
229 processed_count += 1;
230 let term_info = term_range.value();
231 let mut block_segment_postings = inverted_index
232 .read_block_postings_from_terminfo(term_info, IndexRecordOption::Basic)?;
233 loop {
234 let docs = block_segment_postings.docs();
235 if docs.is_empty() {
236 break;
237 }
238 for &doc in block_segment_postings.docs() {
239 doc_bitset.insert(doc);
240 }
241 block_segment_postings.advance();
242 }
243 }
244 let doc_bitset = BitSetDocSet::from(doc_bitset);
245 Ok(Box::new(ConstScorer::new(doc_bitset, boost)))
246 }
247
248 fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
249 let mut scorer = self.scorer(reader, 1.0)?;
250 if scorer.seek(doc) != doc {
251 return Err(does_not_match(doc));
252 }
253 Ok(Explanation::new("RangeQuery", 1.0))
254 }
255}
256
257#[cfg(test)]
258mod tests {
259
260 use std::net::IpAddr;
261 use std::ops::Bound;
262 use std::str::FromStr;
263
264 use rand::seq::SliceRandom;
265
266 use super::RangeQuery;
267 use crate::collector::{Count, TopDocs};
268 use crate::indexer::NoMergePolicy;
269 use crate::query::range_query::fast_field_range_doc_set::RangeDocSet;
270 use crate::query::range_query::range_query::InvertedIndexRangeQuery;
271 use crate::query::{AllScorer, ConstScorer, EmptyScorer, EnableScoring, Query, QueryParser};
272 use crate::schema::{
273 Field, IntoIpv6Addr, Schema, TantivyDocument, FAST, INDEXED, STORED, TEXT,
274 };
275 use crate::{Index, IndexWriter, Term};
276
277 #[test]
278 fn test_range_query_simple() -> crate::Result<()> {
279 let mut schema_builder = Schema::builder();
280 let year_field = schema_builder.add_u64_field("year", INDEXED);
281 let schema = schema_builder.build();
282
283 let index = Index::create_in_ram(schema);
284 {
285 let mut index_writer = index.writer_for_tests()?;
286 for year in 1950u64..2017u64 {
287 let num_docs_within_year = 10 + (year - 1950) * (year - 1950);
288 for _ in 0..num_docs_within_year {
289 index_writer.add_document(doc!(year_field => year))?;
290 }
291 }
292 index_writer.commit()?;
293 }
294 let reader = index.reader()?;
295 let searcher = reader.searcher();
296
297 let docs_in_the_sixties = InvertedIndexRangeQuery::new(
298 Bound::Included(Term::from_field_u64(year_field, 1960)),
299 Bound::Excluded(Term::from_field_u64(year_field, 1970)),
300 );
301
302 let count = searcher.search(&docs_in_the_sixties, &Count)?;
304 assert_eq!(count, 2285);
305 Ok(())
306 }
307
308 #[test]
309 fn test_range_query_with_limit() -> crate::Result<()> {
310 let mut schema_builder = Schema::builder();
311 let year_field = schema_builder.add_u64_field("year", INDEXED);
312 let schema = schema_builder.build();
313
314 let index = Index::create_in_ram(schema);
315 {
316 let mut index_writer = index.writer_for_tests()?;
317 for year in 1950u64..2017u64 {
318 if year == 1963 {
319 continue;
320 }
321 let num_docs_within_year = 10 + (year - 1950) * (year - 1950);
322 for _ in 0..num_docs_within_year {
323 index_writer.add_document(doc!(year_field => year))?;
324 }
325 }
326 index_writer.commit()?;
327 }
328 let reader = index.reader()?;
329 let searcher = reader.searcher();
330
331 let mut docs_in_the_sixties = InvertedIndexRangeQuery::new(
332 Bound::Included(Term::from_field_u64(year_field, 1960)),
333 Bound::Excluded(Term::from_field_u64(year_field, 1970)),
334 );
335 docs_in_the_sixties.limit(5);
336
337 let count = searcher.search(&docs_in_the_sixties, &Count)?;
339 assert_eq!(count, 836);
340 Ok(())
341 }
342
343 #[test]
344 fn test_range_query() -> crate::Result<()> {
345 let int_field: Field;
346 let schema = {
347 let mut schema_builder = Schema::builder();
348 int_field = schema_builder.add_i64_field("intfield", INDEXED);
349 schema_builder.build()
350 };
351
352 let index = Index::create_in_ram(schema);
353 {
354 let mut index_writer = index.writer_with_num_threads(1, 60_000_000)?;
355 index_writer.set_merge_policy(Box::new(NoMergePolicy));
356
357 for i in 1..100 {
358 let mut doc = TantivyDocument::new();
359 for j in 1..100 {
360 if i % j == 0 {
361 doc.add_i64(int_field, j as i64);
362 }
363 }
364 index_writer.add_document(doc)?;
365 if i == 10 {
366 index_writer.commit()?;
367 }
368 }
369
370 index_writer.commit()?;
371 }
372 let reader = index.reader().unwrap();
373 let searcher = reader.searcher();
374 assert_eq!(searcher.segment_readers().len(), 2);
375 let count_multiples =
376 |range_query: RangeQuery| searcher.search(&range_query, &Count).unwrap();
377
378 assert_eq!(
379 count_multiples(RangeQuery::new(
380 Bound::Included(Term::from_field_i64(int_field, 10)),
381 Bound::Excluded(Term::from_field_i64(int_field, 11)),
382 )),
383 9
384 );
385 assert_eq!(
386 count_multiples(RangeQuery::new(
387 Bound::Included(Term::from_field_i64(int_field, 10)),
388 Bound::Included(Term::from_field_i64(int_field, 11)),
389 )),
390 18
391 );
392 assert_eq!(
393 count_multiples(RangeQuery::new(
394 Bound::Excluded(Term::from_field_i64(int_field, 9)),
395 Bound::Included(Term::from_field_i64(int_field, 10)),
396 )),
397 9
398 );
399 assert_eq!(
400 count_multiples(RangeQuery::new(
401 Bound::Included(Term::from_field_i64(int_field, 9)),
402 Bound::Unbounded
403 )),
404 91
405 );
406 Ok(())
407 }
408
409 #[test]
410 fn test_range_float() -> crate::Result<()> {
411 let float_field: Field;
412 let schema = {
413 let mut schema_builder = Schema::builder();
414 float_field = schema_builder.add_f64_field("floatfield", INDEXED);
415 schema_builder.build()
416 };
417
418 let index = Index::create_in_ram(schema);
419 {
420 let mut index_writer = index.writer_with_num_threads(1, 60_000_000).unwrap();
421 let mut docs = Vec::new();
422 for i in 1..100 {
423 let mut doc = TantivyDocument::new();
424 for j in 1..100 {
425 if i % j == 0 {
426 doc.add_f64(float_field, j as f64);
427 }
428 }
429 docs.push(doc);
430 }
431
432 docs.shuffle(&mut rand::rng());
433 let mut docs_it = docs.into_iter();
434 for doc in (&mut docs_it).take(50) {
435 index_writer.add_document(doc)?;
436 }
437 index_writer.commit()?;
438 for doc in docs_it {
439 index_writer.add_document(doc)?;
440 }
441 index_writer.commit()?;
442 }
443 let reader = index.reader()?;
444 let searcher = reader.searcher();
445 assert_eq!(searcher.segment_readers().len(), 2);
446 let count_multiples =
447 |range_query: RangeQuery| searcher.search(&range_query, &Count).unwrap();
448
449 assert_eq!(
450 count_multiples(RangeQuery::new(
451 Bound::Included(Term::from_field_f64(float_field, 10.0)),
452 Bound::Excluded(Term::from_field_f64(float_field, 11.0)),
453 )),
454 9
455 );
456 assert_eq!(
457 count_multiples(RangeQuery::new(
458 Bound::Included(Term::from_field_f64(float_field, 10.0)),
459 Bound::Included(Term::from_field_f64(float_field, 11.0)),
460 )),
461 18
462 );
463 assert_eq!(
464 count_multiples(RangeQuery::new(
465 Bound::Excluded(Term::from_field_f64(float_field, 9.0)),
466 Bound::Included(Term::from_field_f64(float_field, 10.0)),
467 )),
468 9
469 );
470 assert_eq!(
471 count_multiples(RangeQuery::new(
472 Bound::Included(Term::from_field_f64(float_field, 9.0)),
473 Bound::Unbounded
474 )),
475 91
476 );
477 Ok(())
478 }
479
480 #[test]
481 fn test_bug_reproduce_range_query() -> crate::Result<()> {
482 let mut schema_builder = Schema::builder();
483 schema_builder.add_text_field("title", TEXT);
484 schema_builder.add_i64_field("year", INDEXED);
485 let schema = schema_builder.build();
486 let index = Index::create_in_ram(schema.clone());
487 let mut index_writer = index.writer_for_tests()?;
488 let title = schema.get_field("title").unwrap();
489 let year = schema.get_field("year").unwrap();
490 index_writer.add_document(doc!(
491 title => "hemoglobin blood",
492 year => 1990_i64
493 ))?;
494 index_writer.commit()?;
495 let reader = index.reader()?;
496 let searcher = reader.searcher();
497 let query_parser = QueryParser::for_index(&index, vec![title]);
498 let query = query_parser.parse_query("hemoglobin AND year:[1970 TO 1990]")?;
499 let top_docs = searcher.search(&query, &TopDocs::with_limit(10).order_by_score())?;
500 assert_eq!(top_docs.len(), 1);
501 Ok(())
502 }
503
504 #[test]
505 fn search_ip_range_test_posting_list() {
506 search_ip_range_test_opt(false);
507 }
508
509 #[test]
510 fn search_ip_range_test() {
511 search_ip_range_test_opt(true);
512 }
513
514 fn search_ip_range_test_opt(with_fast_field: bool) {
515 let mut schema_builder = Schema::builder();
516 let ip_field = if with_fast_field {
517 schema_builder.add_ip_addr_field("ip", INDEXED | STORED | FAST)
518 } else {
519 schema_builder.add_ip_addr_field("ip", INDEXED | STORED)
520 };
521 let text_field = schema_builder.add_text_field("text", TEXT | STORED);
522 let schema = schema_builder.build();
523 let index = Index::create_in_ram(schema);
524 let ip_addr_1 = IpAddr::from_str("127.0.0.10").unwrap().into_ipv6_addr();
525 let ip_addr_2 = IpAddr::from_str("127.0.0.20").unwrap().into_ipv6_addr();
526
527 {
528 let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
529 for _ in 0..1_000 {
530 index_writer
531 .add_document(doc!(
532 ip_field => ip_addr_1,
533 text_field => "BLUBBER"
534 ))
535 .unwrap();
536 }
537 for _ in 0..1_000 {
538 index_writer
539 .add_document(doc!(
540 ip_field => ip_addr_2,
541 text_field => "BLOBBER"
542 ))
543 .unwrap();
544 }
545 index_writer.commit().unwrap();
546 }
547 let reader = index.reader().unwrap();
548 let searcher = reader.searcher();
549 assert_eq!(searcher.segment_readers().len(), 1);
550
551 let get_num_hits = |query| {
552 let (_top_docs, count) = searcher
553 .search(&query, &(TopDocs::with_limit(10).order_by_score(), Count))
554 .unwrap();
555 count
556 };
557 let query_from_text = |text: &str| {
558 QueryParser::for_index(&index, vec![])
559 .parse_query(text)
560 .unwrap()
561 };
562
563 assert_eq!(
565 get_num_hits(query_from_text("ip:[127.0.0.1 TO 127.0.0.20]")),
566 2000
567 );
568
569 assert_eq!(
570 get_num_hits(query_from_text("ip:[127.0.0.10 TO 127.0.0.20]")),
571 2000
572 );
573
574 assert_eq!(
575 get_num_hits(query_from_text("ip:[127.0.0.11 TO 127.0.0.20]")),
576 1000
577 );
578
579 assert_eq!(
580 get_num_hits(query_from_text("ip:[127.0.0.11 TO 127.0.0.19]")),
581 0
582 );
583
584 assert_eq!(get_num_hits(query_from_text("ip:[127.0.0.11 TO *]")), 1000);
585 assert_eq!(get_num_hits(query_from_text("ip:[127.0.0.21 TO *]")), 0);
586 assert_eq!(get_num_hits(query_from_text("ip:[* TO 127.0.0.9]")), 0);
587 assert_eq!(get_num_hits(query_from_text("ip:[* TO 127.0.0.10]")), 1000);
588
589 assert_eq!(
591 get_num_hits(query_from_text("ip:{127.0.0.1 TO 127.0.0.20}")),
592 1000
593 );
594
595 assert_eq!(
596 get_num_hits(query_from_text("ip:{127.0.0.1 TO 127.0.0.21}")),
597 2000
598 );
599
600 assert_eq!(
601 get_num_hits(query_from_text("ip:{127.0.0.10 TO 127.0.0.20}")),
602 0
603 );
604
605 assert_eq!(
606 get_num_hits(query_from_text("ip:{127.0.0.11 TO 127.0.0.20}")),
607 0
608 );
609
610 assert_eq!(
611 get_num_hits(query_from_text("ip:{127.0.0.11 TO 127.0.0.19}")),
612 0
613 );
614
615 assert_eq!(get_num_hits(query_from_text("ip:{127.0.0.11 TO *}")), 1000);
616 assert_eq!(get_num_hits(query_from_text("ip:{127.0.0.10 TO *}")), 1000);
617 assert_eq!(get_num_hits(query_from_text("ip:{127.0.0.21 TO *}")), 0);
618 assert_eq!(get_num_hits(query_from_text("ip:{127.0.0.20 TO *}")), 0);
619 assert_eq!(get_num_hits(query_from_text("ip:{127.0.0.19 TO *}")), 1000);
620 assert_eq!(get_num_hits(query_from_text("ip:{* TO 127.0.0.9}")), 0);
621 assert_eq!(get_num_hits(query_from_text("ip:{* TO 127.0.0.10}")), 0);
622 assert_eq!(get_num_hits(query_from_text("ip:{* TO 127.0.0.11}")), 1000);
623
624 assert_eq!(
626 get_num_hits(query_from_text("ip:[127.0.0.1 TO 127.0.0.20}")),
627 1000
628 );
629
630 assert_eq!(
631 get_num_hits(query_from_text("ip:{127.0.0.1 TO 127.0.0.20]")),
632 2000
633 );
634
635 assert_eq!(
637 get_num_hits(query_from_text(
638 "text:BLUBBER AND ip:[127.0.0.10 TO 127.0.0.10]"
639 )),
640 1000
641 );
642
643 assert_eq!(
644 get_num_hits(query_from_text(
645 "text:BLOBBER AND ip:[127.0.0.10 TO 127.0.0.10]"
646 )),
647 0
648 );
649
650 assert_eq!(
651 get_num_hits(query_from_text(
652 "text:BLOBBER AND ip:[127.0.0.20 TO 127.0.0.20]"
653 )),
654 1000
655 );
656
657 assert_eq!(
658 get_num_hits(query_from_text(
659 "text:BLUBBER AND ip:[127.0.0.20 TO 127.0.0.20]"
660 )),
661 0
662 );
663 }
664
665 #[test]
666 fn test_range_query_simplified() {
667 let mut schema_builder = Schema::builder();
670 let u64_field = schema_builder.add_u64_field("u64_field", FAST);
671 let schema = schema_builder.build();
672 let index = Index::create_in_ram(schema.clone());
673 let mut index_writer = index.writer_for_tests().unwrap();
674 index_writer.add_document(doc!(u64_field=> 2u64)).unwrap();
675 index_writer.add_document(doc!(u64_field=> 4u64)).unwrap();
676 index_writer.commit().unwrap();
677 let reader = index.reader().unwrap();
678 let searcher = reader.searcher();
679 assert_eq!(searcher.segment_readers().len(), 1);
680 let make_term = |value: u64| Term::from_field_u64(u64_field, value);
681 let make_scorer = move |lower_bound: Bound<u64>, upper_bound: Bound<u64>| {
682 let lower_bound_term = lower_bound.map(make_term);
683 let upper_bound_term = upper_bound.map(make_term);
684 let range_query = RangeQuery::new(lower_bound_term, upper_bound_term);
685 let range_weight = range_query
686 .weight(EnableScoring::disabled_from_schema(&schema))
687 .unwrap();
688 let range_scorer = range_weight
689 .scorer(&searcher.segment_readers()[0], 1.0f32)
690 .unwrap();
691 range_scorer
692 };
693 let range_scorer = make_scorer(Bound::Included(1), Bound::Included(4));
694 assert!(range_scorer.is::<AllScorer>());
695 let range_scorer = make_scorer(Bound::Included(0), Bound::Included(2));
696 assert!(range_scorer.is::<ConstScorer<RangeDocSet<u64>>>());
697 let range_scorer = make_scorer(Bound::Included(3), Bound::Included(10));
698 assert!(range_scorer.is::<ConstScorer<RangeDocSet<u64>>>());
699 let range_scorer = make_scorer(Bound::Included(10), Bound::Included(12));
700 assert!(range_scorer.is::<ConstScorer<RangeDocSet<u64>>>());
701 let range_scorer = make_scorer(Bound::Included(0), Bound::Included(1));
702 assert!(range_scorer.is::<EmptyScorer>());
703 let range_scorer = make_scorer(Bound::Included(0), Bound::Excluded(2));
704 assert!(range_scorer.is::<EmptyScorer>());
705 }
706}