hermes_core/dsl/ql/
mod.rs

1//! Query language parser using pest
2//!
3//! Supports:
4//! - Term queries: `rust` or `title:rust`
5//! - Phrase queries: `"hello world"` or `title:"hello world"`
6//! - Boolean operators: `AND`, `OR`, `NOT` (or `&&`, `||`, `-`)
7//! - Grouping: `(rust OR python) AND programming`
8//! - Default fields for unqualified terms
9
10use pest::Parser;
11use pest_derive::Parser;
12use std::sync::Arc;
13
14use super::query_field_router::{QueryFieldRouter, RoutingMode};
15use super::schema::{Field, Schema};
16use crate::query::{
17    BooleanQuery, DEFAULT_DENSE_RERANK_FACTOR, PhraseQuery, PrefixQuery, Query, TermQuery,
18};
19use crate::tokenizer::{BoxedTokenizer, TokenizerRegistry};
20
21#[derive(Parser)]
22#[grammar = "dsl/ql/ql.pest"]
23struct QueryParser;
24
25/// Parsed query that can be converted to a Query trait object
26#[derive(Debug, Clone)]
27pub enum ParsedQuery {
28    Term {
29        field: Option<String>,
30        term: String,
31    },
32    Phrase {
33        field: Option<String>,
34        phrase: String,
35    },
36    /// Prefix query — matches terms starting with a given prefix
37    Prefix {
38        field: Option<String>,
39        prefix: String,
40    },
41    /// Dense vector ANN query
42    Ann {
43        field: String,
44        vector: Vec<f32>,
45        nprobe: usize,
46        rerank: f32,
47    },
48    /// Sparse vector query
49    Sparse {
50        field: String,
51        vector: Vec<(u32, f32)>,
52    },
53    And(Vec<ParsedQuery>),
54    Or(Vec<ParsedQuery>),
55    Not(Box<ParsedQuery>),
56}
57
58/// Query language parser with schema awareness
59pub struct QueryLanguageParser {
60    schema: Arc<Schema>,
61    default_fields: Vec<Field>,
62    tokenizers: Arc<TokenizerRegistry>,
63    /// Optional query field router for routing queries based on regex patterns
64    field_router: Option<QueryFieldRouter>,
65}
66
67impl QueryLanguageParser {
68    pub fn new(
69        schema: Arc<Schema>,
70        default_fields: Vec<Field>,
71        tokenizers: Arc<TokenizerRegistry>,
72    ) -> Self {
73        Self {
74            schema,
75            default_fields,
76            tokenizers,
77            field_router: None,
78        }
79    }
80
81    /// Create a parser with a query field router
82    pub fn with_router(
83        schema: Arc<Schema>,
84        default_fields: Vec<Field>,
85        tokenizers: Arc<TokenizerRegistry>,
86        router: QueryFieldRouter,
87    ) -> Self {
88        Self {
89            schema,
90            default_fields,
91            tokenizers,
92            field_router: Some(router),
93        }
94    }
95
96    /// Set the query field router
97    pub fn set_router(&mut self, router: QueryFieldRouter) {
98        self.field_router = Some(router);
99    }
100
101    /// Get the query field router
102    pub fn router(&self) -> Option<&QueryFieldRouter> {
103        self.field_router.as_ref()
104    }
105
106    /// Parse a query string into a Query
107    ///
108    /// Supports query language syntax (field:term, AND, OR, NOT, grouping)
109    /// and plain text (tokenized and searched across default fields).
110    ///
111    /// If a query field router is configured, the query is first checked against
112    /// routing rules. If a rule matches:
113    /// - In exclusive mode: only the target field is queried with the substituted value
114    /// - In additional mode: both the target field and default fields are queried
115    pub fn parse(&self, query_str: &str) -> Result<Box<dyn Query>, String> {
116        let query_str = query_str.trim();
117        if query_str.is_empty() {
118            return Err("Empty query".to_string());
119        }
120
121        // Check if query matches any routing rules
122        if let Some(router) = &self.field_router
123            && let Some(routed) = router.route(query_str)
124        {
125            return self.build_routed_query(
126                &routed.query,
127                &routed.target_field,
128                routed.mode,
129                query_str,
130            );
131        }
132
133        // No routing match - parse normally
134        self.parse_normal(query_str)
135    }
136
137    /// Build a query from a routed match
138    fn build_routed_query(
139        &self,
140        routed_query: &str,
141        target_field: &str,
142        mode: RoutingMode,
143        original_query: &str,
144    ) -> Result<Box<dyn Query>, String> {
145        // Validate target field exists
146        let _field_id = self
147            .schema
148            .get_field(target_field)
149            .ok_or_else(|| format!("Unknown target field: {}", target_field))?;
150
151        // Build query for the target field with the substituted value
152        let target_query = self.build_term_query(Some(target_field), routed_query)?;
153
154        match mode {
155            RoutingMode::Exclusive => {
156                // Only query the target field
157                Ok(target_query)
158            }
159            RoutingMode::Additional => {
160                // Query both target field and default fields
161                let mut bool_query = BooleanQuery::new();
162                bool_query = bool_query.should(target_query);
163
164                // Also parse the original query against default fields
165                if let Ok(default_query) = self.parse_normal(original_query) {
166                    bool_query = bool_query.should(default_query);
167                }
168
169                Ok(Box::new(bool_query))
170            }
171        }
172    }
173
174    /// Parse query without routing (normal parsing path)
175    fn parse_normal(&self, query_str: &str) -> Result<Box<dyn Query>, String> {
176        // Try parsing as query language first
177        match self.parse_query_string(query_str) {
178            Ok(parsed) => self.build_query(&parsed),
179            Err(_) => {
180                // If grammar parsing fails, treat as plain text
181                // Split by whitespace and create OR of terms
182                self.parse_plain_text(query_str)
183            }
184        }
185    }
186
187    /// Parse plain text as implicit OR of tokenized terms
188    fn parse_plain_text(&self, text: &str) -> Result<Box<dyn Query>, String> {
189        if self.default_fields.is_empty() {
190            return Err("No default fields configured".to_string());
191        }
192
193        let tokenizer = self.get_tokenizer(self.default_fields[0]);
194        let tokens: Vec<String> = tokenizer
195            .tokenize(text)
196            .into_iter()
197            .map(|t| t.text.to_lowercase())
198            .collect();
199
200        if tokens.is_empty() {
201            return Err("No tokens in query".to_string());
202        }
203
204        let mut bool_query = BooleanQuery::new();
205        for token in &tokens {
206            for &field_id in &self.default_fields {
207                bool_query = bool_query.should(TermQuery::text(field_id, token));
208            }
209        }
210        Ok(Box::new(bool_query))
211    }
212
213    fn parse_query_string(&self, query_str: &str) -> Result<ParsedQuery, String> {
214        let pairs = QueryParser::parse(Rule::query, query_str)
215            .map_err(|e| format!("Parse error: {}", e))?;
216
217        let query_pair = pairs.into_iter().next().ok_or("No query found")?;
218
219        // query = { SOI ~ or_expr ~ EOI }
220        self.parse_or_expr(query_pair.into_inner().next().unwrap())
221    }
222
223    fn parse_or_expr(&self, pair: pest::iterators::Pair<Rule>) -> Result<ParsedQuery, String> {
224        let mut inner = pair.into_inner();
225        let first = self.parse_and_expr(inner.next().unwrap())?;
226
227        let rest: Vec<ParsedQuery> = inner
228            .filter(|p| p.as_rule() == Rule::and_expr)
229            .map(|p| self.parse_and_expr(p))
230            .collect::<Result<Vec<_>, _>>()?;
231
232        if rest.is_empty() {
233            Ok(first)
234        } else {
235            let mut all = vec![first];
236            all.extend(rest);
237            Ok(ParsedQuery::Or(all))
238        }
239    }
240
241    fn parse_and_expr(&self, pair: pest::iterators::Pair<Rule>) -> Result<ParsedQuery, String> {
242        let mut inner = pair.into_inner();
243        let first = self.parse_primary(inner.next().unwrap())?;
244
245        let rest: Vec<ParsedQuery> = inner
246            .filter(|p| p.as_rule() == Rule::primary)
247            .map(|p| self.parse_primary(p))
248            .collect::<Result<Vec<_>, _>>()?;
249
250        if rest.is_empty() {
251            Ok(first)
252        } else {
253            let mut all = vec![first];
254            all.extend(rest);
255            Ok(ParsedQuery::And(all))
256        }
257    }
258
259    fn parse_primary(&self, pair: pest::iterators::Pair<Rule>) -> Result<ParsedQuery, String> {
260        let mut negated = false;
261        let mut inner_query = None;
262
263        for inner in pair.into_inner() {
264            match inner.as_rule() {
265                Rule::not_op => negated = true,
266                Rule::group => {
267                    let or_expr = inner.into_inner().next().unwrap();
268                    inner_query = Some(self.parse_or_expr(or_expr)?);
269                }
270                Rule::ann_query => {
271                    inner_query = Some(self.parse_ann_query(inner)?);
272                }
273                Rule::sparse_query => {
274                    inner_query = Some(self.parse_sparse_query(inner)?);
275                }
276                Rule::phrase_query => {
277                    inner_query = Some(self.parse_phrase_query(inner)?);
278                }
279                Rule::prefix_query => {
280                    inner_query = Some(self.parse_prefix_query(inner)?);
281                }
282                Rule::term_query => {
283                    inner_query = Some(self.parse_term_query(inner)?);
284                }
285                _ => {}
286            }
287        }
288
289        let query = inner_query.ok_or("No query in primary")?;
290
291        if negated {
292            Ok(ParsedQuery::Not(Box::new(query)))
293        } else {
294            Ok(query)
295        }
296    }
297
298    fn parse_term_query(&self, pair: pest::iterators::Pair<Rule>) -> Result<ParsedQuery, String> {
299        let mut field = None;
300        let mut term = String::new();
301
302        for inner in pair.into_inner() {
303            match inner.as_rule() {
304                Rule::field_spec => {
305                    field = Some(inner.into_inner().next().unwrap().as_str().to_string());
306                }
307                Rule::term => {
308                    term = inner.as_str().to_string();
309                }
310                _ => {}
311            }
312        }
313
314        Ok(ParsedQuery::Term { field, term })
315    }
316
317    fn parse_prefix_query(&self, pair: pest::iterators::Pair<Rule>) -> Result<ParsedQuery, String> {
318        let mut field = None;
319        let mut prefix = String::new();
320
321        for inner in pair.into_inner() {
322            match inner.as_rule() {
323                Rule::field_spec => {
324                    field = Some(inner.into_inner().next().unwrap().as_str().to_string());
325                }
326                Rule::prefix_value => {
327                    prefix = inner.as_str().to_string();
328                }
329                _ => {}
330            }
331        }
332
333        Ok(ParsedQuery::Prefix { field, prefix })
334    }
335
336    fn parse_phrase_query(&self, pair: pest::iterators::Pair<Rule>) -> Result<ParsedQuery, String> {
337        let mut field = None;
338        let mut phrase = String::new();
339
340        for inner in pair.into_inner() {
341            match inner.as_rule() {
342                Rule::field_spec => {
343                    field = Some(inner.into_inner().next().unwrap().as_str().to_string());
344                }
345                Rule::quoted_string => {
346                    let s = inner.as_str();
347                    phrase = s[1..s.len() - 1].to_string();
348                }
349                _ => {}
350            }
351        }
352
353        Ok(ParsedQuery::Phrase { field, phrase })
354    }
355
356    /// Parse an ANN query: field:ann([1.0, 2.0, 3.0], nprobe=32, rerank=2)
357    fn parse_ann_query(&self, pair: pest::iterators::Pair<Rule>) -> Result<ParsedQuery, String> {
358        let mut field = String::new();
359        let mut vector = Vec::new();
360        let mut nprobe = 32usize;
361        let mut rerank = DEFAULT_DENSE_RERANK_FACTOR;
362
363        for inner in pair.into_inner() {
364            match inner.as_rule() {
365                Rule::field_spec => {
366                    field = inner.into_inner().next().unwrap().as_str().to_string();
367                }
368                Rule::vector_array => {
369                    for num in inner.into_inner() {
370                        if num.as_rule() == Rule::number
371                            && let Ok(v) = num.as_str().parse::<f32>()
372                        {
373                            vector.push(v);
374                        }
375                    }
376                }
377                Rule::ann_params => {
378                    for param in inner.into_inner() {
379                        if param.as_rule() == Rule::ann_param {
380                            // ann_param = { ("nprobe" | "rerank") ~ "=" ~ number }
381                            let param_str = param.as_str();
382                            if let Some(eq_pos) = param_str.find('=') {
383                                let name = &param_str[..eq_pos];
384                                let value = &param_str[eq_pos + 1..];
385                                match name {
386                                    "nprobe" => nprobe = value.parse().unwrap_or(0),
387                                    "rerank" => rerank = value.parse().unwrap_or(0.0),
388                                    _ => {}
389                                }
390                            }
391                        }
392                    }
393                }
394                _ => {}
395            }
396        }
397
398        Ok(ParsedQuery::Ann {
399            field,
400            vector,
401            nprobe,
402            rerank,
403        })
404    }
405
406    /// Parse a sparse vector query: field:sparse({1: 0.5, 5: 0.3})
407    fn parse_sparse_query(&self, pair: pest::iterators::Pair<Rule>) -> Result<ParsedQuery, String> {
408        let mut field = String::new();
409        let mut vector = Vec::new();
410
411        for inner in pair.into_inner() {
412            match inner.as_rule() {
413                Rule::field_spec => {
414                    field = inner.into_inner().next().unwrap().as_str().to_string();
415                }
416                Rule::sparse_map => {
417                    for entry in inner.into_inner() {
418                        if entry.as_rule() == Rule::sparse_entry {
419                            let mut entry_inner = entry.into_inner();
420                            if let (Some(idx), Some(weight)) =
421                                (entry_inner.next(), entry_inner.next())
422                                && let (Ok(i), Ok(w)) =
423                                    (idx.as_str().parse::<u32>(), weight.as_str().parse::<f32>())
424                            {
425                                vector.push((i, w));
426                            }
427                        }
428                    }
429                }
430                _ => {}
431            }
432        }
433
434        Ok(ParsedQuery::Sparse { field, vector })
435    }
436
437    fn build_query(&self, parsed: &ParsedQuery) -> Result<Box<dyn Query>, String> {
438        use crate::query::{DenseVectorQuery, SparseVectorQuery};
439
440        match parsed {
441            ParsedQuery::Term { field, term } => self.build_term_query(field.as_deref(), term),
442            ParsedQuery::Phrase { field, phrase } => {
443                self.build_phrase_query(field.as_deref(), phrase)
444            }
445            ParsedQuery::Prefix { field, prefix } => {
446                self.build_prefix_query(field.as_deref(), prefix)
447            }
448            ParsedQuery::Ann {
449                field,
450                vector,
451                nprobe,
452                rerank,
453            } => {
454                let field_id = self
455                    .schema
456                    .get_field(field)
457                    .ok_or_else(|| format!("Unknown field: {}", field))?;
458                let query = DenseVectorQuery::new(field_id, vector.clone())
459                    .with_nprobe(*nprobe)
460                    .with_rerank_factor(*rerank);
461                Ok(Box::new(query))
462            }
463            ParsedQuery::Sparse { field, vector } => {
464                let field_id = self
465                    .schema
466                    .get_field(field)
467                    .ok_or_else(|| format!("Unknown field: {}", field))?;
468                let query = SparseVectorQuery::new(field_id, vector.clone());
469                Ok(Box::new(query))
470            }
471            ParsedQuery::And(queries) => {
472                let mut bool_query = BooleanQuery::new();
473                for q in queries {
474                    bool_query = bool_query.must(self.build_query(q)?);
475                }
476                Ok(Box::new(bool_query))
477            }
478            ParsedQuery::Or(queries) => {
479                let mut bool_query = BooleanQuery::new();
480                for q in queries {
481                    bool_query = bool_query.should(self.build_query(q)?);
482                }
483                Ok(Box::new(bool_query))
484            }
485            ParsedQuery::Not(inner) => {
486                // NOT query needs a context - wrap in a match-all with must_not
487                let mut bool_query = BooleanQuery::new();
488                bool_query = bool_query.must_not(self.build_query(inner)?);
489                Ok(Box::new(bool_query))
490            }
491        }
492    }
493
494    fn build_term_query(&self, field: Option<&str>, term: &str) -> Result<Box<dyn Query>, String> {
495        if let Some(field_name) = field {
496            // Field-qualified term: tokenize using field's tokenizer
497            let field_id = self
498                .schema
499                .get_field(field_name)
500                .ok_or_else(|| format!("Unknown field: {}", field_name))?;
501            // Validate field type — TermQuery only works on text fields
502            if let Some(entry) = self.schema.get_field_entry(field_id) {
503                use crate::dsl::FieldType;
504                if entry.field_type != FieldType::Text {
505                    return Err(format!(
506                        "Term query requires a text field, but '{}' is {:?}. Use range query for numeric fields.",
507                        field_name, entry.field_type
508                    ));
509                }
510            }
511            let tokenizer = self.get_tokenizer(field_id);
512            let tokens: Vec<String> = tokenizer
513                .tokenize(term)
514                .into_iter()
515                .map(|t| t.text.to_lowercase())
516                .collect();
517
518            if tokens.is_empty() {
519                return Err("No tokens in term".to_string());
520            }
521
522            if tokens.len() == 1 {
523                Ok(Box::new(TermQuery::text(field_id, &tokens[0])))
524            } else {
525                // Multiple tokens from single term - AND them together
526                let mut bool_query = BooleanQuery::new();
527                for token in &tokens {
528                    bool_query = bool_query.must(TermQuery::text(field_id, token));
529                }
530                Ok(Box::new(bool_query))
531            }
532        } else if !self.default_fields.is_empty() {
533            // Unqualified term: tokenize and search across default fields
534            let tokenizer = self.get_tokenizer(self.default_fields[0]);
535            let tokens: Vec<String> = tokenizer
536                .tokenize(term)
537                .into_iter()
538                .map(|t| t.text.to_lowercase())
539                .collect();
540
541            if tokens.is_empty() {
542                return Err("No tokens in term".to_string());
543            }
544
545            // Build SHOULD query across all default fields for each token
546            let mut bool_query = BooleanQuery::new();
547            for token in &tokens {
548                for &field_id in &self.default_fields {
549                    bool_query = bool_query.should(TermQuery::text(field_id, token));
550                }
551            }
552            Ok(Box::new(bool_query))
553        } else {
554            Err("No field specified and no default fields configured".to_string())
555        }
556    }
557
558    fn build_prefix_query(
559        &self,
560        field: Option<&str>,
561        prefix: &str,
562    ) -> Result<Box<dyn Query>, String> {
563        if let Some(field_name) = field {
564            let field_id = self
565                .schema
566                .get_field(field_name)
567                .ok_or_else(|| format!("Unknown field: {}", field_name))?;
568            Ok(Box::new(PrefixQuery::text(field_id, prefix)))
569        } else if !self.default_fields.is_empty() {
570            // Unqualified prefix: OR across default fields
571            let mut bool_query = BooleanQuery::new();
572            for &field_id in &self.default_fields {
573                bool_query = bool_query.should(PrefixQuery::text(field_id, prefix));
574            }
575            Ok(Box::new(bool_query))
576        } else {
577            Err("No field specified and no default fields configured".to_string())
578        }
579    }
580
581    fn build_phrase_query(
582        &self,
583        field: Option<&str>,
584        phrase: &str,
585    ) -> Result<Box<dyn Query>, String> {
586        // For phrase queries, tokenize and create AND query of terms
587        let field_id = if let Some(field_name) = field {
588            self.schema
589                .get_field(field_name)
590                .ok_or_else(|| format!("Unknown field: {}", field_name))?
591        } else if !self.default_fields.is_empty() {
592            self.default_fields[0]
593        } else {
594            return Err("No field specified and no default fields configured".to_string());
595        };
596
597        let tokenizer = self.get_tokenizer(field_id);
598        let tokens: Vec<(u32, String)> = tokenizer
599            .tokenize(phrase)
600            .into_iter()
601            .map(|t| (t.position, t.text.to_lowercase()))
602            .collect();
603
604        if tokens.is_empty() {
605            return Err("No tokens in phrase".to_string());
606        }
607
608        if tokens.len() == 1 {
609            return Ok(Box::new(TermQuery::text(field_id, &tokens[0].1)));
610        }
611
612        // Positional phrase query; on a field without positions it degrades to
613        // an AND of the terms inside PhraseQuery itself. Token positions come
614        // from the tokenizer so gaps left by dropped stop words survive.
615        let phrase_terms = |tokens: &[(u32, String)]| -> Vec<(u32, Vec<u8>)> {
616            tokens
617                .iter()
618                .map(|(offset, t)| (*offset, t.clone().into_bytes()))
619                .collect()
620        };
621
622        // If no field specified and multiple default fields, wrap in OR
623        if field.is_none() && self.default_fields.len() > 1 {
624            let mut outer = BooleanQuery::new();
625            for &f in &self.default_fields {
626                let tokenizer = self.get_tokenizer(f);
627                let tokens: Vec<(u32, String)> = tokenizer
628                    .tokenize(phrase)
629                    .into_iter()
630                    .map(|t| (t.position, t.text.to_lowercase()))
631                    .collect();
632                if tokens.is_empty() {
633                    continue;
634                }
635                outer = outer.should(PhraseQuery::with_offsets(f, phrase_terms(&tokens)));
636            }
637            return Ok(Box::new(outer));
638        }
639
640        Ok(Box::new(PhraseQuery::with_offsets(
641            field_id,
642            phrase_terms(&tokens),
643        )))
644    }
645
646    fn get_tokenizer(&self, field: Field) -> BoxedTokenizer {
647        // Get tokenizer name from schema field entry, fallback to "simple"
648        let tokenizer_name = self
649            .schema
650            .get_field_entry(field)
651            .and_then(|entry| entry.tokenizer.as_deref())
652            .unwrap_or("simple");
653
654        self.tokenizers
655            .get(tokenizer_name)
656            .unwrap_or_else(|| Box::new(crate::tokenizer::SimpleTokenizer))
657    }
658}
659
660#[cfg(test)]
661mod tests {
662    use super::*;
663    use crate::dsl::SchemaBuilder;
664    use crate::tokenizer::TokenizerRegistry;
665
666    fn setup() -> (Arc<Schema>, Vec<Field>, Arc<TokenizerRegistry>) {
667        let mut builder = SchemaBuilder::default();
668        let title = builder.add_text_field("title", true, true);
669        let body = builder.add_text_field("body", true, true);
670        let schema = Arc::new(builder.build());
671        let tokenizers = Arc::new(TokenizerRegistry::default());
672        (schema, vec![title, body], tokenizers)
673    }
674
675    #[test]
676    fn test_simple_term() {
677        let (schema, default_fields, tokenizers) = setup();
678        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
679
680        // Should parse without error - creates BooleanQuery across default fields
681        let _query = parser.parse("rust").unwrap();
682    }
683
684    #[test]
685    fn test_field_term() {
686        let (schema, default_fields, tokenizers) = setup();
687        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
688
689        // Should parse field:term syntax
690        let _query = parser.parse("title:rust").unwrap();
691    }
692
693    #[test]
694    fn test_boolean_and() {
695        let (schema, default_fields, tokenizers) = setup();
696        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
697
698        // Should parse AND boolean query
699        let _query = parser.parse("rust AND programming").unwrap();
700    }
701
702    #[test]
703    fn test_match_query() {
704        let (schema, default_fields, tokenizers) = setup();
705        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
706
707        // Should tokenize and create boolean query
708        let _query = parser.parse("hello world").unwrap();
709    }
710
711    #[test]
712    fn test_phrase_query() {
713        let (schema, default_fields, tokenizers) = setup();
714        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
715
716        // Should parse quoted phrase
717        let _query = parser.parse("\"hello world\"").unwrap();
718    }
719
720    #[test]
721    fn test_boolean_or() {
722        let (schema, default_fields, tokenizers) = setup();
723        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
724
725        // Should parse OR boolean query
726        let _query = parser.parse("rust OR python").unwrap();
727    }
728
729    #[test]
730    fn test_complex_query() {
731        let (schema, default_fields, tokenizers) = setup();
732        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
733
734        // Should parse complex boolean with grouping
735        let _query = parser.parse("(rust OR python) AND programming").unwrap();
736    }
737
738    #[test]
739    fn test_router_exclusive_mode() {
740        use crate::dsl::query_field_router::{QueryFieldRouter, QueryRouterRule, RoutingMode};
741
742        let mut builder = SchemaBuilder::default();
743        let _title = builder.add_text_field("title", true, true);
744        let _uri = builder.add_text_field("uri", true, true);
745        let schema = Arc::new(builder.build());
746        let tokenizers = Arc::new(TokenizerRegistry::default());
747
748        let router = QueryFieldRouter::from_rules(&[QueryRouterRule {
749            pattern: r"^doi:(10\.\d{4,}/[^\s]+)$".to_string(),
750            substitution: "doi://{1}".to_string(),
751            target_field: "uri".to_string(),
752            mode: RoutingMode::Exclusive,
753        }])
754        .unwrap();
755
756        let parser = QueryLanguageParser::with_router(schema, vec![], tokenizers, router);
757
758        // Should route DOI query to uri field
759        let _query = parser.parse("doi:10.1234/test.123").unwrap();
760    }
761
762    #[test]
763    fn test_router_additional_mode() {
764        use crate::dsl::query_field_router::{QueryFieldRouter, QueryRouterRule, RoutingMode};
765
766        let mut builder = SchemaBuilder::default();
767        let title = builder.add_text_field("title", true, true);
768        let _uri = builder.add_text_field("uri", true, true);
769        let schema = Arc::new(builder.build());
770        let tokenizers = Arc::new(TokenizerRegistry::default());
771
772        let router = QueryFieldRouter::from_rules(&[QueryRouterRule {
773            pattern: r"#(\d+)".to_string(),
774            substitution: "{1}".to_string(),
775            target_field: "uri".to_string(),
776            mode: RoutingMode::Additional,
777        }])
778        .unwrap();
779
780        let parser = QueryLanguageParser::with_router(schema, vec![title], tokenizers, router);
781
782        // Should route to both uri field and default fields
783        let _query = parser.parse("#42").unwrap();
784    }
785
786    #[test]
787    fn test_router_no_match_falls_through() {
788        use crate::dsl::query_field_router::{QueryFieldRouter, QueryRouterRule, RoutingMode};
789
790        let mut builder = SchemaBuilder::default();
791        let title = builder.add_text_field("title", true, true);
792        let _uri = builder.add_text_field("uri", true, true);
793        let schema = Arc::new(builder.build());
794        let tokenizers = Arc::new(TokenizerRegistry::default());
795
796        let router = QueryFieldRouter::from_rules(&[QueryRouterRule {
797            pattern: r"^doi:".to_string(),
798            substitution: "{0}".to_string(),
799            target_field: "uri".to_string(),
800            mode: RoutingMode::Exclusive,
801        }])
802        .unwrap();
803
804        let parser = QueryLanguageParser::with_router(schema, vec![title], tokenizers, router);
805
806        // Should NOT match and fall through to normal parsing
807        let _query = parser.parse("rust programming").unwrap();
808    }
809
810    #[test]
811    fn test_router_invalid_target_field() {
812        use crate::dsl::query_field_router::{QueryFieldRouter, QueryRouterRule, RoutingMode};
813
814        let mut builder = SchemaBuilder::default();
815        let _title = builder.add_text_field("title", true, true);
816        let schema = Arc::new(builder.build());
817        let tokenizers = Arc::new(TokenizerRegistry::default());
818
819        let router = QueryFieldRouter::from_rules(&[QueryRouterRule {
820            pattern: r"test".to_string(),
821            substitution: "{0}".to_string(),
822            target_field: "nonexistent".to_string(),
823            mode: RoutingMode::Exclusive,
824        }])
825        .unwrap();
826
827        let parser = QueryLanguageParser::with_router(schema, vec![], tokenizers, router);
828
829        // Should fail because target field doesn't exist
830        let result = parser.parse("test");
831        assert!(result.is_err());
832        let err = result.err().unwrap();
833        assert!(err.contains("Unknown target field"));
834    }
835
836    #[test]
837    fn test_parse_ann_query() {
838        let mut builder = SchemaBuilder::default();
839        let embedding = builder.add_dense_vector_field("embedding", 128, true, true);
840        let schema = Arc::new(builder.build());
841        let tokenizers = Arc::new(TokenizerRegistry::default());
842
843        let parser = QueryLanguageParser::new(schema, vec![embedding], tokenizers);
844
845        // Parse ANN query
846        let result = parser.parse_query_string("embedding:ann([1.0, 2.0, 3.0], nprobe=32)");
847        assert!(result.is_ok(), "Failed to parse ANN query: {:?}", result);
848
849        if let Ok(ParsedQuery::Ann {
850            field,
851            vector,
852            nprobe,
853            rerank,
854        }) = result
855        {
856            assert_eq!(field, "embedding");
857            assert_eq!(vector, vec![1.0, 2.0, 3.0]);
858            assert_eq!(nprobe, 32);
859            assert_eq!(rerank, 2.0); // default
860        } else {
861            panic!("Expected Ann query, got: {:?}", result);
862        }
863    }
864
865    #[test]
866    fn test_parse_sparse_query() {
867        let mut builder = SchemaBuilder::default();
868        let sparse = builder.add_text_field("sparse", true, true);
869        let schema = Arc::new(builder.build());
870        let tokenizers = Arc::new(TokenizerRegistry::default());
871
872        let parser = QueryLanguageParser::new(schema, vec![sparse], tokenizers);
873
874        // Parse sparse query
875        let result = parser.parse_query_string("sparse:sparse({1: 0.5, 5: 0.3})");
876        assert!(result.is_ok(), "Failed to parse sparse query: {:?}", result);
877
878        if let Ok(ParsedQuery::Sparse { field, vector }) = result {
879            assert_eq!(field, "sparse");
880            assert_eq!(vector, vec![(1, 0.5), (5, 0.3)]);
881        } else {
882            panic!("Expected Sparse query, got: {:?}", result);
883        }
884    }
885
886    #[test]
887    fn test_parse_prefix_simple() {
888        let (schema, default_fields, tokenizers) = setup();
889        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
890
891        // Simple prefix: title:abc*
892        let result = parser.parse_query_string("title:abc*");
893        assert!(result.is_ok(), "Failed to parse prefix query: {:?}", result);
894        if let Ok(ParsedQuery::Prefix { field, prefix }) = result {
895            assert_eq!(field, Some("title".to_string()));
896            assert_eq!(prefix, "abc");
897        } else {
898            panic!("Expected Prefix query, got: {:?}", result);
899        }
900    }
901
902    #[test]
903    fn test_parse_prefix_url() {
904        let mut builder = SchemaBuilder::default();
905        let _site = builder.add_text_field("site", true, true);
906        let schema = Arc::new(builder.build());
907        let tokenizers = Arc::new(TokenizerRegistry::default());
908        let parser = QueryLanguageParser::new(schema, vec![], tokenizers);
909
910        // URL prefix: site:https://reddit.com/r/Transhumanism*
911        let result = parser.parse_query_string("site:https://reddit.com/r/Transhumanism*");
912        assert!(
913            result.is_ok(),
914            "Failed to parse URL prefix query: {:?}",
915            result
916        );
917        if let Ok(ParsedQuery::Prefix { field, prefix }) = result {
918            assert_eq!(field, Some("site".to_string()));
919            assert_eq!(prefix, "https://reddit.com/r/Transhumanism");
920        } else {
921            panic!("Expected Prefix query, got: {:?}", result);
922        }
923    }
924
925    #[test]
926    fn test_parse_prefix_unqualified() {
927        let (schema, default_fields, tokenizers) = setup();
928        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
929
930        // Unqualified prefix: transhuman*
931        let result = parser.parse_query_string("transhuman*");
932        assert!(
933            result.is_ok(),
934            "Failed to parse unqualified prefix: {:?}",
935            result
936        );
937        if let Ok(ParsedQuery::Prefix { field, prefix }) = result {
938            assert_eq!(field, None);
939            assert_eq!(prefix, "transhuman");
940        } else {
941            panic!("Expected Prefix query, got: {:?}", result);
942        }
943    }
944
945    #[test]
946    fn test_prefix_query_builds() {
947        let (schema, default_fields, tokenizers) = setup();
948        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
949
950        // Should build without error
951        let _query = parser.parse("title:abc*").unwrap();
952    }
953
954    #[test]
955    fn test_prefix_in_boolean() {
956        let (schema, default_fields, tokenizers) = setup();
957        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
958
959        // Prefix in boolean: rust AND title:abc*
960        let _query = parser.parse("rust AND title:abc*").unwrap();
961    }
962
963    #[test]
964    fn test_prefix_mixed_with_terms() {
965        let mut builder = SchemaBuilder::default();
966        let title = builder.add_text_field("title", true, true);
967        let _site = builder.add_text_field("site", true, true);
968        let schema = Arc::new(builder.build());
969        let tokenizers = Arc::new(TokenizerRegistry::default());
970        let parser = QueryLanguageParser::new(schema, vec![title], tokenizers);
971
972        // Mixed: prefix + free-text terms (implicit OR)
973        let result =
974            parser.parse_query_string("site:https://reddit.com/r/Transhumanism* longevity drugs");
975        assert!(
976            result.is_ok(),
977            "Failed to parse mixed prefix+terms: {:?}",
978            result
979        );
980        // Should be Or([Prefix, Term, Term])
981        if let Ok(ParsedQuery::Or(parts)) = &result {
982            assert_eq!(parts.len(), 3, "Expected 3 parts, got: {:?}", parts);
983            assert!(
984                matches!(&parts[0], ParsedQuery::And(v) if v.len() == 1 && matches!(&v[0], ParsedQuery::Prefix { .. }))
985                    || matches!(&parts[0], ParsedQuery::Prefix { .. }),
986                "First part should be prefix: {:?}",
987                parts[0]
988            );
989        } else {
990            panic!("Expected Or query, got: {:?}", result);
991        }
992
993        // Should also build into a Query without error
994        let _query = parser
995            .parse("site:https://reddit.com/r/Transhumanism* longevity drugs")
996            .unwrap();
997    }
998
999    #[test]
1000    fn test_implicit_or_plain_terms() {
1001        let (schema, default_fields, tokenizers) = setup();
1002        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
1003
1004        // Space-separated terms: implicit OR
1005        let result = parser.parse_query_string("hello world");
1006        assert!(result.is_ok(), "Failed to parse implicit OR: {:?}", result);
1007        if let Ok(ParsedQuery::Or(parts)) = &result {
1008            assert_eq!(parts.len(), 2);
1009        } else {
1010            panic!("Expected Or query, got: {:?}", result);
1011        }
1012    }
1013}
hermes_core/dsl/ql/mod.rs

hermes_core/dsl/ql/
mod.rs
