hermes_core/dsl/ql/
mod.rs

1//! Query language parser using pest
2//!
3//! Supports:
4//! - Term queries: `rust` or `title:rust`
5//! - Phrase queries: `"hello world"` or `title:"hello world"`
6//! - Boolean operators: `AND`, `OR`, `NOT` (or `&&`, `||`, `-`)
7//! - Grouping: `(rust OR python) AND programming`
8//! - Default fields for unqualified terms
9
10use pest::Parser;
11use pest_derive::Parser;
12use std::sync::Arc;
13
14use super::query_field_router::{QueryFieldRouter, RoutingMode};
15use super::schema::{Field, Schema};
16use crate::query::{
17    BooleanQuery, DEFAULT_DENSE_RERANK_FACTOR, PhraseQuery, PrefixQuery, Query, TermQuery,
18};
19use crate::tokenizer::{BoxedTokenizer, TokenizerRegistry};
20
21#[derive(Parser)]
22#[grammar = "dsl/ql/ql.pest"]
23struct QueryParser;
24
25/// Parsed query that can be converted to a Query trait object
26#[derive(Debug, Clone)]
27pub enum ParsedQuery {
28    Term {
29        field: Option<String>,
30        term: String,
31    },
32    Phrase {
33        field: Option<String>,
34        phrase: String,
35    },
36    /// Prefix query — matches terms starting with a given prefix
37    Prefix {
38        field: Option<String>,
39        prefix: String,
40    },
41    /// Dense vector ANN query
42    Ann {
43        field: String,
44        vector: Vec<f32>,
45        nprobe: usize,
46        rerank: f32,
47    },
48    /// Sparse vector query
49    Sparse {
50        field: String,
51        vector: Vec<(u32, f32)>,
52    },
53    And(Vec<ParsedQuery>),
54    Or(Vec<ParsedQuery>),
55    Not(Box<ParsedQuery>),
56}
57
58/// Query language parser with schema awareness
59pub struct QueryLanguageParser {
60    schema: Arc<Schema>,
61    default_fields: Vec<Field>,
62    tokenizers: Arc<TokenizerRegistry>,
63    /// Optional query field router for routing queries based on regex patterns
64    field_router: Option<QueryFieldRouter>,
65}
66
67impl QueryLanguageParser {
68    pub fn new(
69        schema: Arc<Schema>,
70        default_fields: Vec<Field>,
71        tokenizers: Arc<TokenizerRegistry>,
72    ) -> Self {
73        Self {
74            schema,
75            default_fields,
76            tokenizers,
77            field_router: None,
78        }
79    }
80
81    /// Create a parser with a query field router
82    pub fn with_router(
83        schema: Arc<Schema>,
84        default_fields: Vec<Field>,
85        tokenizers: Arc<TokenizerRegistry>,
86        router: QueryFieldRouter,
87    ) -> Self {
88        Self {
89            schema,
90            default_fields,
91            tokenizers,
92            field_router: Some(router),
93        }
94    }
95
96    /// Set the query field router
97    pub fn set_router(&mut self, router: QueryFieldRouter) {
98        self.field_router = Some(router);
99    }
100
101    /// Get the query field router
102    pub fn router(&self) -> Option<&QueryFieldRouter> {
103        self.field_router.as_ref()
104    }
105
106    /// Parse a query string into a Query
107    ///
108    /// Supports query language syntax (field:term, AND, OR, NOT, grouping)
109    /// and plain text (tokenized and searched across default fields).
110    ///
111    /// If a query field router is configured, the query is first checked against
112    /// routing rules. If a rule matches:
113    /// - In exclusive mode: only the target field is queried with the substituted value
114    /// - In additional mode: both the target field and default fields are queried
115    pub fn parse(&self, query_str: &str) -> Result<Box<dyn Query>, String> {
116        let query_str = query_str.trim();
117        if query_str.is_empty() {
118            return Err("Empty query".to_string());
119        }
120
121        // Check if query matches any routing rules
122        if let Some(router) = &self.field_router
123            && let Some(routed) = router.route(query_str)
124        {
125            return self.build_routed_query(
126                &routed.query,
127                &routed.target_field,
128                routed.mode,
129                query_str,
130            );
131        }
132
133        // No routing match - parse normally
134        self.parse_normal(query_str)
135    }
136
137    /// Build a query from a routed match
138    fn build_routed_query(
139        &self,
140        routed_query: &str,
141        target_field: &str,
142        mode: RoutingMode,
143        original_query: &str,
144    ) -> Result<Box<dyn Query>, String> {
145        // Validate target field exists
146        let _field_id = self
147            .schema
148            .get_field(target_field)
149            .ok_or_else(|| format!("Unknown target field: {}", target_field))?;
150
151        // Build query for the target field with the substituted value
152        let target_query = self.build_term_query(Some(target_field), routed_query)?;
153
154        match mode {
155            RoutingMode::Exclusive => {
156                // Only query the target field
157                Ok(target_query)
158            }
159            RoutingMode::Additional => {
160                // Query both target field and default fields
161                let mut bool_query = BooleanQuery::new();
162                bool_query = bool_query.should(target_query);
163
164                // Also parse the original query against default fields
165                if let Ok(default_query) = self.parse_normal(original_query) {
166                    bool_query = bool_query.should(default_query);
167                }
168
169                Ok(Box::new(bool_query))
170            }
171        }
172    }
173
174    /// Parse query without routing (normal parsing path)
175    fn parse_normal(&self, query_str: &str) -> Result<Box<dyn Query>, String> {
176        // Try parsing as query language first
177        match self.parse_query_string(query_str) {
178            Ok(parsed) => self.build_query(&parsed),
179            Err(_) => {
180                // If grammar parsing fails, treat as plain text
181                // Split by whitespace and create OR of terms
182                self.parse_plain_text(query_str)
183            }
184        }
185    }
186
187    /// Parse plain text as implicit OR of tokenized terms
188    fn parse_plain_text(&self, text: &str) -> Result<Box<dyn Query>, String> {
189        if self.default_fields.is_empty() {
190            return Err("No default fields configured".to_string());
191        }
192
193        let tokenizer = self.get_tokenizer(self.default_fields[0]);
194        let tokens: Vec<String> = tokenizer
195            .tokenize(text)
196            .into_iter()
197            .map(|t| t.text.to_lowercase())
198            .collect();
199
200        if tokens.is_empty() {
201            return Err("No tokens in query".to_string());
202        }
203
204        let mut bool_query = BooleanQuery::new();
205        for token in &tokens {
206            for &field_id in &self.default_fields {
207                bool_query = bool_query.should(TermQuery::text(field_id, token));
208            }
209        }
210        Ok(Box::new(bool_query))
211    }
212
213    fn parse_query_string(&self, query_str: &str) -> Result<ParsedQuery, String> {
214        let pairs = QueryParser::parse(Rule::query, query_str)
215            .map_err(|e| format!("Parse error: {}", e))?;
216
217        let query_pair = pairs.into_iter().next().ok_or("No query found")?;
218
219        // query = { SOI ~ or_expr ~ EOI }
220        self.parse_or_expr(query_pair.into_inner().next().unwrap())
221    }
222
223    fn parse_or_expr(&self, pair: pest::iterators::Pair<Rule>) -> Result<ParsedQuery, String> {
224        let mut inner = pair.into_inner();
225        let first = self.parse_and_expr(inner.next().unwrap())?;
226
227        let rest: Vec<ParsedQuery> = inner
228            .filter(|p| p.as_rule() == Rule::and_expr)
229            .map(|p| self.parse_and_expr(p))
230            .collect::<Result<Vec<_>, _>>()?;
231
232        if rest.is_empty() {
233            Ok(first)
234        } else {
235            let mut all = vec![first];
236            all.extend(rest);
237            Ok(ParsedQuery::Or(all))
238        }
239    }
240
241    fn parse_and_expr(&self, pair: pest::iterators::Pair<Rule>) -> Result<ParsedQuery, String> {
242        let mut inner = pair.into_inner();
243        let first = self.parse_primary(inner.next().unwrap())?;
244
245        let rest: Vec<ParsedQuery> = inner
246            .filter(|p| p.as_rule() == Rule::primary)
247            .map(|p| self.parse_primary(p))
248            .collect::<Result<Vec<_>, _>>()?;
249
250        if rest.is_empty() {
251            Ok(first)
252        } else {
253            let mut all = vec![first];
254            all.extend(rest);
255            Ok(ParsedQuery::And(all))
256        }
257    }
258
259    fn parse_primary(&self, pair: pest::iterators::Pair<Rule>) -> Result<ParsedQuery, String> {
260        let mut negated = false;
261        let mut inner_query = None;
262
263        for inner in pair.into_inner() {
264            match inner.as_rule() {
265                Rule::not_op => negated = true,
266                Rule::group => {
267                    let or_expr = inner.into_inner().next().unwrap();
268                    inner_query = Some(self.parse_or_expr(or_expr)?);
269                }
270                Rule::ann_query => {
271                    inner_query = Some(self.parse_ann_query(inner)?);
272                }
273                Rule::sparse_query => {
274                    inner_query = Some(self.parse_sparse_query(inner)?);
275                }
276                Rule::phrase_query => {
277                    inner_query = Some(self.parse_phrase_query(inner)?);
278                }
279                Rule::prefix_query => {
280                    inner_query = Some(self.parse_prefix_query(inner)?);
281                }
282                Rule::term_query => {
283                    inner_query = Some(self.parse_term_query(inner)?);
284                }
285                _ => {}
286            }
287        }
288
289        let query = inner_query.ok_or("No query in primary")?;
290
291        if negated {
292            Ok(ParsedQuery::Not(Box::new(query)))
293        } else {
294            Ok(query)
295        }
296    }
297
298    fn parse_term_query(&self, pair: pest::iterators::Pair<Rule>) -> Result<ParsedQuery, String> {
299        let mut field = None;
300        let mut term = String::new();
301
302        for inner in pair.into_inner() {
303            match inner.as_rule() {
304                Rule::field_spec => {
305                    field = Some(inner.into_inner().next().unwrap().as_str().to_string());
306                }
307                Rule::term => {
308                    term = inner.as_str().to_string();
309                }
310                _ => {}
311            }
312        }
313
314        Ok(ParsedQuery::Term { field, term })
315    }
316
317    fn parse_prefix_query(&self, pair: pest::iterators::Pair<Rule>) -> Result<ParsedQuery, String> {
318        let mut field = None;
319        let mut prefix = String::new();
320
321        for inner in pair.into_inner() {
322            match inner.as_rule() {
323                Rule::field_spec => {
324                    field = Some(inner.into_inner().next().unwrap().as_str().to_string());
325                }
326                Rule::prefix_value => {
327                    prefix = inner.as_str().to_string();
328                }
329                _ => {}
330            }
331        }
332
333        Ok(ParsedQuery::Prefix { field, prefix })
334    }
335
336    fn parse_phrase_query(&self, pair: pest::iterators::Pair<Rule>) -> Result<ParsedQuery, String> {
337        let mut field = None;
338        let mut phrase = String::new();
339
340        for inner in pair.into_inner() {
341            match inner.as_rule() {
342                Rule::field_spec => {
343                    field = Some(inner.into_inner().next().unwrap().as_str().to_string());
344                }
345                Rule::quoted_string => {
346                    let s = inner.as_str();
347                    phrase = s[1..s.len() - 1].to_string();
348                }
349                _ => {}
350            }
351        }
352
353        Ok(ParsedQuery::Phrase { field, phrase })
354    }
355
356    /// Parse an ANN query: field:ann([1.0, 2.0, 3.0], nprobe=32, rerank=2)
357    fn parse_ann_query(&self, pair: pest::iterators::Pair<Rule>) -> Result<ParsedQuery, String> {
358        let mut field = String::new();
359        let mut vector = Vec::new();
360        let mut nprobe = 32usize;
361        let mut rerank = DEFAULT_DENSE_RERANK_FACTOR;
362
363        for inner in pair.into_inner() {
364            match inner.as_rule() {
365                Rule::field_spec => {
366                    field = inner.into_inner().next().unwrap().as_str().to_string();
367                }
368                Rule::vector_array => {
369                    for num in inner.into_inner() {
370                        if num.as_rule() == Rule::number
371                            && let Ok(v) = num.as_str().parse::<f32>()
372                        {
373                            vector.push(v);
374                        }
375                    }
376                }
377                Rule::ann_params => {
378                    for param in inner.into_inner() {
379                        if param.as_rule() == Rule::ann_param {
380                            // ann_param = { ("nprobe" | "rerank") ~ "=" ~ number }
381                            let param_str = param.as_str();
382                            if let Some(eq_pos) = param_str.find('=') {
383                                let name = &param_str[..eq_pos];
384                                let value = &param_str[eq_pos + 1..];
385                                match name {
386                                    "nprobe" => nprobe = value.parse().unwrap_or(0),
387                                    "rerank" => rerank = value.parse().unwrap_or(0.0),
388                                    _ => {}
389                                }
390                            }
391                        }
392                    }
393                }
394                _ => {}
395            }
396        }
397
398        Ok(ParsedQuery::Ann {
399            field,
400            vector,
401            nprobe,
402            rerank,
403        })
404    }
405
406    /// Parse a sparse vector query: field:sparse({1: 0.5, 5: 0.3})
407    fn parse_sparse_query(&self, pair: pest::iterators::Pair<Rule>) -> Result<ParsedQuery, String> {
408        let mut field = String::new();
409        let mut vector = Vec::new();
410
411        for inner in pair.into_inner() {
412            match inner.as_rule() {
413                Rule::field_spec => {
414                    field = inner.into_inner().next().unwrap().as_str().to_string();
415                }
416                Rule::sparse_map => {
417                    for entry in inner.into_inner() {
418                        if entry.as_rule() == Rule::sparse_entry {
419                            let mut entry_inner = entry.into_inner();
420                            if let (Some(idx), Some(weight)) =
421                                (entry_inner.next(), entry_inner.next())
422                                && let (Ok(i), Ok(w)) =
423                                    (idx.as_str().parse::<u32>(), weight.as_str().parse::<f32>())
424                            {
425                                vector.push((i, w));
426                            }
427                        }
428                    }
429                }
430                _ => {}
431            }
432        }
433
434        Ok(ParsedQuery::Sparse { field, vector })
435    }
436
437    fn build_query(&self, parsed: &ParsedQuery) -> Result<Box<dyn Query>, String> {
438        use crate::query::{DenseVectorQuery, SparseVectorQuery};
439
440        match parsed {
441            ParsedQuery::Term { field, term } => self.build_term_query(field.as_deref(), term),
442            ParsedQuery::Phrase { field, phrase } => {
443                self.build_phrase_query(field.as_deref(), phrase)
444            }
445            ParsedQuery::Prefix { field, prefix } => {
446                self.build_prefix_query(field.as_deref(), prefix)
447            }
448            ParsedQuery::Ann {
449                field,
450                vector,
451                nprobe,
452                rerank,
453            } => {
454                let field_id = self
455                    .schema
456                    .get_field(field)
457                    .ok_or_else(|| format!("Unknown field: {}", field))?;
458                let query = DenseVectorQuery::new(field_id, vector.clone())
459                    .with_nprobe(*nprobe)
460                    .with_rerank_factor(*rerank);
461                Ok(Box::new(query))
462            }
463            ParsedQuery::Sparse { field, vector } => {
464                let field_id = self
465                    .schema
466                    .get_field(field)
467                    .ok_or_else(|| format!("Unknown field: {}", field))?;
468                let query = SparseVectorQuery::new(field_id, vector.clone());
469                Ok(Box::new(query))
470            }
471            ParsedQuery::And(queries) => {
472                let mut bool_query = BooleanQuery::new();
473                for q in queries {
474                    bool_query = bool_query.must(self.build_query(q)?);
475                }
476                Ok(Box::new(bool_query))
477            }
478            ParsedQuery::Or(queries) => {
479                let mut bool_query = BooleanQuery::new();
480                for q in queries {
481                    bool_query = bool_query.should(self.build_query(q)?);
482                }
483                Ok(Box::new(bool_query))
484            }
485            ParsedQuery::Not(inner) => {
486                // NOT query needs a context - wrap in a match-all with must_not
487                let mut bool_query = BooleanQuery::new();
488                bool_query = bool_query.must_not(self.build_query(inner)?);
489                Ok(Box::new(bool_query))
490            }
491        }
492    }
493
494    fn build_term_query(&self, field: Option<&str>, term: &str) -> Result<Box<dyn Query>, String> {
495        if let Some(field_name) = field {
496            // Field-qualified term: tokenize using field's tokenizer
497            let field_id = self
498                .schema
499                .get_field(field_name)
500                .ok_or_else(|| format!("Unknown field: {}", field_name))?;
501            // Validate field type — TermQuery only works on text fields
502            if let Some(entry) = self.schema.get_field_entry(field_id) {
503                use crate::dsl::FieldType;
504                if entry.field_type != FieldType::Text {
505                    return Err(format!(
506                        "Term query requires a text field, but '{}' is {:?}. Use range query for numeric fields.",
507                        field_name, entry.field_type
508                    ));
509                }
510            }
511            let tokenizer = self.get_tokenizer(field_id);
512            let tokens: Vec<String> = tokenizer
513                .tokenize(term)
514                .into_iter()
515                .map(|t| t.text.to_lowercase())
516                .collect();
517
518            if tokens.is_empty() {
519                return Err("No tokens in term".to_string());
520            }
521
522            if tokens.len() == 1 {
523                Ok(Box::new(TermQuery::text(field_id, &tokens[0])))
524            } else {
525                // Multiple tokens from single term - AND them together
526                let mut bool_query = BooleanQuery::new();
527                for token in &tokens {
528                    bool_query = bool_query.must(TermQuery::text(field_id, token));
529                }
530                Ok(Box::new(bool_query))
531            }
532        } else if !self.default_fields.is_empty() {
533            // Unqualified term: tokenize and search across default fields
534            let tokenizer = self.get_tokenizer(self.default_fields[0]);
535            let tokens: Vec<String> = tokenizer
536                .tokenize(term)
537                .into_iter()
538                .map(|t| t.text.to_lowercase())
539                .collect();
540
541            if tokens.is_empty() {
542                return Err("No tokens in term".to_string());
543            }
544
545            // Build SHOULD query across all default fields for each token
546            let mut bool_query = BooleanQuery::new();
547            for token in &tokens {
548                for &field_id in &self.default_fields {
549                    bool_query = bool_query.should(TermQuery::text(field_id, token));
550                }
551            }
552            Ok(Box::new(bool_query))
553        } else {
554            Err("No field specified and no default fields configured".to_string())
555        }
556    }
557
558    fn build_prefix_query(
559        &self,
560        field: Option<&str>,
561        prefix: &str,
562    ) -> Result<Box<dyn Query>, String> {
563        if let Some(field_name) = field {
564            let field_id = self
565                .schema
566                .get_field(field_name)
567                .ok_or_else(|| format!("Unknown field: {}", field_name))?;
568            Ok(Box::new(PrefixQuery::text(field_id, prefix)))
569        } else if !self.default_fields.is_empty() {
570            // Unqualified prefix: OR across default fields
571            let mut bool_query = BooleanQuery::new();
572            for &field_id in &self.default_fields {
573                bool_query = bool_query.should(PrefixQuery::text(field_id, prefix));
574            }
575            Ok(Box::new(bool_query))
576        } else {
577            Err("No field specified and no default fields configured".to_string())
578        }
579    }
580
581    fn build_phrase_query(
582        &self,
583        field: Option<&str>,
584        phrase: &str,
585    ) -> Result<Box<dyn Query>, String> {
586        // For phrase queries, tokenize and create AND query of terms
587        let field_id = if let Some(field_name) = field {
588            self.schema
589                .get_field(field_name)
590                .ok_or_else(|| format!("Unknown field: {}", field_name))?
591        } else if !self.default_fields.is_empty() {
592            self.default_fields[0]
593        } else {
594            return Err("No field specified and no default fields configured".to_string());
595        };
596
597        let tokenizer = self.get_tokenizer(field_id);
598        let tokens: Vec<String> = tokenizer
599            .tokenize(phrase)
600            .into_iter()
601            .map(|t| t.text.to_lowercase())
602            .collect();
603
604        if tokens.is_empty() {
605            return Err("No tokens in phrase".to_string());
606        }
607
608        if tokens.len() == 1 {
609            return Ok(Box::new(TermQuery::text(field_id, &tokens[0])));
610        }
611
612        // Positional phrase query; on a field without positions it degrades to
613        // an AND of the terms inside PhraseQuery itself.
614        let phrase_terms = |tokens: &[String]| -> Vec<Vec<u8>> {
615            tokens.iter().map(|t| t.clone().into_bytes()).collect()
616        };
617
618        // If no field specified and multiple default fields, wrap in OR
619        if field.is_none() && self.default_fields.len() > 1 {
620            let mut outer = BooleanQuery::new();
621            for &f in &self.default_fields {
622                let tokenizer = self.get_tokenizer(f);
623                let tokens: Vec<String> = tokenizer
624                    .tokenize(phrase)
625                    .into_iter()
626                    .map(|t| t.text.to_lowercase())
627                    .collect();
628                if tokens.is_empty() {
629                    continue;
630                }
631                outer = outer.should(PhraseQuery::new(f, phrase_terms(&tokens)));
632            }
633            return Ok(Box::new(outer));
634        }
635
636        Ok(Box::new(PhraseQuery::new(field_id, phrase_terms(&tokens))))
637    }
638
639    fn get_tokenizer(&self, field: Field) -> BoxedTokenizer {
640        // Get tokenizer name from schema field entry, fallback to "simple"
641        let tokenizer_name = self
642            .schema
643            .get_field_entry(field)
644            .and_then(|entry| entry.tokenizer.as_deref())
645            .unwrap_or("simple");
646
647        self.tokenizers
648            .get(tokenizer_name)
649            .unwrap_or_else(|| Box::new(crate::tokenizer::SimpleTokenizer))
650    }
651}
652
653#[cfg(test)]
654mod tests {
655    use super::*;
656    use crate::dsl::SchemaBuilder;
657    use crate::tokenizer::TokenizerRegistry;
658
659    fn setup() -> (Arc<Schema>, Vec<Field>, Arc<TokenizerRegistry>) {
660        let mut builder = SchemaBuilder::default();
661        let title = builder.add_text_field("title", true, true);
662        let body = builder.add_text_field("body", true, true);
663        let schema = Arc::new(builder.build());
664        let tokenizers = Arc::new(TokenizerRegistry::default());
665        (schema, vec![title, body], tokenizers)
666    }
667
668    #[test]
669    fn test_simple_term() {
670        let (schema, default_fields, tokenizers) = setup();
671        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
672
673        // Should parse without error - creates BooleanQuery across default fields
674        let _query = parser.parse("rust").unwrap();
675    }
676
677    #[test]
678    fn test_field_term() {
679        let (schema, default_fields, tokenizers) = setup();
680        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
681
682        // Should parse field:term syntax
683        let _query = parser.parse("title:rust").unwrap();
684    }
685
686    #[test]
687    fn test_boolean_and() {
688        let (schema, default_fields, tokenizers) = setup();
689        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
690
691        // Should parse AND boolean query
692        let _query = parser.parse("rust AND programming").unwrap();
693    }
694
695    #[test]
696    fn test_match_query() {
697        let (schema, default_fields, tokenizers) = setup();
698        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
699
700        // Should tokenize and create boolean query
701        let _query = parser.parse("hello world").unwrap();
702    }
703
704    #[test]
705    fn test_phrase_query() {
706        let (schema, default_fields, tokenizers) = setup();
707        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
708
709        // Should parse quoted phrase
710        let _query = parser.parse("\"hello world\"").unwrap();
711    }
712
713    #[test]
714    fn test_boolean_or() {
715        let (schema, default_fields, tokenizers) = setup();
716        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
717
718        // Should parse OR boolean query
719        let _query = parser.parse("rust OR python").unwrap();
720    }
721
722    #[test]
723    fn test_complex_query() {
724        let (schema, default_fields, tokenizers) = setup();
725        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
726
727        // Should parse complex boolean with grouping
728        let _query = parser.parse("(rust OR python) AND programming").unwrap();
729    }
730
731    #[test]
732    fn test_router_exclusive_mode() {
733        use crate::dsl::query_field_router::{QueryFieldRouter, QueryRouterRule, RoutingMode};
734
735        let mut builder = SchemaBuilder::default();
736        let _title = builder.add_text_field("title", true, true);
737        let _uri = builder.add_text_field("uri", true, true);
738        let schema = Arc::new(builder.build());
739        let tokenizers = Arc::new(TokenizerRegistry::default());
740
741        let router = QueryFieldRouter::from_rules(&[QueryRouterRule {
742            pattern: r"^doi:(10\.\d{4,}/[^\s]+)$".to_string(),
743            substitution: "doi://{1}".to_string(),
744            target_field: "uri".to_string(),
745            mode: RoutingMode::Exclusive,
746        }])
747        .unwrap();
748
749        let parser = QueryLanguageParser::with_router(schema, vec![], tokenizers, router);
750
751        // Should route DOI query to uri field
752        let _query = parser.parse("doi:10.1234/test.123").unwrap();
753    }
754
755    #[test]
756    fn test_router_additional_mode() {
757        use crate::dsl::query_field_router::{QueryFieldRouter, QueryRouterRule, RoutingMode};
758
759        let mut builder = SchemaBuilder::default();
760        let title = builder.add_text_field("title", true, true);
761        let _uri = builder.add_text_field("uri", true, true);
762        let schema = Arc::new(builder.build());
763        let tokenizers = Arc::new(TokenizerRegistry::default());
764
765        let router = QueryFieldRouter::from_rules(&[QueryRouterRule {
766            pattern: r"#(\d+)".to_string(),
767            substitution: "{1}".to_string(),
768            target_field: "uri".to_string(),
769            mode: RoutingMode::Additional,
770        }])
771        .unwrap();
772
773        let parser = QueryLanguageParser::with_router(schema, vec![title], tokenizers, router);
774
775        // Should route to both uri field and default fields
776        let _query = parser.parse("#42").unwrap();
777    }
778
779    #[test]
780    fn test_router_no_match_falls_through() {
781        use crate::dsl::query_field_router::{QueryFieldRouter, QueryRouterRule, RoutingMode};
782
783        let mut builder = SchemaBuilder::default();
784        let title = builder.add_text_field("title", true, true);
785        let _uri = builder.add_text_field("uri", true, true);
786        let schema = Arc::new(builder.build());
787        let tokenizers = Arc::new(TokenizerRegistry::default());
788
789        let router = QueryFieldRouter::from_rules(&[QueryRouterRule {
790            pattern: r"^doi:".to_string(),
791            substitution: "{0}".to_string(),
792            target_field: "uri".to_string(),
793            mode: RoutingMode::Exclusive,
794        }])
795        .unwrap();
796
797        let parser = QueryLanguageParser::with_router(schema, vec![title], tokenizers, router);
798
799        // Should NOT match and fall through to normal parsing
800        let _query = parser.parse("rust programming").unwrap();
801    }
802
803    #[test]
804    fn test_router_invalid_target_field() {
805        use crate::dsl::query_field_router::{QueryFieldRouter, QueryRouterRule, RoutingMode};
806
807        let mut builder = SchemaBuilder::default();
808        let _title = builder.add_text_field("title", true, true);
809        let schema = Arc::new(builder.build());
810        let tokenizers = Arc::new(TokenizerRegistry::default());
811
812        let router = QueryFieldRouter::from_rules(&[QueryRouterRule {
813            pattern: r"test".to_string(),
814            substitution: "{0}".to_string(),
815            target_field: "nonexistent".to_string(),
816            mode: RoutingMode::Exclusive,
817        }])
818        .unwrap();
819
820        let parser = QueryLanguageParser::with_router(schema, vec![], tokenizers, router);
821
822        // Should fail because target field doesn't exist
823        let result = parser.parse("test");
824        assert!(result.is_err());
825        let err = result.err().unwrap();
826        assert!(err.contains("Unknown target field"));
827    }
828
829    #[test]
830    fn test_parse_ann_query() {
831        let mut builder = SchemaBuilder::default();
832        let embedding = builder.add_dense_vector_field("embedding", 128, true, true);
833        let schema = Arc::new(builder.build());
834        let tokenizers = Arc::new(TokenizerRegistry::default());
835
836        let parser = QueryLanguageParser::new(schema, vec![embedding], tokenizers);
837
838        // Parse ANN query
839        let result = parser.parse_query_string("embedding:ann([1.0, 2.0, 3.0], nprobe=32)");
840        assert!(result.is_ok(), "Failed to parse ANN query: {:?}", result);
841
842        if let Ok(ParsedQuery::Ann {
843            field,
844            vector,
845            nprobe,
846            rerank,
847        }) = result
848        {
849            assert_eq!(field, "embedding");
850            assert_eq!(vector, vec![1.0, 2.0, 3.0]);
851            assert_eq!(nprobe, 32);
852            assert_eq!(rerank, 2.0); // default
853        } else {
854            panic!("Expected Ann query, got: {:?}", result);
855        }
856    }
857
858    #[test]
859    fn test_parse_sparse_query() {
860        let mut builder = SchemaBuilder::default();
861        let sparse = builder.add_text_field("sparse", true, true);
862        let schema = Arc::new(builder.build());
863        let tokenizers = Arc::new(TokenizerRegistry::default());
864
865        let parser = QueryLanguageParser::new(schema, vec![sparse], tokenizers);
866
867        // Parse sparse query
868        let result = parser.parse_query_string("sparse:sparse({1: 0.5, 5: 0.3})");
869        assert!(result.is_ok(), "Failed to parse sparse query: {:?}", result);
870
871        if let Ok(ParsedQuery::Sparse { field, vector }) = result {
872            assert_eq!(field, "sparse");
873            assert_eq!(vector, vec![(1, 0.5), (5, 0.3)]);
874        } else {
875            panic!("Expected Sparse query, got: {:?}", result);
876        }
877    }
878
879    #[test]
880    fn test_parse_prefix_simple() {
881        let (schema, default_fields, tokenizers) = setup();
882        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
883
884        // Simple prefix: title:abc*
885        let result = parser.parse_query_string("title:abc*");
886        assert!(result.is_ok(), "Failed to parse prefix query: {:?}", result);
887        if let Ok(ParsedQuery::Prefix { field, prefix }) = result {
888            assert_eq!(field, Some("title".to_string()));
889            assert_eq!(prefix, "abc");
890        } else {
891            panic!("Expected Prefix query, got: {:?}", result);
892        }
893    }
894
895    #[test]
896    fn test_parse_prefix_url() {
897        let mut builder = SchemaBuilder::default();
898        let _site = builder.add_text_field("site", true, true);
899        let schema = Arc::new(builder.build());
900        let tokenizers = Arc::new(TokenizerRegistry::default());
901        let parser = QueryLanguageParser::new(schema, vec![], tokenizers);
902
903        // URL prefix: site:https://reddit.com/r/Transhumanism*
904        let result = parser.parse_query_string("site:https://reddit.com/r/Transhumanism*");
905        assert!(
906            result.is_ok(),
907            "Failed to parse URL prefix query: {:?}",
908            result
909        );
910        if let Ok(ParsedQuery::Prefix { field, prefix }) = result {
911            assert_eq!(field, Some("site".to_string()));
912            assert_eq!(prefix, "https://reddit.com/r/Transhumanism");
913        } else {
914            panic!("Expected Prefix query, got: {:?}", result);
915        }
916    }
917
918    #[test]
919    fn test_parse_prefix_unqualified() {
920        let (schema, default_fields, tokenizers) = setup();
921        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
922
923        // Unqualified prefix: transhuman*
924        let result = parser.parse_query_string("transhuman*");
925        assert!(
926            result.is_ok(),
927            "Failed to parse unqualified prefix: {:?}",
928            result
929        );
930        if let Ok(ParsedQuery::Prefix { field, prefix }) = result {
931            assert_eq!(field, None);
932            assert_eq!(prefix, "transhuman");
933        } else {
934            panic!("Expected Prefix query, got: {:?}", result);
935        }
936    }
937
938    #[test]
939    fn test_prefix_query_builds() {
940        let (schema, default_fields, tokenizers) = setup();
941        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
942
943        // Should build without error
944        let _query = parser.parse("title:abc*").unwrap();
945    }
946
947    #[test]
948    fn test_prefix_in_boolean() {
949        let (schema, default_fields, tokenizers) = setup();
950        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
951
952        // Prefix in boolean: rust AND title:abc*
953        let _query = parser.parse("rust AND title:abc*").unwrap();
954    }
955
956    #[test]
957    fn test_prefix_mixed_with_terms() {
958        let mut builder = SchemaBuilder::default();
959        let title = builder.add_text_field("title", true, true);
960        let _site = builder.add_text_field("site", true, true);
961        let schema = Arc::new(builder.build());
962        let tokenizers = Arc::new(TokenizerRegistry::default());
963        let parser = QueryLanguageParser::new(schema, vec![title], tokenizers);
964
965        // Mixed: prefix + free-text terms (implicit OR)
966        let result =
967            parser.parse_query_string("site:https://reddit.com/r/Transhumanism* longevity drugs");
968        assert!(
969            result.is_ok(),
970            "Failed to parse mixed prefix+terms: {:?}",
971            result
972        );
973        // Should be Or([Prefix, Term, Term])
974        if let Ok(ParsedQuery::Or(parts)) = &result {
975            assert_eq!(parts.len(), 3, "Expected 3 parts, got: {:?}", parts);
976            assert!(
977                matches!(&parts[0], ParsedQuery::And(v) if v.len() == 1 && matches!(&v[0], ParsedQuery::Prefix { .. }))
978                    || matches!(&parts[0], ParsedQuery::Prefix { .. }),
979                "First part should be prefix: {:?}",
980                parts[0]
981            );
982        } else {
983            panic!("Expected Or query, got: {:?}", result);
984        }
985
986        // Should also build into a Query without error
987        let _query = parser
988            .parse("site:https://reddit.com/r/Transhumanism* longevity drugs")
989            .unwrap();
990    }
991
992    #[test]
993    fn test_implicit_or_plain_terms() {
994        let (schema, default_fields, tokenizers) = setup();
995        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
996
997        // Space-separated terms: implicit OR
998        let result = parser.parse_query_string("hello world");
999        assert!(result.is_ok(), "Failed to parse implicit OR: {:?}", result);
1000        if let Ok(ParsedQuery::Or(parts)) = &result {
1001            assert_eq!(parts.len(), 2);
1002        } else {
1003            panic!("Expected Or query, got: {:?}", result);
1004        }
1005    }
1006}
hermes_core/dsl/ql/mod.rs

hermes_core/dsl/ql/
mod.rs
