Skip to main content

summa_core/dsl/ql/
mod.rs

1//! Query language parser using pest
2//!
3//! Supports:
4//! - Term queries: `rust` or `title:rust`
5//! - Phrase queries: `"hello world"` or `title:"hello world"`
6//! - Boolean operators: `AND`, `OR`, `NOT` (or `&&`, `||`, `!`)
7//! - Required/prohibited clauses: `+rust -python`, including phrases and groups
8//! - Grouping: `(rust OR python) AND programming`
9//! - Default fields for unqualified terms
10
11use pest::Parser;
12use pest_derive::Parser;
13use std::sync::Arc;
14
15use super::query_field_router::{QueryFieldRouter, RoutingMode};
16use super::schema::{Field, Schema};
17use crate::query::{
18    BooleanQuery, DEFAULT_DENSE_RERANK_FACTOR, PhraseQuery, PrefixQuery, Query, RegexQuery,
19    TermQuery, WildcardQuery,
20};
21use crate::tokenizer::{BoxedTokenizer, TokenizerRegistry};
22
23#[derive(Parser)]
24#[grammar = "dsl/ql/ql.pest"]
25struct QueryParser;
26
27/// Parsed query that can be converted to a Query trait object
28#[derive(Debug, Clone)]
29pub enum ParsedQuery {
30    Term {
31        field: Option<String>,
32        term: String,
33    },
34    Phrase {
35        field: Option<String>,
36        phrase: String,
37    },
38    /// Prefix query — matches terms starting with a given prefix
39    Prefix {
40        field: Option<String>,
41        prefix: String,
42    },
43    /// Whole-term wildcard filter with explicit pattern syntax.
44    Wildcard {
45        field: Option<String>,
46        pattern: String,
47    },
48    /// Whole-term regular expression; pattern case is preserved.
49    Regex {
50        field: Option<String>,
51        pattern: String,
52    },
53    /// Dense vector ANN query
54    Ann {
55        field: String,
56        vector: Vec<f32>,
57        nprobe: usize,
58        rerank: f32,
59    },
60    /// Sparse vector query
61    Sparse {
62        field: String,
63        vector: Vec<(u32, f32)>,
64    },
65    And(Vec<ParsedQuery>),
66    Or(Vec<ParsedQuery>),
67    Not(Box<ParsedQuery>),
68    Required(Box<ParsedQuery>),
69    Prohibited(Box<ParsedQuery>),
70    /// Preserve the scope of clause modifiers inside parentheses.
71    Group(Box<ParsedQuery>),
72}
73
74/// Query language parser with schema awareness
75pub struct QueryLanguageParser {
76    schema: Arc<Schema>,
77    default_fields: Vec<Field>,
78    tokenizers: Arc<TokenizerRegistry>,
79    /// Optional query field router for routing queries based on regex patterns
80    field_router: Option<QueryFieldRouter>,
81}
82
83impl QueryLanguageParser {
84    pub fn new(
85        schema: Arc<Schema>,
86        default_fields: Vec<Field>,
87        tokenizers: Arc<TokenizerRegistry>,
88    ) -> Self {
89        Self {
90            schema,
91            default_fields,
92            tokenizers,
93            field_router: None,
94        }
95    }
96
97    /// Create a parser with a query field router
98    pub fn with_router(
99        schema: Arc<Schema>,
100        default_fields: Vec<Field>,
101        tokenizers: Arc<TokenizerRegistry>,
102        router: QueryFieldRouter,
103    ) -> Self {
104        Self {
105            schema,
106            default_fields,
107            tokenizers,
108            field_router: Some(router),
109        }
110    }
111
112    /// Set the query field router
113    pub fn set_router(&mut self, router: QueryFieldRouter) {
114        self.field_router = Some(router);
115    }
116
117    /// Get the query field router
118    pub fn router(&self) -> Option<&QueryFieldRouter> {
119        self.field_router.as_ref()
120    }
121
122    /// Parse a query string into a Query
123    ///
124    /// Supports query language syntax (field:term, AND, OR, NOT, grouping)
125    /// and plain text (tokenized and searched across default fields).
126    ///
127    /// If a query field router is configured, the query is first checked against
128    /// routing rules. If a rule matches:
129    /// - In exclusive mode: only the target field is queried with the substituted value
130    /// - In additional mode: both the target field and default fields are queried
131    pub fn parse(&self, query_str: &str) -> Result<Box<dyn Query>, String> {
132        self.parse_with_mode(query_str, false)
133    }
134
135    /// Parse query syntax without falling back to tokenized plain text.
136    /// Configured field routing still applies.
137    pub fn parse_strict(&self, query_str: &str) -> Result<Box<dyn Query>, String> {
138        self.parse_with_mode(query_str, true)
139    }
140
141    fn parse_with_mode(&self, query_str: &str, strict: bool) -> Result<Box<dyn Query>, String> {
142        let query_str = query_str.trim();
143        if query_str.is_empty() {
144            return Err("Empty query".to_string());
145        }
146
147        // Check if query matches any routing rules
148        if let Some(router) = &self.field_router
149            && let Some(routed) = router.route(query_str)
150        {
151            return self.build_routed_query(
152                &routed.query,
153                &routed.target_field,
154                routed.mode,
155                query_str,
156                strict,
157            );
158        }
159
160        // No routing match - parse normally
161        self.parse_normal(query_str, strict)
162    }
163
164    /// Build a query from a routed match
165    fn build_routed_query(
166        &self,
167        routed_query: &str,
168        target_field: &str,
169        mode: RoutingMode,
170        original_query: &str,
171        strict: bool,
172    ) -> Result<Box<dyn Query>, String> {
173        // Validate target field exists
174        let _field_id = self
175            .schema
176            .get_field(target_field)
177            .ok_or_else(|| format!("Unknown target field: {}", target_field))?;
178
179        // Build query for the target field with the substituted value
180        let target_query = self.build_term_query(Some(target_field), routed_query)?;
181
182        match mode {
183            RoutingMode::Exclusive => {
184                // Only query the target field
185                Ok(target_query)
186            }
187            RoutingMode::Additional => {
188                // Query both target field and default fields
189                let mut bool_query = BooleanQuery::new();
190                bool_query = bool_query.should(target_query);
191
192                // Also parse the original query against default fields
193                // An additional route must retain the original query or its
194                // error; dropping it silently changes the requested semantics.
195                bool_query = bool_query.should(self.parse_normal(original_query, strict)?);
196
197                Ok(Box::new(bool_query))
198            }
199        }
200    }
201
202    /// Parse query without routing (normal parsing path)
203    fn parse_normal(&self, query_str: &str, strict: bool) -> Result<Box<dyn Query>, String> {
204        // Try parsing as query language first
205        match self.parse_query_string(query_str) {
206            Ok(parsed) => self.build_query(&parsed),
207            Err(error)
208                if strict
209                    || query_str.contains('\\')
210                    || query_str.match_indices("regex").any(|(at, _)| {
211                        (at == 0
212                            || query_str[..at].ends_with(|ch: char| {
213                                ch.is_whitespace() || matches!(ch, ':' | '+' | '-' | '(')
214                            }))
215                            && query_str[at + "regex".len()..]
216                                .trim_start()
217                                .starts_with('(')
218                    })
219                    || query_str
220                        .split(|c: char| c.is_whitespace() || c == '(')
221                        .any(|word| word.len() > 1 && word.starts_with(['+', '-'])) =>
222            {
223                Err(error)
224            }
225            Err(_) => {
226                // If grammar parsing fails, treat as plain text
227                // Split by whitespace and create OR of terms
228                self.parse_plain_text(query_str)
229            }
230        }
231    }
232
233    /// Parse plain text as implicit OR of tokenized terms
234    fn parse_plain_text(&self, text: &str) -> Result<Box<dyn Query>, String> {
235        if self.default_fields.is_empty() {
236            return Err("No default fields configured".to_string());
237        }
238
239        let mut bool_query = BooleanQuery::new();
240        for &field_id in &self.default_fields {
241            for token in self.get_tokenizer(field_id).tokenize(text) {
242                bool_query =
243                    bool_query.should(TermQuery::text(field_id, &token.text.to_lowercase()));
244            }
245        }
246        if bool_query.should.is_empty() {
247            return Err("No tokens in query".to_string());
248        }
249        Ok(Box::new(bool_query))
250    }
251
252    fn parse_query_string(&self, query_str: &str) -> Result<ParsedQuery, String> {
253        let pairs = QueryParser::parse(Rule::query, query_str)
254            .map_err(|e| format!("Parse error: {}", e))?;
255
256        let query_pair = pairs.into_iter().next().ok_or("No query found")?;
257
258        // query = { SOI ~ or_expr ~ EOI }
259        self.parse_or_expr(query_pair.into_inner().next().unwrap())
260    }
261
262    fn parse_or_expr(&self, pair: pest::iterators::Pair<Rule>) -> Result<ParsedQuery, String> {
263        let mut inner = pair.into_inner();
264        let first = self.parse_and_expr(inner.next().unwrap())?;
265
266        let rest: Vec<ParsedQuery> = inner
267            .filter(|p| p.as_rule() == Rule::and_expr)
268            .map(|p| self.parse_and_expr(p))
269            .collect::<Result<Vec<_>, _>>()?;
270
271        if rest.is_empty() {
272            Ok(first)
273        } else {
274            let mut all = vec![first];
275            all.extend(rest);
276            Ok(ParsedQuery::Or(all))
277        }
278    }
279
280    fn parse_and_expr(&self, pair: pest::iterators::Pair<Rule>) -> Result<ParsedQuery, String> {
281        let mut inner = pair.into_inner();
282        let first = self.parse_primary(inner.next().unwrap())?;
283
284        let rest: Vec<ParsedQuery> = inner
285            .filter(|p| p.as_rule() == Rule::primary)
286            .map(|p| self.parse_primary(p))
287            .collect::<Result<Vec<_>, _>>()?;
288
289        if rest.is_empty() {
290            Ok(first)
291        } else {
292            let mut all = vec![first];
293            all.extend(rest);
294            Ok(ParsedQuery::And(all))
295        }
296    }
297
298    fn parse_primary(&self, pair: pest::iterators::Pair<Rule>) -> Result<ParsedQuery, String> {
299        let mut modifier = None;
300        let mut inner_query = None;
301
302        for inner in pair.into_inner() {
303            match inner.as_rule() {
304                Rule::not_op | Rule::required_op | Rule::prohibited_op => {
305                    modifier = Some(inner.as_rule());
306                }
307                Rule::group => {
308                    let or_expr = inner.into_inner().next().unwrap();
309                    inner_query = Some(ParsedQuery::Group(Box::new(self.parse_or_expr(or_expr)?)));
310                }
311                Rule::ann_query => {
312                    inner_query = Some(self.parse_ann_query(inner)?);
313                }
314                Rule::sparse_query => {
315                    inner_query = Some(self.parse_sparse_query(inner)?);
316                }
317                Rule::phrase_query => {
318                    inner_query = Some(self.parse_phrase_query(inner)?);
319                }
320                Rule::wildcard_query | Rule::regex_query => {
321                    let is_regex = inner.as_rule() == Rule::regex_query;
322                    let mut field = None;
323                    let mut pattern = String::new();
324                    for part in inner.into_inner() {
325                        match part.as_rule() {
326                            Rule::field_spec => {
327                                field = Some(part.into_inner().next().unwrap().as_str().to_owned())
328                            }
329                            Rule::quoted_string => {
330                                pattern = serde_json::from_str(part.as_str())
331                                    .map_err(|error| format!("Invalid pattern string: {error}"))?
332                            }
333                            _ => {}
334                        }
335                    }
336                    inner_query = Some(if is_regex {
337                        ParsedQuery::Regex { field, pattern }
338                    } else {
339                        ParsedQuery::Wildcard { field, pattern }
340                    });
341                }
342                Rule::term_pattern_query => {
343                    inner_query = Some(self.parse_term_pattern(inner)?);
344                }
345                Rule::term_query => {
346                    inner_query = Some(self.parse_term_query(inner)?);
347                }
348                _ => {}
349            }
350        }
351
352        let query = inner_query.ok_or("No query in primary")?;
353
354        Ok(match modifier {
355            Some(Rule::not_op) => ParsedQuery::Not(Box::new(query)),
356            Some(Rule::required_op) => ParsedQuery::Required(Box::new(query)),
357            Some(Rule::prohibited_op) => ParsedQuery::Prohibited(Box::new(query)),
358            _ => query,
359        })
360    }
361
362    fn parse_term_query(&self, pair: pest::iterators::Pair<Rule>) -> Result<ParsedQuery, String> {
363        let mut field = None;
364        let mut term = String::new();
365
366        for inner in pair.into_inner() {
367            match inner.as_rule() {
368                Rule::field_spec => {
369                    field = Some(inner.into_inner().next().unwrap().as_str().to_string());
370                }
371                Rule::term => {
372                    let source = inner.as_str();
373                    if source.contains('\\') {
374                        term.reserve(source.len());
375                        let mut chars = source.chars();
376                        while let Some(ch) = chars.next() {
377                            term.push(if ch == '\\' {
378                                chars.next().ok_or("Dangling term escape")?
379                            } else {
380                                ch
381                            });
382                        }
383                    } else {
384                        term = source.to_owned();
385                    }
386                }
387                _ => {}
388            }
389        }
390
391        Ok(ParsedQuery::Term { field, term })
392    }
393
394    fn parse_term_pattern(&self, pair: pest::iterators::Pair<Rule>) -> Result<ParsedQuery, String> {
395        let mut field = None;
396        let mut prefix = String::new();
397
398        for inner in pair.into_inner() {
399            match inner.as_rule() {
400                Rule::field_spec => {
401                    field = Some(inner.into_inner().next().unwrap().as_str().to_string());
402                }
403                Rule::term_pattern => {
404                    prefix = inner.as_str().to_string();
405                }
406                _ => {}
407            }
408        }
409
410        if let Some(literal) = prefix.strip_suffix('*')
411            && !literal.is_empty()
412            && !literal.contains(['*', '?', '\\'])
413        {
414            Ok(ParsedQuery::Prefix {
415                field,
416                prefix: literal.to_owned(),
417            })
418        } else {
419            Ok(ParsedQuery::Wildcard {
420                field,
421                pattern: prefix,
422            })
423        }
424    }
425
426    fn parse_phrase_query(&self, pair: pest::iterators::Pair<Rule>) -> Result<ParsedQuery, String> {
427        let mut field = None;
428        let mut phrase = String::new();
429
430        for inner in pair.into_inner() {
431            match inner.as_rule() {
432                Rule::field_spec => {
433                    field = Some(inner.into_inner().next().unwrap().as_str().to_string());
434                }
435                Rule::quoted_string => {
436                    let s = inner.as_str();
437                    phrase = s[1..s.len() - 1].to_string();
438                }
439                _ => {}
440            }
441        }
442
443        Ok(ParsedQuery::Phrase { field, phrase })
444    }
445
446    /// Parse an ANN query: field:ann([1.0, 2.0, 3.0], nprobe=32, rerank=2)
447    fn parse_ann_query(&self, pair: pest::iterators::Pair<Rule>) -> Result<ParsedQuery, String> {
448        let mut field = String::new();
449        let mut vector = Vec::new();
450        let mut nprobe = 32usize;
451        let mut rerank = DEFAULT_DENSE_RERANK_FACTOR;
452
453        for inner in pair.into_inner() {
454            match inner.as_rule() {
455                Rule::field_spec => {
456                    field = inner.into_inner().next().unwrap().as_str().to_string();
457                }
458                Rule::vector_array => {
459                    for num in inner.into_inner() {
460                        if num.as_rule() == Rule::number
461                            && let Ok(v) = num.as_str().parse::<f32>()
462                        {
463                            vector.push(v);
464                        }
465                    }
466                }
467                Rule::ann_params => {
468                    for param in inner.into_inner() {
469                        if param.as_rule() == Rule::ann_param {
470                            // ann_param = { ("nprobe" | "rerank") ~ "=" ~ number }
471                            let param_str = param.as_str();
472                            if let Some(eq_pos) = param_str.find('=') {
473                                let name = &param_str[..eq_pos];
474                                let value = &param_str[eq_pos + 1..];
475                                match name {
476                                    "nprobe" => nprobe = value.parse().unwrap_or(0),
477                                    "rerank" => rerank = value.parse().unwrap_or(0.0),
478                                    _ => {}
479                                }
480                            }
481                        }
482                    }
483                }
484                _ => {}
485            }
486        }
487
488        Ok(ParsedQuery::Ann {
489            field,
490            vector,
491            nprobe,
492            rerank,
493        })
494    }
495
496    /// Parse a sparse vector query: field:sparse({1: 0.5, 5: 0.3})
497    fn parse_sparse_query(&self, pair: pest::iterators::Pair<Rule>) -> Result<ParsedQuery, String> {
498        let mut field = String::new();
499        let mut vector = Vec::new();
500
501        for inner in pair.into_inner() {
502            match inner.as_rule() {
503                Rule::field_spec => {
504                    field = inner.into_inner().next().unwrap().as_str().to_string();
505                }
506                Rule::sparse_map => {
507                    for entry in inner.into_inner() {
508                        if entry.as_rule() == Rule::sparse_entry {
509                            let mut entry_inner = entry.into_inner();
510                            if let (Some(idx), Some(weight)) =
511                                (entry_inner.next(), entry_inner.next())
512                                && let (Ok(i), Ok(w)) =
513                                    (idx.as_str().parse::<u32>(), weight.as_str().parse::<f32>())
514                            {
515                                vector.push((i, w));
516                            }
517                        }
518                    }
519                }
520                _ => {}
521            }
522        }
523
524        Ok(ParsedQuery::Sparse { field, vector })
525    }
526
527    fn build_query(&self, parsed: &ParsedQuery) -> Result<Box<dyn Query>, String> {
528        use crate::query::{DenseVectorQuery, SparseVectorQuery};
529
530        match parsed {
531            ParsedQuery::Term { field, term } => self.build_term_query(field.as_deref(), term),
532            ParsedQuery::Phrase { field, phrase } => {
533                self.build_phrase_query(field.as_deref(), phrase)
534            }
535            ParsedQuery::Prefix { field, prefix } => {
536                self.build_prefix_query(field.as_deref(), prefix)
537            }
538            ParsedQuery::Wildcard { field, pattern } | ParsedQuery::Regex { field, pattern } => {
539                let explicit = field
540                    .as_ref()
541                    .map(|name| {
542                        self.schema
543                            .get_field(name)
544                            .ok_or_else(|| format!("Unknown field: {name}"))
545                    })
546                    .transpose()?;
547                let fields = explicit
548                    .as_ref()
549                    .map_or(self.default_fields.as_slice(), std::slice::from_ref);
550                if fields.is_empty() {
551                    return Err("No field specified and no default fields configured".into());
552                }
553                let build = |field| -> Result<Box<dyn Query>, String> {
554                    if matches!(parsed, ParsedQuery::Regex { .. }) {
555                        Ok(Box::new(
556                            RegexQuery::new(field, pattern).map_err(|error| error.to_string())?,
557                        ))
558                    } else {
559                        Ok(Box::new(
560                            WildcardQuery::text(field, pattern)
561                                .map_err(|error| error.to_string())?,
562                        ))
563                    }
564                };
565                if explicit.is_some() {
566                    return build(fields[0]);
567                }
568                let mut query = BooleanQuery::new();
569                for &field in fields {
570                    query.should.push(Arc::from(build(field)?));
571                }
572                Ok(Box::new(query))
573            }
574            ParsedQuery::Ann {
575                field,
576                vector,
577                nprobe,
578                rerank,
579            } => {
580                let field_id = self
581                    .schema
582                    .get_field(field)
583                    .ok_or_else(|| format!("Unknown field: {}", field))?;
584                let query = DenseVectorQuery::new(field_id, vector.clone())
585                    .with_nprobe(*nprobe)
586                    .with_rerank_factor(*rerank);
587                Ok(Box::new(query))
588            }
589            ParsedQuery::Sparse { field, vector } => {
590                let field_id = self
591                    .schema
592                    .get_field(field)
593                    .ok_or_else(|| format!("Unknown field: {}", field))?;
594                let mut query = SparseVectorQuery::new(field_id, vector.clone());
595                if let Some(config) = self
596                    .schema
597                    .get_field_entry(field_id)
598                    .and_then(|entry| entry.sparse_vector_config.as_ref())
599                    .and_then(|config| config.query_config.as_ref())
600                {
601                    if let Some(gamma) = config.lsp_gamma {
602                        query = query.with_lsp_gamma(gamma);
603                    }
604                    query = query
605                        .with_heap_factor(config.heap_factor)
606                        .with_seismic_cut(config.seismic_cut)
607                        .with_seismic_factor(config.seismic_factor)
608                        .with_exhaustive(config.exhaustive);
609                }
610                Ok(Box::new(query))
611            }
612            ParsedQuery::And(queries) => {
613                let mut bool_query = BooleanQuery::new();
614                for q in queries {
615                    bool_query = match q {
616                        ParsedQuery::Prohibited(inner) => {
617                            bool_query.must_not(self.build_query(inner)?)
618                        }
619                        _ => bool_query.must(self.build_query(q)?),
620                    };
621                }
622                Ok(Box::new(bool_query))
623            }
624            ParsedQuery::Or(queries) => {
625                let mut bool_query = BooleanQuery::new();
626                for q in queries {
627                    bool_query = match q {
628                        ParsedQuery::Required(inner) => bool_query.must(self.build_query(inner)?),
629                        ParsedQuery::Prohibited(inner) => {
630                            bool_query.must_not(self.build_query(inner)?)
631                        }
632                        _ => bool_query.should(self.build_query(q)?),
633                    };
634                }
635                Ok(Box::new(bool_query))
636            }
637            ParsedQuery::Required(inner) | ParsedQuery::Group(inner) => self.build_query(inner),
638            ParsedQuery::Not(inner) | ParsedQuery::Prohibited(inner) => {
639                // NOT query needs a context - wrap in a match-all with must_not
640                let mut bool_query = BooleanQuery::new();
641                bool_query = bool_query.must_not(self.build_query(inner)?);
642                Ok(Box::new(bool_query))
643            }
644        }
645    }
646
647    fn build_term_query(&self, field: Option<&str>, term: &str) -> Result<Box<dyn Query>, String> {
648        if let Some(field_name) = field {
649            // Field-qualified term: tokenize using field's tokenizer
650            let field_id = self
651                .schema
652                .get_field(field_name)
653                .ok_or_else(|| format!("Unknown field: {}", field_name))?;
654            // Validate field type — TermQuery only works on text fields
655            if let Some(entry) = self.schema.get_field_entry(field_id) {
656                use crate::dsl::FieldType;
657                if entry.field_type != FieldType::Text {
658                    return Err(format!(
659                        "Term query requires a text field, but '{}' is {:?}. Use range query for numeric fields.",
660                        field_name, entry.field_type
661                    ));
662                }
663            }
664            let tokenizer = self.get_tokenizer(field_id);
665            let tokens: Vec<String> = tokenizer
666                .tokenize(term)
667                .into_iter()
668                .map(|t| t.text.to_lowercase())
669                .collect();
670
671            if tokens.is_empty() {
672                return Err("No tokens in term".to_string());
673            }
674
675            if tokens.len() == 1 {
676                Ok(Box::new(TermQuery::text(field_id, &tokens[0])))
677            } else {
678                // Multiple tokens from single term - AND them together
679                let mut bool_query = BooleanQuery::new();
680                for token in &tokens {
681                    bool_query = bool_query.must(TermQuery::text(field_id, token));
682                }
683                Ok(Box::new(bool_query))
684            }
685        } else if !self.default_fields.is_empty() {
686            // Each default field must use the same tokenizer as its indexed text.
687            // Preserve the unqualified term's OR semantics across fields and tokens.
688            let mut bool_query = BooleanQuery::new();
689            for &field_id in &self.default_fields {
690                let tokens = self.get_tokenizer(field_id).tokenize(term);
691
692                // Preserve term decomposition for the owning planner, including
693                // when nested in a larger Boolean query.
694                if tokens.len() == 1 && self.default_fields.len() == 1 {
695                    return Ok(Box::new(TermQuery::text(
696                        field_id,
697                        &tokens[0].text.to_lowercase(),
698                    )));
699                }
700
701                for token in tokens {
702                    bool_query =
703                        bool_query.should(TermQuery::text(field_id, &token.text.to_lowercase()));
704                }
705            }
706            // An empty token stream in one field must not discard other fields.
707            if bool_query.should.is_empty() {
708                return Err("No tokens in term".to_string());
709            }
710            Ok(Box::new(bool_query))
711        } else {
712            Err("No field specified and no default fields configured".to_string())
713        }
714    }
715
716    fn build_prefix_query(
717        &self,
718        field: Option<&str>,
719        prefix: &str,
720    ) -> Result<Box<dyn Query>, String> {
721        if let Some(field_name) = field {
722            let field_id = self
723                .schema
724                .get_field(field_name)
725                .ok_or_else(|| format!("Unknown field: {}", field_name))?;
726            Ok(Box::new(PrefixQuery::text(field_id, prefix)))
727        } else if !self.default_fields.is_empty() {
728            // Unqualified prefix: OR across default fields
729            let mut bool_query = BooleanQuery::new();
730            for &field_id in &self.default_fields {
731                bool_query = bool_query.should(PrefixQuery::text(field_id, prefix));
732            }
733            Ok(Box::new(bool_query))
734        } else {
735            Err("No field specified and no default fields configured".to_string())
736        }
737    }
738
739    fn build_phrase_query(
740        &self,
741        field: Option<&str>,
742        phrase: &str,
743    ) -> Result<Box<dyn Query>, String> {
744        let explicit = field
745            .map(|name| {
746                self.schema
747                    .get_field(name)
748                    .ok_or_else(|| format!("Unknown field: {name}"))
749            })
750            .transpose()?;
751        let fields = explicit
752            .as_ref()
753            .map_or(self.default_fields.as_slice(), std::slice::from_ref);
754        if fields.is_empty() {
755            return Err("No field specified and no default fields configured".into());
756        }
757        let build = |field_id| -> Result<Option<Box<dyn Query>>, String> {
758            let tokens: Vec<_> = self
759                .get_tokenizer(field_id)
760                .tokenize(phrase)
761                .into_iter()
762                .map(|token| (token.position, token.text.to_lowercase().into_bytes()))
763                .collect();
764            if tokens.is_empty() {
765                return Ok(None);
766            }
767            if tokens.len() == 1 {
768                return Ok(Some(Box::new(TermQuery::new(
769                    field_id,
770                    tokens[0].1.clone(),
771                ))));
772            }
773            PhraseQuery::validate_positions(&self.schema, field_id)
774                .map_err(|error| error.to_string())?;
775            Ok(Some(Box::new(PhraseQuery::with_offsets(field_id, tokens))))
776        };
777        if fields.len() == 1 {
778            return build(fields[0])?.ok_or_else(|| "No tokens in phrase".into());
779        }
780        let mut outer = BooleanQuery::new();
781        let mut populated = false;
782        for &field in fields {
783            if let Some(query) = build(field)? {
784                outer = outer.should(query);
785                populated = true;
786            }
787        }
788        if !populated {
789            return Err("No tokens in phrase".into());
790        }
791        Ok(Box::new(outer))
792    }
793
794    fn get_tokenizer(&self, field: Field) -> BoxedTokenizer {
795        // Get tokenizer name from schema field entry, fallback to "simple"
796        let tokenizer_name = self
797            .schema
798            .get_field_entry(field)
799            .and_then(|entry| entry.tokenizer.as_deref())
800            .unwrap_or("simple");
801
802        self.tokenizers
803            .get(tokenizer_name)
804            .unwrap_or_else(|| Box::new(crate::tokenizer::SimpleTokenizer))
805    }
806}
807
808#[cfg(test)]
809mod tests {
810    use super::*;
811    use crate::dsl::SchemaBuilder;
812    use crate::tokenizer::TokenizerRegistry;
813
814    fn setup() -> (Arc<Schema>, Vec<Field>, Arc<TokenizerRegistry>) {
815        let mut builder = SchemaBuilder::default();
816        let title = builder.add_text_field("title", true, true);
817        let body = builder.add_text_field("body", true, true);
818        builder.set_positions(title, crate::dsl::PositionMode::TokenPosition);
819        builder.set_positions(body, crate::dsl::PositionMode::TokenPosition);
820        let schema = Arc::new(builder.build());
821        let tokenizers = Arc::new(TokenizerRegistry::default());
822        (schema, vec![title, body], tokenizers)
823    }
824
825    #[cfg(feature = "native")]
826    #[tokio::test]
827    async fn unqualified_search_uses_each_fields_tokenizer_in_either_order() {
828        use crate::{Document, Index, IndexConfig, IndexWriter, RamDirectory};
829
830        let cases = [
831            (
832                ["en: text<en_stem>", "zh: text<lex(segmenter: unicode)>"],
833                vec![
834                    [
835                        "The quick brown fox jumps over the lazy dog",
836                        "那只敏捷的棕毛狐狸跃过了那只懒狗",
837                    ],
838                    ["unrelated", "无关"],
839                ],
840                "棕毛狐狸",
841                "zh:棕毛狐狸",
842                vec![0],
843            ),
844            (
845                ["raw: text<simple>", "stemmed: text<en_stem>"],
846                vec![["running", ""], ["", "running"], ["unrelated", "unrelated"]],
847                "running",
848                "raw:running OR stemmed:running",
849                vec![0, 1],
850            ),
851            (
852                [
853                    "filtered: text<lex(default: en, stop_words: true)>",
854                    "raw: text<simple>",
855                ],
856                vec![["", "the"], ["unrelated", "unrelated"]],
857                "the",
858                "raw:the",
859                vec![0],
860            ),
861        ];
862        for (fields, documents, term, qualified, expected) in cases {
863            for reverse in [false, true] {
864                let order = if reverse { [1, 0] } else { [0, 1] };
865                let schema = crate::parse_schema(&format!(
866                    "index test {{ field {} [indexed, stored] field {} [indexed, stored] }}",
867                    fields[order[0]], fields[order[1]],
868                ))
869                .unwrap();
870                let field_ids =
871                    fields.map(|field| schema.get_field(field.split(':').next().unwrap()).unwrap());
872                let directory = RamDirectory::new();
873                let config = IndexConfig::default();
874                let mut writer = IndexWriter::create(directory.clone(), schema, config.clone())
875                    .await
876                    .unwrap();
877                for values in &documents {
878                    let mut document = Document::new();
879                    for (field, value) in field_ids.iter().zip(values) {
880                        if !value.is_empty() {
881                            document.add_text(*field, *value);
882                        }
883                    }
884                    writer.add_document(document).unwrap();
885                }
886                writer.commit().await.unwrap();
887                let index = Index::open(directory, config).await.unwrap();
888                let parser = index.query_parser();
889                // A comma forces the permissive parser's plain-text fallback.
890                let fallback = format!("{term},");
891                assert!(parser.parse_query_string(&fallback).is_err());
892                for text in [qualified, term, fallback.as_str()] {
893                    let result = index.query(text, 10).await.unwrap();
894                    let mut actual: Vec<_> =
895                        result.hits.iter().map(|hit| hit.address.doc_id).collect();
896                    actual.sort_unstable();
897                    assert_eq!(actual, expected, "query={text}, reverse={reverse}");
898                }
899                let strict = parser.parse_strict(term).unwrap();
900                let mut actual: Vec<_> = index
901                    .search(strict.as_ref(), 10)
902                    .await
903                    .unwrap()
904                    .hits
905                    .iter()
906                    .map(|hit| hit.address.doc_id)
907                    .collect();
908                actual.sort_unstable();
909                assert_eq!(actual, expected, "strict query={term}, reverse={reverse}");
910            }
911        }
912    }
913
914    #[test]
915    fn unqualified_search_rejects_queries_with_no_tokens_in_any_default_field() {
916        let schema = Arc::new(
917            crate::parse_schema(
918                "index test {
919                field first: text<en_stem_stop>
920                field second: text<lex(default: en, stop_words: true)>
921            }",
922            )
923            .unwrap(),
924        );
925        let fields = vec![
926            schema.get_field("first").unwrap(),
927            schema.get_field("second").unwrap(),
928        ];
929        let parser =
930            QueryLanguageParser::new(schema, fields, Arc::new(TokenizerRegistry::default()));
931        assert_eq!(parser.parse("the").err().unwrap(), "No tokens in term");
932        assert_eq!(parser.parse("the,").err().unwrap(), "No tokens in query");
933    }
934
935    #[test]
936    fn unqualified_single_field_term_preserves_planner_decomposition() {
937        let (schema, fields, tokenizers) = setup();
938        let parser = QueryLanguageParser::new(schema, vec![fields[0]], tokenizers);
939        let query = parser.parse("RUNNING").unwrap();
940        let crate::query::QueryDecomposition::TextTerm(info) = query.decompose() else {
941            panic!("single-field term must remain decomposable");
942        };
943        assert_eq!(info.field, fields[0]);
944        assert_eq!(info.term, b"running");
945    }
946
947    #[test]
948    fn sparse_query_language_preserves_schema_lsp_gamma_including_exhaustive_zero() {
949        for gamma in [None, Some(0), Some(7)] {
950            let setting = gamma.map_or(String::new(), |gamma| {
951                format!(", query<lsp_gamma: {gamma}, exhaustive: false>")
952            });
953            let schema = crate::parse_schema(&format!(
954                "index test {{ field emb: sparse_vector [indexed<format: bmp, dims: 16{setting}>] }}"
955            )).unwrap();
956            let parser = QueryLanguageParser::new(
957                Arc::new(schema),
958                vec![],
959                Arc::new(TokenizerRegistry::default()),
960            );
961            let query = parser.parse("emb:sparse({0: 1.0})").unwrap();
962            let crate::query::QueryDecomposition::SparseTerms(infos) = query.decompose() else {
963                panic!("sparse syntax did not produce a sparse query");
964            };
965            assert_eq!(infos.len(), 1);
966            assert_eq!(infos[0].lsp_gamma, gamma);
967            assert!(!infos[0].exhaustive);
968        }
969    }
970
971    #[cfg(feature = "native")]
972    #[tokio::test]
973    async fn parsed_sparse_search_uses_the_fields_exhaustive_or_bounded_lsp_policy() {
974        use crate::query::SparseVectorQuery;
975        use crate::{Document, Index, IndexConfig, IndexWriter, RamDirectory};
976        for gamma in [0, 1] {
977            let schema = crate::parse_schema(&format!(
978                "index test {{ field emb: sparse_vector [indexed<format: bmp, dims: 16, bmp_block_size: 1, query<lsp_gamma: {gamma}>>] }}"
979            )).unwrap();
980            let field = schema.get_field("emb").unwrap();
981            let directory = RamDirectory::new();
982            let config = IndexConfig::default();
983            let mut writer = IndexWriter::create(directory.clone(), schema, config.clone())
984                .await
985                .unwrap();
986            // Two superblocks: one of eight weaker documents and one winner.
987            for doc in 0..9 {
988                let mut document = Document::new();
989                document.add_sparse_vector(field, vec![(0, if doc == 8 { 5.0 } else { 0.1 })]);
990                writer.add_document(document).unwrap();
991            }
992            writer.commit().await.unwrap();
993            let index = Index::open(directory, config).await.unwrap();
994            let query = index.query_parser().parse("emb:sparse({0: 1.0})").unwrap();
995            let actual = index.search(query.as_ref(), 9).await.unwrap();
996            let expected = index
997                .search(
998                    &SparseVectorQuery::new(field, vec![(0, 1.0)]).with_lsp_gamma(gamma),
999                    9,
1000                )
1001                .await
1002                .unwrap();
1003            let hits = |result: crate::query::SearchResponse| {
1004                result
1005                    .hits
1006                    .into_iter()
1007                    .map(|hit| (hit.address.doc_id, hit.score.to_bits()))
1008                    .collect::<Vec<_>>()
1009            };
1010            assert_eq!(actual.hits.len(), if gamma == 0 { 9 } else { 1 });
1011            assert_eq!(hits(actual), hits(expected));
1012        }
1013    }
1014
1015    #[test]
1016    fn sparse_query_language_preserves_schema_exhaustive_policy() {
1017        for exhaustive in [false, true] {
1018            let setting = format!(", query<exhaustive: {exhaustive}>");
1019            let schema = crate::parse_schema(&format!(
1020                "index test {{ field emb: sparse_vector [indexed<format: seismic, dims: 16{setting}>] }}"
1021            )).unwrap();
1022            let parser = QueryLanguageParser::new(
1023                Arc::new(schema),
1024                vec![],
1025                Arc::new(TokenizerRegistry::default()),
1026            );
1027            let query = parser.parse("emb:sparse({0: 1.0})").unwrap();
1028            let crate::query::QueryDecomposition::SparseTerms(infos) = query.decompose() else {
1029                panic!("sparse syntax did not produce a sparse query");
1030            };
1031            assert_eq!(infos.len(), 1);
1032            assert_eq!(infos[0].exhaustive, exhaustive);
1033        }
1034    }
1035
1036    #[cfg(feature = "native")]
1037    #[tokio::test]
1038    async fn parsed_sparse_search_uses_the_fields_exhaustive_policy() {
1039        use crate::query::SparseVectorQuery;
1040        use crate::{Document, Index, IndexConfig, IndexWriter, RamDirectory};
1041        for exhaustive in [false, true] {
1042            let schema = crate::parse_schema(&format!(
1043                "index test {{ field emb: sparse_vector [indexed<format: seismic, dims: 16, seismic_postings: 1, seismic_cluster_size: 1, query<exhaustive: {exhaustive}>>] }}"
1044            )).unwrap();
1045            let field = schema.get_field("emb").unwrap();
1046            let directory = RamDirectory::new();
1047            let config = IndexConfig::default();
1048            let mut writer = IndexWriter::create(directory.clone(), schema, config.clone())
1049                .await
1050                .unwrap();
1051            // Two superblocks: one of eight weaker documents and one winner.
1052            for doc in 0..9 {
1053                let mut document = Document::new();
1054                document.add_sparse_vector(field, vec![(0, if doc == 8 { 5.0 } else { 0.1 })]);
1055                writer.add_document(document).unwrap();
1056            }
1057            writer.commit().await.unwrap();
1058            let index = Index::open(directory, config).await.unwrap();
1059            let query = index.query_parser().parse("emb:sparse({0: 1.0})").unwrap();
1060            let actual = index.search(query.as_ref(), 9).await.unwrap();
1061            let expected = index
1062                .search(
1063                    &SparseVectorQuery::new(field, vec![(0, 1.0)]).with_exhaustive(exhaustive),
1064                    9,
1065                )
1066                .await
1067                .unwrap();
1068            let hits = |result: crate::query::SearchResponse| {
1069                result
1070                    .hits
1071                    .into_iter()
1072                    .map(|hit| (hit.address.doc_id, hit.score.to_bits()))
1073                    .collect::<Vec<_>>()
1074            };
1075            assert_eq!(actual.hits.len(), if exhaustive { 9 } else { 1 });
1076            assert_eq!(hits(actual), hits(expected));
1077        }
1078    }
1079
1080    #[test]
1081    fn test_simple_term() {
1082        let (schema, default_fields, tokenizers) = setup();
1083        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
1084
1085        // Should parse without error - creates BooleanQuery across default fields
1086        let _query = parser.parse("rust").unwrap();
1087    }
1088
1089    #[test]
1090    fn test_field_term() {
1091        let (schema, default_fields, tokenizers) = setup();
1092        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
1093
1094        // Should parse field:term syntax
1095        let _query = parser.parse("title:rust").unwrap();
1096    }
1097
1098    #[test]
1099    fn test_boolean_and() {
1100        let (schema, default_fields, tokenizers) = setup();
1101        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
1102
1103        // Should parse AND boolean query
1104        let _query = parser.parse("rust AND programming").unwrap();
1105    }
1106
1107    #[test]
1108    fn test_match_query() {
1109        let (schema, default_fields, tokenizers) = setup();
1110        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
1111
1112        // Should tokenize and create boolean query
1113        let _query = parser.parse("hello world").unwrap();
1114    }
1115
1116    #[test]
1117    fn test_phrase_query() {
1118        let (schema, default_fields, tokenizers) = setup();
1119        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
1120
1121        // Should parse quoted phrase
1122        let _query = parser.parse("\"hello world\"").unwrap();
1123    }
1124
1125    #[test]
1126    fn test_boolean_or() {
1127        let (schema, default_fields, tokenizers) = setup();
1128        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
1129
1130        // Should parse OR boolean query
1131        let _query = parser.parse("rust OR python").unwrap();
1132    }
1133
1134    #[test]
1135    fn test_complex_query() {
1136        let (schema, default_fields, tokenizers) = setup();
1137        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
1138
1139        // Should parse complex boolean with grouping
1140        let _query = parser.parse("(rust OR python) AND programming").unwrap();
1141    }
1142
1143    #[test]
1144    fn test_router_exclusive_mode() {
1145        use crate::dsl::query_field_router::{QueryFieldRouter, QueryRouterRule, RoutingMode};
1146
1147        let mut builder = SchemaBuilder::default();
1148        let _title = builder.add_text_field("title", true, true);
1149        let _uri = builder.add_text_field("uri", true, true);
1150        let schema = Arc::new(builder.build());
1151        let tokenizers = Arc::new(TokenizerRegistry::default());
1152
1153        let router = QueryFieldRouter::from_rules(&[QueryRouterRule {
1154            pattern: r"^doi:(10\.\d{4,}/[^\s]+)$".to_string(),
1155            substitution: "doi://{1}".to_string(),
1156            target_field: "uri".to_string(),
1157            mode: RoutingMode::Exclusive,
1158        }])
1159        .unwrap();
1160
1161        let parser = QueryLanguageParser::with_router(schema, vec![], tokenizers, router);
1162
1163        // Should route DOI query to uri field
1164        let _query = parser.parse("doi:10.1234/test.123").unwrap();
1165    }
1166
1167    #[test]
1168    fn test_router_additional_mode() {
1169        use crate::dsl::query_field_router::{QueryFieldRouter, QueryRouterRule, RoutingMode};
1170
1171        let mut builder = SchemaBuilder::default();
1172        let title = builder.add_text_field("title", true, true);
1173        let _uri = builder.add_text_field("uri", true, true);
1174        let schema = Arc::new(builder.build());
1175        let tokenizers = Arc::new(TokenizerRegistry::default());
1176
1177        let router = QueryFieldRouter::from_rules(&[QueryRouterRule {
1178            pattern: r"#(\d+)".to_string(),
1179            substitution: "{1}".to_string(),
1180            target_field: "uri".to_string(),
1181            mode: RoutingMode::Additional,
1182        }])
1183        .unwrap();
1184
1185        let parser = QueryLanguageParser::with_router(schema, vec![title], tokenizers, router);
1186
1187        // Should route to both uri field and default fields
1188        let _query = parser.parse("#42").unwrap();
1189    }
1190
1191    #[test]
1192    fn test_router_no_match_falls_through() {
1193        use crate::dsl::query_field_router::{QueryFieldRouter, QueryRouterRule, RoutingMode};
1194
1195        let mut builder = SchemaBuilder::default();
1196        let title = builder.add_text_field("title", true, true);
1197        let _uri = builder.add_text_field("uri", true, true);
1198        let schema = Arc::new(builder.build());
1199        let tokenizers = Arc::new(TokenizerRegistry::default());
1200
1201        let router = QueryFieldRouter::from_rules(&[QueryRouterRule {
1202            pattern: r"^doi:".to_string(),
1203            substitution: "{0}".to_string(),
1204            target_field: "uri".to_string(),
1205            mode: RoutingMode::Exclusive,
1206        }])
1207        .unwrap();
1208
1209        let parser = QueryLanguageParser::with_router(schema, vec![title], tokenizers, router);
1210
1211        // Should NOT match and fall through to normal parsing
1212        let _query = parser.parse("rust programming").unwrap();
1213    }
1214
1215    #[test]
1216    fn test_router_invalid_target_field() {
1217        use crate::dsl::query_field_router::{QueryFieldRouter, QueryRouterRule, RoutingMode};
1218
1219        let mut builder = SchemaBuilder::default();
1220        let _title = builder.add_text_field("title", true, true);
1221        let schema = Arc::new(builder.build());
1222        let tokenizers = Arc::new(TokenizerRegistry::default());
1223
1224        let router = QueryFieldRouter::from_rules(&[QueryRouterRule {
1225            pattern: r"test".to_string(),
1226            substitution: "{0}".to_string(),
1227            target_field: "nonexistent".to_string(),
1228            mode: RoutingMode::Exclusive,
1229        }])
1230        .unwrap();
1231
1232        let parser = QueryLanguageParser::with_router(schema, vec![], tokenizers, router);
1233
1234        // Should fail because target field doesn't exist
1235        let result = parser.parse("test");
1236        assert!(result.is_err());
1237        let err = result.err().unwrap();
1238        assert!(err.contains("Unknown target field"));
1239    }
1240
1241    #[test]
1242    fn test_parse_ann_query() {
1243        let mut builder = SchemaBuilder::default();
1244        let embedding = builder.add_dense_vector_field("embedding", 128, true, true);
1245        let schema = Arc::new(builder.build());
1246        let tokenizers = Arc::new(TokenizerRegistry::default());
1247
1248        let parser = QueryLanguageParser::new(schema, vec![embedding], tokenizers);
1249
1250        // Parse ANN query
1251        let result = parser.parse_query_string("embedding:ann([1.0, 2.0, 3.0], nprobe=32)");
1252        assert!(result.is_ok(), "Failed to parse ANN query: {:?}", result);
1253
1254        if let Ok(ParsedQuery::Ann {
1255            field,
1256            vector,
1257            nprobe,
1258            rerank,
1259        }) = result
1260        {
1261            assert_eq!(field, "embedding");
1262            assert_eq!(vector, vec![1.0, 2.0, 3.0]);
1263            assert_eq!(nprobe, 32);
1264            assert_eq!(rerank, 2.0); // default
1265        } else {
1266            panic!("Expected Ann query, got: {:?}", result);
1267        }
1268    }
1269
1270    #[test]
1271    fn test_parse_sparse_query() {
1272        let mut builder = SchemaBuilder::default();
1273        let sparse = builder.add_text_field("sparse", true, true);
1274        let schema = Arc::new(builder.build());
1275        let tokenizers = Arc::new(TokenizerRegistry::default());
1276
1277        let parser = QueryLanguageParser::new(schema, vec![sparse], tokenizers);
1278
1279        // Parse sparse query
1280        let result = parser.parse_query_string("sparse:sparse({1: 0.5, 5: 0.3})");
1281        assert!(result.is_ok(), "Failed to parse sparse query: {:?}", result);
1282
1283        if let Ok(ParsedQuery::Sparse { field, vector }) = result {
1284            assert_eq!(field, "sparse");
1285            assert_eq!(vector, vec![(1, 0.5), (5, 0.3)]);
1286        } else {
1287            panic!("Expected Sparse query, got: {:?}", result);
1288        }
1289    }
1290
1291    #[test]
1292    fn test_parse_prefix_simple() {
1293        let (schema, default_fields, tokenizers) = setup();
1294        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
1295
1296        // Simple prefix: title:abc*
1297        let result = parser.parse_query_string("title:abc*");
1298        assert!(result.is_ok(), "Failed to parse prefix query: {:?}", result);
1299        if let Ok(ParsedQuery::Prefix { field, prefix }) = result {
1300            assert_eq!(field, Some("title".to_string()));
1301            assert_eq!(prefix, "abc");
1302        } else {
1303            panic!("Expected Prefix query, got: {:?}", result);
1304        }
1305    }
1306
1307    #[test]
1308    fn test_parse_prefix_url() {
1309        let mut builder = SchemaBuilder::default();
1310        let _site = builder.add_text_field("site", true, true);
1311        let schema = Arc::new(builder.build());
1312        let tokenizers = Arc::new(TokenizerRegistry::default());
1313        let parser = QueryLanguageParser::new(schema, vec![], tokenizers);
1314
1315        // URL prefix: site:https://reddit.com/r/Transhumanism*
1316        let result = parser.parse_query_string("site:https://reddit.com/r/Transhumanism*");
1317        assert!(
1318            result.is_ok(),
1319            "Failed to parse URL prefix query: {:?}",
1320            result
1321        );
1322        if let Ok(ParsedQuery::Prefix { field, prefix }) = result {
1323            assert_eq!(field, Some("site".to_string()));
1324            assert_eq!(prefix, "https://reddit.com/r/Transhumanism");
1325        } else {
1326            panic!("Expected Prefix query, got: {:?}", result);
1327        }
1328    }
1329
1330    #[test]
1331    fn test_parse_prefix_unqualified() {
1332        let (schema, default_fields, tokenizers) = setup();
1333        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
1334
1335        // Unqualified prefix: transhuman*
1336        let result = parser.parse_query_string("transhuman*");
1337        assert!(
1338            result.is_ok(),
1339            "Failed to parse unqualified prefix: {:?}",
1340            result
1341        );
1342        if let Ok(ParsedQuery::Prefix { field, prefix }) = result {
1343            assert_eq!(field, None);
1344            assert_eq!(prefix, "transhuman");
1345        } else {
1346            panic!("Expected Prefix query, got: {:?}", result);
1347        }
1348    }
1349
1350    #[test]
1351    fn test_prefix_query_builds() {
1352        let (schema, default_fields, tokenizers) = setup();
1353        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
1354
1355        // Should build without error
1356        let _query = parser.parse("title:abc*").unwrap();
1357    }
1358
1359    #[test]
1360    fn test_prefix_in_boolean() {
1361        let (schema, default_fields, tokenizers) = setup();
1362        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
1363
1364        // Prefix in boolean: rust AND title:abc*
1365        let _query = parser.parse("rust AND title:abc*").unwrap();
1366    }
1367
1368    #[test]
1369    fn test_prefix_mixed_with_terms() {
1370        let mut builder = SchemaBuilder::default();
1371        let title = builder.add_text_field("title", true, true);
1372        let _site = builder.add_text_field("site", true, true);
1373        let schema = Arc::new(builder.build());
1374        let tokenizers = Arc::new(TokenizerRegistry::default());
1375        let parser = QueryLanguageParser::new(schema, vec![title], tokenizers);
1376
1377        // Mixed: prefix + free-text terms (implicit OR)
1378        let result =
1379            parser.parse_query_string("site:https://reddit.com/r/Transhumanism* longevity drugs");
1380        assert!(
1381            result.is_ok(),
1382            "Failed to parse mixed prefix+terms: {:?}",
1383            result
1384        );
1385        // Should be Or([Prefix, Term, Term])
1386        if let Ok(ParsedQuery::Or(parts)) = &result {
1387            assert_eq!(parts.len(), 3, "Expected 3 parts, got: {:?}", parts);
1388            assert!(
1389                matches!(&parts[0], ParsedQuery::And(v) if v.len() == 1 && matches!(&v[0], ParsedQuery::Prefix { .. }))
1390                    || matches!(&parts[0], ParsedQuery::Prefix { .. }),
1391                "First part should be prefix: {:?}",
1392                parts[0]
1393            );
1394        } else {
1395            panic!("Expected Or query, got: {:?}", result);
1396        }
1397
1398        // Should also build into a Query without error
1399        let _query = parser
1400            .parse("site:https://reddit.com/r/Transhumanism* longevity drugs")
1401            .unwrap();
1402    }
1403
1404    #[test]
1405    fn boolean_keywords_do_not_split_longer_terms() {
1406        let (schema, default_fields, tokenizers) = setup();
1407        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
1408        for word in ["NOTHING", "NOT_name", "NOT-thing", "NOT42", "NOTé"] {
1409            let parsed = parser.parse_query_string(word).unwrap();
1410            assert!(
1411                matches!(&parsed, ParsedQuery::Term { term, .. } if term == word),
1412                "{word}: {parsed:?}"
1413            );
1414        }
1415        for word in ["ANDROID", "ORCHID", "AND_name", "OR-thing", "AND42", "ORé"] {
1416            let parsed = parser.parse_query_string(&format!("alpha {word}")).unwrap();
1417            let ParsedQuery::Or(parts) = &parsed else {
1418                panic!("{word}: {parsed:?}");
1419            };
1420            assert_eq!(parts.len(), 2, "{word}: {parsed:?}");
1421            assert!(
1422                matches!(&parts[1], ParsedQuery::Term { term, .. } if term == word),
1423                "{word}: {parsed:?}"
1424            );
1425        }
1426        assert!(
1427            matches!(parser.parse_query_string("NOT:term").unwrap(), ParsedQuery::Term { field: Some(field), term } if field == "NOT" && term == "term")
1428        );
1429        for prefix in ["NOT", "NOT/path", "NOT@example", "NOT.thing"] {
1430            assert!(
1431                matches!(parser.parse_query_string(&format!("{prefix}*")).unwrap(), ParsedQuery::Prefix { prefix: value, .. } if value == prefix)
1432            );
1433        }
1434        assert!(matches!(
1435            parser.parse_query_string("alpha AND(beta)").unwrap(),
1436            ParsedQuery::And(_)
1437        ));
1438        assert!(matches!(
1439            parser.parse_query_string("NOT(alpha)").unwrap(),
1440            ParsedQuery::Not(_)
1441        ));
1442        assert!(matches!(
1443            parser.parse_query_string("alpha OR(beta)").unwrap(),
1444            ParsedQuery::Or(_)
1445        ));
1446    }
1447
1448    #[test]
1449    fn test_implicit_or_plain_terms() {
1450        let (schema, default_fields, tokenizers) = setup();
1451        let parser = QueryLanguageParser::new(schema, default_fields, tokenizers);
1452
1453        // Space-separated terms: implicit OR
1454        let result = parser.parse_query_string("hello world");
1455        assert!(result.is_ok(), "Failed to parse implicit OR: {:?}", result);
1456        if let Ok(ParsedQuery::Or(parts)) = &result {
1457            assert_eq!(parts.len(), 2);
1458        } else {
1459            panic!("Expected Or query, got: {:?}", result);
1460        }
1461    }
1462}