1use crate::parser::ast::stmt::Stmt;
7use crate::parser::ast::stmt::{OrderByClause, OrderByItem};
8use crate::parser::ast::types::{LimitClause, OrderDirection, SkipClause};
9use crate::parser::ast::vector::{
10 CreateVectorIndex, DropVectorIndex, LookupVector, MatchVector, SearchVectorStatement,
11 VectorDistance, VectorIndexConfig, VectorMatchCondition, VectorQueryExpr, VectorQueryType,
12 VectorYieldClause, VectorYieldItem,
13};
14use crate::parser::parsing::expr_parser::parse_expression_with_context;
15use crate::parser::parsing::parse_context::ParseContext;
16use crate::parser::TokenKind;
17use linkrs_core::types::expr::{create_contextual_expression, Expression};
18
19pub fn parse_vector(ctx: &mut ParseContext) -> Result<Stmt, crate::parser::ParseError> {
21 if ctx.check_keyword("CREATE") {
22 return parse_create_vector_index(ctx);
23 } else if ctx.check_keyword("DROP") {
24 return parse_drop_vector_index(ctx);
25 } else if ctx.check_keyword("SEARCH") {
26 return parse_search_vector_statement(ctx);
27 } else if ctx.check_keyword("LOOKUP") {
28 return parse_lookup_vector(ctx);
29 } else if ctx.check_keyword("MATCH") {
30 return parse_match_vector(ctx);
31 }
32
33 Err(crate::parser::ParseError::new(
34 crate::parser::core::error::ParseErrorKind::SyntaxError,
35 "Not a vector search statement".to_string(),
36 ctx.current_position(),
37 ))
38}
39
40pub fn parse_create_vector_index(
42 ctx: &mut ParseContext,
43) -> Result<Stmt, crate::parser::ParseError> {
44 ctx.consume_keyword("CREATE")?;
45 parse_create_vector_index_after_create(ctx)
46}
47
48pub fn parse_create_vector_index_after_create(
49 ctx: &mut ParseContext,
50) -> Result<Stmt, crate::parser::ParseError> {
51 let if_not_exists = if ctx.check_keyword("IF") {
52 ctx.consume_keyword("IF")?;
53 ctx.consume_keyword("NOT")?;
54 ctx.consume_keyword("EXISTS")?;
55 true
56 } else {
57 false
58 };
59
60 ctx.consume_keyword("VECTOR")?;
61 ctx.consume_keyword("INDEX")?;
62
63 let index_name = ctx.consume_identifier()?;
64 ctx.consume_keyword("ON")?;
65 let schema_name = ctx.consume_identifier()?;
66
67 ctx.expect_token(TokenKind::LParen)?;
68 let field_name = ctx.consume_identifier()?;
69 ctx.expect_token(TokenKind::RParen)?;
70
71 ctx.expect_token(TokenKind::With)?;
72 let config = parse_vector_index_config(ctx)?;
73
74 let mut create = CreateVectorIndex::new(
75 ctx.current_span(),
76 index_name,
77 schema_name,
78 field_name,
79 config,
80 );
81 create.if_not_exists = if_not_exists;
82
83 Ok(Stmt::CreateVectorIndex(create))
84}
85
86fn parse_vector_index_config(
88 ctx: &mut ParseContext,
89) -> Result<VectorIndexConfig, crate::parser::ParseError> {
90 ctx.expect_token(TokenKind::LParen)?;
91
92 let mut vector_size = None;
93 let mut distance = VectorDistance::Cosine;
94 let mut hnsw_m = None;
95 let mut hnsw_ef_construct = None;
96 let mut quantization: Option<crate::parser::ast::vector::QuantizationKind> = None;
97 let mut quantile: Option<f32> = None;
98 let mut compression: Option<crate::parser::ast::vector::CompressionRatioKind> = None;
99 let mut always_ram: Option<bool> = None;
100
101 loop {
102 let key = ctx.consume_identifier()?;
103 ctx.expect_token(TokenKind::Assign)?;
104
105 match key.to_lowercase().as_str() {
106 "vector_size" => {
107 vector_size = Some(ctx.consume_int()? as usize);
108 }
109 "distance" => {
110 let dist_str = if matches!(ctx.current_token().kind, TokenKind::StringLiteral(_)) {
112 ctx.consume_string()?
113 } else {
114 ctx.consume_identifier()?
115 };
116 distance = match dist_str.to_lowercase().as_str() {
117 "cosine" => VectorDistance::Cosine,
118 "euclidean" => VectorDistance::Euclidean,
119 "dot" => VectorDistance::Dot,
120 "manhattan" => VectorDistance::Manhattan,
121 _ => {
122 return Err(crate::parser::ParseError::new(
123 crate::parser::core::error::ParseErrorKind::SyntaxError,
124 format!("Unknown distance metric '{}'", dist_str),
125 ctx.current_position(),
126 ))
127 }
128 };
129 }
130 "hnsw_m" => {
131 hnsw_m = Some(ctx.consume_int()? as usize);
132 }
133 "hnsw_ef_construct" => {
134 hnsw_ef_construct = Some(ctx.consume_int()? as usize);
135 }
136 "quantization" | "quantization_type" | "quant_type" => {
137 let q_str = if matches!(ctx.current_token().kind, TokenKind::StringLiteral(_)) {
139 ctx.consume_string()?
140 } else {
141 ctx.consume_identifier()?
142 };
143 match q_str.to_lowercase().as_str() {
144 "none" | "disabled" | "off" => quantization = None,
145 "scalar" => {
146 quantization = Some(crate::parser::ast::vector::QuantizationKind::Scalar)
147 }
148 "binary" => {
149 quantization = Some(crate::parser::ast::vector::QuantizationKind::Binary)
150 }
151 "product" | "pq" => {
152 quantization = Some(crate::parser::ast::vector::QuantizationKind::Product)
153 }
154 _ => {
155 return Err(crate::parser::ParseError::new(
156 crate::parser::core::error::ParseErrorKind::SyntaxError,
157 format!(
158 "Unknown quantization '{}', expected scalar/binary/product/none",
159 q_str
160 ),
161 ctx.current_position(),
162 ))
163 }
164 }
165 }
166 "quantile" => {
167 let v = ctx.consume_float()? as f32;
168 if !v.is_finite() || v <= 0.0 || v > 1.0 {
169 return Err(crate::parser::ParseError::new(
170 crate::parser::core::error::ParseErrorKind::SyntaxError,
171 format!("quantile {} must be in (0,1]", v),
172 ctx.current_position(),
173 ));
174 }
175 quantile = Some(v);
176 }
177 "compression" | "pq_compression" => {
178 let c_str = if matches!(ctx.current_token().kind, TokenKind::StringLiteral(_)) {
180 ctx.consume_string()?
181 } else if matches!(
182 ctx.current_token().kind,
183 TokenKind::FloatLiteral(_) | TokenKind::IntegerLiteral(_)
184 ) {
185 let n = ctx.consume_int()?;
187 format!("x{}", n)
188 } else {
189 ctx.consume_identifier()?
190 };
191 compression = Some(match c_str.to_lowercase().as_str() {
192 "x4" | "4" => crate::parser::ast::vector::CompressionRatioKind::X4,
193 "x8" | "8" => crate::parser::ast::vector::CompressionRatioKind::X8,
194 "x16" | "16" => crate::parser::ast::vector::CompressionRatioKind::X16,
195 "x32" | "32" => crate::parser::ast::vector::CompressionRatioKind::X32,
196 "x64" | "64" => crate::parser::ast::vector::CompressionRatioKind::X64,
197 _ => {
198 return Err(crate::parser::ParseError::new(
199 crate::parser::core::error::ParseErrorKind::SyntaxError,
200 format!(
201 "Unknown compression '{}', expected x4/x8/x16/x32/x64",
202 c_str
203 ),
204 ctx.current_position(),
205 ))
206 }
207 });
208 }
209 "always_ram" => {
210 let b = match ctx.current_token().kind.clone() {
211 TokenKind::BooleanLiteral(v) => {
212 ctx.next_token();
213 v
214 }
215 TokenKind::StringLiteral(s) => {
216 ctx.next_token();
217 match s.to_lowercase().as_str() {
218 "true" | "1" | "yes" | "on" => true,
219 "false" | "0" | "no" | "off" => false,
220 _ => {
221 return Err(crate::parser::ParseError::new(
222 crate::parser::core::error::ParseErrorKind::SyntaxError,
223 format!("Unknown always_ram '{}', expected true/false", s),
224 ctx.current_position(),
225 ))
226 }
227 }
228 }
229 TokenKind::Identifier(s) => {
230 let lower = s.to_lowercase();
231 ctx.next_token();
232 match lower.as_str() {
233 "true" | "1" | "yes" | "on" => true,
234 "false" | "0" | "no" | "off" => false,
235 _ => {
236 return Err(crate::parser::ParseError::new(
237 crate::parser::core::error::ParseErrorKind::SyntaxError,
238 format!("Unknown always_ram '{}', expected true/false", s),
239 ctx.current_position(),
240 ))
241 }
242 }
243 }
244 _ => {
245 return Err(crate::parser::ParseError::new(
246 crate::parser::core::error::ParseErrorKind::SyntaxError,
247 "always_ram expects true/false".to_string(),
248 ctx.current_position(),
249 ))
250 }
251 };
252 always_ram = Some(b);
253 }
254 _ => {
255 return Err(crate::parser::ParseError::new(
256 crate::parser::core::error::ParseErrorKind::SyntaxError,
257 format!("Unknown config option '{}'", key),
258 ctx.current_position(),
259 ))
260 }
261 }
262
263 if !ctx.consume_optional_token(",") {
264 break;
265 }
266 }
267
268 ctx.expect_token(TokenKind::RParen)?;
269
270 let vector_size = vector_size.ok_or_else(|| {
271 crate::parser::ParseError::new(
272 crate::parser::core::error::ParseErrorKind::SyntaxError,
273 "vector_size is required".to_string(),
274 ctx.current_position(),
275 )
276 })?;
277
278 let mut config = VectorIndexConfig::new(vector_size, distance);
279 if let Some(m) = hnsw_m {
280 config.hnsw_m = Some(m);
281 }
282 if let Some(ef) = hnsw_ef_construct {
283 config.hnsw_ef_construct = Some(ef);
284 }
285 if let Some(q) = quantization {
287 config.quantization = Some(q);
288 config.quantile = quantile;
289 config.compression = compression;
290 config.always_ram = always_ram;
291 } else if quantile.is_some() || compression.is_some() || always_ram.is_some() {
292 return Err(crate::parser::ParseError::new(
293 crate::parser::core::error::ParseErrorKind::SyntaxError,
294 "quantile/compression/always_ram require quantization='scalar'/'binary'/'product'"
295 .to_string(),
296 ctx.current_position(),
297 ));
298 }
299
300 Ok(config)
301}
302
303pub fn parse_drop_vector_index(ctx: &mut ParseContext) -> Result<Stmt, crate::parser::ParseError> {
305 ctx.consume_keyword("DROP")?;
306 parse_drop_vector_index_after_drop(ctx)
307}
308
309pub fn parse_drop_vector_index_after_drop(
310 ctx: &mut ParseContext,
311) -> Result<Stmt, crate::parser::ParseError> {
312 ctx.consume_keyword("VECTOR")?;
313 ctx.consume_keyword("INDEX")?;
314
315 let if_exists = if ctx.check_keyword("IF") {
316 ctx.consume_keyword("IF")?;
317 ctx.consume_keyword("EXISTS")?;
318 true
319 } else {
320 false
321 };
322
323 let index_name = ctx.consume_identifier()?;
324
325 let drop = DropVectorIndex {
326 span: ctx.current_span(),
327 index_name,
328 if_exists,
329 };
330
331 Ok(Stmt::DropVectorIndex(drop))
332}
333
334pub fn parse_search_vector_statement(
336 ctx: &mut ParseContext,
337) -> Result<Stmt, crate::parser::ParseError> {
338 let span = ctx.current_span();
339
340 ctx.consume_keyword("SEARCH")?;
341 ctx.consume_keyword("VECTOR")?;
342
343 let index_name = ctx.consume_identifier()?;
344
345 ctx.consume_keyword("WITH")?;
346 let query = parse_vector_query_expr(ctx)?;
347
348 let mut threshold = None;
349 if ctx.check_keyword("THRESHOLD") {
350 ctx.consume_keyword("THRESHOLD")?;
351 threshold = Some(ctx.consume_float()? as f32);
352 }
353
354 let mut where_clause = None;
355 if ctx.check_keyword("WHERE") {
356 ctx.consume_keyword("WHERE")?;
357 where_clause = Some(parse_expression_with_context(
358 ctx,
359 ctx.expression_context_clone(),
360 )?);
361 }
362
363 let mut order_clause = None;
364 if ctx.check_keyword("ORDER") {
365 ctx.consume_keyword("ORDER")?;
366 ctx.consume_keyword("BY")?;
367 order_clause = Some(parse_order_clause(ctx)?);
368 }
369
370 let mut limit = None;
371 if ctx.check_keyword("LIMIT") {
372 ctx.consume_keyword("LIMIT")?;
373 let count = ctx.consume_int()? as usize;
374 limit = Some(LimitClause {
375 span: ctx.current_span(),
376 count,
377 });
378 }
379
380 let mut skip = None;
381 if ctx.check_keyword("OFFSET") {
382 ctx.consume_keyword("OFFSET")?;
383 let count = ctx.consume_int()? as usize;
384 skip = Some(SkipClause {
385 span: ctx.current_span(),
386 count,
387 });
388 }
389
390 let mut yield_clause = None;
391 if ctx.check_keyword("YIELD") || ctx.check_keyword("RETURN") {
392 if ctx.check_keyword("YIELD") {
393 ctx.consume_keyword("YIELD")?;
394 } else {
395 ctx.consume_keyword("RETURN")?;
396 }
397 yield_clause = Some(parse_vector_yield_clause(ctx)?);
398 }
399
400 Ok(Stmt::SearchVector(SearchVectorStatement {
401 span,
402 index_name,
403 query,
404 threshold,
405 where_clause,
406 order_clause,
407 limit,
408 skip,
409 yield_clause,
410 }))
411}
412
413fn parse_vector_query_expr(
415 ctx: &mut ParseContext,
416) -> Result<VectorQueryExpr, crate::parser::ParseError> {
417 let span = ctx.current_span();
418
419 let keyword = ctx.consume_identifier()?;
420 if ctx.expect_token(TokenKind::Eq).is_err() {
421 ctx.expect_token(TokenKind::Assign)?;
422 }
423
424 let (query_type, query_data) = if keyword.to_lowercase() == "vector" {
425 let vector_str = parse_vector_literal(ctx)?;
427 (VectorQueryType::Vector, vector_str)
428 } else if keyword.to_lowercase() == "text" {
429 let text = ctx.consume_string()?;
431 (VectorQueryType::Text, text)
432 } else if keyword.to_lowercase() == "param" || keyword.to_lowercase() == "parameter" {
433 ctx.expect_token(TokenKind::Dollar)?;
435 let param = ctx.expect_identifier()?;
436 (VectorQueryType::Parameter, param)
437 } else {
438 return Err(crate::parser::ParseError::new(
439 crate::parser::core::error::ParseErrorKind::SyntaxError,
440 format!("Expected 'vector', 'text', or 'param', found '{}'", keyword),
441 ctx.current_position(),
442 ));
443 };
444
445 Ok(VectorQueryExpr {
446 span,
447 query_type,
448 query_data,
449 })
450}
451
452fn parse_vector_literal(ctx: &mut ParseContext) -> Result<String, crate::parser::ParseError> {
454 ctx.expect_token(TokenKind::LBracket)?;
455 let mut elements = Vec::new();
456
457 loop {
458 let num = ctx.consume_float()?;
459 elements.push(format!("{}", num));
460
461 if !ctx.consume_optional_token(",") {
462 break;
463 }
464 }
465
466 ctx.expect_token(TokenKind::RBracket)?;
467 Ok(format!("[{}]", elements.join(", ")))
468}
469
470fn parse_order_clause(ctx: &mut ParseContext) -> Result<OrderByClause, crate::parser::ParseError> {
472 let span = ctx.current_span();
473 let mut items = Vec::new();
474
475 loop {
476 let expression = parse_expression_with_context(ctx, ctx.expression_context_clone())?;
477 let direction = if ctx.check_keyword("DESC") {
478 ctx.consume_keyword("DESC")?;
479 OrderDirection::Desc
480 } else {
481 let _ = ctx.consume_keyword("ASC");
482 OrderDirection::Asc
483 };
484
485 items.push(OrderByItem {
486 expression,
487 direction,
488 });
489
490 if !ctx.consume_optional_token(",") {
491 break;
492 }
493 }
494
495 Ok(OrderByClause { span, items })
496}
497
498fn parse_vector_yield_clause(
500 ctx: &mut ParseContext,
501) -> Result<VectorYieldClause, crate::parser::ParseError> {
502 let mut items = Vec::new();
503
504 loop {
505 let expr = if ctx.consume_optional_token("*") {
506 create_contextual_expression(Expression::variable("*"))
507 } else {
508 parse_expression_with_context(ctx, ctx.expression_context_clone())?
509 };
510 let alias = if ctx.check_keyword("AS") {
511 ctx.consume_keyword("AS")?;
512 Some(ctx.consume_identifier()?)
513 } else {
514 None
515 };
516
517 items.push(VectorYieldItem { expr, alias });
518
519 if !ctx.consume_optional_token(",") {
520 break;
521 }
522 }
523
524 Ok(VectorYieldClause { items })
525}
526
527pub fn parse_lookup_vector(ctx: &mut ParseContext) -> Result<Stmt, crate::parser::ParseError> {
529 let span = ctx.current_span();
530
531 ctx.consume_keyword("LOOKUP")?;
532 ctx.consume_keyword("VECTOR")?;
533
534 let schema_name = ctx.consume_identifier()?;
535 let index_name = ctx.consume_identifier()?;
536
537 ctx.consume_keyword("WITH")?;
538 let query = parse_vector_query_expr(ctx)?;
539
540 let mut yield_clause = None;
541 if ctx.check_keyword("YIELD") || ctx.check_keyword("RETURN") {
542 if ctx.check_keyword("YIELD") {
543 ctx.consume_keyword("YIELD")?;
544 } else {
545 ctx.consume_keyword("RETURN")?;
546 }
547 yield_clause = Some(parse_vector_yield_clause(ctx)?);
548 }
549
550 let mut limit = None;
551 if ctx.check_keyword("LIMIT") {
552 ctx.consume_keyword("LIMIT")?;
553 limit = Some(ctx.consume_int()? as usize);
554 }
555
556 Ok(Stmt::LookupVector(LookupVector {
557 span,
558 schema_name,
559 index_name,
560 query,
561 yield_clause,
562 limit,
563 }))
564}
565
566pub fn parse_match_vector(ctx: &mut ParseContext) -> Result<Stmt, crate::parser::ParseError> {
568 let span = ctx.current_span();
569
570 ctx.consume_keyword("MATCH")?;
571 ctx.consume_keyword("VECTOR")?;
574
575 let pattern = ctx.consume_string()?;
577
578 ctx.consume_keyword("WHERE")?;
579
580 let field = ctx.consume_identifier()?;
582 let query = parse_vector_query_expr(ctx)?;
583
584 let mut threshold = None;
585 if ctx.check_keyword("THRESHOLD") {
586 ctx.consume_keyword("THRESHOLD")?;
587 threshold = Some(ctx.consume_float()? as f32);
588 }
589
590 let vector_condition = VectorMatchCondition {
591 field,
592 query,
593 threshold,
594 };
595
596 let mut yield_clause = None;
597 if ctx.check_keyword("YIELD") || ctx.check_keyword("RETURN") {
598 if ctx.check_keyword("YIELD") {
599 ctx.consume_keyword("YIELD")?;
600 } else {
601 ctx.consume_keyword("RETURN")?;
602 }
603 yield_clause = Some(parse_vector_yield_clause(ctx)?);
604 }
605
606 Ok(Stmt::MatchVector(MatchVector {
607 span,
608 pattern,
609 vector_condition,
610 yield_clause,
611 }))
612}
613
614#[cfg(test)]
615mod tests {
616 use crate::parser::ast::stmt::Stmt;
617 use crate::parser::parsing::parser::Parser;
618
619 #[test]
620 fn test_parse_search_vector_statement() {
621 let sql = r#"SEARCH VECTOR idx_product_embedding WITH vector=[0.1, 0.2, 0.3]
622 WHERE price < 500 AND score > 0.5
623 ORDER BY price DESC
624 YIELD product_id, name, price
625 LIMIT 10"#;
626
627 let mut parser = Parser::new(sql);
628 let result = parser.parse().expect("SEARCH VECTOR should parse");
629 let stmt = result.ast.stmt();
630 let search = match stmt {
631 Stmt::SearchVector(stmt) => stmt,
632 _ => panic!("expected SEARCH VECTOR statement"),
633 };
634
635 assert!(search.where_clause.is_some());
636 assert!(search.where_clause.as_ref().unwrap().is_binary());
637 assert!(search.order_clause.is_some());
638 assert_eq!(search.order_clause.as_ref().unwrap().items.len(), 1);
639
640 let yield_items = search.yield_clause.as_ref().expect("YIELD should parse");
641 assert_eq!(yield_items.items.len(), 3);
642 }
643
644 #[test]
645 fn test_parse_drop_vector_index_statement() {
646 for (sql, if_exists) in [
647 ("DROP VECTOR INDEX idx_product_embedding", false),
648 ("DROP VECTOR INDEX IF EXISTS idx_product_embedding", true),
649 ] {
650 let mut parser = Parser::new(sql);
651 let result = parser.parse().expect("DROP VECTOR INDEX should parse");
652 match result.ast.stmt() {
653 Stmt::DropVectorIndex(drop) => {
654 assert_eq!(drop.index_name, "idx_product_embedding");
655 assert_eq!(drop.if_exists, if_exists);
656 }
657 other => panic!("expected DROP VECTOR INDEX statement, got {:?}", other),
658 }
659 }
660 }
661
662 #[test]
663 fn test_parse_lookup_vector_statement() {
664 let sql = r#"LOOKUP VECTOR basketball idx_product_embedding WITH vector=[0.1, 0.2]
665 YIELD product_id
666 LIMIT 5"#;
667
668 let mut parser = Parser::new(sql);
669 let result = parser.parse().expect("LOOKUP VECTOR should parse");
670 match result.ast.stmt() {
671 Stmt::LookupVector(lookup) => {
672 assert_eq!(lookup.schema_name, "basketball");
673 assert_eq!(lookup.index_name, "idx_product_embedding");
674 assert_eq!(lookup.limit, Some(5));
675 assert!(lookup.yield_clause.is_some());
676 }
677 other => panic!("expected LOOKUP VECTOR statement, got {:?}", other),
678 }
679 }
680
681 #[test]
682 fn test_parse_match_vector_statement() {
683 let sql = r#"MATCH VECTOR "(n:Person)" WHERE embedding vector=[0.1, 0.2]
684 THRESHOLD 0.8"#;
685
686 let mut parser = Parser::new(sql);
687 let result = parser.parse().expect("MATCH VECTOR should parse");
688 match result.ast.stmt() {
689 Stmt::MatchVector(match_stmt) => {
690 assert_eq!(match_stmt.pattern, "(n:Person)");
691 assert_eq!(match_stmt.vector_condition.field, "embedding");
692 assert_eq!(match_stmt.vector_condition.threshold, Some(0.8));
693 }
694 other => panic!("expected MATCH VECTOR statement, got {:?}", other),
695 }
696 }
697
698 #[test]
699 fn test_plain_match_and_lookup_still_parse() {
700 let mut parser = Parser::new("MATCH (n:Person) RETURN n");
702 let match_result = parser.parse().expect("plain MATCH should parse");
703 assert!(
704 matches!(match_result.ast.stmt(), Stmt::Match(_)),
705 "plain MATCH stays a traversal"
706 );
707
708 let mut parser = Parser::new("LOOKUP ON TAG player");
710 let lookup_result = parser.parse().expect("plain LOOKUP ON TAG should parse");
711 assert!(
712 matches!(lookup_result.ast.stmt(), Stmt::Lookup(_)),
713 "plain LOOKUP stays a util lookup"
714 );
715 }
716}