1use crate::parser::lexing::LexError;
4use crate::parser::{Token, TokenKind as Tk};
5
6use super::Lexer;
7
8impl<'a> Lexer<'a> {
9 fn lookup_keyword(&self, identifier: &str) -> Tk {
10 match identifier.to_uppercase().as_str() {
11 "CREATE" => Tk::Create,
12 "MATCH" => Tk::Match,
13 "RETURN" => Tk::Return,
14 "WHERE" => Tk::Where,
15 "DELETE" => Tk::Delete,
16 "DETACH" => Tk::Detach,
17 "UPDATE" => Tk::Update,
18 "INSERT" => Tk::Insert,
19 "UPSERT" => Tk::Upsert,
20 "VALUES" => Tk::Values,
21 "FROM" => Tk::From,
22 "TO" => Tk::To,
23 "AS" => Tk::As,
24 "WITH" => Tk::With,
25 "YIELD" => Tk::Yield,
26 "GO" => Tk::Go,
27 "OVER" => Tk::Over,
28 "STEPS" | "STEP" => Tk::Step,
29 "UPTO" => Tk::Upto,
30 "LIMIT" => Tk::Limit,
31 "ASC" => Tk::Asc,
32 "DESC" | "DESCRIBE" => Tk::Desc,
33 "ORDER" => Tk::Order,
34 "BY" => Tk::By,
35 "SKIP" => Tk::Skip,
36 "UNWIND" => Tk::Unwind,
37 "OPTIONAL" => Tk::Optional,
38 "DISTINCT" => Tk::Distinct,
39 "ALL" => Tk::All,
40 "NULL" => Tk::Null,
41 "IS" => Tk::Is,
42 "NOT" => Tk::Not,
43 "AND" => Tk::And,
44 "OR" => Tk::Or,
45 "XOR" => Tk::Xor,
46 "CONTAINS" => Tk::Contains,
47 "STARTS" | "STARTS WITH" => Tk::StartsWith,
48 "ENDS" | "ENDS WITH" => Tk::EndsWith,
49 "CASE" => Tk::Case,
50 "WHEN" => Tk::When,
51 "THEN" => Tk::Then,
52 "ELSE" => Tk::Else,
53 "END" => Tk::End,
54 "UNION" => Tk::Union,
55 "INTERSECT" => Tk::Intersect,
56 "MINUS" => Tk::SetMinus,
57 "GROUP" => Tk::Group,
58 "HAVING" => Tk::Having,
59 "FILTER" => Tk::Filter,
60 "ROLLUP" => Tk::Rollup,
61 "CUBE" => Tk::Cube,
62 "GROUPING" => Tk::Grouping,
63 "SETS" => Tk::Sets,
64 "BETWEEN" => Tk::Between,
65 "ADMIN" => Tk::Admin,
66 "EDGE" => Tk::Edge,
67 "EDGES" => Tk::Edges,
68 "VERTEX" => Tk::Vertex,
69 "VERTICES" => Tk::Vertices,
70 "TAG" => Tk::Tag,
71 "TAGS" => Tk::Tags,
72 "INDEX" => Tk::Index,
73 "INDEXES" => Tk::Indexes,
74 "LOOKUP" => Tk::Lookup,
75 "FIND" => Tk::Find,
76 "WEIGHT" => Tk::Weight,
77 "PATH" => Tk::Path,
78 "SHORTEST" => Tk::Shortest,
79 "ALLSHORTESTPATHS" => Tk::AllShortestPaths,
80 "TRAIL" => Tk::Trail,
81 "WALK" => Tk::Walk,
82 "ACYCLIC" => Tk::Acyclic,
83 "WEIGHTED" => Tk::Weighted,
84 "LOOP" => Tk::Loop,
85 "CYCLE" => Tk::Cycle,
86 "SUBGRAPH" => Tk::Subgraph,
87 "BOTH" => Tk::Both,
88 "OUT" => Tk::Out,
89 "IN" => Tk::In,
90 "REVERSELY" => Tk::Reversely,
91 "RECURSIVE" => Tk::Recursive,
92 "NO" => Tk::No,
93 "OVERWRITE" => Tk::Overwrite,
94 "SHOW" => Tk::Show,
95 "ADD" => Tk::Add,
96 "DROP" => Tk::Drop,
97 "REMOVE" => Tk::Remove,
98 "ALTER" => Tk::Alter,
99 "IF" => Tk::If,
100 "EXISTS" => Tk::Exists,
101 "SUBQUERY" => Tk::Subquery,
102 "CHANGE" => Tk::Change,
103 "CREATEUSER" => Tk::CreateUser,
104 "ALTERUSER" => Tk::AlterUser,
105 "DROPUSER" => Tk::DropUser,
106 "CHANGEPASSWORD" => Tk::ChangePassword,
107 "GRANT" => Tk::Grant,
108 "REVOKE" => Tk::Revoke,
109 "ON" => Tk::On,
110 "OF" => Tk::Of,
111 "GET" => Tk::Get,
112 "SET" => Tk::Set,
113 "HOST" => Tk::Host,
114 "HOSTS" => Tk::Hosts,
115 "SPACE" => Tk::Space,
116 "SPACES" => Tk::Spaces,
117 "SEQUENCE" => Tk::Sequence,
118 "SEQUENCES" => Tk::Sequences,
119 "USER" => Tk::User,
120 "USERS" => Tk::Users,
121 "PASSWORD" => Tk::Password,
122 "ROLE" => Tk::Role,
123 "ROLES" => Tk::Roles,
124 "FUNCTIONS" => Tk::Functions,
125 "GRAPHS" => Tk::Graphs,
126 "MACROS" => Tk::Macros,
127 "LOCKED" => Tk::Locked,
128 "GOD" => Tk::God,
129 "DBA" => Tk::Dba,
130 "GUEST" => Tk::Guest,
131 "COMMENT" => Tk::Comment,
132 "CHARSET" => Tk::Charset,
133 "COLLATE" => Tk::Collate,
134 "COLLATION" => Tk::Collation,
135 "VID_TYPE" => Tk::VIdType,
136 "PARTITION_NUM" => Tk::PartitionNum,
137 "REPLICA_FACTOR" => Tk::ReplicaFactor,
138 "REBUILD" => Tk::Rebuild,
139 "BOOL" => Tk::Bool,
140 "INT" => Tk::Int,
141 "INT8" => Tk::Int8,
142 "INT16" => Tk::Int16,
143 "INT32" => Tk::Int32,
144 "INT64" => Tk::Int64,
145 "FLOAT" => Tk::Float,
146 "DOUBLE" => Tk::Double,
147 "STRING" => Tk::String,
148 "FIXED_STRING" => Tk::FixedString,
149 "TIMESTAMP" => Tk::Timestamp,
150 "DATE" => Tk::Date,
151 "TIME" => Tk::Time,
152 "DATETIME" => Tk::Datetime,
153 "SERIAL" => Tk::Serial,
154 "DURATION" => Tk::Duration,
155 "GEOGRAPHY" => Tk::Geography,
156 "POINT" => Tk::Point,
157 "LINESTRING" => Tk::Linestring,
158 "POLYGON" => Tk::Polygon,
159 "LIST" => Tk::List,
160 "MAP" => Tk::Map,
161 "STRUCT" => Tk::Struct,
162 "ARRAY" => Tk::Array,
163 "DOWNLOAD" => Tk::Download,
164 "HDFS" => Tk::HDFS,
165 "UUID" => Tk::UUID,
166 "CONFIGS" => Tk::Configs,
167 "FORCE" => Tk::Force,
168 "PART" => Tk::Part,
169 "PARTS" => Tk::Parts,
170 "DATA" => Tk::Data,
171 "LEADER" => Tk::Leader,
172 "JOBS" => Tk::Jobs,
173 "JOB" => Tk::Job,
174 "BIDIRECT" => Tk::Bidirect,
175 "STATS" => Tk::Stats,
176 "STATUS" => Tk::Status,
177 "RECOVER" => Tk::Recover,
178 "EXPLAIN" => Tk::Explain,
179 "PROFILE" => Tk::Profile,
180 "ANALYZE" => Tk::Analyze,
181 "FORMAT" => Tk::Format,
182 "ATOMIC_EDGE" => Tk::AtomicEdge,
183 "DEFAULT" => Tk::Default,
184 "FLUSH" => Tk::Flush,
185 "COMPACT" => Tk::Compact,
186 "SUBMIT" => Tk::Submit,
187 "ASCENDING" => Tk::Ascending,
188 "DESCENDING" => Tk::Descending,
189 "FETCH" => Tk::Fetch,
190 "PROP" => Tk::Prop,
191 "BALANCE" => Tk::Identifier("balance".to_string()),
192 "STOP" => Tk::Stop,
193 "REVERT" => Tk::Revert,
194 "USE" => Tk::Use,
195 "BEGIN" => Tk::Begin,
196 "COMMIT" => Tk::Commit,
197 "ROLLBACK" => Tk::Rollback,
198 "SAVEPOINT" => Tk::Savepoint,
199 "RELEASE" => Tk::Release,
200 "TRANSACTION" => Tk::Transaction,
201 "LET" => Tk::Let,
202 "READ" => Tk::Read,
203 "ONLY" => Tk::Only,
204 "WRITE" => Tk::Write,
205 "SETLIST" => Tk::SetList,
206 "CLEAR" => Tk::Clear,
207 "MERGE" => Tk::Merge,
208 "DIVIDE" => Tk::Divide,
209 "RENAME" => Tk::Rename,
210 "LOCAL" => Tk::Local,
211 "SESSIONS" => Tk::Sessions,
212 "SESSION" => Tk::Session,
213 "SAMPLE" => Tk::Sample,
214 "QUERIES" => Tk::Queries,
215 "QUERY" => Tk::Query,
216 "KILL" => Tk::Kill,
217 "TOP" => Tk::Top,
218 "TEXT" => Tk::Text,
219 "SEARCH" => Tk::Search,
220 "VECTOR" => Tk::KeywordVector,
221 "CLIENT" => Tk::Client,
222 "CLIENTS" => Tk::Clients,
223 "SIGN" => Tk::Sign,
224 "SERVICE" => Tk::Service,
225 "COUNT" => Tk::Count,
226 "SUM" => Tk::Sum,
227 "AVG" => Tk::Avg,
228 "MIN" => Tk::Min,
229 "MAX" => Tk::Max,
230 "TTL_DURATION" => Tk::TtlDuration,
231 "TTL_COL" => Tk::TtlCol,
232 "SOURCE" => Tk::Source,
233 "DESTINATION" => Tk::Destination,
234 "RANK" => Tk::Rank,
235 "INPUT" => Tk::Input,
236 "COPY" => Tk::Copy,
237 "HEADER" => Tk::Header,
238 "DELIMITER" => Tk::Delimiter,
239 "CSV" => Tk::Csv,
240 "TRUE" => Tk::BooleanLiteral(true),
241 "FALSE" => Tk::BooleanLiteral(false),
242 _ => Tk::Identifier(identifier.to_string()),
243 }
244 }
245 pub fn next_token(&mut self) -> Token {
246 self.skip_whitespace();
247
248 if let Some(&ch) = self.peek_char() {
249 if ch == '/' || ch == '-' {
250 if let Ok(()) = self.skip_comment() {
251 if let Some(&ch) = self.peek_char() {
252 if ch == '\n' || ch == '\0' {
253 return Token::new(Tk::Eof, String::new(), self.line, self.column);
254 }
255 }
256 }
257 }
258 }
259
260 self.skip_whitespace();
261
262 let token = match self.peek_char() {
263 Some(&'=') => {
264 self.read_char();
265 if let Some(&'=') = self.peek_char() {
266 self.read_char();
267 Token::new(Tk::Eq, "==".to_string(), self.line, self.column)
268 } else {
269 Token::new(Tk::Assign, "=".to_string(), self.line, self.column)
270 }
271 }
272 Some(&'+') => {
273 self.read_char();
274 Token::new(Tk::Plus, "+".to_string(), self.line, self.column)
275 }
276 Some(&'-') => {
277 self.read_char();
278 if let Some(&'>') = self.peek_char() {
279 self.read_char();
280 if let Some(&'>') = self.peek_char() {
281 self.read_char();
282 Token::new(Tk::ArrowRight, "->>".to_string(), self.line, self.column)
283 } else {
284 Token::new(Tk::Arrow, "->".to_string(), self.line, self.column)
285 }
286 } else {
287 Token::new(Tk::Minus, "-".to_string(), self.line, self.column)
288 }
289 }
290 Some(&'*') => {
291 self.read_char();
292 if let Some(&'*') = self.peek_char() {
293 self.read_char();
294 Token::new(Tk::Exp, "**".to_string(), self.line, self.column)
295 } else {
296 Token::new(Tk::Star, "*".to_string(), self.line, self.column)
297 }
298 }
299 Some(&'/') => {
300 self.read_char();
301 Token::new(Tk::Div, "/".to_string(), self.line, self.column)
302 }
303 Some(&'%') => {
304 self.read_char();
305 Token::new(Tk::Mod, "%".to_string(), self.line, self.column)
306 }
307 Some(&'!') => {
308 self.read_char();
309 if let Some(&'=') = self.peek_char() {
310 self.read_char();
311 Token::new(Tk::Ne, "!=".to_string(), self.line, self.column)
312 } else {
313 Token::new(Tk::NotOp, "!".to_string(), self.line, self.column)
314 }
315 }
316 Some(&'<') => {
317 self.read_char();
318 match self.peek_char() {
319 Some(&'-') => {
320 self.read_char();
321 Token::new(Tk::BackArrow, "<-".to_string(), self.line, self.column)
322 }
323 Some(&'=') => {
324 self.read_char();
325 Token::new(Tk::Le, "<=".to_string(), self.line, self.column)
326 }
327 Some(&'>') => {
328 self.read_char();
329 Token::new(Tk::Ne, "<>".to_string(), self.line, self.column)
330 }
331 Some(&'<') => {
332 self.read_char();
333 Token::new(Tk::ShiftLeft, "<<".to_string(), self.line, self.column)
334 }
335 _ => Token::new(Tk::Lt, "<".to_string(), self.line, self.column),
336 }
337 }
338 Some(&'>') => {
339 self.read_char();
340 if let Some(&'=') = self.peek_char() {
341 self.read_char();
342 Token::new(Tk::Ge, ">=".to_string(), self.line, self.column)
343 } else if let Some(&'>') = self.peek_char() {
344 self.read_char();
345 Token::new(Tk::ShiftRight, ">>".to_string(), self.line, self.column)
346 } else {
347 Token::new(Tk::Gt, ">".to_string(), self.line, self.column)
348 }
349 }
350 Some(&'~') => {
351 self.read_char();
352 if let Some(&'=') = self.peek_char() {
353 self.read_char();
354 Token::new(Tk::Regex, "=~".to_string(), self.line, self.column)
355 } else {
356 self.read_char();
357 Token::new(Tk::NotOp, "~".to_string(), self.line, self.column)
358 }
359 }
360 Some(&'(') => {
361 self.read_char();
362 Token::new(Tk::LParen, "(".to_string(), self.line, self.column)
363 }
364 Some(&')') => {
365 self.read_char();
366 Token::new(Tk::RParen, ")".to_string(), self.line, self.column)
367 }
368 Some(&'[') => {
369 self.read_char();
370 Token::new(Tk::LBracket, "[".to_string(), self.line, self.column)
371 }
372 Some(&']') => {
373 self.read_char();
374 Token::new(Tk::RBracket, "]".to_string(), self.line, self.column)
375 }
376 Some(&'{') => {
377 self.read_char();
378 Token::new(Tk::LBrace, "{".to_string(), self.line, self.column)
379 }
380 Some(&'}') => {
381 self.read_char();
382 Token::new(Tk::RBrace, "}".to_string(), self.line, self.column)
383 }
384 Some(&',') => {
385 self.read_char();
386 Token::new(Tk::Comma, ",".to_string(), self.line, self.column)
387 }
388 Some(&'.') => {
389 self.read_char();
390 if let Some(&'.') = self.peek_char() {
391 self.read_char();
392 Token::new(Tk::DotDot, "..".to_string(), self.line, self.column)
393 } else {
394 Token::new(Tk::Dot, ".".to_string(), self.line, self.column)
395 }
396 }
397 Some(&':') => {
398 self.read_char();
399 if let Some(&':') = self.peek_char() {
400 self.read_char();
401 Token::new(Tk::DoubleColon, "::".to_string(), self.line, self.column)
402 } else {
403 Token::new(Tk::Colon, ":".to_string(), self.line, self.column)
404 }
405 }
406 Some(&';') => {
407 self.read_char();
408 Token::new(Tk::Semicolon, ";".to_string(), self.line, self.column)
409 }
410 Some(&'?') => {
411 self.read_char();
412 Token::new(Tk::QMark, "?".to_string(), self.line, self.column)
413 }
414 Some(&'|') => {
415 self.read_char();
416 if let Some(&'|') = self.peek_char() {
417 self.read_char();
418 Token::new(Tk::DoublePipe, "||".to_string(), self.line, self.column)
419 } else {
420 Token::new(Tk::Pipe, "|".to_string(), self.line, self.column)
421 }
422 }
423 Some(&'@') => {
424 self.read_char();
425 Token::new(Tk::At, "@".to_string(), self.line, self.column)
426 }
427 Some(&'$') => {
428 self.read_char();
429 match self.peek_char() {
430 Some(&'$') => {
431 self.read_char();
432 Token::new(Tk::DstRef, "$$".to_string(), self.line, self.column)
433 }
434 Some(&'^') => {
435 self.read_char();
436 Token::new(Tk::SrcRef, "$^".to_string(), self.line, self.column)
437 }
438 Some(&'-') => {
439 self.read_char();
440 Token::new(Tk::InputRef, "$-".to_string(), self.line, self.column)
441 }
442 _ => Token::new(Tk::Dollar, "$".to_string(), self.line, self.column),
443 }
444 }
445 Some(&'"') | Some(&'\'') => {
446 let start_col = self.column;
447 let start_line = self.line;
448 match self.read_string() {
449 Ok(literal) => Token::new(
450 Tk::StringLiteral(literal.clone()),
451 literal,
452 start_line,
453 start_col,
454 ),
455 Err(e) => {
456 self.add_error(e);
457 Token::new(
458 Tk::StringLiteral(String::new()),
459 String::new(),
460 start_line,
461 start_col,
462 )
463 }
464 }
465 }
466 Some(&ch) if ch.is_ascii_digit() => {
467 let start_col = self.column;
468 let start_line = self.line;
469 let literal = self.read_number();
470 if literal.contains('.') || literal.contains('e') || literal.contains('E') {
471 match literal.parse::<f64>() {
472 Ok(float_val) => {
473 Token::new(Tk::FloatLiteral(float_val), literal, start_line, start_col)
474 }
475 Err(_) => {
476 let error =
477 LexError::invalid_number(literal.clone(), self.current_position());
478 self.add_error(error);
479 Token::new(Tk::FloatLiteral(0.0), literal, start_line, start_col)
480 }
481 }
482 } else {
483 match literal.parse::<i64>() {
484 Ok(int_val) => {
485 Token::new(Tk::IntegerLiteral(int_val), literal, start_line, start_col)
486 }
487 Err(_) => {
488 let error =
489 LexError::invalid_number(literal.clone(), self.current_position());
490 self.add_error(error);
491 Token::new(Tk::IntegerLiteral(0), literal, start_line, start_col)
492 }
493 }
494 }
495 }
496 Some(&ch) if ch.is_alphabetic() || ch == '_' => {
497 let start_col = self.column;
498 let start_line = self.line;
499 let literal = self.read_identifier();
500 match literal.as_str() {
501 "_id" => Token::new(Tk::IdProp, literal, start_line, start_col),
502 "_type" => Token::new(Tk::TypeProp, literal, start_line, start_col),
503 "_src" => Token::new(Tk::SrcIdProp, literal, start_line, start_col),
504 "_dst" => Token::new(Tk::DstIdProp, literal, start_line, start_col),
505 "_rank" => Token::new(Tk::RankProp, literal, start_line, start_col),
506 _ => {
507 let token_kind = self.lookup_keyword(&literal);
508 match token_kind {
509 Tk::KeywordVector => {
510 self.skip_whitespace();
512 if let Some(&'[') = self.peek_char() {
513 self.read_char(); let vector_data = self.parse_vector_elements();
516 return Token::new(
517 Tk::VectorLiteral(vector_data.clone()),
518 format!(
519 "VECTOR[{}]",
520 vector_data
521 .iter()
522 .map(|f| f.to_string())
523 .collect::<Vec<_>>()
524 .join(", ")
525 ),
526 start_line,
527 start_col,
528 );
529 } else {
530 Token::new(token_kind, literal, start_line, start_col)
531 }
532 }
533 Tk::Not => {
534 if self.peek_word() == "IN" {
535 self.skip_word();
536 Token::new(
537 Tk::NotIn,
538 "NOT IN".to_string(),
539 start_line,
540 start_col,
541 )
542 } else {
543 Token::new(token_kind, literal, start_line, start_col)
544 }
545 }
546 Tk::Is => match self.peek_word().as_str() {
547 "NULL" => {
548 self.skip_word();
549 Token::new(
550 Tk::IsNull,
551 "IS NULL".to_string(),
552 start_line,
553 start_col,
554 )
555 }
556 "NOT" => match self.peek_word_after().as_str() {
557 "NULL" => {
558 self.skip_word();
559 self.skip_word();
560 Token::new(
561 Tk::IsNotNull,
562 "IS NOT NULL".to_string(),
563 start_line,
564 start_col,
565 )
566 }
567 "EMPTY" => {
568 self.skip_word();
569 self.skip_word();
570 Token::new(
571 Tk::IsNotEmpty,
572 "IS NOT EMPTY".to_string(),
573 start_line,
574 start_col,
575 )
576 }
577 _ => Token::new(token_kind, literal, start_line, start_col),
578 },
579 "EMPTY" => {
580 self.skip_word();
581 Token::new(
582 Tk::IsEmpty,
583 "IS EMPTY".to_string(),
584 start_line,
585 start_col,
586 )
587 }
588 _ => Token::new(token_kind, literal, start_line, start_col),
589 },
590 _ => Token::new(token_kind, literal, start_line, start_col),
591 }
592 }
593 }
594 }
595 Some(&'`') => {
596 let start_col = self.column;
597 let start_line = self.line;
598 self.read_char();
599 let literal = self.read_backtick_identifier();
600 Token::new(
601 Tk::Identifier(literal.clone()),
602 literal,
603 start_line,
604 start_col,
605 )
606 }
607 Some(&'#') => {
608 self.read_char();
609 if let Some(&'>') = self.peek_char() {
610 self.read_char();
611 if let Some(&'>') = self.peek_char() {
612 self.read_char();
613 Token::new(
614 Tk::HashArrowRight,
615 "#>>".to_string(),
616 self.line,
617 self.column,
618 )
619 } else {
620 Token::new(Tk::HashArrow, "#>".to_string(), self.line, self.column)
621 }
622 } else {
623 let start_col = self.column;
624 let start_line = self.line;
625 self.add_error(LexError::unexpected_character('#', self.current_position()));
626 Token::new(
627 Tk::Identifier("#".to_string()),
628 "#".to_string(),
629 start_line,
630 start_col,
631 )
632 }
633 }
634 Some(&'&') => {
635 self.read_char();
636 Token::new(Tk::Ampersand, "&".to_string(), self.line, self.column)
637 }
638 Some(&ch) => {
639 let start_col = self.column;
640 let start_line = self.line;
641 let unexpected = ch.to_string();
642 self.read_char();
643 self.add_error(LexError::unexpected_character(ch, self.current_position()));
644 Token::new(
645 Tk::Identifier(unexpected.clone()),
646 unexpected,
647 start_line,
648 start_col,
649 )
650 }
651 None => Token::new(Tk::Eof, String::new(), self.line, self.column),
652 };
653
654 token
655 }
656 fn peek_word(&mut self) -> String {
657 let mut temp_lexer = self.clone();
658 temp_lexer.skip_whitespace();
659 temp_lexer.read_identifier()
660 }
661
662 fn peek_word_after(&mut self) -> String {
663 let mut temp_lexer = self.clone();
664 temp_lexer.skip_whitespace();
665 temp_lexer.read_identifier();
666 temp_lexer.read_identifier()
667 }
668
669 fn skip_word(&mut self) {
670 self.skip_whitespace();
671 while let Some(&ch) = self.peek_char() {
672 if ch.is_whitespace() {
673 break;
674 }
675 self.read_char();
676 }
677 }
678}