1pub mod compiler;
14pub mod context;
15pub mod executor;
16
17pub use compiler::{CompiledFilter, FilterCompiler, OptimizationLevel};
18pub use context::{CompilationContext, ErrorMode, FilterContext, FunctionBody, FunctionDef};
19pub use dsq_functions::BuiltinRegistry;
20pub use executor::{
21 ExecutionMode, ExecutionResult, ExecutionStats, ExecutorConfig, FilterExecutor,
22};
23
24pub fn execute_filter(
26 filter: &str,
27 value: &dsq_shared::value::Value,
28) -> anyhow::Result<dsq_shared::value::Value> {
29 let mut executor = FilterExecutor::new();
30 let result = executor.execute_str(filter, value.clone())?;
31 Ok(result.value)
32}
33
34pub fn compile_filter(filter: &str) -> anyhow::Result<CompiledFilter> {
36 let compiler = FilterCompiler::new();
37 compiler.compile_str(filter)
38}
39
40pub use dsq_shared::value::Value;
42pub use dsq_shared::Result;
43
44#[cfg(test)]
45mod tests {
46 use super::*;
47
48 #[test]
49 fn test_compile_example_002() {
50 let query = r#"group_by(.genre) | map({
51 genre: .[0].genre,
52 count: length,
53 avg_price: (map(.price) | add / length)
54})"#;
55 let result = compile_filter(query);
56 assert!(
57 result.is_ok(),
58 "Failed to compile example_002 query: {:?}",
59 result.err()
60 );
61 }
62
63 #[test]
64 fn test_compile_example_second() {
65 let query = r#"filter(.year > 1900) | sort_by(.year) | map({title, author})"#;
66 let result = compile_filter(query);
67 assert!(
68 result.is_ok(),
69 "Failed to compile second query: {:?}",
70 result.err()
71 );
72 }
73
74 #[test]
75 fn test_compile_join_example() {
76 let query = r#"join("departments.csv", .dept_id == .id) | map({employee_name: .name, salary, department: .name_right, location})"#;
77 let result = compile_filter(query);
78 assert!(
79 result.is_ok(),
80 "Failed to compile join query: {:?}",
81 result.err()
82 );
83 }
84
85 #[test]
86 fn test_select_on_array() {
87 use dsq_shared::value::Value;
88 let input = Value::Array(vec![
89 Value::Int(100),
90 Value::Int(200),
91 Value::Int(50),
92 Value::Int(300),
93 ]);
94 let result = execute_filter("select(. > 100)", &input);
95 assert!(
96 result.is_ok(),
97 "Failed to execute select on array: {:?}",
98 result.err()
99 );
100 let expected = Value::Array(vec![Value::Int(200), Value::Int(300)]);
101 assert_eq!(result.unwrap(), expected);
102 }
103
104 #[test]
105 fn test_select_on_object() {
106 use dsq_shared::value::Value;
107 use std::collections::HashMap;
108 let mut obj = HashMap::new();
109 obj.insert("age".to_string(), Value::Int(35));
110 obj.insert("name".to_string(), Value::String("John".to_string()));
111 obj.insert(
112 "US City Name".to_string(),
113 Value::String("New York".to_string()),
114 );
115 let input = Value::Object(obj.clone());
116
117 let result = execute_filter("select(.age > 30)", &input);
119 assert!(
120 result.is_ok(),
121 "Failed to execute select on object: {:?}",
122 result.err()
123 );
124 assert_eq!(result.unwrap(), Value::Object(obj.clone()));
125
126 let result = execute_filter("select(.age < 30)", &input);
128 assert!(
129 result.is_ok(),
130 "Failed to execute select on object: {:?}",
131 result.err()
132 );
133 assert_eq!(result.unwrap(), Value::Null);
134
135 let result = execute_filter(".[\"US City Name\"]", &input);
137 assert!(
138 result.is_ok(),
139 "Failed to execute field access with brackets: {:?}",
140 result.err()
141 );
142 assert_eq!(result.unwrap(), Value::String("New York".to_string()));
143 }
144
145 #[test]
146 fn test_stress_009_execution() {
147 use dsq_shared::value::Value;
149 use std::collections::HashMap;
150
151 let employees = vec![
153 {
154 let mut obj = HashMap::new();
155 obj.insert("id".to_string(), Value::Int(1));
156 obj.insert(
157 "name".to_string(),
158 Value::String("Alice Johnson".to_string()),
159 );
160 obj.insert("age".to_string(), Value::Int(28));
161 obj.insert("city".to_string(), Value::String("New York".to_string()));
162 obj.insert("salary".to_string(), Value::Int(75000));
163 obj.insert(
164 "department".to_string(),
165 Value::String("Engineering".to_string()),
166 );
167 Value::Object(obj)
168 },
169 {
170 let mut obj = HashMap::new();
171 obj.insert("id".to_string(), Value::Int(2));
172 obj.insert("name".to_string(), Value::String("Bob Smith".to_string()));
173 obj.insert("age".to_string(), Value::Int(34));
174 obj.insert("city".to_string(), Value::String("Los Angeles".to_string()));
175 obj.insert("salary".to_string(), Value::Int(82000));
176 obj.insert("department".to_string(), Value::String("Sales".to_string()));
177 Value::Object(obj)
178 },
179 ];
180
181 let input = Value::Array(employees);
182
183 let query = r#"map(select(.age > 30 and .department == "IT" or .salary < 60000)) | map({name, age, department, salary})"#;
184 let result = execute_filter(query, &input);
185
186 assert!(
188 result.is_ok(),
189 "Failed to execute stress_009 query: {:?}",
190 result.err()
191 );
192 assert_eq!(result.unwrap(), Value::Array(vec![]));
193 }
194
195 #[test]
196 fn test_if_then_else() {
197 use dsq_shared::value::Value;
198
199 let result = execute_filter("if . > 5 then \"big\" else \"small\" end", &Value::Int(10));
201 assert!(
202 result.is_ok(),
203 "Failed to execute if-then-else: {:?}",
204 result.err()
205 );
206 assert_eq!(result.unwrap(), Value::String("big".to_string()));
207
208 let result = execute_filter("if . > 5 then \"big\" else \"small\" end", &Value::Int(3));
209 assert!(result.is_ok());
210 assert_eq!(result.unwrap(), Value::String("small".to_string()));
211
212 let result = execute_filter(
214 "if type == \"integer\" then . * 2 else . end",
215 &Value::Int(5),
216 );
217 assert!(
218 result.is_ok(),
219 "Failed to execute if with type: {:?}",
220 result.err()
221 );
222 assert_eq!(result.unwrap(), Value::Int(10));
223
224 let result = execute_filter(
225 "if type == \"integer\" then . * 2 else . end",
226 &Value::String("hello".to_string()),
227 );
228 assert!(result.is_ok());
229 assert_eq!(result.unwrap(), Value::String("hello".to_string()));
230 }
231
232 #[test]
233 fn test_max_by() {
234 use dsq_shared::value::Value;
235 use std::collections::HashMap;
236
237 let data = vec![
239 {
240 let mut obj = HashMap::new();
241 obj.insert("name".to_string(), Value::String("Laptop".to_string()));
242 obj.insert("price".to_string(), Value::Int(1200));
243 Value::Object(obj)
244 },
245 {
246 let mut obj = HashMap::new();
247 obj.insert("name".to_string(), Value::String("Phone".to_string()));
248 obj.insert("price".to_string(), Value::Int(800));
249 Value::Object(obj)
250 },
251 {
252 let mut obj = HashMap::new();
253 obj.insert("name".to_string(), Value::String("Book".to_string()));
254 obj.insert("price".to_string(), Value::Int(20));
255 Value::Object(obj)
256 },
257 ];
258
259 let input = Value::Array(data);
260
261 let result = execute_filter("max_by(.price)", &input);
262 assert!(
263 result.is_ok(),
264 "Failed to execute max_by: {:?}",
265 result.err()
266 );
267
268 let expected = {
269 let mut obj = HashMap::new();
270 obj.insert("name".to_string(), Value::String("Laptop".to_string()));
271 obj.insert("price".to_string(), Value::Int(1200));
272 Value::Object(obj)
273 };
274
275 assert_eq!(result.unwrap(), expected);
276 }
277
278 #[test]
279 fn test_max_by_dataframe() {
280 use dsq_shared::value::Value;
281 use polars::prelude::*;
282
283 let df = DataFrame::new(vec![
285 Series::new("name".into(), &["Laptop", "Phone", "Book", "Shoes"]).into(),
286 Series::new("price".into(), &[1200, 800, 20, 150]).into(),
287 Series::new(
288 "category".into(),
289 &["Electronics", "Electronics", "Books", "Clothing"],
290 )
291 .into(),
292 ])
293 .unwrap();
294
295 let input = Value::DataFrame(df);
296
297 let result = execute_filter("max_by(.price)", &input);
298 assert!(
299 result.is_ok(),
300 "Failed to execute max_by on DataFrame: {:?}",
301 result.err()
302 );
303
304 let expected = {
305 let mut obj = std::collections::HashMap::new();
306 obj.insert("name".to_string(), Value::String("Laptop".to_string()));
307 obj.insert("price".to_string(), Value::Int(1200));
308 obj.insert(
309 "category".to_string(),
310 Value::String("Electronics".to_string()),
311 );
312 Value::Object(obj)
313 };
314
315 assert_eq!(result.unwrap(), expected);
316 }
317
318 #[test]
319 fn test_add_function() {
320 use dsq_shared::value::Value;
321
322 let input = Value::Array(vec![
324 Value::Int(1),
325 Value::Int(2),
326 Value::Int(3),
327 Value::Int(4),
328 ]);
329
330 let result = execute_filter("add", &input);
331 assert!(
332 result.is_ok(),
333 "Failed to execute add on array: {:?}",
334 result.err()
335 );
336 assert_eq!(result.unwrap(), Value::Int(10));
337
338 let input_float = Value::Array(vec![
340 Value::Float(1.5),
341 Value::Float(2.5),
342 Value::Float(3.0),
343 ]);
344
345 let result_float = execute_filter("add", &input_float);
346 assert!(
347 result_float.is_ok(),
348 "Failed to execute add on float array: {:?}",
349 result_float.err()
350 );
351 assert_eq!(result_float.unwrap(), Value::Float(7.0));
352
353 let input_string = Value::Array(vec![
355 Value::String("hello".to_string()),
356 Value::String(" ".to_string()),
357 Value::String("world".to_string()),
358 ]);
359
360 let result_string = execute_filter("add", &input_string);
361 assert!(
362 result_string.is_ok(),
363 "Failed to execute add on string array: {:?}",
364 result_string.err()
365 );
366 assert_eq!(
367 result_string.unwrap(),
368 Value::String("hello world".to_string())
369 );
370 }
371
372 #[test]
373 fn test_filter_on_array() {
374 use dsq_shared::value::Value;
375
376 let input = Value::Array(vec![
378 Value::Int(1),
379 Value::Int(2),
380 Value::Int(3),
381 Value::Int(4),
382 Value::Int(5),
383 ]);
384
385 let result = execute_filter("filter(. > 3)", &input);
386 assert!(
387 result.is_ok(),
388 "Failed to execute filter on array: {:?}",
389 result.err()
390 );
391 let expected = Value::Array(vec![Value::Int(4), Value::Int(5)]);
392 assert_eq!(result.unwrap(), expected);
393
394 let result = execute_filter("filter(. >= 2 and . <= 4)", &input);
396 assert!(
397 result.is_ok(),
398 "Failed to execute filter with range: {:?}",
399 result.err()
400 );
401 let expected_range = Value::Array(vec![Value::Int(2), Value::Int(3), Value::Int(4)]);
402 assert_eq!(result.unwrap(), expected_range);
403 }
404
405 #[test]
406 fn test_filter_on_dataframe() {
407 use dsq_shared::value::Value;
408 use polars::prelude::*;
409
410 let df = DataFrame::new(vec![
412 Series::new("id".into(), &[1, 2, 3, 4]).into(),
413 Series::new("age".into(), &[25, 30, 35, 28]).into(),
414 Series::new("salary".into(), &[50000, 60000, 70000, 55000]).into(),
415 ])
416 .unwrap();
417
418 let input = Value::DataFrame(df);
419
420 let result = execute_filter("filter(.id)", &input); assert!(
423 result.is_ok(),
424 "Failed to execute basic filter on DataFrame: {:?}",
425 result.err()
426 );
427
428 if let Value::DataFrame(filtered_df) = result.unwrap() {
429 assert_eq!(
430 filtered_df.height(),
431 4,
432 "Should have all 4 rows (all ids are truthy)"
433 );
434 } else {
435 panic!("Expected DataFrame result");
436 }
437
438 let result = execute_filter("filter(.id == 1)", &input);
440 assert!(
441 result.is_ok(),
442 "Failed to execute equality filter on DataFrame: {:?}",
443 result.err()
444 );
445
446 if let Value::DataFrame(filtered_df) = result.unwrap() {
447 assert_eq!(filtered_df.height(), 1, "Should have 1 row with id == 1");
448
449 let ids = filtered_df.column("id").unwrap().i32().unwrap();
450 assert_eq!(ids.get(0).unwrap(), 1, "ID should be 1");
451 } else {
452 panic!("Expected DataFrame result");
453 }
454 }
455
456 #[test]
457 fn test_filter_with_function_calls() {
458 use dsq_shared::value::Value;
459 use polars::prelude::*;
460
461 let df = DataFrame::new(vec![
463 Series::new(
464 "date".into(),
465 &[
466 "2023-10-01",
467 "2023-10-02",
468 "2023-10-03",
469 "2023-10-04",
470 "2023-10-05",
471 "2023-10-06",
472 "2023-10-07",
473 ],
474 )
475 .into(),
476 Series::new("value".into(), &[100, 200, 150, 300, 250, 175, 225]).into(),
477 ])
478 .unwrap();
479
480 let input = Value::DataFrame(df);
481
482 let result = execute_filter("filter(end_of_week(.date))", &input);
486 if let Ok(Value::DataFrame(filtered_df)) = result {
489 assert_eq!(filtered_df.height(), 1, "Should have 1 row (Sunday)");
491 } else if let Err(e) = result {
492 assert!(
495 e.to_string().contains("end_of_week") || e.to_string().contains("function"),
496 "Error should be about function availability, not filter execution: {:?}",
497 e
498 );
499 } else {
500 }
503 }
504
505 #[test]
506 fn test_filter_edge_cases() {
507 use dsq_shared::value::Value;
508
509 let input = Value::Array(vec![]);
511 let result = execute_filter("filter(. > 0)", &input);
512 assert!(
513 result.is_ok(),
514 "Failed to execute filter on empty array: {:?}",
515 result.err()
516 );
517 assert_eq!(result.unwrap(), Value::Array(vec![]));
518
519 let input = Value::Array(vec![Value::Int(1), Value::Int(2), Value::Int(3)]);
521 let result = execute_filter("filter(. > 10)", &input);
522 assert!(
523 result.is_ok(),
524 "Failed to execute filter with no matches: {:?}",
525 result.err()
526 );
527 assert_eq!(result.unwrap(), Value::Array(vec![]));
528
529 let result = execute_filter("filter(. <= 10)", &input);
531 assert!(
532 result.is_ok(),
533 "Failed to execute filter matching all: {:?}",
534 result.err()
535 );
536 assert_eq!(result.unwrap(), input);
537
538 let input_with_null = Value::Array(vec![Value::Int(1), Value::Null, Value::Int(3)]);
540 let result = execute_filter("filter(. != null)", &input_with_null);
541 assert!(
542 result.is_ok(),
543 "Failed to execute filter with null check: {:?}",
544 result.err()
545 );
546 let expected = Value::Array(vec![Value::Int(1), Value::Int(3)]);
547 assert_eq!(result.unwrap(), expected);
548 }
549
550 #[test]
551 fn test_filter_compilation_errors() {
552 let result = compile_filter("filter(");
554 assert!(
555 result.is_err(),
556 "Should fail to compile incomplete filter expression"
557 );
558
559 let result = compile_filter("filter(. > )");
560 assert!(
561 result.is_err(),
562 "Should fail to compile malformed filter condition"
563 );
564
565 let result = compile_filter("filter(.field)");
566 assert!(result.is_ok(), "Valid filter expression should compile");
568 }
569
570 #[test]
571 fn test_assign_update_compilation() {
572 use dsq_shared::value::Value;
573 use std::collections::HashMap;
574
575 let result = compile_filter(".field |= 42");
577 assert!(
578 result.is_ok(),
579 "Should compile |= assignment: {:?}",
580 result.err()
581 );
582
583 let mut obj = HashMap::new();
585 obj.insert("field".to_string(), Value::Int(10));
586 obj.insert("other".to_string(), Value::String("unchanged".to_string()));
587 let input = Value::Object(obj);
588
589 let result = execute_filter(".field |= 42", &input);
590 assert!(
591 result.is_ok(),
592 "Should execute |= assignment: {:?}",
593 result.err()
594 );
595
596 if let Value::Object(result_obj) = result.unwrap() {
597 assert_eq!(result_obj.get("field"), Some(&Value::Int(42)));
598 assert_eq!(
599 result_obj.get("other"),
600 Some(&Value::String("unchanged".to_string()))
601 );
602 } else {
603 panic!("Expected object result");
604 }
605 }
606
607 #[test]
608 fn test_assign_update_compilation_with_expression() {
609 use dsq_shared::value::Value;
610 use std::collections::HashMap;
611
612 let result = compile_filter(".total |= .price + .tax");
614 assert!(
615 result.is_ok(),
616 "Should compile |= assignment with expression: {:?}",
617 result.err()
618 );
619
620 let mut obj = HashMap::new();
622 obj.insert("price".to_string(), Value::Int(100));
623 obj.insert("tax".to_string(), Value::Int(10));
624 let input = Value::Object(obj);
625
626 let result = execute_filter(".total |= .price + .tax", &input);
627 assert!(
628 result.is_ok(),
629 "Should execute |= assignment with expression: {:?}",
630 result.err()
631 );
632
633 if let Value::Object(result_obj) = result.unwrap() {
634 assert_eq!(result_obj.get("price"), Some(&Value::Int(100)));
635 assert_eq!(result_obj.get("tax"), Some(&Value::Int(10)));
636 assert_eq!(result_obj.get("total"), Some(&Value::Int(110)));
637 } else {
638 panic!("Expected object result");
639 }
640 }
641
642 #[test]
643 fn test_assign_update_compilation_string_field() {
644 use dsq_shared::value::Value;
645 use std::collections::HashMap;
646
647 let mut obj = HashMap::new();
649 obj.insert("status".to_string(), Value::String("pending".to_string()));
650 let input = Value::Object(obj);
651
652 let result = execute_filter(".status |= \"completed\"", &input);
653 assert!(
654 result.is_ok(),
655 "Should execute |= assignment with string: {:?}",
656 result.err()
657 );
658
659 if let Value::Object(result_obj) = result.unwrap() {
660 assert_eq!(
661 result_obj.get("status"),
662 Some(&Value::String("completed".to_string()))
663 );
664 } else {
665 panic!("Expected object result");
666 }
667 }
668
669 #[test]
670 fn test_assign_update_compilation_array_field() {
671 use dsq_shared::value::Value;
672 use std::collections::HashMap;
673
674 let mut obj = HashMap::new();
676 obj.insert(
677 "tags".to_string(),
678 Value::Array(vec![Value::String("old".to_string())]),
679 );
680 let input = Value::Object(obj);
681
682 let result = execute_filter(".tags |= [\"new\", \"tags\"]", &input);
683 assert!(
684 result.is_ok(),
685 "Should execute |= assignment with array: {:?}",
686 result.err()
687 );
688
689 let result_val = result.unwrap();
690 println!("Result: {:?}", result_val);
691
692 if let Value::Object(result_obj) = result_val {
693 if let Some(Value::Array(tags)) = result_obj.get("tags") {
694 println!("Tags: {:?}", tags);
695 assert_eq!(tags.len(), 2);
696 assert_eq!(tags[0], Value::String("new".to_string()));
697 assert_eq!(tags[1], Value::String("tags".to_string()));
698 } else {
699 panic!(
700 "Expected array for tags field, got: {:?}",
701 result_obj.get("tags")
702 );
703 }
704 } else {
705 panic!("Expected object result, got: {:?}", result_val);
706 }
707 }
708
709 #[test]
710 fn test_assign_update_compilation_in_pipeline() {
711 use dsq_shared::value::Value;
712 use std::collections::HashMap;
713
714 let mut obj = HashMap::new();
716 obj.insert("salary".to_string(), Value::Int(50000));
717 obj.insert("name".to_string(), Value::String("Alice".to_string()));
718 let input = Value::Object(obj);
719
720 let result = execute_filter(".salary |= .salary + 5000 | .name", &input);
721 assert!(
722 result.is_ok(),
723 "Should execute |= in pipeline: {:?}",
724 result.err()
725 );
726
727 assert_eq!(result.unwrap(), Value::String("Alice".to_string()));
729 }
730
731 #[test]
732 fn test_assign_update_compilation_nested_object() {
733 use dsq_shared::value::Value;
734 use std::collections::HashMap;
735
736 let mut address = HashMap::new();
738 address.insert("city".to_string(), Value::String("NYC".to_string()));
739
740 let mut obj = HashMap::new();
741 obj.insert("address".to_string(), Value::Object(address));
742 let input = Value::Object(obj);
743
744 let result = execute_filter(".address.city |= \"Boston\"", &input);
745 assert!(
746 result.is_ok(),
747 "Should execute |= on nested field: {:?}",
748 result.err()
749 );
750
751 if let Value::Object(result_obj) = result.unwrap() {
752 if let Some(Value::Object(addr_obj)) = result_obj.get("address") {
753 assert_eq!(
754 addr_obj.get("city"),
755 Some(&Value::String("Boston".to_string()))
756 );
757 } else {
758 panic!("Expected nested address object");
759 }
760 } else {
761 panic!("Expected object result");
762 }
763 }
764
765 #[test]
766 fn test_assign_update_compilation_error_cases() {
767 let input = Value::Array(vec![Value::Int(1), Value::Int(2)]);
769
770 let result = execute_filter(".field |= 42", &input);
771 assert!(
773 result.is_ok(),
774 "Should handle |= on non-object gracefully: {:?}",
775 result.err()
776 );
777 assert_eq!(result.unwrap(), Value::Int(42));
778 }
779
780 #[test]
781 fn test_execute_filter_map_identity_preserves_nulls() {
782 let input = Value::Array(vec![Value::Int(1), Value::Null, Value::Int(2)]);
784
785 let result = execute_filter("map(.)", &input);
786 assert!(result.is_ok());
787 let expected = Value::Array(vec![Value::Int(1), Value::Null, Value::Int(2)]);
788 assert_eq!(result.unwrap(), expected);
789 }
790}