1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
use crate::Value;
/// Check if a value supports a specific type of operation
///
/// This function provides a way to determine compatibility before
/// attempting to apply operations.
#[must_use]
pub fn supports_operation(value: &Value, operation_type: OperationType) -> bool {
matches!(
(value, operation_type),
(Value::DataFrame(_) | Value::LazyFrame(_), _)
| (
Value::Array(_),
OperationType::Basic
| OperationType::Aggregate
| OperationType::Transform
| OperationType::Filter
| OperationType::Join,
)
| (
Value::Object(_),
OperationType::Basic | OperationType::Transform | OperationType::Filter,
)
| (
Value::Series(_),
OperationType::Basic | OperationType::Transform
)
)
}
/// Types of operations supported by dsq
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum OperationType {
/// Basic operations like select, filter, sort
Basic,
/// Aggregation operations like group by, sum, mean
Aggregate,
/// Join operations for combining datasets
Join,
/// Transformation operations like pivot, transpose
Transform,
/// Filter operations for jq-style expressions
Filter,
}
/// Get the recommended batch size for operations on large datasets
///
/// This function provides guidance on how to chunk large datasets
/// for efficient processing.
#[must_use]
pub fn recommended_batch_size(value: &Value, operation_type: OperationType) -> Option<usize> {
match value {
Value::DataFrame(df) => {
let rows = df.height();
match operation_type {
OperationType::Aggregate => {
if rows > 500_000 {
Some(50_000)
} else {
None
}
}
OperationType::Join => {
if rows > 100_000 {
Some(10_000)
} else {
None
}
}
OperationType::Basic | OperationType::Filter | OperationType::Transform => {
if rows > 1_000_000 {
Some(100_000)
} else {
None
}
}
}
}
Value::Array(arr) => {
let len = arr.len();
match operation_type {
OperationType::Aggregate => {
if len > 50_000 {
Some(5_000)
} else {
None
}
}
OperationType::Join => {
if len > 10_000 {
Some(1_000)
} else {
None
}
}
OperationType::Basic | OperationType::Filter | OperationType::Transform => {
if len > 100_000 {
Some(10_000)
} else {
None
}
}
}
}
_ => None,
}
}