1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
//! Core structure definition and basic functionality for OptimizedDataFrame
use crate::column::{Column, ColumnTrait};
use crate::error::Result;
use crate::index::{DataFrameIndex, Index};
use std::collections::HashMap;
use std::fmt::{self, Debug, Display};
/// Optimized DataFrame implementation
/// Uses column-oriented storage for fast data processing
#[derive(Clone)]
pub struct OptimizedDataFrame {
// Column data
pub(crate) columns: Vec<Column>,
// Column name → index mapping
pub(crate) column_indices: HashMap<String, usize>,
// Column order
pub(crate) column_names: Vec<String>,
// Row count
pub(crate) row_count: usize,
// Index (optional)
pub(crate) index: Option<DataFrameIndex<String>>,
}
/// Structure representing a view (reference) to a column
#[derive(Clone)]
pub struct ColumnView {
pub(crate) column: Column,
}
impl Display for OptimizedDataFrame {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
<Self as Debug>::fmt(self, f)
}
}
impl Debug for OptimizedDataFrame {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
// Maximum display rows
const MAX_ROWS: usize = 10;
if self.columns.is_empty() {
return write!(f, "OptimizedDataFrame (0 rows x 0 columns)");
}
writeln!(
f,
"OptimizedDataFrame ({} rows x {} columns):",
self.row_count,
self.columns.len()
)?;
// Display column headers
write!(f, "{:<5} |", "idx")?;
for name in &self.column_names {
write!(f, " {:<15} |", name)?;
}
writeln!(f)?;
// Separator line
write!(f, "{:-<5}-+", "")?;
for _ in &self.column_names {
write!(f, "-{:-<15}-+", "")?;
}
writeln!(f)?;
// Display up to MAX_ROWS rows
let display_rows = std::cmp::min(self.row_count, MAX_ROWS);
for i in 0..display_rows {
if let Some(ref idx) = self.index {
let idx_value = match idx {
DataFrameIndex::Simple(ref simple_idx) => {
if i < simple_idx.len() {
simple_idx
.get_value(i)
.map(|s| s.to_string())
.unwrap_or_else(|| i.to_string())
} else {
i.to_string()
}
}
DataFrameIndex::Multi(_) => i.to_string(),
};
write!(f, "{:<5} |", idx_value)?;
} else {
write!(f, "{:<5} |", i)?;
}
for col_idx in 0..self.columns.len() {
let col = &self.columns[col_idx];
let value = match col {
Column::Int64(col) => {
if let Ok(Some(val)) = col.get(i) {
format!("{}", val)
} else {
"NULL".to_string()
}
}
Column::Float64(col) => {
if let Ok(Some(val)) = col.get(i) {
format!("{:.3}", val)
} else {
"NULL".to_string()
}
}
Column::String(col) => {
if let Ok(Some(val)) = col.get(i) {
format!("\"{}\"", val)
} else {
"NULL".to_string()
}
}
Column::Boolean(col) => {
if let Ok(Some(val)) = col.get(i) {
format!("{}", val)
} else {
"NULL".to_string()
}
}
};
write!(f, " {:<15} |", value)?;
}
writeln!(f)?;
}
// Ellipsis for additional rows
if self.row_count > MAX_ROWS {
writeln!(f, "... ({} more rows)", self.row_count - MAX_ROWS)?;
}
Ok(())
}
}
impl OptimizedDataFrame {
/// Create a new empty DataFrame
pub fn new() -> Self {
Self {
columns: Vec::new(),
column_indices: HashMap::new(),
column_names: Vec::new(),
row_count: 0,
index: None,
}
}
/// Create DataFrame with string index
pub fn with_index(index: Index<String>) -> Self {
Self {
columns: Vec::new(),
column_indices: HashMap::new(),
column_names: Vec::new(),
row_count: index.len(),
index: Some(DataFrameIndex::<String>::from_simple(index)),
}
}
/// Create DataFrame with multi-index
pub fn with_multi_index(index: crate::index::MultiIndex<String>) -> Self {
Self {
columns: Vec::new(),
column_indices: HashMap::new(),
column_names: Vec::new(),
row_count: index.len(),
index: Some(DataFrameIndex::<String>::from_multi(index)),
}
}
/// Create DataFrame with range index
pub fn with_range_index(range: std::ops::Range<usize>) -> Result<Self> {
let range_idx = Index::<usize>::from_range(range)?;
// Convert numeric index to string index
let string_values: Vec<String> = range_idx.values().iter().map(|i| i.to_string()).collect();
let string_idx = Index::<String>::new(string_values)?;
Ok(Self::with_index(string_idx))
}
/// Get row count
pub fn row_count(&self) -> usize {
self.row_count
}
/// Get column count
pub fn column_count(&self) -> usize {
self.columns.len()
}
/// Get list of column names
pub fn column_names(&self) -> &[String] {
&self.column_names
}
/// Check if specified column exists
pub fn contains_column(&self, name: &str) -> bool {
self.column_indices.contains_key(name)
}
/// Optimize string columns using global string pool
pub fn optimize_strings(&mut self) -> Result<()> {
use crate::column::string_pool::GLOBAL_STRING_POOL;
for (_i, column) in self.columns.iter_mut().enumerate() {
if let Column::String(string_col) = column {
// Get all string values
let values: Vec<String> = (0..string_col.len())
.filter_map(|idx| string_col.get(idx).ok().flatten().map(|s| s.to_string()))
.collect();
// Add to global string pool
for s in &values {
// `get_or_insert` now returns `Result` (propagates a
// poisoned-lock error instead of silently aliasing
// index 0 -- see `column::string_pool`); this loop
// only wants the interning side effect and is
// superseded by `StringColumn::new_with_global_pool`
// just below anyway, so the outcome is intentionally
// discarded here, same as before.
let _ = GLOBAL_STRING_POOL.get_or_insert(s);
}
// Create optimized column with pooled strings
*string_col = crate::column::StringColumn::new_with_global_pool(values);
}
}
Ok(())
}
/// Get memory usage statistics
pub fn memory_usage(&self) -> std::collections::HashMap<String, usize> {
let mut usage = std::collections::HashMap::new();
// Calculate column memory usage
for (name, &idx) in &self.column_indices {
let column_size = match &self.columns[idx] {
Column::Int64(col) => col.len() * std::mem::size_of::<Option<i64>>(),
Column::Float64(col) => col.len() * std::mem::size_of::<Option<f64>>(),
Column::String(col) => {
// Estimate string memory usage
let mut size = col.len() * std::mem::size_of::<Option<String>>();
for i in 0..col.len() {
if let Ok(Some(s)) = col.get(i) {
size += s.len();
}
}
size
}
Column::Boolean(col) => col.len() * std::mem::size_of::<Option<bool>>(),
};
usage.insert(name.clone(), column_size);
}
// Add metadata overhead
usage.insert(
"metadata".to_string(),
std::mem::size_of::<Self>()
+ self.column_names.capacity() * std::mem::size_of::<String>()
+ self.column_indices.capacity() * std::mem::size_of::<(String, usize)>(),
);
usage
}
}