sql_cli/data/
datatable.rs

1use crate::api_client::QueryResponse;
2use crate::data::data_provider::DataProvider;
3use crate::data::type_inference::{InferredType, TypeInference};
4use serde::de::{VariantAccess, Visitor};
5use serde::{Deserialize, Serialize};
6use serde_json::Value as JsonValue;
7use std::collections::HashMap;
8use std::fmt;
9use std::sync::Arc;
10use tracing::debug;
11
12/// Represents the data type of a column
13#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
14pub enum DataType {
15    String,
16    Integer,
17    Float,
18    Boolean,
19    DateTime,
20    Null,
21    Mixed, // For columns with mixed types
22}
23
24impl DataType {
25    /// Infer type from a string value
26    #[must_use]
27    pub fn infer_from_string(value: &str) -> Self {
28        // Handle explicit null string
29        if value.eq_ignore_ascii_case("null") {
30            return DataType::Null;
31        }
32
33        // Use the shared type inference logic
34        match TypeInference::infer_from_string(value) {
35            InferredType::Null => DataType::Null,
36            InferredType::Boolean => DataType::Boolean,
37            InferredType::Integer => DataType::Integer,
38            InferredType::Float => DataType::Float,
39            InferredType::DateTime => DataType::DateTime,
40            InferredType::String => DataType::String,
41        }
42    }
43
44    /// Check if a string looks like a datetime value
45    /// Delegates to shared type inference logic
46    fn looks_like_datetime(value: &str) -> bool {
47        TypeInference::looks_like_datetime(value)
48    }
49
50    /// Merge two types (for columns with mixed types)
51    #[must_use]
52    pub fn merge(&self, other: &DataType) -> DataType {
53        if self == other {
54            return self.clone();
55        }
56
57        match (self, other) {
58            (DataType::Null, t) | (t, DataType::Null) => t.clone(),
59            (DataType::Integer, DataType::Float) | (DataType::Float, DataType::Integer) => {
60                DataType::Float
61            }
62            _ => DataType::Mixed,
63        }
64    }
65}
66
67/// Column metadata and definition
68#[derive(Debug, Clone, Serialize, Deserialize)]
69pub struct DataColumn {
70    pub name: String,
71    pub data_type: DataType,
72    pub nullable: bool,
73    pub unique_values: Option<usize>,
74    pub null_count: usize,
75    pub metadata: HashMap<String, String>,
76    /// Qualified name with table prefix (e.g., "messages.field_name")
77    pub qualified_name: Option<String>,
78    /// Source table or CTE name
79    pub source_table: Option<String>,
80}
81
82impl DataColumn {
83    pub fn new(name: impl Into<String>) -> Self {
84        Self {
85            name: name.into(),
86            data_type: DataType::String,
87            nullable: true,
88            unique_values: None,
89            null_count: 0,
90            metadata: HashMap::new(),
91            qualified_name: None,
92            source_table: None,
93        }
94    }
95
96    #[must_use]
97    pub fn with_type(mut self, data_type: DataType) -> Self {
98        self.data_type = data_type;
99        self
100    }
101
102    /// Set the qualified name (table.column format)
103    #[must_use]
104    pub fn with_qualified_name(mut self, table_name: &str) -> Self {
105        self.qualified_name = Some(format!("{}.{}", table_name, self.name));
106        self.source_table = Some(table_name.to_string());
107        self
108    }
109
110    /// Get the qualified name if available, otherwise return the simple name
111    pub fn get_qualified_or_simple_name(&self) -> &str {
112        self.qualified_name.as_deref().unwrap_or(&self.name)
113    }
114
115    #[must_use]
116    pub fn with_nullable(mut self, nullable: bool) -> Self {
117        self.nullable = nullable;
118        self
119    }
120}
121
122/// A single cell value in the table
123#[derive(Debug, Clone, PartialEq, PartialOrd)]
124pub enum DataValue {
125    String(String),
126    InternedString(Arc<String>), // For repeated strings (e.g., status, trader names)
127    Integer(i64),
128    Float(f64),
129    Boolean(bool),
130    DateTime(String), // Store as ISO 8601 string for now
131    Vector(Vec<f64>), // For vector mathematics (physics, geometry, etc.)
132    Null,
133}
134
135// Custom Hash implementation for DataValue to handle f64
136impl std::hash::Hash for DataValue {
137    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
138        match self {
139            DataValue::String(s) => {
140                0u8.hash(state);
141                s.hash(state);
142            }
143            DataValue::InternedString(s) => {
144                1u8.hash(state);
145                s.hash(state);
146            }
147            DataValue::Integer(i) => {
148                2u8.hash(state);
149                i.hash(state);
150            }
151            DataValue::Float(f) => {
152                3u8.hash(state);
153                // Hash the bits of the float for consistency
154                f.to_bits().hash(state);
155            }
156            DataValue::Boolean(b) => {
157                4u8.hash(state);
158                b.hash(state);
159            }
160            DataValue::DateTime(dt) => {
161                5u8.hash(state);
162                dt.hash(state);
163            }
164            DataValue::Vector(v) => {
165                6u8.hash(state);
166                // Hash each float's bits
167                for f in v {
168                    f.to_bits().hash(state);
169                }
170            }
171            DataValue::Null => {
172                7u8.hash(state);
173            }
174        }
175    }
176}
177
178// Custom Serialize implementation for DataValue to handle Arc<String>
179impl Serialize for DataValue {
180    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
181    where
182        S: serde::Serializer,
183    {
184        match self {
185            DataValue::String(s) => {
186                serializer.serialize_newtype_variant("DataValue", 0, "String", s)
187            }
188            DataValue::InternedString(arc_s) => {
189                // Serialize the Arc<String> as just the String content
190                serializer.serialize_newtype_variant(
191                    "DataValue",
192                    1,
193                    "InternedString",
194                    arc_s.as_ref(),
195                )
196            }
197            DataValue::Integer(i) => {
198                serializer.serialize_newtype_variant("DataValue", 2, "Integer", i)
199            }
200            DataValue::Float(f) => serializer.serialize_newtype_variant("DataValue", 3, "Float", f),
201            DataValue::Boolean(b) => {
202                serializer.serialize_newtype_variant("DataValue", 4, "Boolean", b)
203            }
204            DataValue::DateTime(dt) => {
205                serializer.serialize_newtype_variant("DataValue", 5, "DateTime", dt)
206            }
207            DataValue::Vector(v) => {
208                serializer.serialize_newtype_variant("DataValue", 6, "Vector", v)
209            }
210            DataValue::Null => serializer.serialize_unit_variant("DataValue", 7, "Null"),
211        }
212    }
213}
214
215// Custom Deserialize implementation for DataValue to handle Arc<String>
216impl<'de> Deserialize<'de> for DataValue {
217    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
218    where
219        D: serde::Deserializer<'de>,
220    {
221        #[derive(Deserialize)]
222        #[serde(field_identifier, rename_all = "PascalCase")]
223        enum Field {
224            String,
225            InternedString,
226            Integer,
227            Float,
228            Boolean,
229            DateTime,
230            Vector,
231            Null,
232        }
233
234        struct DataValueVisitor;
235
236        impl<'de> Visitor<'de> for DataValueVisitor {
237            type Value = DataValue;
238
239            fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
240                formatter.write_str("enum DataValue")
241            }
242
243            fn visit_enum<A>(self, data: A) -> Result<Self::Value, A::Error>
244            where
245                A: serde::de::EnumAccess<'de>,
246            {
247                let (field, variant) = data.variant()?;
248                match field {
249                    Field::String => {
250                        let s: String = variant.newtype_variant()?;
251                        Ok(DataValue::String(s))
252                    }
253                    Field::InternedString => {
254                        let s: String = variant.newtype_variant()?;
255                        Ok(DataValue::InternedString(Arc::new(s)))
256                    }
257                    Field::Integer => {
258                        let i: i64 = variant.newtype_variant()?;
259                        Ok(DataValue::Integer(i))
260                    }
261                    Field::Float => {
262                        let f: f64 = variant.newtype_variant()?;
263                        Ok(DataValue::Float(f))
264                    }
265                    Field::Boolean => {
266                        let b: bool = variant.newtype_variant()?;
267                        Ok(DataValue::Boolean(b))
268                    }
269                    Field::DateTime => {
270                        let dt: String = variant.newtype_variant()?;
271                        Ok(DataValue::DateTime(dt))
272                    }
273                    Field::Vector => {
274                        let v: Vec<f64> = variant.newtype_variant()?;
275                        Ok(DataValue::Vector(v))
276                    }
277                    Field::Null => {
278                        variant.unit_variant()?;
279                        Ok(DataValue::Null)
280                    }
281                }
282            }
283        }
284
285        deserializer.deserialize_enum(
286            "DataValue",
287            &[
288                "String",
289                "InternedString",
290                "Integer",
291                "Float",
292                "Boolean",
293                "DateTime",
294                "Vector",
295                "Null",
296            ],
297            DataValueVisitor,
298        )
299    }
300}
301
302// Custom Eq implementation for DataValue
303impl Eq for DataValue {}
304
305impl DataValue {
306    pub fn from_string(s: &str, data_type: &DataType) -> Self {
307        if s.is_empty() || s.eq_ignore_ascii_case("null") {
308            return DataValue::Null;
309        }
310
311        // Numbers and booleans carry no meaningful surrounding whitespace, so a
312        // padded field from a column-aligned CSV (` 1732`) parses like the bare
313        // value. String columns below deliberately use `s`, not `t` — there the
314        // spaces may well be the data.
315        let t = s.trim();
316
317        match data_type {
318            DataType::String => DataValue::String(s.to_string()),
319            DataType::Integer => t.parse::<i64>().map_or_else(
320                // The column was inferred as Integer (type inference only samples
321                // the first N rows, so a fractional value further down can be
322                // missed). Promote to Float rather than demoting to String, which
323                // would corrupt numeric sorting (String sorts after all numbers).
324                // The final infer_column_types() pass re-merges the column to Float.
325                |_| {
326                    t.parse::<f64>()
327                        .map_or_else(|_| DataValue::String(s.to_string()), DataValue::Float)
328                },
329                DataValue::Integer,
330            ),
331            DataType::Float => t
332                .parse::<f64>()
333                .map_or_else(|_| DataValue::String(s.to_string()), DataValue::Float),
334            DataType::Boolean => {
335                let lower = t.to_lowercase();
336                DataValue::Boolean(lower == "true" || lower == "1" || lower == "yes")
337            }
338            DataType::DateTime => DataValue::DateTime(s.to_string()),
339            DataType::Null => DataValue::Null,
340            DataType::Mixed => {
341                // Try to infer for mixed columns
342                let inferred = DataType::infer_from_string(s);
343                Self::from_string(s, &inferred)
344            }
345        }
346    }
347
348    #[must_use]
349    pub fn is_null(&self) -> bool {
350        matches!(self, DataValue::Null)
351    }
352
353    #[must_use]
354    pub fn data_type(&self) -> DataType {
355        match self {
356            DataValue::String(_) | DataValue::InternedString(_) => DataType::String,
357            DataValue::Integer(_) => DataType::Integer,
358            DataValue::Float(_) => DataType::Float,
359            DataValue::Boolean(_) => DataType::Boolean,
360            DataValue::DateTime(_) => DataType::DateTime,
361            DataValue::Vector(_) => DataType::String, // Display as string "[x,y,z]"
362            DataValue::Null => DataType::Null,
363        }
364    }
365
366    /// Get string representation without allocation when possible
367    /// Returns owned String for compatibility but tries to reuse existing strings
368    #[must_use]
369    pub fn to_string_optimized(&self) -> String {
370        match self {
371            DataValue::String(s) => s.clone(), // Clone existing string
372            DataValue::InternedString(s) => s.as_ref().clone(), // Clone from Rc
373            DataValue::DateTime(s) => s.clone(), // Clone existing string
374            DataValue::Integer(i) => i.to_string(),
375            DataValue::Float(f) => f.to_string(),
376            DataValue::Boolean(b) => {
377                if *b {
378                    "true".to_string()
379                } else {
380                    "false".to_string()
381                }
382            }
383            DataValue::Vector(v) => {
384                // Format as "[x,y,z]"
385                let components: Vec<String> = v.iter().map(|f| f.to_string()).collect();
386                format!("[{}]", components.join(","))
387            }
388            DataValue::Null => String::new(), // Empty string, minimal allocation
389        }
390    }
391}
392
393impl fmt::Display for DataValue {
394    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
395        match self {
396            DataValue::String(s) => write!(f, "{s}"),
397            DataValue::InternedString(s) => write!(f, "{s}"),
398            DataValue::Integer(i) => write!(f, "{i}"),
399            DataValue::Float(fl) => write!(f, "{fl}"),
400            DataValue::Boolean(b) => write!(f, "{b}"),
401            DataValue::DateTime(dt) => write!(f, "{dt}"),
402            DataValue::Vector(v) => {
403                let components: Vec<String> = v.iter().map(|fl| fl.to_string()).collect();
404                write!(f, "[{}]", components.join(","))
405            }
406            DataValue::Null => write!(f, ""),
407        }
408    }
409}
410
411/// A row of data in the table
412#[derive(Debug, Clone, Serialize, Deserialize)]
413pub struct DataRow {
414    pub values: Vec<DataValue>,
415}
416
417impl DataRow {
418    #[must_use]
419    pub fn new(values: Vec<DataValue>) -> Self {
420        Self { values }
421    }
422
423    #[must_use]
424    pub fn get(&self, index: usize) -> Option<&DataValue> {
425        self.values.get(index)
426    }
427
428    pub fn get_mut(&mut self, index: usize) -> Option<&mut DataValue> {
429        self.values.get_mut(index)
430    }
431
432    #[must_use]
433    pub fn len(&self) -> usize {
434        self.values.len()
435    }
436
437    #[must_use]
438    pub fn is_empty(&self) -> bool {
439        self.values.is_empty()
440    }
441}
442
443/// The main `DataTable` structure
444#[derive(Debug, Clone, Serialize, Deserialize)]
445pub struct DataTable {
446    pub name: String,
447    pub columns: Vec<DataColumn>,
448    pub rows: Vec<DataRow>,
449    pub metadata: HashMap<String, String>,
450}
451
452impl DataTable {
453    pub fn new(name: impl Into<String>) -> Self {
454        Self {
455            name: name.into(),
456            columns: Vec::new(),
457            rows: Vec::new(),
458            metadata: HashMap::new(),
459        }
460    }
461
462    /// Create a DUAL table (similar to Oracle's DUAL) with one row and one column
463    /// Used for evaluating expressions without a data source
464    #[must_use]
465    pub fn dual() -> Self {
466        let mut table = DataTable::new("DUAL");
467        table.add_column(DataColumn::new("DUMMY").with_type(DataType::String));
468        table
469            .add_row(DataRow::new(vec![DataValue::String("X".to_string())]))
470            .unwrap();
471        table
472    }
473
474    pub fn add_column(&mut self, column: DataColumn) -> &mut Self {
475        self.columns.push(column);
476        self
477    }
478
479    pub fn add_row(&mut self, row: DataRow) -> Result<(), String> {
480        if row.len() != self.columns.len() {
481            return Err(format!(
482                "Row has {} values but table has {} columns",
483                row.len(),
484                self.columns.len()
485            ));
486        }
487        self.rows.push(row);
488        Ok(())
489    }
490
491    #[must_use]
492    pub fn get_column(&self, name: &str) -> Option<&DataColumn> {
493        self.columns.iter().find(|c| c.name == name)
494    }
495
496    #[must_use]
497    pub fn get_column_index(&self, name: &str) -> Option<usize> {
498        self.columns.iter().position(|c| c.name == name)
499    }
500
501    /// Find column index by qualified name (e.g., "messages.field_name")
502    #[must_use]
503    pub fn find_column_by_qualified_name(&self, qualified_name: &str) -> Option<usize> {
504        self.columns
505            .iter()
506            .position(|c| c.qualified_name.as_deref() == Some(qualified_name))
507    }
508
509    /// Find column by either qualified or simple name
510    /// First tries qualified match, then falls back to simple name
511    #[must_use]
512    pub fn find_column_flexible(&self, name: &str, table_prefix: Option<&str>) -> Option<usize> {
513        // If table prefix provided, try qualified match first
514        if let Some(prefix) = table_prefix {
515            let qualified = format!("{}.{}", prefix, name);
516            if let Some(idx) = self.find_column_by_qualified_name(&qualified) {
517                return Some(idx);
518            }
519        }
520
521        // Fall back to simple name match
522        self.get_column_index(name)
523    }
524
525    /// Enrich all columns with qualified names based on the table name
526    pub fn enrich_columns_with_qualified_names(&mut self, table_name: &str) {
527        for column in &mut self.columns {
528            column.qualified_name = Some(format!("{}.{}", table_name, column.name));
529            column.source_table = Some(table_name.to_string());
530        }
531    }
532
533    #[must_use]
534    pub fn column_count(&self) -> usize {
535        self.columns.len()
536    }
537
538    #[must_use]
539    pub fn row_count(&self) -> usize {
540        self.rows.len()
541    }
542
543    #[must_use]
544    pub fn is_empty(&self) -> bool {
545        self.rows.is_empty()
546    }
547
548    /// Get column names as a vector
549    #[must_use]
550    pub fn column_names(&self) -> Vec<String> {
551        self.columns.iter().map(|c| c.name.clone()).collect()
552    }
553
554    /// Get mutable access to columns for enrichment
555    pub fn columns_mut(&mut self) -> &mut [DataColumn] {
556        &mut self.columns
557    }
558
559    /// Infer and update column types based on data
560    pub fn infer_column_types(&mut self) {
561        for (col_idx, column) in self.columns.iter_mut().enumerate() {
562            let mut inferred_type = DataType::Null;
563            let mut null_count = 0;
564            let mut unique_values = std::collections::HashSet::new();
565
566            for row in &self.rows {
567                if let Some(value) = row.get(col_idx) {
568                    if value.is_null() {
569                        null_count += 1;
570                    } else {
571                        let value_type = value.data_type();
572                        inferred_type = inferred_type.merge(&value_type);
573                        unique_values.insert(value.to_string());
574                    }
575                }
576            }
577
578            column.data_type = inferred_type;
579            column.null_count = null_count;
580            column.nullable = null_count > 0;
581            column.unique_values = Some(unique_values.len());
582        }
583    }
584
585    /// Get a value at specific row and column
586    #[must_use]
587    pub fn get_value(&self, row: usize, col: usize) -> Option<&DataValue> {
588        self.rows.get(row)?.get(col)
589    }
590
591    /// Get a value by row index and column name
592    #[must_use]
593    pub fn get_value_by_name(&self, row: usize, col_name: &str) -> Option<&DataValue> {
594        let col_idx = self.get_column_index(col_name)?;
595        self.get_value(row, col_idx)
596    }
597
598    /// Convert to a vector of string vectors (for display/compatibility)
599    #[must_use]
600    pub fn to_string_table(&self) -> Vec<Vec<String>> {
601        self.rows
602            .iter()
603            .map(|row| {
604                row.values
605                    .iter()
606                    .map(DataValue::to_string_optimized)
607                    .collect()
608            })
609            .collect()
610    }
611
612    /// Get table statistics
613    #[must_use]
614    pub fn get_stats(&self) -> DataTableStats {
615        DataTableStats {
616            row_count: self.row_count(),
617            column_count: self.column_count(),
618            memory_size: self.estimate_memory_size(),
619            null_count: self.columns.iter().map(|c| c.null_count).sum(),
620        }
621    }
622
623    /// Generate a debug dump string for display
624    #[must_use]
625    pub fn debug_dump(&self) -> String {
626        let mut output = String::new();
627
628        output.push_str(&format!("DataTable: {}\n", self.name));
629        output.push_str(&format!(
630            "Rows: {} | Columns: {}\n",
631            self.row_count(),
632            self.column_count()
633        ));
634
635        if !self.metadata.is_empty() {
636            output.push_str("Metadata:\n");
637            for (key, value) in &self.metadata {
638                output.push_str(&format!("  {key}: {value}\n"));
639            }
640        }
641
642        output.push_str("\nColumns:\n");
643        for column in &self.columns {
644            output.push_str(&format!("  {} ({:?})", column.name, column.data_type));
645            if column.nullable {
646                output.push_str(&format!(" - nullable, {} nulls", column.null_count));
647            }
648            if let Some(unique) = column.unique_values {
649                output.push_str(&format!(", {unique} unique"));
650            }
651            output.push('\n');
652        }
653
654        // Show first few rows
655        if self.row_count() > 0 {
656            let sample_size = 5.min(self.row_count());
657            output.push_str(&format!("\nFirst {sample_size} rows:\n"));
658
659            for row_idx in 0..sample_size {
660                output.push_str(&format!("  [{row_idx}]: "));
661                for (col_idx, value) in self.rows[row_idx].values.iter().enumerate() {
662                    if col_idx > 0 {
663                        output.push_str(", ");
664                    }
665                    output.push_str(&value.to_string());
666                }
667                output.push('\n');
668            }
669        }
670
671        output
672    }
673
674    #[must_use]
675    pub fn estimate_memory_size(&self) -> usize {
676        // Base structure size
677        let mut size = std::mem::size_of::<Self>();
678
679        // Column metadata
680        size += self.columns.len() * std::mem::size_of::<DataColumn>();
681        for col in &self.columns {
682            size += col.name.len();
683        }
684
685        // Row structure overhead
686        size += self.rows.len() * std::mem::size_of::<DataRow>();
687
688        // Actual data values
689        for row in &self.rows {
690            for value in &row.values {
691                // Base enum size
692                size += std::mem::size_of::<DataValue>();
693                // Add string content size
694                match value {
695                    DataValue::String(s) | DataValue::DateTime(s) => size += s.len(),
696                    DataValue::Vector(v) => size += v.len() * std::mem::size_of::<f64>(),
697                    _ => {} // Numbers and booleans are inline
698                }
699            }
700        }
701
702        size
703    }
704
705    /// Convert DataTable to CSV format
706    pub fn to_csv(&self) -> String {
707        let mut csv_output = String::new();
708
709        // Write headers
710        let headers: Vec<String> = self
711            .columns
712            .iter()
713            .map(|col| {
714                if col.name.contains(',') || col.name.contains('"') || col.name.contains('\n') {
715                    format!("\"{}\"", col.name.replace('"', "\"\""))
716                } else {
717                    col.name.clone()
718                }
719            })
720            .collect();
721        csv_output.push_str(&headers.join(","));
722        csv_output.push('\n');
723
724        // Write data rows
725        for row in &self.rows {
726            let row_values: Vec<String> = row
727                .values
728                .iter()
729                .map(|value| {
730                    let str_val = value.to_string();
731                    if str_val.contains(',') || str_val.contains('"') || str_val.contains('\n') {
732                        format!("\"{}\"", str_val.replace('"', "\"\""))
733                    } else {
734                        str_val
735                    }
736                })
737                .collect();
738            csv_output.push_str(&row_values.join(","));
739            csv_output.push('\n');
740        }
741
742        csv_output
743    }
744
745    /// V46: Create `DataTable` from `QueryResponse`
746    /// This is the key conversion function that bridges old and new systems
747    pub fn from_query_response(response: &QueryResponse, table_name: &str) -> Result<Self, String> {
748        debug!(
749            "V46: Converting QueryResponse to DataTable for table '{}'",
750            table_name
751        );
752
753        // Track memory before conversion
754        crate::utils::memory_tracker::track_memory("start_from_query_response");
755
756        let mut table = DataTable::new(table_name);
757
758        // Extract column names and types from first row
759        if let Some(first_row) = response.data.first() {
760            if let Some(obj) = first_row.as_object() {
761                // Create columns based on the keys in the JSON object
762                for key in obj.keys() {
763                    let column = DataColumn::new(key.clone());
764                    table.add_column(column);
765                }
766
767                // Now convert all rows
768                for json_row in &response.data {
769                    if let Some(row_obj) = json_row.as_object() {
770                        let mut values = Vec::new();
771
772                        // Ensure we get values in the same order as columns
773                        for column in &table.columns {
774                            let value = row_obj
775                                .get(&column.name)
776                                .map_or(DataValue::Null, json_value_to_data_value);
777                            values.push(value);
778                        }
779
780                        table.add_row(DataRow::new(values))?;
781                    }
782                }
783
784                // Infer column types from the data
785                table.infer_column_types();
786
787                // Add metadata
788                if let Some(source) = &response.source {
789                    table.metadata.insert("source".to_string(), source.clone());
790                }
791                if let Some(cached) = response.cached {
792                    table
793                        .metadata
794                        .insert("cached".to_string(), cached.to_string());
795                }
796                table
797                    .metadata
798                    .insert("original_count".to_string(), response.count.to_string());
799
800                debug!(
801                    "V46: Created DataTable with {} columns and {} rows",
802                    table.column_count(),
803                    table.row_count()
804                );
805            } else {
806                // Handle non-object JSON (single values)
807                table.add_column(DataColumn::new("value"));
808                for json_value in &response.data {
809                    let value = json_value_to_data_value(json_value);
810                    table.add_row(DataRow::new(vec![value]))?;
811                }
812            }
813        }
814
815        Ok(table)
816    }
817
818    /// Get a single row by index
819    #[must_use]
820    pub fn get_row(&self, index: usize) -> Option<&DataRow> {
821        self.rows.get(index)
822    }
823
824    /// V50: Get a single row as strings
825    #[must_use]
826    pub fn get_row_as_strings(&self, index: usize) -> Option<Vec<String>> {
827        self.rows.get(index).map(|row| {
828            row.values
829                .iter()
830                .map(DataValue::to_string_optimized)
831                .collect()
832        })
833    }
834
835    /// Pretty print the `DataTable` with a nice box drawing
836    #[must_use]
837    pub fn pretty_print(&self) -> String {
838        let mut output = String::new();
839
840        // Header
841        output.push_str("╔═══════════════════════════════════════════════════════╗\n");
842        output.push_str(&format!("║ DataTable: {:^41} ║\n", self.name));
843        output.push_str("╠═══════════════════════════════════════════════════════╣\n");
844
845        // Summary stats
846        output.push_str(&format!(
847            "║ Rows: {:6} | Columns: {:3} | Memory: ~{:6} bytes ║\n",
848            self.row_count(),
849            self.column_count(),
850            self.get_stats().memory_size
851        ));
852
853        // Metadata if any
854        if !self.metadata.is_empty() {
855            output.push_str("╠═══════════════════════════════════════════════════════╣\n");
856            output.push_str("║ Metadata:                                             ║\n");
857            for (key, value) in &self.metadata {
858                let truncated_value = if value.len() > 35 {
859                    format!("{}...", &value[..32])
860                } else {
861                    value.clone()
862                };
863                output.push_str(&format!(
864                    "║   {:15} : {:35} ║\n",
865                    Self::truncate_string(key, 15),
866                    truncated_value
867                ));
868            }
869        }
870
871        // Column details
872        output.push_str("╠═══════════════════════════════════════════════════════╣\n");
873        output.push_str("║ Columns:                                              ║\n");
874        output.push_str("╟───────────────────┬──────────┬─────────┬──────┬──────╢\n");
875        output.push_str("║ Name              │ Type     │ Nullable│ Nulls│Unique║\n");
876        output.push_str("╟───────────────────┼──────────┼─────────┼──────┼──────╢\n");
877
878        for column in &self.columns {
879            let type_str = match &column.data_type {
880                DataType::String => "String",
881                DataType::Integer => "Integer",
882                DataType::Float => "Float",
883                DataType::Boolean => "Boolean",
884                DataType::DateTime => "DateTime",
885                DataType::Null => "Null",
886                DataType::Mixed => "Mixed",
887            };
888
889            output.push_str(&format!(
890                "║ {:17} │ {:8} │ {:7} │ {:4} │ {:4} ║\n",
891                Self::truncate_string(&column.name, 17),
892                type_str,
893                if column.nullable { "Yes" } else { "No" },
894                column.null_count,
895                column.unique_values.unwrap_or(0)
896            ));
897        }
898
899        output.push_str("╚═══════════════════════════════════════════════════════╝\n");
900
901        // Sample data (first 5 rows)
902        output.push_str("\nSample Data (first 5 rows):\n");
903        let sample_count = self.rows.len().min(5);
904
905        if sample_count > 0 {
906            // Column headers
907            output.push('┌');
908            for (i, _col) in self.columns.iter().enumerate() {
909                if i > 0 {
910                    output.push('┬');
911                }
912                output.push_str(&"─".repeat(20));
913            }
914            output.push_str("┐\n");
915
916            output.push('│');
917            for col in &self.columns {
918                output.push_str(&format!(" {:^18} │", Self::truncate_string(&col.name, 18)));
919            }
920            output.push('\n');
921
922            output.push('├');
923            for (i, _) in self.columns.iter().enumerate() {
924                if i > 0 {
925                    output.push('┼');
926                }
927                output.push_str(&"─".repeat(20));
928            }
929            output.push_str("┤\n");
930
931            // Data rows
932            for row_idx in 0..sample_count {
933                if let Some(row) = self.rows.get(row_idx) {
934                    output.push('│');
935                    for value in &row.values {
936                        let value_str = value.to_string();
937                        output
938                            .push_str(&format!(" {:18} │", Self::truncate_string(&value_str, 18)));
939                    }
940                    output.push('\n');
941                }
942            }
943
944            output.push('└');
945            for (i, _) in self.columns.iter().enumerate() {
946                if i > 0 {
947                    output.push('┴');
948                }
949                output.push_str(&"─".repeat(20));
950            }
951            output.push_str("┘\n");
952        }
953
954        output
955    }
956
957    fn truncate_string(s: &str, max_len: usize) -> String {
958        if s.len() > max_len {
959            format!("{}...", &s[..max_len - 3])
960        } else {
961            s.to_string()
962        }
963    }
964
965    /// Get a schema summary of the `DataTable`
966    #[must_use]
967    pub fn get_schema_summary(&self) -> String {
968        let mut summary = String::new();
969        summary.push_str(&format!(
970            "DataTable Schema ({} columns, {} rows):\n",
971            self.columns.len(),
972            self.rows.len()
973        ));
974
975        for (idx, column) in self.columns.iter().enumerate() {
976            let type_str = match &column.data_type {
977                DataType::String => "String",
978                DataType::Integer => "Integer",
979                DataType::Float => "Float",
980                DataType::Boolean => "Boolean",
981                DataType::DateTime => "DateTime",
982                DataType::Null => "Null",
983                DataType::Mixed => "Mixed",
984            };
985
986            let nullable_str = if column.nullable {
987                "nullable"
988            } else {
989                "not null"
990            };
991            let null_info = if column.null_count > 0 {
992                format!(", {} nulls", column.null_count)
993            } else {
994                String::new()
995            };
996
997            summary.push_str(&format!(
998                "  [{:3}] {} : {} ({}{})\n",
999                idx, column.name, type_str, nullable_str, null_info
1000            ));
1001        }
1002
1003        summary
1004    }
1005
1006    /// Get detailed schema information as a structured format
1007    #[must_use]
1008    pub fn get_schema_info(&self) -> Vec<(String, String, bool, usize)> {
1009        self.columns
1010            .iter()
1011            .map(|col| {
1012                let type_name = format!("{:?}", col.data_type);
1013                (col.name.clone(), type_name, col.nullable, col.null_count)
1014            })
1015            .collect()
1016    }
1017
1018    /// Reserve capacity for rows to avoid reallocations
1019    pub fn reserve_rows(&mut self, additional: usize) {
1020        self.rows.reserve(additional);
1021    }
1022
1023    /// Shrink vectors to fit actual data (removes excess capacity)
1024    pub fn shrink_to_fit(&mut self) {
1025        self.rows.shrink_to_fit();
1026        for _column in &mut self.columns {
1027            // Shrink any column-specific data if needed
1028        }
1029    }
1030
1031    /// Get actual memory usage estimate (more accurate than `estimate_memory_size`)
1032    #[must_use]
1033    pub fn get_memory_usage(&self) -> usize {
1034        let mut size = std::mem::size_of::<Self>();
1035
1036        // Account for string allocations
1037        size += self.name.capacity();
1038
1039        // Account for columns
1040        size += self.columns.capacity() * std::mem::size_of::<DataColumn>();
1041        for col in &self.columns {
1042            size += col.name.capacity();
1043        }
1044
1045        // Account for rows and their capacity
1046        size += self.rows.capacity() * std::mem::size_of::<DataRow>();
1047
1048        // Account for actual data values
1049        for row in &self.rows {
1050            size += row.values.capacity() * std::mem::size_of::<DataValue>();
1051            for value in &row.values {
1052                match value {
1053                    DataValue::String(s) => size += s.capacity(),
1054                    DataValue::InternedString(_) => size += std::mem::size_of::<Arc<String>>(),
1055                    DataValue::DateTime(s) => size += s.capacity(),
1056                    DataValue::Vector(v) => size += v.capacity() * std::mem::size_of::<f64>(),
1057                    _ => {} // Other types are inline
1058                }
1059            }
1060        }
1061
1062        // Account for metadata
1063        size += self.metadata.capacity() * std::mem::size_of::<(String, String)>();
1064        for (k, v) in &self.metadata {
1065            size += k.capacity() + v.capacity();
1066        }
1067
1068        size
1069    }
1070
1071    /// Serialize DataTable to bytes for caching (using MessagePack for now, can be upgraded to Parquet)
1072    pub fn to_parquet_bytes(&self) -> Result<Vec<u8>, String> {
1073        // For now, use MessagePack which is binary-safe and fast
1074        // Later we can upgrade to actual Parquet format
1075        rmp_serde::to_vec(self).map_err(|e| format!("Failed to serialize DataTable: {}", e))
1076    }
1077
1078    /// Deserialize DataTable from cached bytes
1079    pub fn from_parquet_bytes(bytes: &[u8]) -> Result<Self, String> {
1080        // For now, use MessagePack
1081        // Later we can upgrade to actual Parquet format
1082        rmp_serde::from_slice(bytes).map_err(|e| format!("Failed to deserialize DataTable: {}", e))
1083    }
1084}
1085
1086/// V46: Helper function to convert JSON value to `DataValue`
1087fn json_value_to_data_value(json: &JsonValue) -> DataValue {
1088    match json {
1089        JsonValue::Null => DataValue::Null,
1090        JsonValue::Bool(b) => DataValue::Boolean(*b),
1091        JsonValue::Number(n) => {
1092            if let Some(i) = n.as_i64() {
1093                DataValue::Integer(i)
1094            } else if let Some(f) = n.as_f64() {
1095                DataValue::Float(f)
1096            } else {
1097                DataValue::String(n.to_string())
1098            }
1099        }
1100        JsonValue::String(s) => {
1101            // Try to detect if it's a date/time
1102            if s.contains('-') && s.len() >= 8 && s.len() <= 30 {
1103                // Simple heuristic for dates
1104                DataValue::DateTime(s.clone())
1105            } else {
1106                DataValue::String(s.clone())
1107            }
1108        }
1109        JsonValue::Array(_) | JsonValue::Object(_) => {
1110            // Store complex types as JSON string
1111            DataValue::String(json.to_string())
1112        }
1113    }
1114}
1115
1116/// Statistics about a `DataTable`
1117#[derive(Debug, Clone)]
1118pub struct DataTableStats {
1119    pub row_count: usize,
1120    pub column_count: usize,
1121    pub memory_size: usize,
1122    pub null_count: usize,
1123}
1124
1125/// Implementation of `DataProvider` for `DataTable`
1126/// This allows `DataTable` to be used wherever `DataProvider` trait is expected
1127impl DataProvider for DataTable {
1128    fn get_row(&self, index: usize) -> Option<Vec<String>> {
1129        self.rows.get(index).map(|row| {
1130            row.values
1131                .iter()
1132                .map(DataValue::to_string_optimized)
1133                .collect()
1134        })
1135    }
1136
1137    fn get_column_names(&self) -> Vec<String> {
1138        self.column_names()
1139    }
1140
1141    fn get_row_count(&self) -> usize {
1142        self.row_count()
1143    }
1144
1145    fn get_column_count(&self) -> usize {
1146        self.column_count()
1147    }
1148}
1149
1150#[cfg(test)]
1151mod tests {
1152    use super::*;
1153
1154    #[test]
1155    fn test_data_type_inference() {
1156        assert_eq!(DataType::infer_from_string("123"), DataType::Integer);
1157        assert_eq!(DataType::infer_from_string("123.45"), DataType::Float);
1158        assert_eq!(DataType::infer_from_string("true"), DataType::Boolean);
1159        assert_eq!(DataType::infer_from_string("hello"), DataType::String);
1160        assert_eq!(DataType::infer_from_string(""), DataType::Null);
1161        assert_eq!(
1162            DataType::infer_from_string("2024-01-01"),
1163            DataType::DateTime
1164        );
1165    }
1166
1167    #[test]
1168    fn test_datatable_creation() {
1169        let mut table = DataTable::new("test");
1170
1171        table.add_column(DataColumn::new("id").with_type(DataType::Integer));
1172        table.add_column(DataColumn::new("name").with_type(DataType::String));
1173        table.add_column(DataColumn::new("active").with_type(DataType::Boolean));
1174
1175        assert_eq!(table.column_count(), 3);
1176        assert_eq!(table.row_count(), 0);
1177
1178        let row = DataRow::new(vec![
1179            DataValue::Integer(1),
1180            DataValue::String("Alice".to_string()),
1181            DataValue::Boolean(true),
1182        ]);
1183
1184        table.add_row(row).unwrap();
1185        assert_eq!(table.row_count(), 1);
1186
1187        let value = table.get_value_by_name(0, "name").unwrap();
1188        assert_eq!(value.to_string(), "Alice");
1189    }
1190
1191    #[test]
1192    fn test_type_inference() {
1193        let mut table = DataTable::new("test");
1194
1195        // Add columns without types
1196        table.add_column(DataColumn::new("mixed"));
1197
1198        // Add rows with different types
1199        table
1200            .add_row(DataRow::new(vec![DataValue::Integer(1)]))
1201            .unwrap();
1202        table
1203            .add_row(DataRow::new(vec![DataValue::Float(2.5)]))
1204            .unwrap();
1205        table.add_row(DataRow::new(vec![DataValue::Null])).unwrap();
1206
1207        table.infer_column_types();
1208
1209        // Should infer Float since we have both Integer and Float
1210        assert_eq!(table.columns[0].data_type, DataType::Float);
1211        assert_eq!(table.columns[0].null_count, 1);
1212        assert!(table.columns[0].nullable);
1213    }
1214
1215    #[test]
1216    fn test_from_query_response() {
1217        use crate::api_client::{QueryInfo, QueryResponse};
1218        use serde_json::json;
1219
1220        let response = QueryResponse {
1221            query: QueryInfo {
1222                select: vec!["id".to_string(), "name".to_string(), "age".to_string()],
1223                where_clause: None,
1224                order_by: None,
1225            },
1226            data: vec![
1227                json!({
1228                    "id": 1,
1229                    "name": "Alice",
1230                    "age": 30
1231                }),
1232                json!({
1233                    "id": 2,
1234                    "name": "Bob",
1235                    "age": 25
1236                }),
1237                json!({
1238                    "id": 3,
1239                    "name": "Carol",
1240                    "age": null
1241                }),
1242            ],
1243            count: 3,
1244            source: Some("test.csv".to_string()),
1245            table: Some("test".to_string()),
1246            cached: Some(false),
1247        };
1248
1249        let table = DataTable::from_query_response(&response, "test").unwrap();
1250
1251        assert_eq!(table.name, "test");
1252        assert_eq!(table.row_count(), 3);
1253        assert_eq!(table.column_count(), 3);
1254
1255        // Check column names
1256        let col_names = table.column_names();
1257        assert!(col_names.contains(&"id".to_string()));
1258        assert!(col_names.contains(&"name".to_string()));
1259        assert!(col_names.contains(&"age".to_string()));
1260
1261        // Check metadata
1262        assert_eq!(table.metadata.get("source"), Some(&"test.csv".to_string()));
1263        assert_eq!(table.metadata.get("cached"), Some(&"false".to_string()));
1264
1265        // Check first row values
1266        assert_eq!(
1267            table.get_value_by_name(0, "id"),
1268            Some(&DataValue::Integer(1))
1269        );
1270        assert_eq!(
1271            table.get_value_by_name(0, "name"),
1272            Some(&DataValue::String("Alice".to_string()))
1273        );
1274        assert_eq!(
1275            table.get_value_by_name(0, "age"),
1276            Some(&DataValue::Integer(30))
1277        );
1278
1279        // Check null handling
1280        assert_eq!(table.get_value_by_name(2, "age"), Some(&DataValue::Null));
1281    }
1282}
sql_cli/data/datatable.rs

sql_cli/data/
datatable.rs
