tellaro-query-language 1.3.8

//! TQL CLI - Command-line interface for Tellaro Query Language
//!
//! This binary provides a command-line interface for querying structured data files
//! using TQL syntax, with full feature parity to the Python implementation.

use clap::{Parser, ValueEnum};
use serde_json::Value as JsonValue;
use std::collections::HashMap;
use std::fs::{self, File};
use std::io::{self, BufRead, BufReader, BufWriter, Write};
use std::path::Path;
use tellaro_query_language::Tql;

#[derive(Debug, Clone, ValueEnum)]
enum InputFormat {
    Json,
    Jsonl,
    Csv,
    Auto,
}

#[derive(Debug, Clone, ValueEnum)]
enum OutputFormat {
    Json,
    Jsonl,
    Table,
}

/// TQL - Tellaro Query Language CLI for querying structured data files
#[derive(Parser, Debug)]
#[command(author, version, about, long_about = None)]
struct Args {
    /// TQL query string
    query: String,

    /// Path to file or folder (reads from stdin if not provided)
    file_or_folder: Option<String>,

    /// Input file format (default: auto-detect from extension)
    #[arg(long, value_enum, default_value = "auto")]
    format: InputFormat,

    /// CSV delimiter character (default: ,)
    #[arg(long, default_value = ",")]
    csv_delimiter: String,

    /// Comma-separated CSV header names (overrides auto-detection)
    #[arg(long)]
    csv_headers: Option<String>,

    /// CSV has no header row (generates column1, column2, etc.)
    #[arg(long, default_value_t = false)]
    no_header: bool,

    /// JSON string mapping field names to types (e.g., '{"age":"integer"}')
    #[arg(long)]
    field_types: Option<String>,

    /// Process folders recursively
    #[arg(long, default_value_t = false)]
    recursive: bool,

    /// File pattern for folder processing (default: *)
    #[arg(long, default_value = "*")]
    pattern: String,

    /// Output file path (default: stdout)
    #[arg(long, short = 'o')]
    output: Option<String>,

    /// Output format
    #[arg(long, value_enum, default_value = "table")]
    output_format: OutputFormat,

    /// Maximum number of records to output
    #[arg(long, short = 'n')]
    limit: Option<usize>,

    /// Only output statistics, no records
    #[arg(long, default_value_t = false)]
    stats_only: bool,

    /// Number of parallel workers for folder processing (default: 4)
    #[arg(long, default_value_t = 4)]
    parallel: usize,

    /// Number of records to sample for type inference (default: 100)
    #[arg(long, default_value_t = 100)]
    sample_size: usize,

    /// Verbose output with progress information
    #[arg(long, short = 'v')]
    verbose: bool,

    /// Suppress informational messages
    #[arg(long, short = 'q')]
    quiet: bool,
}

fn main() {
    let args = Args::parse();

    if let Err(e) = run(args) {
        eprintln!("Error: {}", e);
        std::process::exit(1);
    }
}

fn run(args: Args) -> Result<(), Box<dyn std::error::Error>> {
    // Read input data
    let records = read_input(&args)?;

    if args.verbose && !args.quiet {
        eprintln!("Loaded {} records", records.len());
        eprintln!("Executing query: {}", args.query);
    }

    // Create TQL instance
    let tql = Tql::new();

    // Check if this is a stats query
    let is_stats = tql.is_stats_query(&args.query)?;

    if is_stats {
        // Execute stats query
        let stats_result = tql.evaluate_stats(&records, &args.query)?;

        if args.verbose && !args.quiet {
            eprintln!("Stats query executed successfully");
        }

        // Output stats results
        write_stats_output(&args, &stats_result)?;

        if !args.quiet {
            eprintln!("✓ Processed {} records", records.len());
        }
    } else {
        // Execute filter query with enrichment (applies mutators)
        let results: Vec<JsonValue> = tql.query_enriched(&records, &args.query)?;

        // Apply limit if specified
        let output_records: Vec<JsonValue> = if let Some(limit) = args.limit {
            results.into_iter().take(limit).collect()
        } else {
            results
        };

        if args.verbose && !args.quiet {
            eprintln!("Found {} matching records", output_records.len());
        }

        // Write output
        write_output(&args, &output_records)?;

        if !args.quiet {
            eprintln!(
                "✓ Processed {} records, {} matched",
                records.len(),
                output_records.len()
            );
        }
    }

    Ok(())
}

fn read_input(args: &Args) -> Result<Vec<JsonValue>, Box<dyn std::error::Error>> {
    match &args.file_or_folder {
        Some(path) => read_from_path(path, args),
        None => read_from_stdin(&args.format, args),
    }
}

fn read_from_path(path: &str, args: &Args) -> Result<Vec<JsonValue>, Box<dyn std::error::Error>> {
    let path_obj = Path::new(path);

    if !path_obj.exists() {
        return Err(format!("File or folder not found: {}", path).into());
    }

    if path_obj.is_file() {
        if args.verbose && !args.quiet {
            eprintln!("Reading file: {}", path);
        }
        read_single_file(path_obj, args)
    } else if path_obj.is_dir() {
        if args.verbose && !args.quiet {
            eprintln!("Reading directory: {}", path);
        }
        read_directory(path_obj, args)
    } else {
        Err(format!("Invalid path: {}", path).into())
    }
}

fn read_single_file(
    path: &Path,
    args: &Args,
) -> Result<Vec<JsonValue>, Box<dyn std::error::Error>> {
    // Detect format from extension if auto
    let format = match &args.format {
        InputFormat::Auto => detect_format_from_path(path)?,
        f => f.clone(),
    };

    match format {
        InputFormat::Json => read_json_file(path),
        InputFormat::Jsonl => read_jsonl_file(path),
        InputFormat::Csv => read_csv_file(path, args),
        InputFormat::Auto => unreachable!(),
    }
}

fn read_directory(
    dir_path: &Path,
    args: &Args,
) -> Result<Vec<JsonValue>, Box<dyn std::error::Error>> {
    use walkdir::WalkDir;

    let mut all_records = Vec::new();
    let pattern = glob::Pattern::new(&args.pattern)?;

    let walker = if args.recursive {
        WalkDir::new(dir_path)
    } else {
        WalkDir::new(dir_path).max_depth(1)
    };

    let matching_files: Vec<_> = walker
        .into_iter()
        .filter_map(|e| e.ok())
        .filter(|e| e.path().is_file())
        .filter(|e| {
            e.path()
                .file_name()
                .and_then(|n| n.to_str())
                .map(|name| pattern.matches(name))
                .unwrap_or(false)
        })
        .collect();

    if args.verbose && !args.quiet {
        eprintln!("Found {} matching files", matching_files.len());
    }

    // Use rayon for parallel processing if multiple files
    use rayon::prelude::*;

    if matching_files.len() > 1 && args.parallel > 1 {
        let pool = rayon::ThreadPoolBuilder::new()
            .num_threads(args.parallel)
            .build()?;

        let results: Vec<Vec<JsonValue>> = pool.install(|| {
            matching_files
                .par_iter()
                .filter_map(|entry| {
                    let file_path = entry.path();
                    if args.verbose && !args.quiet {
                        eprintln!("  Processing: {}", file_path.display());
                    }
                    read_single_file(file_path, args).ok()
                })
                .collect()
        });

        for records in results {
            all_records.extend(records);
        }
    } else {
        for entry in matching_files {
            let file_path = entry.path();
            if args.verbose && !args.quiet {
                eprintln!("  Processing: {}", file_path.display());
            }
            match read_single_file(file_path, args) {
                Ok(records) => all_records.extend(records),
                Err(e) => eprintln!("Warning: Failed to read {}: {}", file_path.display(), e),
            }
        }
    }

    Ok(all_records)
}

fn detect_format_from_path(path: &Path) -> Result<InputFormat, Box<dyn std::error::Error>> {
    match path.extension().and_then(|s| s.to_str()) {
        Some("json") => Ok(InputFormat::Json),
        Some("jsonl") | Some("ndjson") => Ok(InputFormat::Jsonl),
        Some("csv") => Ok(InputFormat::Csv),
        _ => Ok(InputFormat::Jsonl), // Default to JSONL
    }
}

fn read_json_file(path: &Path) -> Result<Vec<JsonValue>, Box<dyn std::error::Error>> {
    let content = fs::read_to_string(path)?;
    let value: JsonValue = serde_json::from_str(&content)?;

    match value {
        JsonValue::Array(arr) => Ok(arr),
        single => Ok(vec![single]),
    }
}

fn read_jsonl_file(path: &Path) -> Result<Vec<JsonValue>, Box<dyn std::error::Error>> {
    let file = File::open(path)?;
    let reader = BufReader::new(file);
    let mut records = Vec::new();

    for (line_num, line) in reader.lines().enumerate() {
        let line = line?;
        if line.trim().is_empty() {
            continue;
        }

        match serde_json::from_str::<JsonValue>(&line) {
            Ok(record) => records.push(record),
            Err(e) => {
                eprintln!("Warning: Failed to parse line {}: {}", line_num + 1, e);
            }
        }
    }

    Ok(records)
}

fn read_csv_file(path: &Path, args: &Args) -> Result<Vec<JsonValue>, Box<dyn std::error::Error>> {
    use csv::ReaderBuilder;

    let delimiter = args.csv_delimiter.as_bytes()[0];
    let mut reader = ReaderBuilder::new()
        .delimiter(delimiter)
        .has_headers(!args.no_header)
        .from_path(path)?;

    let mut records = Vec::new();

    // Get or generate headers
    let headers: Vec<String> = if let Some(ref header_str) = args.csv_headers {
        // User-specified headers
        header_str
            .split(',')
            .map(|s| s.trim().to_string())
            .collect()
    } else if args.no_header {
        // Generate headers: column1, column2, etc.
        let first_record = if let Some(result) = reader.records().next() {
            result?
        } else {
            return Ok(records); // Empty file
        };
        (1..=first_record.len())
            .map(|i| format!("column{}", i))
            .collect()
    } else {
        // Read headers from file
        reader.headers()?.iter().map(|s| s.to_string()).collect()
    };

    // Process records
    for result in reader.records() {
        let record = result?;
        let mut json_obj = serde_json::Map::new();

        for (i, field) in record.iter().enumerate() {
            if i < headers.len() {
                let value = infer_value_type(field, args);
                json_obj.insert(headers[i].clone(), value);
            }
        }

        records.push(JsonValue::Object(json_obj));
    }

    Ok(records)
}

fn infer_value_type(field: &str, _args: &Args) -> JsonValue {
    // Try integer
    if let Ok(i) = field.parse::<i64>() {
        return JsonValue::Number(serde_json::Number::from(i));
    }

    // Try float
    if let Ok(f) = field.parse::<f64>() {
        if let Some(num) = serde_json::Number::from_f64(f) {
            return JsonValue::Number(num);
        }
    }

    // Try boolean
    match field.to_lowercase().as_str() {
        "true" => return JsonValue::Bool(true),
        "false" => return JsonValue::Bool(false),
        _ => {}
    }

    // Default to string
    JsonValue::String(field.to_string())
}

fn read_from_stdin(
    format: &InputFormat,
    _args: &Args,
) -> Result<Vec<JsonValue>, Box<dyn std::error::Error>> {
    let stdin = io::stdin();
    let mut records = Vec::new();

    match format {
        InputFormat::Json | InputFormat::Auto => {
            let mut buffer = String::new();
            for line in stdin.lock().lines() {
                buffer.push_str(&line?);
                buffer.push('\n');
            }

            let value: JsonValue = serde_json::from_str(&buffer)?;
            match value {
                JsonValue::Array(arr) => Ok(arr),
                single => Ok(vec![single]),
            }
        }
        InputFormat::Jsonl => {
            for line in stdin.lock().lines() {
                let line = line?;
                if line.trim().is_empty() {
                    continue;
                }

                match serde_json::from_str::<JsonValue>(&line) {
                    Ok(record) => records.push(record),
                    Err(e) => {
                        eprintln!("Warning: Failed to parse line: {}", e);
                    }
                }
            }
            Ok(records)
        }
        InputFormat::Csv => {
            // CSV from stdin not fully implemented yet
            Err("CSV from stdin not yet supported".into())
        }
    }
}

fn write_stats_output(
    _args: &Args,
    stats_result: &JsonValue,
) -> Result<(), Box<dyn std::error::Error>> {
    // For now, print stats in a readable format
    println!("Statistics:");
    println!("{}", "-".repeat(50));
    println!("{}", serde_json::to_string_pretty(stats_result)?);
    Ok(())
}

fn write_output(args: &Args, records: &[JsonValue]) -> Result<(), Box<dyn std::error::Error>> {
    match &args.output {
        Some(path) => write_to_file(path, records, &args.output_format),
        None => write_to_stdout(records, &args.output_format),
    }
}

fn write_to_file(
    path: &str,
    records: &[JsonValue],
    format: &OutputFormat,
) -> Result<(), Box<dyn std::error::Error>> {
    let mut file = File::create(path)?;

    match format {
        OutputFormat::Json => {
            let json = serde_json::to_string_pretty(records)?;
            file.write_all(json.as_bytes())?;
        }
        OutputFormat::Jsonl => {
            for record in records {
                let json = serde_json::to_string(record)?;
                writeln!(file, "{}", json)?;
            }
        }
        OutputFormat::Table => {
            // For file output with table format, write JSON instead
            let json = serde_json::to_string_pretty(records)?;
            file.write_all(json.as_bytes())?;
        }
    }

    Ok(())
}

fn write_to_stdout(
    records: &[JsonValue],
    format: &OutputFormat,
) -> Result<(), Box<dyn std::error::Error>> {
    let stdout = io::stdout();
    let mut handle = BufWriter::new(stdout.lock());

    match format {
        OutputFormat::Json => {
            let json = serde_json::to_string_pretty(records)?;
            writeln!(handle, "{}", json)?;
        }
        OutputFormat::Jsonl => {
            for record in records {
                let json = serde_json::to_string(record)?;
                writeln!(handle, "{}", json)?;
            }
        }
        OutputFormat::Table => {
            print_table(records)?;
        }
    }

    handle.flush()?;
    Ok(())
}

fn print_table(records: &[JsonValue]) -> Result<(), Box<dyn std::error::Error>> {
    if records.is_empty() {
        println!("No matching records");
        return Ok(());
    }

    // Flatten all records to get all possible keys
    let mut all_keys = Vec::new();
    let flattened_records: Vec<HashMap<String, String>> = records
        .iter()
        .map(|record| {
            let flat = flatten_json(record, "");
            for key in flat.keys() {
                if !all_keys.contains(key) {
                    all_keys.push(key.clone());
                }
            }
            flat
        })
        .collect();

    all_keys.sort();

    // Calculate column widths
    let mut col_widths: HashMap<String, usize> = HashMap::new();
    for key in &all_keys {
        let mut max_width = key.len();
        for record in &flattened_records {
            if let Some(value) = record.get(key) {
                max_width = max_width.max(value.len());
            }
        }
        col_widths.insert(key.clone(), max_width.min(50)); // Max 50 chars per column
    }

    // Print header
    print!("|");
    for key in &all_keys {
        let width = col_widths.get(key).unwrap_or(&10);
        print!(" {:<width$} |", key, width = width);
    }
    println!();

    // Print separator
    print!("|");
    for key in &all_keys {
        let width = col_widths.get(key).unwrap_or(&10);
        print!("{}", "-".repeat(width + 2));
        print!("|");
    }
    println!();

    // Print rows
    for record in &flattened_records {
        print!("|");
        for key in &all_keys {
            let width = col_widths.get(key).unwrap_or(&10);
            let value = record.get(key).map(|s| s.as_str()).unwrap_or("");
            let truncated = if value.len() > *width {
                format!("{}...", &value[..width.saturating_sub(3)])
            } else {
                value.to_string()
            };
            print!(" {:<width$} |", truncated, width = width);
        }
        println!();
    }

    Ok(())
}

fn flatten_json(value: &JsonValue, prefix: &str) -> HashMap<String, String> {
    let mut result = HashMap::new();

    match value {
        JsonValue::Object(map) => {
            for (key, val) in map {
                let new_prefix = if prefix.is_empty() {
                    key.clone()
                } else {
                    format!("{}.{}", prefix, key)
                };
                let flattened = flatten_json(val, &new_prefix);
                result.extend(flattened);
            }
        }
        JsonValue::Array(arr) => {
            result.insert(prefix.to_string(), format!("{:?}", arr));
        }
        JsonValue::String(s) => {
            result.insert(prefix.to_string(), s.clone());
        }
        JsonValue::Number(n) => {
            result.insert(prefix.to_string(), n.to_string());
        }
        JsonValue::Bool(b) => {
            result.insert(prefix.to_string(), b.to_string());
        }
        JsonValue::Null => {
            result.insert(prefix.to_string(), "null".to_string());
        }
    }

    result
}