file_kitty/

encoding.rs

//! Module for file encoding detection and conversion
//! 
//! This module provides functionality to:
//! - Detect file encodings
//! - Convert files to UTF-8
//! - Skip binary files automatically
//! - Process directories recursively

use anyhow::Result;
use chardetng::EncodingDetector;
use encoding_rs::Encoding;
use std::path::Path;
use tokio::fs;
use walkdir::WalkDir;
use std::io::{self, Write};

/// Checks if a file should be processed based on allowed file extensions
///
/// # Arguments
///
/// * `path` - The path to check
/// * `allowed_extensions` - The list of allowed file extensions
///
/// # Returns
///
/// `true` if the file should be processed, `false` otherwise
fn should_process_file(path: &Path, allowed_extensions: &[String]) -> bool {
    if let Some(extension) = path.extension() {
        if let Some(ext) = extension.to_str() {
            return allowed_extensions.iter().any(|allowed| allowed.eq_ignore_ascii_case(ext));
        }
    }
    false
}

/// Returns the default allowed file extensions for text files
fn default_allowed_extensions() -> Vec<String> {
    vec![
        "c", "h", "cpp", "hpp", "cc", "cxx",  // C/C++
        "rs",  // Rust
        "ts", "tsx", "js", "jsx",  // TypeScript/JavaScript
        "txt",  // Plain text
        "html", "htm", "xml",  // Markup
        "py",  // Python
        "java",  // Java
        "go",  // Go
        "md", "markdown",  // Markdown
        "json", "yaml", "yml", "toml",  // Config files
        "sh", "bash",  // Shell scripts
        "sql",  // SQL
        "css", "scss", "sass",  // Stylesheets
    ]
    .into_iter()
    .map(String::from)
    .collect()
}

/// Scans a directory for non-UTF-8 encoded files and optionally converts them to UTF-8
///
/// # Arguments
///
/// * `dir_path` - The directory path to scan
/// * `convert` - Whether to automatically convert files to UTF-8
/// * `verbose` - Whether to show detailed encoding information
/// * `types` - Optional list of file extensions to process. If None, uses default list.
///
/// # Returns
///
/// Returns `Ok(())` if the operation was successful, or an error if something went wrong
///
/// # Example
///
/// ```no_run
/// use file_kitty::encoding::scan_directory;
///
/// #[tokio::main]
/// async fn main() -> anyhow::Result<()> {
///     scan_directory("./my_project", false, false, None).await?;
///     Ok(())
/// }
/// ```
pub async fn scan_directory(dir_path: &str, convert: bool, verbose: bool, types: Option<Vec<String>>) -> Result<()> {
    let allowed_extensions = types.unwrap_or_else(default_allowed_extensions);
    let mut found_non_utf8 = false;
    
    // First scan and display all non-UTF-8 files
    for entry in WalkDir::new(dir_path).into_iter().filter_map(|e| e.ok()) {
        if entry.file_type().is_file() {
            let path = entry.path();
            if !should_process_file(path, &allowed_extensions) {
                if verbose {
                    println!("Skip: {}", path.display());
                }
                continue;
            }
            
            if let Some(encoding_info) = detect_file_encoding(path).await? {
                if encoding_info.0 != "UTF-8" {
                    found_non_utf8 = true;
                    if verbose {
                        let content = fs::read(path).await?;
                        println!(
                            "File: {}\nEncoding: {}\nSize: {} bytes\n",
                            path.display(),
                            encoding_info.0,
                            content.len(),
                        );
                    } else {
                        println!("{} {}", path.display(), encoding_info.0);
                    }
                }
            }
        }
    }
    
    if !found_non_utf8 {
        return Ok(());
    }
    
    let should_convert = if convert {
        true
    } else {
        print!("\nConvert to UTF-8? (y/n): ");
        io::stdout().flush()?;

        let mut input = String::new();
        io::stdin().read_line(&mut input)?;

        input.trim().eq_ignore_ascii_case("y")
    };
    
    if should_convert {
        // Iterate again and perform conversion
        for entry in WalkDir::new(dir_path).into_iter().filter_map(|e| e.ok()) {
            if entry.file_type().is_file() {
                let path = entry.path();
                if !should_process_file(path, &allowed_extensions) {
                    continue;
                }
                if let Some(encoding_info) = detect_file_encoding(path).await? {
                    if encoding_info.0 != "UTF-8" {
                        convert_to_utf8(path, encoding_info.1).await?;
                        println!("Converted: {}", path.display());
                    }
                }
            }
        }
    }
    
    Ok(())
}

/// Detects the encoding of a file
/// 
/// # Arguments
/// 
/// * `path` - Path to the file to check
/// 
/// # Returns
/// 
/// Returns `Some((encoding_name, encoding))` if non-UTF-8, `None` if UTF-8
async fn detect_file_encoding(path: &Path) -> Result<Option<(&'static str, &'static Encoding)>> {
    let content = fs::read(path).await?;
    let mut detector = EncodingDetector::new();
    detector.feed(&content, true);
    let encoding = detector.guess(None, true);
    
    if encoding.name() != "UTF-8" {
        Ok(Some((encoding.name(), encoding)))
    } else {
        Ok(None)
    }
}

/// Converts a file to UTF-8 encoding
/// 
/// # Arguments
/// 
/// * `path` - Path to the file to convert
/// * `encoding` - The current encoding of the file
/// 
/// # Returns
/// 
/// Returns `Ok(())` if conversion was successful, or an error if something went wrong
async fn convert_to_utf8(path: &Path, encoding: &'static Encoding) -> Result<()> {
    let content = fs::read(path).await?;
    let (decoded, _, had_errors) = encoding.decode(&content);
    if had_errors {
        println!("Warning: decoding error in {}", path.display());
    }

    fs::write(path, decoded.as_bytes()).await?;
    Ok(())
}