coderlib 0.1.0

//! Text processing utilities for CoderLib

/// Count the number of tokens in a text (rough estimation)
pub fn estimate_tokens(text: &str) -> u32 {
    // Rough estimation: 1 token ≈ 4 characters for English text
    // This is a very basic estimation and should be replaced with proper tokenization
    (text.len() as f32 / 4.0).ceil() as u32
}

/// Split text into chunks of approximately the given token count
pub fn chunk_text(text: &str, max_tokens: u32) -> Vec<String> {
    let max_chars = (max_tokens * 4) as usize; // Rough conversion
    let mut chunks = Vec::new();
    let mut current_chunk = String::new();
    
    for line in text.lines() {
        if current_chunk.len() + line.len() + 1 > max_chars && !current_chunk.is_empty() {
            chunks.push(current_chunk);
            current_chunk = String::new();
        }
        
        if !current_chunk.is_empty() {
            current_chunk.push('\n');
        }
        current_chunk.push_str(line);
    }
    
    if !current_chunk.is_empty() {
        chunks.push(current_chunk);
    }
    
    chunks
}

/// Extract code blocks from markdown text
pub fn extract_code_blocks(text: &str) -> Vec<CodeBlock> {
    let mut blocks = Vec::new();
    let mut in_code_block = false;
    let mut current_block = CodeBlock {
        language: None,
        code: String::new(),
        start_line: 0,
        end_line: 0,
    };
    
    for (line_num, line) in text.lines().enumerate() {
        if line.starts_with("```") {
            if in_code_block {
                // End of code block
                current_block.end_line = line_num;
                blocks.push(current_block);
                current_block = CodeBlock {
                    language: None,
                    code: String::new(),
                    start_line: 0,
                    end_line: 0,
                };
                in_code_block = false;
            } else {
                // Start of code block
                let language = line.strip_prefix("```").unwrap_or("").trim();
                current_block.language = if language.is_empty() { None } else { Some(language.to_string()) };
                current_block.start_line = line_num;
                in_code_block = true;
            }
        } else if in_code_block {
            if !current_block.code.is_empty() {
                current_block.code.push('\n');
            }
            current_block.code.push_str(line);
        }
    }
    
    blocks
}

/// A code block extracted from markdown
#[derive(Debug, Clone)]
pub struct CodeBlock {
    pub language: Option<String>,
    pub code: String,
    pub start_line: usize,
    pub end_line: usize,
}

/// Clean up text by removing extra whitespace and normalizing line endings
pub fn normalize_text(text: &str) -> String {
    text.lines()
        .map(|line| line.trim_end())
        .collect::<Vec<_>>()
        .join("\n")
        .trim()
        .to_string()
}

/// Convert text to a safe identifier (for use in filenames, variable names, etc.)
pub fn to_identifier(text: &str) -> String {
    text.chars()
        .map(|c| {
            if c.is_alphanumeric() {
                c.to_ascii_lowercase()
            } else if c.is_whitespace() || c == '-' || c == '_' {
                '_'
            } else {
                '_'
            }
        })
        .collect::<String>()
        .trim_matches('_')
        .to_string()
}

/// Count lines in text
pub fn count_lines(text: &str) -> usize {
    if text.is_empty() {
        0
    } else {
        text.lines().count()
    }
}

/// Get the line at a specific index (0-based)
pub fn get_line(text: &str, line_index: usize) -> Option<&str> {
    text.lines().nth(line_index)
}

/// Get lines in a range (inclusive, 0-based)
pub fn get_lines_range(text: &str, start: usize, end: usize) -> Vec<&str> {
    text.lines()
        .skip(start)
        .take(end.saturating_sub(start) + 1)
        .collect()
}

/// Insert text at a specific line (0-based)
pub fn insert_at_line(text: &str, line_index: usize, new_text: &str) -> String {
    let mut lines: Vec<&str> = text.lines().collect();
    
    if line_index <= lines.len() {
        lines.insert(line_index, new_text);
    } else {
        lines.push(new_text);
    }
    
    lines.join("\n")
}

/// Replace a line at a specific index (0-based)
pub fn replace_line(text: &str, line_index: usize, new_text: &str) -> String {
    let mut lines: Vec<&str> = text.lines().collect();
    
    if line_index < lines.len() {
        lines[line_index] = new_text;
    }
    
    lines.join("\n")
}

/// Remove a line at a specific index (0-based)
pub fn remove_line(text: &str, line_index: usize) -> String {
    let mut lines: Vec<&str> = text.lines().collect();
    
    if line_index < lines.len() {
        lines.remove(line_index);
    }
    
    lines.join("\n")
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_estimate_tokens() {
        assert_eq!(estimate_tokens("hello"), 2); // 5 chars / 4 = 1.25 -> 2
        assert_eq!(estimate_tokens("hello world"), 3); // 11 chars / 4 = 2.75 -> 3
        assert_eq!(estimate_tokens(""), 0);
    }

    #[test]
    fn test_chunk_text() {
        let text = "line1\nline2\nline3\nline4";
        let chunks = chunk_text(text, 2); // 8 chars max per chunk
        assert!(chunks.len() > 1);
    }

    #[test]
    fn test_extract_code_blocks() {
        let text = r#"
Some text
```rust
fn main() {
    println!("Hello");
}
```
More text
```
plain code
```
"#;
        let blocks = extract_code_blocks(text);
        assert_eq!(blocks.len(), 2);
        assert_eq!(blocks[0].language, Some("rust".to_string()));
        assert_eq!(blocks[1].language, None);
    }

    #[test]
    fn test_normalize_text() {
        let text = "  hello  \n  world  \n\n";
        assert_eq!(normalize_text(text), "hello\n  world");
    }

    #[test]
    fn test_to_identifier() {
        assert_eq!(to_identifier("Hello World!"), "hello_world");
        assert_eq!(to_identifier("test-file.txt"), "test_file_txt");
        assert_eq!(to_identifier("123abc"), "123abc");
    }

    #[test]
    fn test_line_operations() {
        let text = "line1\nline2\nline3";
        
        assert_eq!(count_lines(text), 3);
        assert_eq!(get_line(text, 1), Some("line2"));
        assert_eq!(get_lines_range(text, 0, 1), vec!["line1", "line2"]);
        
        let inserted = insert_at_line(text, 1, "new_line");
        assert!(inserted.contains("new_line"));
        
        let replaced = replace_line(text, 1, "replaced");
        assert!(replaced.contains("replaced"));
        assert!(!replaced.contains("line2"));
        
        let removed = remove_line(text, 1);
        assert!(!removed.contains("line2"));
    }
}