txt-cleaner 0.1.0

/// Options for `txt-cleaner` operations.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct CleanOptions {
    /// When true, preserves paragraph structure by collapsing consecutive blank
    /// lines to a single blank line (`\n\n`).
    pub preserve_paragraphs: bool,

    /// When true, remove common markdown/html artifacts from the edges of the text.
    pub strip_markdown_artifacts: bool,

    /// When true, remove invisible characters like BOM and zero-width spaces.
    pub strip_invisible_chars: bool,

    /// When true, collapse all whitespace sequences to a single space.
    pub collapse_all_whitespace: bool,
}

impl CleanOptions {
    /// Create a new builder for `CleanOptions`.
    pub fn builder() -> CleanBuilder {
        CleanBuilder::default()
    }
}

impl Default for CleanOptions {
    fn default() -> Self {
        Self {
            preserve_paragraphs: true,
            strip_markdown_artifacts: true,
            strip_invisible_chars: true,
            collapse_all_whitespace: false,
        }
    }
}

/// Builder for cleaning options.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct CleanBuilder {
    preserve_paragraphs: bool,
    strip_markdown_artifacts: bool,
    strip_invisible_chars: bool,
    collapse_all_whitespace: bool,
}

impl Default for CleanBuilder {
    fn default() -> Self {
        Self {
            preserve_paragraphs: true,
            strip_markdown_artifacts: true,
            strip_invisible_chars: true,
            collapse_all_whitespace: false,
        }
    }
}

impl CleanBuilder {
    pub fn new() -> Self {
        Self::default()
    }

    pub fn preserve_paragraphs(mut self, preserve: bool) -> Self {
        self.preserve_paragraphs = preserve;
        self
    }

    pub fn strip_markdown_artifacts(mut self, strip: bool) -> Self {
        self.strip_markdown_artifacts = strip;
        self
    }

    pub fn strip_invisible_chars(mut self, strip: bool) -> Self {
        self.strip_invisible_chars = strip;
        self
    }

    pub fn collapse_all_whitespace(mut self, collapse: bool) -> Self {
        self.collapse_all_whitespace = collapse;
        self
    }

    pub fn build(self) -> CleanOptions {
        CleanOptions {
            preserve_paragraphs: self.preserve_paragraphs,
            strip_markdown_artifacts: self.strip_markdown_artifacts,
            strip_invisible_chars: self.strip_invisible_chars,
            collapse_all_whitespace: self.collapse_all_whitespace,
        }
    }

    pub fn clean(self, text: &str) -> String {
        clean_with_options(text, self.build())
    }
}

/// Clean text with the default safe options.
///
/// This includes:
/// - removing BOM and zero-width characters,
/// - stripping common markdown/html edge artifacts,
/// - normalizing whitespace and collapsing consecutive spaces/tabs/newlines.
pub fn clean(text: &str) -> String {
    clean_with_options(text, CleanOptions::default())
}

/// Clean text using a builder configuration.
pub fn clean_with_builder(text: &str, builder: CleanBuilder) -> String {
    builder.clean(text)
}

/// Clean text using explicit options.
pub fn clean_with_options(text: &str, options: CleanOptions) -> String {
    let mut result = text.to_string();

    if options.strip_invisible_chars {
        result = trim_bom_and_zero_width(&result);
    }

    if options.strip_markdown_artifacts {
        result = trim_html_markdown_artifacts(&result);
    }

    if options.collapse_all_whitespace {
        collapse_whitespace_to_single_space(&result)
    } else if options.preserve_paragraphs {
        trim_consecutive_whitespaces(&result)
    } else {
        collapse_whitespace_to_single_space(&result)
    }
}

/// Remove BOM and zero-width characters from the input text.
pub fn trim_bom_and_zero_width(text: &str) -> String {
    const REMOVED: [char; 5] = ['\u{feff}', '\u{200b}', '\u{200c}', '\u{200d}', '\u{2060}'];

    text.chars()
        .filter(|ch| !REMOVED.contains(ch))
        .collect()
}

/// Collapse whitespace while preserving paragraph structure.
///
/// - consecutive spaces/tabs become a single space
/// - a single newline stays a newline
/// - multiple blank lines collapse to a single blank line (`\n\n`)
/// - leading/trailing whitespace is removed
pub fn trim_consecutive_whitespaces(text: &str) -> String {
    let normalized = normalize_newlines(text);
    let mut output = String::with_capacity(normalized.len());
    let mut newline_count = 0;
    let mut pending_space = false;

    for ch in normalized.chars() {
        match ch {
            '\n' => {
                newline_count += 1;
                pending_space = false;
            }
            ' ' | '\t' => {
                if newline_count == 0 {
                    pending_space = true;
                }
            }
            _ => {
                if newline_count > 0 {
                    if !output.ends_with('\n') {
                        if newline_count == 1 {
                            output.push('\n');
                        } else {
                            output.push_str("\n\n");
                        }
                    }
                    newline_count = 0;
                } else if pending_space && !output.ends_with(' ') && !output.ends_with('\n') {
                    output.push(' ');
                }

                pending_space = false;
                output.push(ch);
            }
        }
    }

    output.trim_matches(|c: char| c.is_whitespace()).to_string()
}

fn collapse_whitespace_to_single_space(text: &str) -> String {
    let mut output = String::with_capacity(text.len());
    let mut pending_space = false;

    for ch in normalize_newlines(text).chars() {
        if ch.is_whitespace() {
            pending_space = true;
            continue;
        }

        if pending_space && !output.is_empty() {
            output.push(' ');
        }

        pending_space = false;
        output.push(ch);
    }

    output.trim().to_string()
}

fn normalize_newlines(text: &str) -> String {
    text.replace("\r\n", "\n").replace('\r', "\n")
}

/// Remove stray markdown/html artifacts from the text edges.
///
/// This is heuristic cleanup for broken input such as:
/// - `** broken sentence`
/// - `# heading text`
/// - `<div>lorem ipsum</div>`
/// - stray backticks, asterisks, underscores, or tildes at the text edges
pub fn trim_html_markdown_artifacts(text: &str) -> String {
    let mut current = text.trim().to_string();

    loop {
        let before = current.clone();
        current = strip_edge_artifacts(&current);
        current = current.trim().to_string();
        if current == before {
            break;
        }
    }

    current
}

fn strip_edge_artifacts(text: &str) -> String {
    if let Some(stripped) = strip_html_tag_edge(text) {
        return stripped;
    }

    if let Some(stripped) = strip_heading_prefix(text) {
        return stripped.trim_start().to_string();
    }

    if let Some(stripped) = strip_blockquote_or_list_prefix(text) {
        return stripped.trim_start().to_string();
    }

    if let Some(stripped) = strip_unpaired_edge_markers(text, true) {
        return stripped;
    }

    if let Some(stripped) = strip_unpaired_edge_markers(text, false) {
        return stripped;
    }

    if let Some(stripped) = strip_matching_wrappers(text) {
        return stripped.trim().to_string();
    }

    text.to_string()
}

fn strip_html_tag_edge(text: &str) -> Option<String> {
    let lower = text.to_lowercase();

    let html_tags = ["<div>", "<p>", "<span>", "<strong>", "<em>", "<b>", "<i>"];
    for tag in html_tags {
        if lower.starts_with(tag) {
            return Some(text[tag.len()..].trim_start().to_string());
        }
    }

    let trimmed_end = text.trim_end();
    let lower_end = trimmed_end.to_lowercase();
    for tag in html_tags {
        if lower_end.ends_with(tag) {
            return Some(trimmed_end[..trimmed_end.len() - tag.len()].trim_end().to_string());
        }
    }

    let closing_tags = ["</div>", "</p>", "</span>", "</strong>", "</em>", "</b>", "</i>"];
    for tag in closing_tags {
        if lower.starts_with(tag) {
            return Some(text[tag.len()..].trim_start().to_string());
        }
    }

    for tag in closing_tags {
        if lower_end.ends_with(tag) {
            return Some(trimmed_end[..trimmed_end.len() - tag.len()].trim_end().to_string());
        }
    }

    if lower.starts_with("<!--") {
        return Some(text[4..].trim_start().to_string());
    }

    if lower_end.ends_with("-->") {
        return Some(trimmed_end[..trimmed_end.len() - 3].trim_end().to_string());
    }

    None
}

fn strip_blockquote_or_list_prefix(text: &str) -> Option<String> {
    let trimmed = text.trim_start();

    if trimmed.starts_with("> ") {
        return Some(trimmed[2..].to_string());
    }

    if trimmed == ">" {
        return Some(String::new());
    }

    for prefix in ["- ", "* ", "+ "] {
        if trimmed.starts_with(prefix) {
            return Some(trimmed[prefix.len()..].to_string());
        }
    }

    None
}

fn strip_matching_wrappers(text: &str) -> Option<&str> {
    const MARKERS: [char; 4] = ['*', '_', '~', '`'];

    for marker in MARKERS {
        let prefix = count_leading_chars(text, marker);
        let suffix = count_trailing_chars(text, marker);

        if prefix > 0 && suffix > 0 {
            let matched = prefix.min(suffix);
            if matched > 0 {
                let start = text.char_indices().nth(matched).map(|(idx, _)| idx).unwrap_or(text.len());
                let end = text.len() - text.chars().rev().take(matched).map(|c| c.len_utf8()).sum::<usize>();
                if start < end {
                    return Some(&text[start..end]);
                }
            }
        }
    }

    None
}

fn strip_unpaired_edge_markers(text: &str, leading: bool) -> Option<String> {
    const MARKERS: [char; 4] = ['*', '_', '~', '`'];

    if leading {
        let count = count_leading_chars_set(text, &MARKERS);
        if count > 0 {
            let after = &text[text.char_indices().nth(count).map(|(idx, _)| idx).unwrap_or(text.len())..];
            if after.is_empty()
                || after.chars().next().map_or(false, |c| {
                    c.is_whitespace() || c == '#' || c == '>' || c == '<'
                })
            {
                return Some(after.trim_start().to_string());
            }
        }
    } else {
        let count = count_trailing_chars_set(text, &MARKERS);
        if count > 0 {
            let before_end = text.char_indices().rev().nth(count - 1).map(|(idx, _ch)| idx).unwrap_or(0);
            let before = &text[..before_end];
            if before.is_empty()
                || before.chars().rev().next().map_or(false, |c| {
                    c.is_whitespace() || c == '#' || c == '>' || c == '<'
                })
            {
                return Some(before.trim_end().to_string());
            }
        }
    }

    None
}

fn count_leading_chars(text: &str, marker: char) -> usize {
    text.chars().take_while(|&c| c == marker).count()
}

fn count_trailing_chars(text: &str, marker: char) -> usize {
    text.chars().rev().take_while(|&c| c == marker).count()
}

fn count_leading_chars_set(text: &str, markers: &[char]) -> usize {
    text.chars().take_while(|c| markers.contains(c)).count()
}

fn count_trailing_chars_set(text: &str, markers: &[char]) -> usize {
    text.chars().rev().take_while(|c| markers.contains(c)).count()
}

fn strip_heading_prefix(text: &str) -> Option<&str> {
    for level in (1..=6).rev() {
        let prefix = "#".repeat(level);
        if let Some(stripped) = text.strip_prefix(&prefix) {
            if stripped.starts_with(char::is_whitespace) {
                return Some(stripped);
            }
        }
    }

    None
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn trim_bom_and_zero_width_removes_invisible_chars() {
        let input = "\u{feff}Hello\u{200B} world\u{200C}!";
        assert_eq!(trim_bom_and_zero_width(input), "Hello world!");
    }

    #[test]
    fn trim_consecutive_whitespaces_collapses_space_tab_newline() {
        let input = "  foo   bar\t\tbaz\n\n\nqux   ";
        assert_eq!(trim_consecutive_whitespaces(input), "foo bar baz\n\nqux");
    }

    #[test]
    fn collapse_whitespace_to_single_space_works() {
        let input = "foo\n\nbar\t\t baz";
        assert_eq!(collapse_whitespace_to_single_space(input), "foo bar baz");
    }

    #[test]
    fn trim_html_markdown_artifacts_removes_edge_tokens() {
        let input = "  **# Hello *world* <div>  ";
        assert_eq!(trim_html_markdown_artifacts(input), "Hello *world*");
    }

    #[test]
    fn trim_html_markdown_artifacts_removes_heading_and_html_wrappers() {
        let input = "## <div>**Hello**</div>";
        assert_eq!(trim_html_markdown_artifacts(input), "Hello");
    }

    #[test]
    fn clean_with_builder_can_collapse_all_whitespace() {
        let input = "foo\n\nbar\t baz";
        let output = CleanBuilder::new().collapse_all_whitespace(true).clean(input);
        assert_eq!(output, "foo bar baz");
    }

    #[test]
    fn builder_api_allows_full_configuration() {
        let input = "\u{feff}  **# Hello  \n\n\n  world!  **\u{200B} ";
        let options = CleanOptions::builder()
            .strip_invisible_chars(true)
            .strip_markdown_artifacts(true)
            .preserve_paragraphs(false)
            .collapse_all_whitespace(true)
            .build();

        assert_eq!(clean_with_options(input, options), "Hello world!");
    }

    #[test]
    fn clean_applies_all_steps() {
        let input = "\u{feff}**\n  Hello   \n\n\n  world!  **\u{200B} ";
        assert_eq!(clean(input), "Hello\n\nworld!");
    }
}