kibble 0.1.0

use crate::clean::clean_text;
use serde_json::Value;
use std::fs;
use std::path::Path;

fn strip_js_prefix(raw: &str) -> &str {
    // Archive files look like `window.YTD.tweets.part0 = [ ... ]`.
    match raw.find('[') {
        Some(i) => &raw[i..],
        None => raw,
    }
}

pub fn ingest_twitter(archive_dir: &Path, out_dir: &Path) -> std::io::Result<usize> {
    fs::create_dir_all(out_dir)?;
    let tweets_path = archive_dir.join("tweets.js");
    let raw = fs::read_to_string(&tweets_path)?;
    let json = strip_js_prefix(&raw);
    let parsed: Value = serde_json::from_str(json)
        .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;

    let mut count = 0usize;
    if let Some(arr) = parsed.as_array() {
        for entry in arr {
            let tweet = entry.get("tweet").unwrap_or(entry);
            let id = tweet.get("id_str").and_then(|v| v.as_str()).unwrap_or("");
            let text = tweet.get("full_text").and_then(|v| v.as_str()).unwrap_or("");
            if id.is_empty() {
                continue;
            }
            let cleaned = clean_text(text);
            if cleaned.is_empty() {
                continue;
            }
            fs::write(out_dir.join(format!("tweet_{id}.txt")), &cleaned)?;
            count += 1;
        }
    }
    let manifest = serde_json::json!({
        "count": count,
        "archive": archive_dir.to_string_lossy(),
    });
    fs::write(out_dir.join("manifest.json"), serde_json::to_string_pretty(&manifest)?)?;
    Ok(count)
}

fn walk(dir: &Path, files: &mut Vec<std::path::PathBuf>) -> std::io::Result<()> {
    for entry in fs::read_dir(dir)? {
        let path = entry?.path();
        if path.is_dir() {
            walk(&path, files)?;
        } else if matches!(
            path.extension().and_then(|e| e.to_str()),
            Some("txt") | Some("md")
        ) {
            files.push(path);
        }
    }
    Ok(())
}

pub fn ingest_textfiles(src_root: &Path, out_dir: &Path) -> std::io::Result<usize> {
    fs::create_dir_all(out_dir)?;
    let mut files = Vec::new();
    walk(src_root, &mut files)?;
    let mut count = 0usize;
    for path in files {
        let rel = path.strip_prefix(src_root).unwrap_or(&path);
        let flat = rel.to_string_lossy().replace(['/', '\\'], "_");
        let raw = fs::read_to_string(&path).unwrap_or_default();
        let cleaned = clean_text(&raw);
        if cleaned.is_empty() {
            continue;
        }
        let name = if flat.ends_with(".txt") {
            flat.clone()
        } else {
            format!("{flat}.txt")
        };
        fs::write(out_dir.join(name), &cleaned)?;
        count += 1;
    }
    Ok(count)
}

#[cfg(test)]
mod tests {
    use super::ingest_twitter;
    use std::fs;

    #[test]
    fn copies_and_cleans_textfiles() {
        use super::ingest_textfiles;
        let dir = std::env::temp_dir().join("kibble_tf_test");
        let _ = std::fs::remove_dir_all(&dir);
        let src = dir.join("src");
        let out = dir.join("out");
        std::fs::create_dir_all(src.join("sub")).unwrap();
        std::fs::write(src.join("a.txt"), "clean @me text").unwrap();
        std::fs::write(src.join("sub").join("b.md"), "# heading\n\n@only").unwrap();
        std::fs::write(src.join("ignore.bin"), "nope").unwrap();

        let n = ingest_textfiles(&src, &out).unwrap();
        assert_eq!(n, 2);
        assert_eq!(std::fs::read_to_string(out.join("a.txt")).unwrap(), "clean text");
    }

    #[test]
    fn parses_and_cleans_tweets() {
        let dir = std::env::temp_dir().join("kibble_tw_test");
        let _ = fs::remove_dir_all(&dir);
        let archive = dir.join("archive");
        let out = dir.join("out");
        fs::create_dir_all(&archive).unwrap();
        // Twitter archive wraps JSON in an assignment prefix.
        let js = r#"window.YTD.tweets.part0 = [
          {"tweet": {"id_str": "111", "full_text": "hello @bob #rust world"}},
          {"tweet": {"id_str": "222", "full_text": "@only mentions"}}
        ]"#;
        fs::write(archive.join("tweets.js"), js).unwrap();

        let n = ingest_twitter(&archive, &out).unwrap();
        // tweet 111 cleans to "hello world"; tweet 222 cleans to "mentions"
        assert_eq!(n, 2);
        let t1 = fs::read_to_string(out.join("tweet_111.txt")).unwrap();
        assert_eq!(t1, "hello world");
    }
}