kibble 0.1.0

chew through any source into clean datasets — a fast ingestion, RAG & fine-tuning toolkit
Documentation
use std::path::Path;
use std::process::Command;

pub fn proxy_env(proxy: Option<&str>) -> Vec<(String, String)> {
    match proxy {
        Some(p) => vec![
            ("ALL_PROXY".to_string(), p.to_string()),
            ("https_proxy".to_string(), p.to_string()),
            ("http_proxy".to_string(), p.to_string()),
        ],
        None => Vec::new(),
    }
}

pub fn cache_slug(url: &str) -> String {
    url.chars()
        .map(|c| if c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-') { c } else { '_' })
        .collect()
}

/// Update an existing clone via `git pull --ff-only`, through the configured proxy.
pub fn pull_repo(dest: &Path, proxy: Option<&str>) -> std::io::Result<()> {
    let mut cmd = Command::new("git");
    cmd.arg("-C").arg(dest).args(["pull", "--ff-only"]);
    for (k, v) in proxy_env(proxy) {
        cmd.env(k, v);
    }
    let output = cmd.output()?;
    if !output.status.success() {
        return Err(std::io::Error::other(format!(
            "git pull failed in {}: {}",
            dest.display(),
            String::from_utf8_lossy(&output.stderr).trim()
        )));
    }
    Ok(())
}

/// Clone a git repository into `dest`.
///
/// When `proxy` is `None`, the git child process inherits the ambient
/// environment, including `http_proxy`, `https_proxy`, and `all_proxy`.
/// KIBBLE intentionally does **not** scrub those variables; callers that
/// need a proxy-free clone should unset them before calling this function.
pub fn clone_repo(url: &str, dest: &Path, proxy: Option<&str>) -> std::io::Result<()> {
    if url.starts_with('-') {
        return Err(std::io::Error::new(
            std::io::ErrorKind::InvalidInput,
            format!("refusing to clone url starting with '-': {url}"),
        ));
    }
    if dest.exists() {
        return Ok(()); // cached clone — reuse
    }
    if let Some(parent) = dest.parent() {
        std::fs::create_dir_all(parent)?;
    }
    let mut cmd = Command::new("git");
    cmd.args(["clone", "--depth", "1", "--single-branch", "--"])
        .arg(url)
        .arg(dest);
    for (k, v) in proxy_env(proxy) {
        cmd.env(k, v);
    }
    let output = cmd.output()?;
    if !output.status.success() {
        return Err(std::io::Error::other(format!(
            "git clone failed for {url}: {}",
            String::from_utf8_lossy(&output.stderr).trim()
        )));
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::fs;
    use std::io::ErrorKind;
    use std::process::Command;

    #[test]
    fn proxy_env_empty_when_none() {
        assert!(proxy_env(None).is_empty());
    }

    #[test]
    fn proxy_env_sets_all_three() {
        let env = proxy_env(Some("socks5://127.0.0.1:9050"));
        let keys: Vec<&str> = env.iter().map(|(k, _)| k.as_str()).collect();
        assert!(keys.contains(&"ALL_PROXY"));
        assert!(keys.contains(&"https_proxy"));
        assert!(keys.contains(&"http_proxy"));
        assert!(env.iter().all(|(_, v)| v == "socks5://127.0.0.1:9050"));
    }

    #[test]
    fn cache_slug_sanitizes() {
        assert_eq!(cache_slug("https://github.com/u/r"), "https___github.com_u_r");
    }

    #[test]
    fn clone_repo_rejects_dash_leading_url() {
        let dest = std::env::temp_dir()
            .join(format!("kibble_dash_test_{}", std::process::id()));
        let result = clone_repo("--upload-pack=evil", &dest, None);
        assert!(result.is_err());
        assert_eq!(result.unwrap_err().kind(), ErrorKind::InvalidInput);
        assert!(!dest.exists(), "dest must not be created for rejected url");
    }

    #[test]
    fn pull_repo_fast_forwards_a_clone() {
        let base = std::env::temp_dir().join(format!("kibble_pull_{}", std::process::id()));
        let _ = fs::remove_dir_all(&base);
        let src = base.join("src");
        fs::create_dir_all(&src).unwrap();
        let run = |args: &[&str], cwd: &std::path::Path| {
            let o = Command::new("git").args(args).current_dir(cwd).output().unwrap();
            assert!(o.status.success(), "git {:?}: {}", args, String::from_utf8_lossy(&o.stderr));
        };
        run(&["init", "-q"], &src);
        run(&["config", "user.email", "t@t"], &src);
        run(&["config", "user.name", "t"], &src);
        fs::write(src.join("f.txt"), "v1").unwrap();
        run(&["add", "."], &src);
        run(&["commit", "-q", "-m", "c1"], &src);

        let dest = base.join("clone");
        clone_repo(src.to_str().unwrap(), &dest, None).unwrap();
        assert_eq!(fs::read_to_string(dest.join("f.txt")).unwrap(), "v1");

        // advance the source, then pull into the clone
        fs::write(src.join("f.txt"), "v2").unwrap();
        run(&["commit", "-qam", "c2"], &src);
        super::pull_repo(&dest, None).unwrap();
        assert_eq!(fs::read_to_string(dest.join("f.txt")).unwrap(), "v2");
        let _ = fs::remove_dir_all(&base);
    }

    #[test]
    fn clone_repo_clones_local_repo_and_reuses() {
        // Build a real source git repo in a temp dir (no network).
        let base = std::env::temp_dir().join(format!("kibble_git_{}", std::process::id()));
        let _ = fs::remove_dir_all(&base);
        let src = base.join("src_repo");
        fs::create_dir_all(&src).unwrap();
        let run = |args: &[&str], cwd: &Path| {
            let ok = Command::new("git").args(args).current_dir(cwd).output().unwrap();
            assert!(ok.status.success(), "git {:?} failed: {}", args, String::from_utf8_lossy(&ok.stderr));
        };
        run(&["init", "-q"], &src);
        run(&["config", "user.email", "t@t"], &src);
        run(&["config", "user.name", "t"], &src);
        fs::write(src.join("main.rs"), "fn main() {}").unwrap();
        run(&["add", "."], &src);
        run(&["commit", "-q", "-m", "init"], &src);

        let dest = base.join("clone");
        // clone via filesystem path (offline)
        clone_repo(src.to_str().unwrap(), &dest, None).unwrap();
        assert!(dest.join("main.rs").is_file());

        // second call reuses (no error, dest already present)
        clone_repo(src.to_str().unwrap(), &dest, None).unwrap();
        let _ = fs::remove_dir_all(&base);
    }
}