pub const MAX_CODE_CHUNK_CHARS: usize = 5500;
pub const MIN_CODE_CHARS: usize = 40;
pub const CODE_USER_PROMPT_TEMPLATES: &[&str] = &[
"Walk through what this code does: {topic}",
"Explain what this code does (technical, precise): {topic}",
"Document this file clearly: {topic}",
];
pub fn lang_for_path(path: &str) -> &'static str {
let p = std::path::Path::new(path);
let name = p.file_name().and_then(|n| n.to_str()).unwrap_or("");
let ext = p.extension().and_then(|e| e.to_str()).unwrap_or("").to_lowercase();
if ext.is_empty() && name.eq_ignore_ascii_case("dockerfile") {
return "dockerfile";
}
match ext.as_str() {
"rs" => "rust",
"go" | "mod" => "go",
"py" => "python",
"sh" | "bash" | "hook" => "bash",
"zsh" => "zsh",
"fish" => "fish",
"toml" => "toml",
"yaml" | "yml" => "yaml",
"c" | "h" => "c",
"cpp" | "cc" => "cpp",
"ts" => "typescript",
"tsx" => "tsx",
"js" => "javascript",
"lua" => "lua",
"nix" => "nix",
_ => "",
}
}
fn comment_prefix(lang: &str) -> &'static str {
match lang {
"rust" | "go" | "c" | "cpp" | "typescript" | "tsx" | "javascript" => "// ",
"lua" => "-- ",
_ => "# ", }
}
pub fn is_mostly_text(content: &str) -> bool {
if content.is_empty() {
return false;
}
let sample: String = content.chars().take(8000).collect();
if sample.contains('\0') {
return false;
}
let printable = sample
.chars()
.filter(|c| !c.is_control() || matches!(c, '\n' | '\t' | '\r'))
.count();
printable as f64 / sample.chars().count() as f64 >= 0.92
}
pub fn format_code_block(code: &str, lang: &str, rel_path: &str) -> String {
let code = code.trim();
let prefix = comment_prefix(lang);
let first_line = code.split('\n').next().unwrap_or("");
let body = if !first_line.trim_start().starts_with(prefix.trim()) {
format!("{prefix}{rel_path}\n{code}")
} else {
code.to_string()
};
let fence_lang = if lang.is_empty() { "text" } else { lang };
format!("```{fence_lang}\n{body}\n```")
}
use crate::build::{Msg, Row};
use crate::config::SourceConfig;
use crate::text::{chunk_text_raw, pick_template};
use std::path::Path;
const SKIP_DIRS: &[&str] = &[
".git", "node_modules", "target", ".venv", "venv", "dist", "build", ".next",
];
pub struct CodebaseLoad {
pub rows: Vec<(String, Row)>,
pub files: usize,
}
fn collect_files(dir: &Path, out: &mut Vec<std::path::PathBuf>) -> std::io::Result<()> {
for entry in std::fs::read_dir(dir)? {
let path = entry?.path();
if path.is_dir() {
let name = path.file_name().and_then(|n| n.to_str()).unwrap_or("");
if SKIP_DIRS.contains(&name) {
continue;
}
collect_files(&path, out)?;
} else {
out.push(path);
}
}
Ok(())
}
pub fn scan_codebase(source: &SourceConfig, root: &Path) -> std::io::Result<CodebaseLoad> {
if !root.is_dir() {
return Err(std::io::Error::new(
std::io::ErrorKind::NotFound,
format!("codebase path is not a directory: {}", root.display()),
));
}
let name = source.source_name();
let mut files = Vec::new();
collect_files(root, &mut files)?;
let mut rows: Vec<(String, Row)> = Vec::new();
let mut file_count = 0usize;
for path in files {
let rel = match path.strip_prefix(root) {
Ok(r) => r,
Err(_) => continue,
};
let rel_path = rel.to_string_lossy().replace('\\', "/");
let lang = lang_for_path(&rel_path);
if lang.is_empty() {
continue;
}
let raw = match std::fs::read_to_string(&path) {
Ok(s) => s,
Err(_) => continue,
};
let trimmed = raw.trim();
if trimmed.chars().filter(|c| !c.is_whitespace()).count() < MIN_CODE_CHARS {
continue;
}
if !is_mostly_text(trimmed) {
continue;
}
let mut produced = false;
let row_key_base = format!("{name}:{rel_path}");
for (i, chunk) in chunk_text_raw(trimmed, MAX_CODE_CHUNK_CHARS).into_iter().enumerate() {
if chunk.chars().count() < MIN_CODE_CHARS {
continue;
}
let block = format_code_block(&chunk, lang, &rel_path);
let user = pick_template(&row_key_base, i, CODE_USER_PROMPT_TEMPLATES)
.replace("{topic}", &rel_path);
let row = Row {
messages: vec![
Msg { role: "system".to_string(), content: crate::build::SYSTEM_PROMPT.to_string() },
Msg { role: "user".to_string(), content: user },
Msg { role: "assistant".to_string(), content: block },
],
};
rows.push((format!("{row_key_base}#{i}"), row));
produced = true;
}
if produced {
file_count += 1;
}
}
let load = CodebaseLoad { rows, files: file_count };
eprintln!("kibble: scanned {} code files under {}", load.files, root.display());
Ok(load)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn lang_detection() {
assert_eq!(lang_for_path("src/main.rs"), "rust");
assert_eq!(lang_for_path("a/b.py"), "python");
assert_eq!(lang_for_path("go.mod"), "go");
assert_eq!(lang_for_path("Dockerfile"), "dockerfile");
assert_eq!(lang_for_path("notes.unknown"), "");
}
#[test]
fn mostly_text_rejects_binary() {
assert!(is_mostly_text("fn main() {}"));
assert!(!is_mostly_text("ab\0cd")); }
#[test]
fn fenced_block_adds_path_comment() {
let out = format_code_block("fn main() {}", "rust", "src/main.rs");
assert_eq!(out, "```rust\n// src/main.rs\nfn main() {}\n```");
}
#[test]
fn fenced_block_unknown_lang_uses_text_fence_and_hash_prefix() {
let out = format_code_block("data: 1", "", "x.unknown");
assert_eq!(out, "```text\n# x.unknown\ndata: 1\n```");
}
#[test]
fn fenced_block_keeps_existing_leading_comment() {
let out = format_code_block("// src/main.rs\nfn main() {}", "rust", "src/main.rs");
assert_eq!(out, "```rust\n// src/main.rs\nfn main() {}\n```");
}
#[test]
fn scan_builds_code_rows_and_skips_noise() {
use super::scan_codebase;
use crate::config::SourceConfig;
use std::fs;
let root = std::env::temp_dir().join(format!("kibble_cb_{}", std::process::id()));
let _ = fs::remove_dir_all(&root);
fs::create_dir_all(root.join("src")).unwrap();
fs::create_dir_all(root.join("target")).unwrap(); fs::create_dir_all(root.join("node_modules")).unwrap();
let body = "fn main() {\n println!(\"hello world from a real file\");\n}\n".repeat(2);
fs::write(root.join("src/main.rs"), &body).unwrap();
fs::write(root.join("target/gen.rs"), "fn x() { /* generated */ }").unwrap();
fs::write(root.join("README.txt"), "not code, unmapped extension").unwrap();
let source = SourceConfig {
path: root.to_string_lossy().to_string(),
r#type: Some("codebase".to_string()),
name: Some("repo".to_string()),
max_rows: None,
system_prompt: None,
user_prefix: None,
role: None,
preserve_code: None, refresh: None,
};
let load = scan_codebase(&source, &root).unwrap();
assert_eq!(load.files, 1); assert!(!load.rows.is_empty());
let (id, row) = &load.rows[0];
assert!(id.starts_with("repo:src/main.rs#"));
assert_eq!(row.messages[0].role, "system");
assert_eq!(row.messages[2].role, "assistant");
assert!(row.messages[2].content.contains("```rust"));
assert!(row.messages[2].content.contains("// src/main.rs"));
}
#[test]
fn scan_errors_when_not_a_directory() {
use super::scan_codebase;
use crate::config::SourceConfig;
let source = SourceConfig {
path: "/no/such/dir".to_string(),
r#type: Some("codebase".to_string()),
name: None, max_rows: None, system_prompt: None, user_prefix: None, role: None, preserve_code: None, refresh: None,
};
assert!(scan_codebase(&source, std::path::Path::new("/no/such/dir")).is_err());
}
}