use std::collections::BTreeSet;
use std::path::Path;
use std::sync::Arc;
use std::time::{Duration, Instant};
use indexmap::IndexMap;
use serde_json::json;
use crate::indexing::index::{
load_or_build_index, source_fingerprint, CspIndex, LoadOrBuildOptions, QueryOptions,
};
use crate::stats::save_search_stats;
use crate::types::{CallType, ContentType};
use crate::utils::{format_results, is_git_url, resolve_chunk};
pub const SERVER_INSTRUCTIONS: &str = concat!(
"Instant code search for any local or remote git repository. ",
"Call `search` to find relevant code; call `find_related` on a result to discover similar code elsewhere. ",
"Pass `content` (`code`, `docs`, `config`, or `all`) to choose what a single call searches; ",
"it defaults to the server's configured content. ",
"Prefer these tools over Grep, Glob, or Read for any question about how code works."
);
const ALL_CONTENT: [ContentType; 3] = [ContentType::Code, ContentType::Docs, ContentType::Config];
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[cfg_attr(feature = "cli", derive(schemars::JsonSchema))]
#[serde(rename_all = "lowercase")]
pub enum ContentSelection {
Code,
Docs,
Config,
All,
}
pub fn normalize_content(content: &[ContentType]) -> Vec<ContentType> {
content
.iter()
.copied()
.collect::<BTreeSet<_>>()
.into_iter()
.collect()
}
pub fn resolve_content_selection(
selection: Option<ContentSelection>,
default_content: &[ContentType],
) -> Vec<ContentType> {
match selection {
None => normalize_content(default_content),
Some(ContentSelection::All) => ALL_CONTENT.to_vec(),
Some(ContentSelection::Code) => vec![ContentType::Code],
Some(ContentSelection::Docs) => vec![ContentType::Docs],
Some(ContentSelection::Config) => vec![ContentType::Config],
}
}
const CACHE_MAX_SIZE: usize = 10;
const MIN_REVALIDATE_FACTOR: u32 = 3;
const MIN_REVALIDATE_COOLDOWN: Duration = Duration::from_secs(2);
pub trait LoadOrBuild {
fn load_or_build(
&self,
source: &str,
content: &[ContentType],
git_ref: Option<&str>,
) -> Result<CspIndex, String>;
fn fingerprint(&self, source: &str, content: &[ContentType]) -> Option<String>;
}
pub struct DiskLoadOrBuild;
impl LoadOrBuild for DiskLoadOrBuild {
fn load_or_build(
&self,
source: &str,
content: &[ContentType],
git_ref: Option<&str>,
) -> Result<CspIndex, String> {
load_or_build_index(
source,
&LoadOrBuildOptions {
content: Some(content.to_vec()),
git_ref: git_ref.map(str::to_string),
..Default::default()
},
)
}
fn fingerprint(&self, source: &str, content: &[ContentType]) -> Option<String> {
source_fingerprint(source, content)
}
}
struct CacheEntry {
index: Arc<CspIndex>,
fingerprint: Option<String>,
revalidate_after: Instant,
revalidate_cooldown: std::time::Duration,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
struct CacheKey {
source: String,
content: Vec<ContentType>,
}
pub struct IndexCache<S: LoadOrBuild = DiskLoadOrBuild> {
tasks: IndexMap<CacheKey, CacheEntry>,
seam: S,
}
impl IndexCache<DiskLoadOrBuild> {
pub fn new() -> Self {
Self::with_seam(DiskLoadOrBuild)
}
}
impl Default for IndexCache<DiskLoadOrBuild> {
fn default() -> Self {
Self::new()
}
}
impl<S: LoadOrBuild> IndexCache<S> {
pub fn with_seam(seam: S) -> Self {
Self {
tasks: IndexMap::new(),
seam,
}
}
fn compute_key(
&self,
source: &str,
git_ref: Option<&str>,
content: &[ContentType],
) -> CacheKey {
let source = if is_git_url(source) {
match git_ref {
Some(r) if !r.is_empty() => format!("{source}@{r}"),
_ => source.to_string(),
}
} else {
crate::indexing::cache::normalize_source(source)
};
CacheKey {
source,
content: normalize_content(content),
}
}
pub fn get(
&mut self,
source: &str,
git_ref: Option<&str>,
content: &[ContentType],
) -> Result<Arc<CspIndex>, String> {
let key = self.compute_key(source, git_ref, content);
let content = key.content.as_slice();
let mut entry = self.tasks.shift_remove(&key);
let stale = if let Some(entry) = entry.as_mut() {
if entry.fingerprint.is_some() && Instant::now() >= entry.revalidate_after {
if self.seam.fingerprint(source, content) != entry.fingerprint {
true
} else {
entry.revalidate_after = Instant::now() + entry.revalidate_cooldown;
false
}
} else {
false
}
} else {
false
};
if stale {
entry = None;
}
if let Some(entry) = entry {
let index = entry.index.clone();
self.tasks.insert(key, entry);
return Ok(index);
}
if self.tasks.len() >= CACHE_MAX_SIZE {
self.tasks.shift_remove_index(0);
}
let start = Instant::now();
let index = Arc::new(self.seam.load_or_build(source, content, git_ref)?);
let build_elapsed = start.elapsed();
let fingerprint = self.seam.fingerprint(source, content);
let revalidate_cooldown =
(build_elapsed * MIN_REVALIDATE_FACTOR).max(MIN_REVALIDATE_COOLDOWN);
self.tasks.insert(
key,
CacheEntry {
index: index.clone(),
fingerprint,
revalidate_after: Instant::now() + revalidate_cooldown,
revalidate_cooldown,
},
);
Ok(index)
}
pub fn evict(&mut self, source: &str, git_ref: Option<&str>, content: &[ContentType]) {
let key = self.compute_key(source, git_ref, content);
self.tasks.shift_remove(&key);
}
pub fn size(&self) -> usize {
self.tasks.len()
}
}
pub fn get_index<S: LoadOrBuild>(
repo: Option<&str>,
default_source: Option<&str>,
default_ref: Option<&str>,
content: &[ContentType],
cache: &mut IndexCache<S>,
) -> Result<Arc<CspIndex>, String> {
if let Some(r) = repo {
if is_git_url(r) && !r.starts_with("https://") && !r.starts_with("http://") {
return Err(format!(
"Only https://, http://, or local directory paths are accepted as `repo`. Got: {}",
json!(r)
));
}
}
let use_default = repo.filter(|s| !s.is_empty()).is_none();
let source = repo.or(default_source).filter(|s| !s.is_empty());
let Some(source) = source else {
return Err("No repo specified and no default index. \
Pass an https:// or http:// git URL or local directory path as `repo`."
.to_string());
};
let git_ref = if use_default { default_ref } else { None };
cache
.get(source, git_ref, content)
.map_err(|e| format!("Failed to index {}: {e}", json!(source)))
}
#[allow(clippy::too_many_arguments)]
pub fn search_tool<S: LoadOrBuild>(
cache: &mut IndexCache<S>,
default_source: Option<&str>,
default_ref: Option<&str>,
query: &str,
repo: Option<&str>,
content: &[ContentType],
top_k: usize,
max_snippet_lines: Option<usize>,
stats_file: Option<&Path>,
) -> String {
let index = match get_index(repo, default_source, default_ref, content, cache) {
Ok(idx) => idx,
Err(e) => return e,
};
let results = index.search(
query,
&QueryOptions {
top_k: Some(top_k),
..Default::default()
},
);
if let Some(stats_file) = stats_file {
save_search_stats(
stats_file,
&results,
CallType::Search,
&index.file_sizes,
max_snippet_lines,
);
}
if results.is_empty() {
json!({ "error": "No results found." }).to_string()
} else {
format_results(query, &results, max_snippet_lines).to_string()
}
}
#[allow(clippy::too_many_arguments)]
pub fn find_related_tool<S: LoadOrBuild>(
cache: &mut IndexCache<S>,
default_source: Option<&str>,
default_ref: Option<&str>,
file_path: &str,
line: i64,
repo: Option<&str>,
content: &[ContentType],
top_k: usize,
max_snippet_lines: Option<usize>,
stats_file: Option<&Path>,
) -> String {
let index = match get_index(repo, default_source, default_ref, content, cache) {
Ok(idx) => idx,
Err(e) => return e,
};
let chunk = if (0..=i64::from(u32::MAX)).contains(&line) {
resolve_chunk(&index.chunks, file_path, line as u32)
} else {
None
};
let Some(chunk) = chunk else {
return format!(
"No chunk found at {file_path}:{line}. \
Make sure the file is indexed and the line number is within a known chunk."
);
};
let results = index.find_related(
&chunk.clone(),
&QueryOptions {
top_k: Some(top_k),
..Default::default()
},
);
if let Some(stats_file) = stats_file {
save_search_stats(
stats_file,
&results,
CallType::FindRelated,
&index.file_sizes,
max_snippet_lines,
);
}
if results.is_empty() {
json!({ "error": format!("No related chunks found for {file_path}:{line}.") }).to_string()
} else {
format_results(
&format!("Chunks related to {file_path}:{line}"),
&results,
max_snippet_lines,
)
.to_string()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::indexing::dense::make_stub_model;
use crate::indexing::dense::SelectableBasicBackend;
use crate::indexing::index::CspIndexState;
use crate::indexing::sparse::Bm25Index;
use crate::types::Chunk;
use std::cell::RefCell;
const CODE: &[ContentType] = &[ContentType::Code];
fn empty_index() -> CspIndex {
CspIndex::new(CspIndexState {
model: make_stub_model(4),
bm25_index: Bm25Index::build(&[]),
semantic_index: SelectableBasicBackend::from_vectors(vec![]).unwrap(),
chunks: vec![],
model_path: "test".to_string(),
root: None,
content: vec![ContentType::Code],
files: Default::default(),
})
}
fn index_with_chunk() -> CspIndex {
let chunk = Chunk {
content: "fn main() {}".to_string(),
file_path: "a.ts".to_string(),
start_line: 1,
end_line: 10,
language: Some("typescript".to_string()),
};
CspIndex::new(CspIndexState {
model: make_stub_model(4),
bm25_index: Bm25Index::build(&[vec!["main".to_string()]]),
semantic_index: SelectableBasicBackend::from_vectors(vec![vec![1.0, 0.0, 0.0, 0.0]])
.unwrap(),
chunks: vec![chunk],
model_path: "test".to_string(),
root: None,
content: vec![ContentType::Code],
files: Default::default(),
})
}
struct Stub {
git_calls: RefCell<usize>,
path_calls: RefCell<usize>,
fail: bool,
fingerprint: RefCell<Option<String>>,
}
impl Stub {
fn new() -> Self {
Self {
git_calls: RefCell::new(0),
path_calls: RefCell::new(0),
fail: false,
fingerprint: RefCell::new(Some("fp1".to_string())),
}
}
}
impl LoadOrBuild for Stub {
fn load_or_build(
&self,
source: &str,
_c: &[ContentType],
_r: Option<&str>,
) -> Result<CspIndex, String> {
if self.fail {
return Err("boom".to_string());
}
if is_git_url(source) {
*self.git_calls.borrow_mut() += 1;
} else {
*self.path_calls.borrow_mut() += 1;
}
Ok(empty_index())
}
fn fingerprint(&self, source: &str, _c: &[ContentType]) -> Option<String> {
if is_git_url(source) {
None
} else {
self.fingerprint.borrow().clone()
}
}
}
#[test]
fn cache_reuses_second_call() {
let mut cache = IndexCache::with_seam(Stub::new());
let first = cache.get("/tmp/repo", None, CODE).unwrap();
let second = cache.get("/tmp/repo", None, CODE).unwrap();
assert!(Arc::ptr_eq(&first, &second));
assert_eq!(*cache.seam.path_calls.borrow(), 1);
}
#[test]
fn cache_keys_on_content() {
let mut cache = IndexCache::with_seam(Stub::new());
let code = cache.get("/tmp/repo", None, CODE).unwrap();
let docs = cache
.get("/tmp/repo", None, &[ContentType::Code, ContentType::Docs])
.unwrap();
assert!(!Arc::ptr_eq(&code, &docs));
assert_eq!(cache.size(), 2);
assert_eq!(*cache.seam.path_calls.borrow(), 2);
let again = cache
.get(
"/tmp/repo",
None,
&[ContentType::Docs, ContentType::Code, ContentType::Docs],
)
.unwrap();
assert!(Arc::ptr_eq(&docs, &again));
assert_eq!(*cache.seam.path_calls.borrow(), 2);
cache.evict("/tmp/repo", None, CODE);
assert_eq!(cache.size(), 1);
assert!(Arc::ptr_eq(
&docs,
&cache
.get("/tmp/repo", None, &[ContentType::Code, ContentType::Docs])
.unwrap()
));
}
#[test]
fn resolve_content_selection_maps_default_all_and_single() {
let default = [ContentType::Docs, ContentType::Code];
assert_eq!(
resolve_content_selection(None, &default),
vec![ContentType::Code, ContentType::Docs]
);
assert_eq!(
resolve_content_selection(Some(ContentSelection::All), &default),
vec![ContentType::Code, ContentType::Docs, ContentType::Config]
);
assert_eq!(
resolve_content_selection(Some(ContentSelection::Config), &default),
vec![ContentType::Config]
);
}
#[test]
fn content_selection_deserializes_lowercase_only() {
for (raw, want) in [
("code", ContentSelection::Code),
("docs", ContentSelection::Docs),
("config", ContentSelection::Config),
("all", ContentSelection::All),
] {
let got: ContentSelection = serde_json::from_value(json!(raw)).unwrap();
assert_eq!(got, want);
}
assert!(serde_json::from_value::<ContentSelection>(json!("Docs")).is_err());
assert!(serde_json::from_value::<ContentSelection>(json!("tests")).is_err());
}
#[test]
fn cache_evict_forces_rebuild() {
let mut cache = IndexCache::with_seam(Stub::new());
cache.get("/tmp/repo", None, CODE).unwrap();
assert_eq!(*cache.seam.path_calls.borrow(), 1);
cache.evict("/tmp/repo", None, CODE);
assert_eq!(cache.size(), 0);
cache.get("/tmp/repo", None, CODE).unwrap();
assert_eq!(*cache.seam.path_calls.borrow(), 2);
}
#[test]
fn cache_lru_evicts_oldest() {
let mut cache = IndexCache::with_seam(Stub::new());
for i in 0..10 {
cache.get(&format!("/tmp/repo-{i}"), None, CODE).unwrap();
}
assert_eq!(cache.size(), 10);
cache.get("/tmp/repo-10", None, CODE).unwrap();
assert_eq!(cache.size(), 10);
let before = *cache.seam.path_calls.borrow();
cache.get("/tmp/repo-0", None, CODE).unwrap();
assert_eq!(*cache.seam.path_calls.borrow(), before + 1);
}
#[test]
fn cache_git_vs_path_routing() {
let mut cache = IndexCache::with_seam(Stub::new());
cache
.get("https://github.com/org/repo.git", None, CODE)
.unwrap();
assert_eq!(*cache.seam.git_calls.borrow(), 1);
assert_eq!(*cache.seam.path_calls.borrow(), 0);
cache.get("/tmp/local", None, CODE).unwrap();
assert_eq!(*cache.seam.path_calls.borrow(), 1);
}
#[test]
fn cache_revalidates_stale_local_path() {
let mut cache = IndexCache::with_seam(Stub::new());
let key = cache.compute_key("/tmp/repo", None, CODE);
cache.get("/tmp/repo", None, CODE).unwrap();
assert_eq!(*cache.seam.path_calls.borrow(), 1);
assert!(cache.tasks.get(&key).unwrap().revalidate_cooldown >= MIN_REVALIDATE_COOLDOWN);
*cache.seam.fingerprint.borrow_mut() = Some("fp2".to_string());
cache.get("/tmp/repo", None, CODE).unwrap();
assert_eq!(*cache.seam.path_calls.borrow(), 1);
cache.tasks.get_mut(&key).unwrap().revalidate_after = Instant::now();
cache.get("/tmp/repo", None, CODE).unwrap();
assert_eq!(*cache.seam.path_calls.borrow(), 2);
assert_eq!(cache.size(), 1);
cache.tasks.get_mut(&key).unwrap().revalidate_after = Instant::now();
cache.get("/tmp/repo", None, CODE).unwrap();
assert_eq!(*cache.seam.path_calls.borrow(), 2);
assert!(cache.tasks.get(&key).unwrap().revalidate_after > Instant::now());
}
#[test]
fn cache_git_url_not_revalidated() {
let mut cache = IndexCache::with_seam(Stub::new());
let url = "https://github.com/org/repo.git";
cache.get(url, None, CODE).unwrap();
assert_eq!(*cache.seam.git_calls.borrow(), 1);
*cache.seam.fingerprint.borrow_mut() = Some("fp2".to_string());
cache.get(url, None, CODE).unwrap();
assert_eq!(*cache.seam.git_calls.borrow(), 1);
}
#[test]
fn cache_failure_not_poisoned() {
let mut seam = Stub::new();
seam.fail = true;
let mut cache = IndexCache::with_seam(seam);
assert!(cache.get("/tmp/will-fail", None, CODE).is_err());
assert_eq!(cache.size(), 0);
}
#[test]
fn get_index_rejects_unsafe_schemes() {
let mut cache = IndexCache::with_seam(Stub::new());
for url in [
"ssh://git@github.com/o/r.git",
"git://github.com/o/r.git",
"file:///tmp/x",
] {
let err = get_index(Some(url), None, None, CODE, &mut cache).unwrap_err();
assert!(err.contains("Only https://, http://"), "{url}: {err}");
}
}
#[test]
fn get_index_requires_source() {
let mut cache = IndexCache::with_seam(Stub::new());
let err = get_index(None, None, None, CODE, &mut cache).unwrap_err();
assert!(err.contains("No repo specified"));
}
#[test]
fn get_index_allows_https_and_path() {
let mut cache = IndexCache::with_seam(Stub::new());
assert!(get_index(
Some("https://github.com/o/r.git"),
None,
None,
CODE,
&mut cache
)
.is_ok());
assert!(get_index(None, Some("/tmp/default"), None, CODE, &mut cache).is_ok());
}
#[test]
fn search_tool_no_results() {
let mut cache = IndexCache::with_seam(Stub::new());
let out = search_tool(
&mut cache,
Some("/tmp/repo"),
None,
"anything",
None,
CODE,
5,
None,
None,
);
assert_eq!(out, json!({ "error": "No results found." }).to_string());
}
struct OneChunkSeam;
impl LoadOrBuild for OneChunkSeam {
fn load_or_build(
&self,
_s: &str,
_c: &[ContentType],
_r: Option<&str>,
) -> Result<CspIndex, String> {
Ok(index_with_chunk())
}
fn fingerprint(&self, _s: &str, _c: &[ContentType]) -> Option<String> {
None
}
}
#[test]
fn search_tool_returns_results_json() {
let mut cache = IndexCache::with_seam(OneChunkSeam);
let out = search_tool(
&mut cache,
Some("/tmp/repo"),
None,
"main",
None,
CODE,
5,
None,
None,
);
let value: serde_json::Value = serde_json::from_str(&out).unwrap();
assert!(value.get("query").is_some());
assert!(value["results"].as_array().is_some());
assert!(value["results"][0].get("content").is_some());
}
#[test]
fn search_tool_records_savings_when_stats_file_given() {
let mut cache = IndexCache::with_seam(OneChunkSeam);
let dir = tempfile::tempdir().unwrap();
let stats_file = dir.path().join("savings.jsonl");
let _ = search_tool(
&mut cache,
Some("/tmp/repo"),
None,
"main",
None,
CODE,
5,
None,
Some(&stats_file),
);
let content = std::fs::read_to_string(&stats_file).unwrap();
let lines: Vec<&str> = content.lines().filter(|l| !l.is_empty()).collect();
assert_eq!(lines.len(), 1);
assert!(lines[0].contains("\"call\":\"search\""));
}
#[test]
fn search_tool_respects_max_snippet_lines_zero() {
let mut cache = IndexCache::with_seam(OneChunkSeam);
let out = search_tool(
&mut cache,
Some("/tmp/repo"),
None,
"main",
None,
CODE,
5,
Some(0),
None,
);
let value: serde_json::Value = serde_json::from_str(&out).unwrap();
let entry = &value["results"][0];
assert!(entry.get("content").is_none());
assert_eq!(entry["file_path"], "a.ts");
}
#[test]
fn find_related_no_chunk_message() {
let mut cache = IndexCache::with_seam(OneChunkSeam);
let out = find_related_tool(
&mut cache,
Some("/tmp/repo"),
None,
"nope.ts",
1,
None,
CODE,
5,
None,
None,
);
assert!(out.contains("No chunk found at nope.ts:1"));
}
#[test]
fn find_related_returns_json_for_known_chunk() {
let mut cache = IndexCache::with_seam(OneChunkSeam);
let out = find_related_tool(
&mut cache,
Some("/tmp/repo"),
None,
"a.ts",
5,
None,
CODE,
5,
None,
None,
);
let value: serde_json::Value = serde_json::from_str(&out).unwrap();
assert!(value.get("query").is_some() || value.get("error").is_some());
}
}