semtree-rag 0.5.0

//! The index lifecycle: open, refresh, persist, reopen.
//!
//! These run against deterministic fake backends rather than fastembed, so
//! they cover the parts that actually break - what survives a reopen, when a
//! rebuild is forced, what an incremental pass leaves behind - without a model
//! download.

mod fakes;

use std::path::Path;
use std::sync::Arc;

use fakes::{BagOfWords, MemStore};
use semtree_embed::Embedder;
use semtree_rag::{IndexSession, RagError, RebuildReason, SearchFilters, SearchMode};
use semtree_store::VectorStore;
use tempfile::TempDir;

/// A project with two Rust files and one Python file.
fn fixture() -> TempDir {
    let dir = tempfile::tempdir().unwrap();
    let src = dir.path().join("src");
    std::fs::create_dir_all(&src).unwrap();

    std::fs::write(
        src.join("auth.rs"),
        "pub fn verify_session_token(token: &str) -> bool {\n    !token.is_empty()\n}\n",
    )
    .unwrap();
    std::fs::write(
        src.join("math.rs"),
        "pub fn add_two_numbers(a: i64, b: i64) -> i64 {\n    a + b\n}\n",
    )
    .unwrap();
    std::fs::write(
        src.join("shapes.py"),
        "def area_of_circle(radius):\n    return 3.14 * radius * radius\n",
    )
    .unwrap();

    dir
}

fn backends(model_id: &str) -> (Arc<dyn Embedder>, Arc<dyn VectorStore>) {
    (
        Arc::new(BagOfWords::new(model_id)),
        Arc::new(MemStore::default()),
    )
}

fn open(model_id: &str, index_dir: &Path) -> IndexSession {
    let (embedder, store) = backends(model_id);
    IndexSession::open(embedder, store, index_dir).unwrap()
}

/// Index `project` into `index_dir` from scratch and persist it.
async fn build_index(model_id: &str, project: &Path, index_dir: &Path) -> IndexSession {
    let mut session = open(model_id, index_dir);
    session.index(project, false, |_, _| {}).await.unwrap();
    session.save().unwrap();
    session
}

async fn names_for(session: &IndexSession, query: &str) -> Vec<String> {
    session
        .search(query, 10, SearchMode::Semantic, &SearchFilters::default())
        .await
        .unwrap()
        .iter()
        .filter_map(|r| r.chunk.name.clone())
        .collect()
}

#[tokio::test]
async fn a_fresh_directory_reports_a_missing_index() {
    let index_dir = tempfile::tempdir().unwrap();
    let session = open("fake", index_dir.path());

    assert_eq!(session.pending_rebuild(), Some(&RebuildReason::Missing));
    assert_eq!(session.stats().chunks, 0);
}

#[tokio::test]
async fn read_only_callers_are_told_there_is_no_index() {
    let index_dir = tempfile::tempdir().unwrap();
    let (embedder, store) = backends("fake");

    let result = IndexSession::open_existing(embedder, store, index_dir.path());
    match result {
        Err(RagError::NoIndex(dir)) => assert_eq!(dir, index_dir.path()),
        Err(other) => panic!("expected a missing-index error, got {other}"),
        Ok(_) => panic!("an unindexed project must not look like an empty one"),
    }
}

#[tokio::test]
async fn an_index_survives_being_closed_and_reopened() {
    let project = fixture();
    let index_dir = tempfile::tempdir().unwrap();

    let built = build_index("fake", project.path(), index_dir.path()).await;
    let chunks = built.stats().chunks;
    assert!(chunks >= 3, "each file contributed at least one chunk");
    drop(built);

    let reopened = open("fake", index_dir.path());
    assert_eq!(reopened.pending_rebuild(), None, "nothing to rebuild");
    assert_eq!(reopened.stats().chunks, chunks);
    assert_eq!(
        reopened.stats().vectors,
        Some(chunks),
        "the vectors came back with the metadata"
    );
}

#[tokio::test]
async fn an_incremental_pass_keeps_what_was_already_indexed() {
    // The regression that matters: an update used to persist only the chunks it
    // had just touched, silently emptying the index of everything else.
    let project = fixture();
    let index_dir = tempfile::tempdir().unwrap();

    let before = build_index("fake", project.path(), index_dir.path())
        .await
        .stats()
        .chunks;

    std::fs::write(
        project.path().join("src/cache.rs"),
        "pub fn evict_oldest_entry() -> Option<String> {\n    None\n}\n",
    )
    .unwrap();

    let mut session = open("fake", index_dir.path());
    let report = session
        .index(project.path(), false, |_, _| {})
        .await
        .unwrap();
    session.save().unwrap();

    assert!(report.was_incremental(), "an added file is not a rebuild");
    assert_eq!(report.chunks_indexed, 1, "only the new file was embedded");

    let stats = session.stats();
    assert_eq!(stats.chunks, before + 1);
    assert_eq!(stats.vectors, Some(before + 1), "no vectors were dropped");

    let found = names_for(&session, "verify session token").await;
    assert!(
        found.iter().any(|n| n == "verify_session_token"),
        "code indexed before the update is still searchable, got {found:?}"
    );
}

#[tokio::test]
async fn an_unchanged_tree_costs_nothing_to_refresh() {
    let project = fixture();
    let index_dir = tempfile::tempdir().unwrap();
    build_index("fake", project.path(), index_dir.path()).await;

    let mut session = open("fake", index_dir.path());
    let report = session
        .index(project.path(), false, |_, _| {})
        .await
        .unwrap();

    assert!(report.was_incremental());
    assert_eq!(
        report.chunks_indexed, 0,
        "nothing changed, so nothing was re-embedded"
    );
}

#[tokio::test]
async fn a_deleted_file_leaves_the_index() {
    let project = fixture();
    let index_dir = tempfile::tempdir().unwrap();
    let before = build_index("fake", project.path(), index_dir.path())
        .await
        .stats()
        .chunks;

    std::fs::remove_file(project.path().join("src/math.rs")).unwrap();

    let mut session = open("fake", index_dir.path());
    session
        .index(project.path(), false, |_, _| {})
        .await
        .unwrap();

    assert!(session.stats().chunks < before);
    let found = names_for(&session, "add two numbers").await;
    assert!(
        !found.iter().any(|n| n == "add_two_numbers"),
        "the deleted function is gone from search, got {found:?}"
    );
}

#[tokio::test]
async fn switching_embedder_forces_a_rebuild_instead_of_mixing_vectors() {
    let project = fixture();
    let index_dir = tempfile::tempdir().unwrap();
    let chunks = build_index("model-a", project.path(), index_dir.path())
        .await
        .stats()
        .chunks;

    // Same code, different model: the stored vectors are not comparable to what
    // this embedder produces, so they must not be reused.
    let mut session = open("model-b", index_dir.path());
    let reason = session.pending_rebuild().cloned();
    assert!(
        matches!(reason, Some(RebuildReason::Incompatible { .. })),
        "expected an incompatibility, got {reason:?}"
    );
    assert_eq!(
        session.stats().chunks,
        0,
        "the unusable index is not loaded"
    );

    let report = session
        .index(project.path(), false, |_, _| {})
        .await
        .unwrap();
    assert!(!report.was_incremental());
    assert_eq!(
        session.stats().vectors,
        Some(chunks),
        "rebuilt clean, with no leftovers from the previous model"
    );
}

#[tokio::test]
async fn a_full_rebuild_does_not_accumulate_duplicates() {
    let project = fixture();
    let index_dir = tempfile::tempdir().unwrap();
    let chunks = build_index("fake", project.path(), index_dir.path())
        .await
        .stats()
        .chunks;

    let mut session = open("fake", index_dir.path());
    let report = session
        .index(project.path(), true, |_, _| {})
        .await
        .unwrap();

    assert_eq!(report.rebuilt, Some(RebuildReason::Requested));
    assert_eq!(session.stats().chunks, chunks);
    assert_eq!(
        session.stats().vectors,
        Some(chunks),
        "the store was emptied before being refilled"
    );
}

#[tokio::test]
async fn filters_narrow_results_by_language() {
    let project = fixture();
    let index_dir = tempfile::tempdir().unwrap();
    let session = build_index("fake", project.path(), index_dir.path()).await;

    let filters = SearchFilters::default()
        .with_language_names(["python"])
        .unwrap();
    let results = session
        .search("area", 10, SearchMode::Semantic, &filters)
        .await
        .unwrap();

    assert!(!results.is_empty(), "the Python file is indexed");
    assert!(
        results
            .iter()
            .all(|r| r.chunk.path.extension().unwrap() == "py"),
        "a language filter admits nothing else"
    );
}

#[tokio::test]
async fn context_carries_the_matched_source() {
    let project = fixture();
    let index_dir = tempfile::tempdir().unwrap();
    let session = build_index("fake", project.path(), index_dir.path()).await;

    let window = session
        .context("verify session token", 3, SearchMode::Semantic)
        .await
        .unwrap();

    assert!(!window.snippets.is_empty());
    assert!(
        window.prompt.contains("verify_session_token"),
        "the prompt carries real code, not just chunk ids"
    );
}