scryer-engine 0.3.0

//! Indexing the same tree must store the same references and edges however the writer batches
//! its work and whatever order the files are processed in.
//!
//! Before Plan 18 a reference whose target sat in a later writer batch was silently dropped, so
//! results varied with batch boundaries (5% of references on cargo, `short_hash` 13 vs 8).

mod support;

use std::collections::BTreeMap;
use std::fs;
use std::path::Path;

use scryer_db::ScryerDb;
use scryer_engine::IndexOptions;
use support::{
    fingerprint::{Fingerprint, fingerprint},
    fixture::{index_dir_keeping_service, index_files},
};

const MODULES: usize = 12;

/// A ring of modules (`mNN::fNN` calls the next one forward and wraps), two same-named
/// `helper` functions, a `crate::` path call and a re-export, plus a Python and a TypeScript
/// file that call each other forward.
fn fixture() -> Vec<(String, String)> {
    let mut files = Vec::new();
    for i in 0..MODULES {
        let next = (i + 1) % MODULES;
        let mut src = format!(
            "pub fn f{i:02}() -> u32 {{\n    let _ = crate::m{next:02}::f{next:02}();\n    helper() + shared()\n}}\n"
        );
        if i % 6 == 0 {
            // Two modules define `helper`, so a bare call to it needs the tie-break rule.
            src.push_str("pub fn helper() -> u32 {\n    1\n}\n");
        }
        if i == 3 {
            src.push_str("pub use crate::m11::f11 as reexported;\n");
        }
        if i == 11 {
            src.push_str("pub fn shared() -> u32 {\n    f00()\n}\n");
        }
        files.push((format!("src/m{i:02}.rs"), src));
    }
    files.push((
        "src/lib.rs".into(),
        (0..MODULES)
            .map(|i| format!("pub mod m{i:02};\n"))
            .collect::<String>(),
    ));
    files.push((
        "a.py".into(),
        "def first():\n    return zed()\n\n\ndef again():\n    return first()\n".into(),
    ));
    files.push(("z.py".into(), "def zed():\n    return 1\n".into()));
    files.push((
        "a.ts".into(),
        "export function first() { return zed(); }\n".into(),
    ));
    files.push((
        "z.ts".into(),
        "export function zed() { return 1; }\n".into(),
    ));
    files
}

async fn index(options: IndexOptions) -> anyhow::Result<Fingerprint> {
    let files = fixture();
    let borrowed: Vec<(&str, &str)> = files
        .iter()
        .map(|(p, c)| (p.as_str(), c.as_str()))
        .collect();
    let (db, project) = index_files(&borrowed, options).await?;
    fingerprint(&db, project).await
}

#[tokio::test]
async fn indexing_does_not_depend_on_batching_or_file_order() -> anyhow::Result<()> {
    let reference = index(IndexOptions::default()).await?;
    // Golden counts, worked out by hand: per module three calls (next module, `helper`,
    // `shared`), `shared` calls `f00`, two Python calls, one TypeScript call and one
    // re-export; the edges are the same calls plus the re-export. Every reference resolves.
    assert_eq!(
        reference.references.len(),
        41,
        "{:#?}",
        reference.references
    );
    assert_eq!(reference.edges.len(), 41, "{:#?}", reference.edges);

    for cap in [1, 7, 500] {
        for seed in [None, Some(1), Some(2)] {
            let got = index(IndexOptions {
                writer_batch_cap: cap,
                file_order_seed: seed,
                ..IndexOptions::default()
            })
            .await?;
            let missing: Vec<_> = reference
                .references
                .iter()
                .filter(|r| !got.references.contains(r))
                .collect();
            assert_eq!(
                got.digest(),
                reference.digest(),
                "cap={cap} seed={seed:?}: {} refs vs {}, {} edges vs {}; missing refs: {missing:#?}",
                got.references.len(),
                reference.references.len(),
                got.edges.len(),
                reference.edges.len(),
            );
        }
    }
    Ok(())
}

/// One change to the tree: new content for a path, or `None` to delete it.
type Edit = (&'static str, Option<String>);

/// The fixture plus a file that calls a function nothing defines yet.
fn tree() -> BTreeMap<String, String> {
    let mut files: BTreeMap<String, String> = fixture().into_iter().collect();
    files.insert(
        "src/caller.rs".into(),
        "pub fn calls_later() {\n    later_defined();\n}\n".into(),
    );
    files
}

/// Edits of every kind: a header and a body change that move symbols, a symbol removed, a
/// symbol added that an unresolved reference names, a new function with calls, a deleted file.
fn edits(files: &BTreeMap<String, String>) -> Vec<Edit> {
    vec![
        (
            "src/m03.rs",
            Some(
                "// header\n// more header\n".to_string()
                    + &files["src/m03.rs"]
                        .replace("helper() + shared()", "helper() + shared() + 0"),
            ),
        ),
        (
            "src/m11.rs",
            Some(files["src/m11.rs"].replace("pub fn shared() -> u32 {\n    f00()\n}\n", "")),
        ),
        (
            "src/later.rs",
            Some("pub fn later_defined() -> u32 {\n    1\n}\n".into()),
        ),
        (
            "src/m07.rs",
            Some(
                files["src/m07.rs"].clone() + "pub fn extra() -> u32 {\n    helper() + f08()\n}\n",
            ),
        ),
        ("z.py", None),
    ]
}

fn apply(root: &Path, edit: &Edit) -> anyhow::Result<()> {
    let path = root.join(edit.0);
    match &edit.1 {
        Some(content) => fs::write(path, content)?,
        None => fs::remove_file(path)?,
    }
    Ok(())
}

/// Index `files`, change them as `edits` say with `edit_one` and return what is stored.
async fn indexed_then_edited(
    files: &BTreeMap<String, String>,
    edits: &[Edit],
    incremental_by_file: bool,
) -> anyhow::Result<Fingerprint> {
    let temp = tempfile::tempdir()?;
    for (path, content) in files {
        let full = temp.path().join(path);
        fs::create_dir_all(full.parent().unwrap())?;
        fs::write(full, content)?;
    }
    let db = ScryerDb::new_in_memory().await?;
    let (project, service, _) =
        index_dir_keeping_service(&db, temp.path(), IndexOptions::default()).await?;
    for edit in edits {
        apply(temp.path(), edit)?;
        if incremental_by_file {
            if edit.1.is_some() {
                service
                    .index_file(project, temp.path(), Path::new(edit.0))
                    .await?;
            } else {
                service.remove_file(project, Path::new(edit.0)).await?;
            }
        }
    }
    if !incremental_by_file {
        service.index_project(project, temp.path()).await?;
    }
    fingerprint(&db, project).await
}

/// `file:start:role[via]->target` as `file:start:role->target`.
fn without_via(references: &[String]) -> Vec<String> {
    references
        .iter()
        .map(|r| match (r.find('['), r.find(']')) {
            (Some(open), Some(close)) => format!("{}{}", &r[..open], &r[close + 1..]),
            _ => r.clone(),
        })
        .collect()
}

/// Re-indexing after edits stores what a fresh index of the edited tree stores: the same
/// symbols (spans included), references, edges and unresolved references, whether the edits
/// arrive as one incremental run or file by file through the watcher path.
#[tokio::test]
async fn incremental_indexing_matches_a_fresh_index_of_the_edited_tree() -> anyhow::Result<()> {
    let files = tree();
    let edits = edits(&files);

    let mut edited = files.clone();
    for (path, content) in &edits {
        match content {
            Some(content) => edited.insert(path.to_string(), content.clone()),
            None => edited.remove(*path),
        };
    }
    let borrowed: Vec<(&str, &str)> = edited
        .iter()
        .map(|(p, c)| (p.as_str(), c.as_str()))
        .collect();
    let (db, project) = index_files(&borrowed, IndexOptions::default()).await?;
    let fresh = fingerprint(&db, project).await?;
    // The edits matter: the late definition links, the removed one does not.
    assert!(
        fresh
            .references
            .iter()
            .any(|r| r.ends_with("::crate::later::later_defined")),
        "{:#?}",
        fresh.references
    );
    assert!(!fresh.unresolved.is_empty(), "{:#?}", fresh.unresolved);

    for by_file in [false, true] {
        let got = indexed_then_edited(&files, &edits, by_file).await?;
        assert_eq!(got.symbol_rows, fresh.symbol_rows, "by_file={by_file}");
        assert_eq!(got.unresolved, fresh.unresolved, "by_file={by_file}");
        // `via` is left out: an incremental run resolves a changed file against the changed
        // files' symbols only, so a reference into an unchanged file is a name match where a
        // fresh index finds it exactly (docs/learnings/indexing-freshness.md).
        assert_eq!(
            without_via(&got.references),
            without_via(&fresh.references),
            "by_file={by_file}"
        );
        assert_eq!(got.edges, fresh.edges, "by_file={by_file}");
    }
    Ok(())
}