relay_knowledge/code/

mod.rs

//! Git snapshot and tree-sitter code index construction.
//!
//! This module owns blocking Git, filesystem, and parser work. Application
//! methods run these workflows behind explicit blocking-worker boundaries.

use std::{
    collections::BTreeMap,
    error::Error,
    fmt, fs,
    path::{Path, PathBuf},
};

mod changes;
mod git;
mod identity;
mod ids;
mod languages;
mod parser;
mod pipeline;
mod scope;
mod snapshot;

#[cfg(test)]
mod tests;

use crate::domain::{
    CodeFileFingerprint, CodeIndexMode, CodeIndexSnapshot, CodePathTombstone,
    CodeRepositoryRegistration, CodeRepositorySelector,
};

use changes::{GitChange, diff_changes, tracked_paths, worktree_changed_paths};
use git::{
    git_bytes, git_object_exists, git_optional, resolve_git_root, resolve_ref, resolve_tree,
};
use ids::{stable_content_hash, stable_hash64, stable_id};
use parser::parse_indexed_file;
pub use pipeline::{CodeIndexPlan, prepare_full_index_plan};
use scope::{
    load_ignore_rules, load_ignore_rules_from_commit, path_is_selected_with_rules,
    path_scope_overlaps, selection_exclusion_reason,
};
pub use scope::{partition_changed_paths_for_selector, preview_repository_scope};
use snapshot::SnapshotBuild;

#[cfg(test)]
use identity::resolve_reference_targets;

#[cfg(test)]
use languages::language_id;

#[cfg(test)]
use scope::{path_is_selected, path_scope_allows};

/// Blocking code index failure.
#[derive(Debug)]
pub enum CodeIndexError {
    Io(std::io::Error),
    Git { args: Vec<String>, message: String },
    TreeSitter(String),
    InvalidInput(String),
}

impl fmt::Display for CodeIndexError {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Self::Io(error) => write!(formatter, "code index I/O failed: {error}"),
            Self::Git { args, message } => {
                write!(formatter, "git command failed ({args:?}): {message}")
            }
            Self::TreeSitter(message) => write!(formatter, "tree-sitter parse failed: {message}"),
            Self::InvalidInput(message) => write!(formatter, "invalid code index input: {message}"),
        }
    }
}

impl Error for CodeIndexError {}

impl From<std::io::Error> for CodeIndexError {
    fn from(error: std::io::Error) -> Self {
        Self::Io(error)
    }
}

/// Validates a Git worktree and creates a stable repository registration.
pub fn register_repository(
    path: impl AsRef<Path>,
    alias: impl Into<String>,
    path_filters: Vec<String>,
    language_filters: Vec<String>,
) -> Result<CodeRepositoryRegistration, CodeIndexError> {
    let root = resolve_git_root(path.as_ref())?;
    let root_identity = root.display().to_string();
    let origin = git_optional(&root, ["config", "--get", "remote.origin.url"])?
        .unwrap_or_else(|| root_identity.clone());
    let repository_id = stable_id("repo", [origin.as_str(), root_identity.as_str()]);

    CodeRepositoryRegistration::new(
        repository_id,
        alias,
        root_identity,
        path_filters,
        language_filters,
    )
    .map_err(|error| CodeIndexError::InvalidInput(error.to_string()))
}

/// Builds a code index snapshot from a clean Git commit or incremental diff.
pub fn build_index_snapshot(
    registration: &CodeRepositoryRegistration,
    selector: &CodeRepositorySelector,
    mode: CodeIndexMode,
    previous_hashes: Vec<CodeFileFingerprint>,
) -> Result<CodeIndexSnapshot, CodeIndexError> {
    let root = PathBuf::from(&registration.root_path);
    let previous_hashes = previous_hashes
        .into_iter()
        .map(|fingerprint| (fingerprint.path, fingerprint.blob_hash))
        .collect::<BTreeMap<_, _>>();

    match mode {
        CodeIndexMode::Full => build_full_snapshot(registration, selector, &root),
        CodeIndexMode::Incremental { base_ref, head_ref } => build_incremental_snapshot(
            registration,
            selector,
            &root,
            &base_ref,
            &head_ref,
            &previous_hashes,
        ),
        CodeIndexMode::WorktreeOverlay => {
            build_worktree_overlay_snapshot(registration, selector, &root, &previous_hashes)
        }
    }
}

/// Returns changed paths for impact analysis without mutating the code index.
pub fn changed_paths_for_diff(
    root_path: impl AsRef<Path>,
    base_ref: &str,
    head_ref: &str,
) -> Result<Vec<String>, CodeIndexError> {
    let changes = diff_changes(root_path.as_ref(), base_ref, head_ref)?;

    Ok(impact_paths_from_changes(changes))
}

fn impact_paths_from_changes(changes: Vec<GitChange>) -> Vec<String> {
    let mut paths = Vec::new();
    for change in changes {
        match change {
            GitChange::AddedOrModified { path }
            | GitChange::Deleted { path }
            | GitChange::TypeChanged { path } => paths.push(path),
            GitChange::Renamed { old_path, new_path } => {
                paths.push(old_path);
                paths.push(new_path);
            }
            GitChange::Copied { new_path, .. } => paths.push(new_path),
        }
    }
    paths.sort();
    paths.dedup();

    paths
}

/// Extracts symbol names removed by a diff so impact can include deleted APIs.
pub fn deleted_symbol_names_for_diff(
    registration: &CodeRepositoryRegistration,
    selector: &CodeRepositorySelector,
    base_ref: &str,
    head_ref: &str,
) -> Result<Vec<String>, CodeIndexError> {
    let root = PathBuf::from(&registration.root_path);
    let base_commit = resolve_ref(&root, base_ref)?;
    let head_commit = resolve_ref(&root, head_ref)?;
    let changes = diff_changes(&root, base_ref, head_ref)?;
    let ignore_rules = load_ignore_rules_from_commit(&root, &head_commit)?;
    let mut names = Vec::new();

    for change in changes {
        let deleted_path = match change {
            GitChange::Deleted { path } | GitChange::Renamed { old_path: path, .. } => path,
            GitChange::AddedOrModified { .. }
            | GitChange::Copied { .. }
            | GitChange::TypeChanged { .. } => continue,
        };
        if !path_is_selected_with_rules(&deleted_path, registration, selector, &ignore_rules) {
            continue;
        }
        let bytes = git_bytes(&root, ["show", &format!("{base_commit}:{deleted_path}")])?;
        let mut build = SnapshotBuild::new_with_selector(
            registration,
            selector,
            base_commit.clone(),
            "deleted-symbol-seed".to_owned(),
            true,
            1,
            0,
        );
        parse_indexed_file(&mut build, &deleted_path, &bytes)?;
        names.extend(build.symbols.into_iter().map(|symbol| symbol.name));
    }
    names.sort();
    names.dedup();

    Ok(names)
}

/// Resolves a repository ref selector to the exact commit used by storage.
pub fn resolve_repository_ref(
    root_path: impl AsRef<Path>,
    ref_selector: &str,
) -> Result<String, CodeIndexError> {
    let root = resolve_git_root(root_path.as_ref())?;

    resolve_ref(&root, ref_selector)
}

/// Resolves a repository ref selector to the exact commit and tree hash.
pub fn resolve_repository_snapshot(
    root_path: impl AsRef<Path>,
    ref_selector: &str,
) -> Result<(String, String), CodeIndexError> {
    let root = resolve_git_root(root_path.as_ref())?;
    let commit = resolve_ref(&root, ref_selector)?;
    let tree_hash = resolve_tree(&root, &commit)?;

    Ok((commit, tree_hash))
}

fn build_full_snapshot(
    registration: &CodeRepositoryRegistration,
    selector: &CodeRepositorySelector,
    root: &Path,
) -> Result<CodeIndexSnapshot, CodeIndexError> {
    let commit = resolve_ref(root, &selector.ref_selector)?;
    let tree_hash = resolve_tree(root, &commit)?;
    let ignore_rules = load_ignore_rules_from_commit(root, &commit)?;
    let paths = tracked_paths(root, &commit)?
        .into_iter()
        .filter(|path| {
            selection_exclusion_reason(path, registration, selector, &ignore_rules).is_none()
        })
        .collect::<Vec<_>>();
    let mut build = SnapshotBuild::new_with_selector(
        registration,
        selector,
        commit,
        tree_hash,
        true,
        paths.len(),
        0,
    );

    for path in paths {
        let bytes = git_bytes(root, ["show", &format!("{}:{path}", build.commit)])?;
        parse_indexed_file(&mut build, &path, &bytes)?;
    }

    Ok(build.finish())
}

fn build_incremental_snapshot(
    registration: &CodeRepositoryRegistration,
    selector: &CodeRepositorySelector,
    root: &Path,
    base_ref: &str,
    head_ref: &str,
    previous_hashes: &BTreeMap<String, String>,
) -> Result<CodeIndexSnapshot, CodeIndexError> {
    let base_commit = resolve_ref(root, base_ref)?;
    let commit = resolve_ref(root, head_ref)?;
    let tree_hash = resolve_tree(root, &commit)?;
    let changes = diff_changes(root, base_ref, head_ref)?;
    let base_ignore_rules = load_ignore_rules_from_commit(root, &base_commit)?;
    let ignore_rules = load_ignore_rules_from_commit(root, &commit)?;
    let mut build = SnapshotBuild::new_with_selector(
        registration,
        selector,
        commit,
        tree_hash,
        false,
        changes.len(),
        0,
    );
    build.base_resolved_commit_sha = Some(base_commit.clone());

    for change in changes {
        match change {
            GitChange::Deleted { path } => {
                if path_is_selected_with_rules(&path, registration, selector, &base_ignore_rules) {
                    build.deleted_paths.push(path);
                }
            }
            GitChange::Renamed { old_path, new_path } => {
                if path_is_selected_with_rules(
                    &old_path,
                    registration,
                    selector,
                    &base_ignore_rules,
                ) {
                    build.deleted_paths.push(old_path.clone());
                    build.tombstones.push(CodePathTombstone {
                        repository_id: registration.repository_id.clone(),
                        source_scope: build.source_scope.clone(),
                        old_path,
                        new_path: Some(new_path.clone()),
                        base_ref: base_ref.to_owned(),
                        head_ref: head_ref.to_owned(),
                    });
                }
                parse_changed_path(
                    &mut build,
                    registration,
                    selector,
                    root,
                    &new_path,
                    previous_hashes,
                    &ignore_rules,
                )?;
            }
            GitChange::Copied { old_path, new_path } => {
                if path_is_selected_with_rules(&new_path, registration, selector, &ignore_rules) {
                    build.tombstones.push(CodePathTombstone {
                        repository_id: registration.repository_id.clone(),
                        source_scope: build.source_scope.clone(),
                        old_path,
                        new_path: Some(new_path.clone()),
                        base_ref: base_ref.to_owned(),
                        head_ref: head_ref.to_owned(),
                    });
                }
                parse_changed_path(
                    &mut build,
                    registration,
                    selector,
                    root,
                    &new_path,
                    previous_hashes,
                    &ignore_rules,
                )?;
            }
            GitChange::AddedOrModified { path } | GitChange::TypeChanged { path } => {
                parse_changed_path(
                    &mut build,
                    registration,
                    selector,
                    root,
                    &path,
                    previous_hashes,
                    &ignore_rules,
                )?;
            }
        }
    }

    Ok(build.finish())
}

fn build_worktree_overlay_snapshot(
    registration: &CodeRepositoryRegistration,
    selector: &CodeRepositorySelector,
    root: &Path,
    previous_hashes: &BTreeMap<String, String>,
) -> Result<CodeIndexSnapshot, CodeIndexError> {
    let commit = resolve_ref(root, &selector.ref_selector)?;
    let head_commit = resolve_ref(root, "HEAD")?;
    if commit != head_commit {
        return Err(CodeIndexError::InvalidInput(format!(
            "worktree overlay ref '{}' resolves to {}, but checked-out HEAD is {}",
            selector.ref_selector, commit, head_commit
        )));
    }
    let status = git_bytes(
        root,
        ["status", "--porcelain=v1", "-z", "--untracked-files=all"],
    )?;
    let changes = worktree_changed_paths(&status);
    if changes.is_empty() {
        return build_full_snapshot(registration, selector, root);
    }
    let mut overlay_hash_input = Vec::new();
    let mut deleted_paths = Vec::new();
    let mut files_to_parse = Vec::new();
    let mut skipped_unchanged_count = 0;
    let ignore_rules = load_ignore_rules(root)?;

    for change in &changes {
        if let Some(deleted_path) = &change.deleted_source {
            if path_is_selected_with_rules(deleted_path, registration, selector, &ignore_rules) {
                overlay_hash_input.extend_from_slice(b"D\0");
                overlay_hash_input.extend_from_slice(deleted_path.as_bytes());
                overlay_hash_input.push(0);
                deleted_paths.push(deleted_path.clone());
            }
        }
        let path = &change.path;
        if !path_scope_overlaps(path, registration, selector) {
            continue;
        }
        let full_path = root.join(path);
        let metadata = match fs::symlink_metadata(&full_path) {
            Ok(metadata) => metadata,
            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
                if path_is_selected_with_rules(path, registration, selector, &ignore_rules) {
                    overlay_hash_input.extend_from_slice(b"D\0");
                    overlay_hash_input.extend_from_slice(path.as_bytes());
                    overlay_hash_input.push(0);
                    deleted_paths.push(path.clone());
                }
                continue;
            }
            Err(error) => return Err(error.into()),
        };
        let file_type = metadata.file_type();
        if file_type.is_symlink() {
            if path_is_selected_with_rules(path, registration, selector, &ignore_rules) {
                record_worktree_status_marker(path, &mut overlay_hash_input);
            }
            continue;
        }
        if file_type.is_dir() {
            if !change.is_untracked() || !worktree_directory_is_expandable(root, path)? {
                if path_is_selected_with_rules(path, registration, selector, &ignore_rules) {
                    record_worktree_status_marker(path, &mut overlay_hash_input);
                }
                continue;
            }
            for nested_path in worktree_directory_files(root, path)? {
                if path_is_selected_with_rules(&nested_path, registration, selector, &ignore_rules)
                {
                    record_worktree_file(
                        root,
                        &nested_path,
                        previous_hashes,
                        &mut overlay_hash_input,
                        &mut files_to_parse,
                        &mut skipped_unchanged_count,
                    )?;
                }
            }
            continue;
        }
        if !file_type.is_file() {
            if path_is_selected_with_rules(path, registration, selector, &ignore_rules) {
                record_worktree_status_marker(path, &mut overlay_hash_input);
            }
            continue;
        }
        if path_is_selected_with_rules(path, registration, selector, &ignore_rules) {
            record_worktree_file(
                root,
                path,
                previous_hashes,
                &mut overlay_hash_input,
                &mut files_to_parse,
                &mut skipped_unchanged_count,
            )?;
        }
    }
    if overlay_hash_input.is_empty() {
        return build_full_snapshot(registration, selector, root);
    }

    let overlay_hash = format!("{:016x}", stable_hash64(&overlay_hash_input));
    let tree_hash = format!("worktree:{overlay_hash}");
    let overlay_commit = format!("worktree:{commit}:{overlay_hash}");
    let mut build = SnapshotBuild::new_with_selector(
        registration,
        selector,
        overlay_commit,
        tree_hash,
        false,
        changes.len(),
        skipped_unchanged_count,
    );
    build.base_resolved_commit_sha = Some(commit);
    build.deleted_paths = deleted_paths;

    for (path, bytes) in files_to_parse {
        parse_indexed_file(&mut build, &path, &bytes)?;
    }

    Ok(build.finish())
}

fn record_worktree_status_marker(path: &str, overlay_hash_input: &mut Vec<u8>) {
    overlay_hash_input.extend_from_slice(b"S\0");
    overlay_hash_input.extend_from_slice(path.as_bytes());
    overlay_hash_input.push(0);
}

fn record_worktree_file(
    root: &Path,
    path: &str,
    previous_hashes: &BTreeMap<String, String>,
    overlay_hash_input: &mut Vec<u8>,
    files_to_parse: &mut Vec<(String, Vec<u8>)>,
    skipped_unchanged_count: &mut usize,
) -> Result<(), CodeIndexError> {
    let bytes = fs::read(root.join(path))?;
    let blob_hash = stable_content_hash(&bytes);
    overlay_hash_input.extend_from_slice(b"F\0");
    overlay_hash_input.extend_from_slice(path.as_bytes());
    overlay_hash_input.push(0);
    overlay_hash_input.extend_from_slice(blob_hash.as_bytes());
    overlay_hash_input.push(0);
    if previous_hashes.get(path) == Some(&blob_hash) {
        *skipped_unchanged_count += 1;
        return Ok(());
    }
    files_to_parse.push((path.to_owned(), bytes));

    Ok(())
}

fn worktree_directory_files(
    root: &Path,
    relative_dir: &str,
) -> Result<Vec<String>, CodeIndexError> {
    if !worktree_directory_is_expandable(root, relative_dir)? {
        return Ok(Vec::new());
    }
    let mut files = Vec::new();
    collect_worktree_directory_files(root, Path::new(relative_dir), &mut files)?;
    files.sort();

    Ok(files)
}

fn worktree_directory_is_expandable(
    root: &Path,
    relative_dir: &str,
) -> Result<bool, CodeIndexError> {
    let full_path = root.join(relative_dir);
    let metadata = fs::symlink_metadata(&full_path)?;
    if !metadata.file_type().is_dir() {
        return Ok(false);
    }

    Ok(!contains_git_metadata(root, Path::new(relative_dir))?)
}

fn contains_git_metadata(root: &Path, relative: &Path) -> Result<bool, CodeIndexError> {
    match fs::symlink_metadata(root.join(relative).join(".git")) {
        Ok(_) => Ok(true),
        Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false),
        Err(error) => Err(error.into()),
    }
}

fn collect_worktree_directory_files(
    root: &Path,
    relative: &Path,
    files: &mut Vec<String>,
) -> Result<(), CodeIndexError> {
    for entry in fs::read_dir(root.join(relative))? {
        let entry = entry?;
        let path = relative.join(entry.file_name());
        let file_type = entry.file_type()?;
        if file_type.is_dir() {
            if entry.file_name() == ".git" || contains_git_metadata(root, &path)? {
                continue;
            }
            collect_worktree_directory_files(root, &path, files)?;
        } else if file_type.is_file() {
            files.push(path.to_string_lossy().replace('\\', "/"));
        }
    }

    Ok(())
}

fn parse_changed_path(
    build: &mut SnapshotBuild,
    registration: &CodeRepositoryRegistration,
    selector: &CodeRepositorySelector,
    root: &Path,
    path: &str,
    previous_hashes: &BTreeMap<String, String>,
    ignore_rules: &[scope::IgnoreRule],
) -> Result<(), CodeIndexError> {
    if !path_is_selected_with_rules(path, registration, selector, ignore_rules) {
        return Ok(());
    }
    let object = format!("{}:{path}", build.commit);
    let bytes = git_bytes(root, ["show", &object])?;
    let blob_hash = stable_content_hash(&bytes);
    if previous_hashes.get(path) == Some(&blob_hash) {
        build.skipped_unchanged_count += 1;
        return Ok(());
    }

    parse_indexed_file(build, path, &bytes)
}