semlith 0.12.0

//! Narrowing a search to part of the corpus.
//!
//! A [`Filter`] is resolved to SQLite `GLOB` patterns and then to a set of
//! chunk ids by [`crate::store::filtered_chunk_ids`]. That one id set drives
//! both halves of the hybrid search — the allowlist handed to the vector index
//! and the predicate inside the FTS5 query — so the two can never disagree
//! about which chunks were eligible.

use anyhow::{Result, bail};

/// One `--lang` name and everything that counts as it.
pub struct Language {
    pub name: &'static str,
    /// Lowercase, without the dot, sorted.
    pub extensions: &'static [&'static str],
    /// Whole filenames, lowercase, for the languages whose files carry no
    /// extension at all. Glob patterns, so `dockerfile.*` covers
    /// `Dockerfile.prod` alongside the bare name.
    pub filenames: &'static [&'static str],
}

/// What makes up each language name accepted by `--lang`.
///
/// The name is the signal, and for almost every language the name is the
/// extension. The two exceptions are the reason `filenames` exists: a
/// Dockerfile and a Makefile are identified by being called that, and a filter
/// that only knew extensions would answer `--lang dockerfile` with an empty
/// result on a repository full of them — which is indistinguishable from
/// "there are none", the worst answer available.
///
/// Contents are never read. Telling Perl from anything else by looking at it is
/// a research project, and a store is queried far more often than it is built,
/// so guessing at query time would cost on every search.
pub const LANGUAGES: &[Language] = &[
    lang("c", &["c", "h"], &[]),
    lang("clojure", &["clj", "cljc", "cljs", "edn"], &[]),
    lang("cpp", &["cc", "cpp", "cxx", "hh", "hpp", "hxx"], &[]),
    lang("csharp", &["cs"], &[]),
    lang("css", &["css", "sass", "scss"], &[]),
    lang("dart", &["dart"], &[]),
    lang(
        "dockerfile",
        &["dockerfile"],
        &["dockerfile", "dockerfile.*"],
    ),
    lang("elixir", &["ex", "exs"], &[]),
    lang("elm", &["elm"], &[]),
    lang("erlang", &["erl", "hrl"], &[]),
    lang("fortran", &["f", "f03", "f90", "f95"], &[]),
    lang("go", &["go"], &[]),
    lang("graphql", &["gql", "graphql"], &[]),
    lang("groovy", &["gradle", "groovy"], &[]),
    lang("haskell", &["hs"], &[]),
    lang("html", &["htm", "html"], &[]),
    lang("java", &["java"], &[]),
    lang("javascript", &["cjs", "js", "jsx", "mjs"], &[]),
    lang("json", &["json"], &[]),
    lang("julia", &["jl"], &[]),
    lang("kotlin", &["kt", "kts"], &[]),
    lang("lua", &["lua"], &[]),
    lang("makefile", &["mk"], &["gnumakefile", "makefile"]),
    lang("markdown", &["markdown", "md"], &[]),
    lang("nix", &["nix"], &[]),
    // `.m` is Objective-C and it is also MATLAB. Objective-C is the one with a
    // header file beside it, which is the only signal an extension-based filter
    // has, so it takes the extension.
    lang("objective-c", &["m", "mm"], &[]),
    lang("ocaml", &["ml", "mli"], &[]),
    lang("perl", &["pl", "pm", "t"], &[]),
    lang("php", &["php"], &[]),
    lang("powershell", &["ps1", "psd1", "psm1"], &[]),
    lang("proto", &["proto"], &[]),
    lang("python", &["py", "pyi"], &[]),
    lang("r", &["r"], &[]),
    lang("ruby", &["rb"], &[]),
    lang("rust", &["rs"], &[]),
    lang("scala", &["sc", "scala"], &[]),
    lang("shell", &["bash", "sh", "zsh"], &[]),
    lang("sql", &["sql"], &[]),
    lang("svelte", &["svelte"], &[]),
    lang("swift", &["swift"], &[]),
    lang("terraform", &["tf", "tfvars"], &[]),
    lang("toml", &["toml"], &[]),
    lang("typescript", &["cts", "mts", "ts", "tsx"], &[]),
    lang("vue", &["vue"], &[]),
    lang("yaml", &["yaml", "yml"], &[]),
    lang("zig", &["zig"], &[]),
];

/// One row of [`LANGUAGES`], so the table reads as a table.
const fn lang(
    name: &'static str,
    extensions: &'static [&'static str],
    filenames: &'static [&'static str],
) -> Language {
    Language {
        name,
        extensions,
        filenames,
    }
}

/// The entry for a `--lang` name, matched case-insensitively.
pub fn language(name: &str) -> Option<&'static Language> {
    let wanted = name.to_ascii_lowercase();
    LANGUAGES.iter().find(|entry| entry.name == wanted)
}

/// Which chunks a search is allowed to see.
///
/// Patterns are grouped: within a group they union, across groups they
/// intersect. So `--ext rs --ext toml` is "Rust or TOML", while
/// `--path 'src/**' --ext md` is "Markdown, under src".
#[derive(Debug, Default, Clone, PartialEq)]
pub struct Filter {
    groups: Vec<Vec<String>>,
}

impl Filter {
    /// Build a filter from the three user-facing kinds.
    ///
    /// Fails on a language name that is not in [`LANGUAGES`]. A silent miss
    /// there is indistinguishable from an empty corpus, which is the worst
    /// possible answer to give an agent.
    pub fn new(paths: &[String], exts: &[String], langs: &[String]) -> Result<Self> {
        let mut groups = Vec::new();

        if !paths.is_empty() {
            groups.push(paths.iter().map(|p| anchor(p)).collect());
        }

        // Extensions and languages are both extension sets, so they share one
        // group: `--ext rs --lang markdown` means "Rust or Markdown", the same
        // way two `--ext` flags do.
        let mut patterns: Vec<String> = exts
            .iter()
            .map(|e| format!("*.{}", e.trim_start_matches('.').to_ascii_lowercase()))
            .collect();
        for name in langs {
            let Some(entry) = language(name) else {
                bail!("unknown language {name:?}; run `semlith languages` for the list");
            };
            patterns.extend(entry.extensions.iter().map(|e| format!("*.{e}")));
            // A filename pattern resolves into the same group as an extension
            // one, so `--lang dockerfile --ext md` unions exactly the way two
            // extensions do and nothing downstream has a second case to handle.
            patterns.extend(entry.filenames.iter().map(|f| f.to_string()));
        }
        if !patterns.is_empty() {
            groups.push(patterns.iter().map(|p| anchor(p)).collect());
        }

        Ok(Self { groups })
    }

    pub fn is_empty(&self) -> bool {
        self.groups.is_empty()
    }

    /// AND-groups of OR-patterns, ready for the SQL builder.
    pub fn groups(&self) -> &[Vec<String>] {
        &self.groups
    }
}

/// Turn a user's pattern into one matchable against the canonical absolute
/// path a store holds.
///
/// A relative pattern gains a `*/` prefix, so `src/**` finds
/// `/home/me/proj/src/lib.rs` no matter which directory the search runs from.
/// An absolute pattern is used verbatim, and therefore matches only what it
/// literally covers.
///
/// Lowercased, because the query side compares against `lower(files.path)`:
/// `README.MD` and `readme.md` are the same file to anyone typing `--ext md`.
fn anchor(pattern: &str) -> String {
    let pattern = pattern.to_lowercase();
    // ponytail: paths on Windows are stored with backslashes, so a pattern
    // written with forward slashes would match nothing. Translate rather than
    // asking the user to write a platform-specific glob.
    #[cfg(windows)]
    let pattern = pattern.replace('/', "\\");

    if is_absolute(&pattern) {
        pattern
    } else {
        format!("{}{pattern}", separator_prefix())
    }
}

fn is_absolute(pattern: &str) -> bool {
    if pattern.starts_with(std::path::MAIN_SEPARATOR) {
        return true;
    }
    // `c:\...` on Windows. Elsewhere a colon is an ordinary filename character.
    cfg!(windows) && pattern.as_bytes().get(1) == Some(&b':')
}

fn separator_prefix() -> String {
    format!("*{}", std::path::MAIN_SEPARATOR)
}

#[cfg(test)]
mod tests {
    use super::*;

    fn s(items: &[&str]) -> Vec<String> {
        items.iter().map(|i| i.to_string()).collect()
    }

    /// The rule the README documents: a relative glob is anchored so it works
    /// from any directory; an absolute one is not, so it means exactly itself.
    #[test]
    fn a_relative_pattern_is_anchored_and_an_absolute_one_is_not() {
        let sep = std::path::MAIN_SEPARATOR;
        let f = Filter::new(&s(&["src/**"]), &[], &[]).unwrap();
        assert_eq!(f.groups(), [[format!("*{sep}src{sep}**")]]);

        let absolute = format!("{sep}home{sep}me{sep}proj{sep}src{sep}*");
        let f = Filter::new(&s(&[&absolute]), &[], &[]).unwrap();
        assert_eq!(f.groups(), [[absolute]]);
    }

    #[test]
    fn extensions_and_languages_share_one_group_and_union() {
        let sep = std::path::MAIN_SEPARATOR;
        let f = Filter::new(&[], &s(&["toml"]), &s(&["rust"])).unwrap();
        assert_eq!(f.groups().len(), 1, "one group means they union");
        assert_eq!(
            f.groups()[0],
            [format!("*{sep}*.toml"), format!("*{sep}*.rs")]
        );
    }

    /// `--path 'src/**' --ext md` must mean "Markdown under src", not
    /// "Markdown, or anything under src".
    #[test]
    fn paths_and_extensions_are_separate_groups_and_intersect() {
        let f = Filter::new(&s(&["src/**"]), &s(&["md"]), &[]).unwrap();
        assert_eq!(f.groups().len(), 2);
    }

    #[test]
    fn a_leading_dot_on_an_extension_is_accepted() {
        let plain = Filter::new(&[], &s(&["rs"]), &[]).unwrap();
        let dotted = Filter::new(&[], &s(&[".rs"]), &[]).unwrap();
        assert_eq!(plain, dotted);
    }

    #[test]
    fn an_unknown_language_names_the_command_that_lists_them() {
        let err = Filter::new(&[], &[], &s(&["klingon"]))
            .unwrap_err()
            .to_string();
        assert!(err.contains("semlith languages"), "unhelpful error: {err}");
    }

    /// The two languages whose files carry no extension. Without the filename
    /// column `--lang dockerfile` resolves to `*.dockerfile` alone, which
    /// matches none of the Dockerfiles anyone actually has.
    #[test]
    fn a_language_with_no_extension_matches_by_filename() {
        let sep = std::path::MAIN_SEPARATOR;
        let f = Filter::new(&[], &[], &s(&["dockerfile"])).unwrap();
        assert_eq!(
            f.groups()[0],
            [
                format!("*{sep}*.dockerfile"),
                format!("*{sep}dockerfile"),
                format!("*{sep}dockerfile.*"),
            ],
            "a bare Dockerfile and a Dockerfile.prod are both the language"
        );

        let f = Filter::new(&[], &[], &s(&["makefile"])).unwrap();
        assert_eq!(
            f.groups()[0],
            [
                format!("*{sep}*.mk"),
                format!("*{sep}gnumakefile"),
                format!("*{sep}makefile"),
            ]
        );

        // Filenames land in the same group extensions do, so the two union
        // rather than intersecting — one code path, so the vector allowlist and
        // the FTS5 predicate cannot come to different answers about them.
        let f = Filter::new(&[], &s(&["md"]), &s(&["dockerfile"])).unwrap();
        assert_eq!(f.groups().len(), 1);
        assert_eq!(f.groups()[0].len(), 4);
    }

    /// `--lang` is case-insensitive on the way in, the way `--ext` is.
    #[test]
    fn a_language_name_is_matched_however_it_is_typed() {
        let lower = Filter::new(&[], &[], &s(&["dockerfile"])).unwrap();
        let shouted = Filter::new(&[], &[], &s(&["DockerFile"])).unwrap();
        assert_eq!(lower, shouted);
    }

    #[test]
    fn no_flags_is_no_filter() {
        assert!(Filter::new(&[], &[], &[]).unwrap().is_empty());
    }

    /// Every entry must be lowercase and its extensions sorted, since `--lang`
    /// lowercases what it is given and `semlith languages` prints the table
    /// verbatim.
    #[test]
    fn the_language_table_is_normalised_and_sorted() {
        let mut previous = "";
        for entry in LANGUAGES {
            let name = entry.name;
            assert!(name > previous, "language table is out of order at {name}");
            previous = name;
            assert_eq!(name, name.to_ascii_lowercase());
            for set in [entry.extensions, entry.filenames] {
                let mut sorted = set.to_vec();
                sorted.sort_unstable();
                assert_eq!(&sorted, &set, "{name}'s entries are out of order");
                assert!(
                    set.iter().all(|e| *e == e.to_ascii_lowercase()),
                    "{name} has an uppercase entry"
                );
            }
        }
    }
}