github-mcp 0.5.7

// GitHub v3 REST API MCP server — generated by mcpify. Do not hand-edit.
//
// Populates a store db's semantic_endpoints vec0 table with real
// embedding vectors. mcpify's Rust generator only creates this table's
// schema (see the plan's embeddings decision) — this binary is the single
// place vectors get computed, reusing the exact same embedding function the
// `search` tool calls at runtime (`services::embedding_service::embed`), so
// indexing-time and live-query embeddings are guaranteed to share the same
// model and vector space. Vectors are bound as a raw little-endian `f32`
// blob (sqlite-vec's native `FLOAT[N]` wire format) rather than a JSON
// array string — `services::embedding_service`'s `search`-side query code
// (Story R6) must encode/decode with this exact same byte layout.

use std::fs::File;
use std::io::Write;
use std::path::{Path, PathBuf};

use anyhow::Context;
use github_mcp::data::store::{VERSION_STORE_FILES, open_store_read_write};
use github_mcp::services::embedding_service::embed;

/// Compression level for the `.db.zst` sibling this binary leaves behind —
/// matches the level `store.rs`'s embedded `VERSION_STORE_BYTES` were
/// produced with, so re-running this binary never silently regresses the
/// published package size.
const ZSTD_LEVEL: i32 = 19;

/// The `semantic_endpoints` vec0 embedding column's dimension — must match
/// `services::embedding_service::embed`'s output width for the model
/// currently in use (`all-MiniLM-L6-v2`, 384 dims). Every store's
/// `semantic_endpoints` table is dropped and recreated with this dimension
/// before repopulating, since sqlite-vec's vec0 tables are fixed-width and
/// reject inserting a vector of a different length than the column was
/// declared with — mcpify's own generator always creates it at `FLOAT[768]`
/// (matching `all-mpnet-base-v2`, the default for every target), so this
/// must run on every generation/regeneration, not just the first one.
const EMBEDDING_DIM: usize = 384;

/// Returns `path`'s zstd sibling, e.g. `mcp_store.db` -> `mcp_store.db.zst`.
fn zst_sibling(path: &Path) -> PathBuf {
    let mut name = path.as_os_str().to_owned();
    name.push(".zst");
    PathBuf::from(name)
}

/// Ensures a real, uncompressed `.db` file exists at `path` for SQLite to
/// open read-write: if only the `.db.zst` sibling exists (the normal state
/// once a store has been compressed and committed), decompresses it into
/// place first. If both somehow exist, the raw `.db` is left untouched and
/// treated as the working copy — `.zst` is regenerated from it below.
fn ensure_raw_db(path: &Path) -> anyhow::Result<()> {
    if path.exists() {
        return Ok(());
    }
    let zst_path = zst_sibling(path);
    let compressed = std::fs::read(&zst_path).with_context(|| {
        format!(
            "neither '{}' nor '{}' exists",
            path.display(),
            zst_path.display()
        )
    })?;
    let decompressed = zstd::stream::decode_all(compressed.as_slice())
        .with_context(|| format!("failed to decompress '{}'", zst_path.display()))?;
    std::fs::write(path, decompressed)
        .with_context(|| format!("failed to write decompressed '{}'", path.display()))?;
    Ok(())
}

/// Re-compresses the raw `.db` at `path` into its `.db.zst` sibling, then
/// removes the raw file — the working copy this binary operates on must
/// never linger on disk afterward (it would otherwise risk getting
/// accidentally committed, and the raw file is exactly what pushes the
/// published crate package over crates.io's 10MiB limit).
fn recompress_and_remove_raw(path: &Path) -> anyhow::Result<()> {
    let raw =
        std::fs::read(path).with_context(|| format!("failed to read '{}'", path.display()))?;
    let compressed = zstd::stream::encode_all(raw.as_slice(), ZSTD_LEVEL)
        .with_context(|| format!("failed to zstd-compress '{}'", path.display()))?;
    let zst_path = zst_sibling(path);
    let mut file = File::create(&zst_path)
        .with_context(|| format!("failed to create '{}'", zst_path.display()))?;
    file.write_all(&compressed)
        .with_context(|| format!("failed to write '{}'", zst_path.display()))?;
    std::fs::remove_file(path).with_context(|| {
        format!(
            "failed to remove raw '{}' after recompressing",
            path.display()
        )
    })?;
    Ok(())
}

struct EndpointRow {
    operation_id: String,
    path: String,
    method: String,
    summary: Option<String>,
    description: Option<String>,
}

fn vector_to_le_bytes(vector: &[f32]) -> Vec<u8> {
    vector
        .iter()
        .flat_map(|value| value.to_le_bytes())
        .collect()
}

/// Populates one store file's `semantic_endpoints` table in place,
/// returning how many operations were (re)indexed. Any single row's
/// embedding computation/insert failing propagates as a hard error via `?`
/// (never silently skipped) — a partially-embedded store is worse than a
/// script that stops and reports exactly which operation broke.
fn populate_one(path: &Path) -> anyhow::Result<usize> {
    let conn = open_store_read_write(path)?;

    // The `semantic_endpoints` vec0 table's `embedding` column is a
    // fixed-width `FLOAT[N]`. mcpify's generator always creates it at
    // 768-dim (matching `all-mpnet-base-v2`); a full rebuild against the
    // current model (`all-MiniLM-L6-v2`, `EMBEDDING_DIM`-dim) requires
    // dropping and recreating the table at the new width rather than just
    // deleting rows, since vec0 rejects inserting a vector whose length
    // doesn't match the column's declared dimension.
    conn.execute("DROP TABLE IF EXISTS semantic_endpoints", [])
        .with_context(|| {
            format!(
                "failed to drop 'semantic_endpoints' in '{}'",
                path.display()
            )
        })?;
    conn.execute(
        &format!(
            "CREATE VIRTUAL TABLE semantic_endpoints USING vec0(
                operation_id TEXT PRIMARY KEY,
                embedding FLOAT[{EMBEDDING_DIM}]
            )"
        ),
        [],
    )
    .with_context(|| {
        format!(
            "failed to recreate 'semantic_endpoints' in '{}'",
            path.display()
        )
    })?;

    let mut select =
        conn.prepare("SELECT operation_id, path, method, summary, description FROM endpoints")?;
    let rows: Vec<EndpointRow> = select
        .query_map([], |row| {
            Ok(EndpointRow {
                operation_id: row.get(0)?,
                path: row.get(1)?,
                method: row.get(2)?,
                summary: row.get(3)?,
                description: row.get(4)?,
            })
        })?
        .collect::<Result<_, _>>()?;

    // sqlite-vec's vec0 virtual tables don't implement conflict resolution
    // (no ON CONFLICT / INSERT OR REPLACE support — a duplicate primary key
    // always raises a UNIQUE constraint error, regardless of the conflict
    // clause used), so re-running this script against a db that already has
    // rows requires an explicit delete before each insert to stay idempotent.
    let mut delete = conn.prepare("DELETE FROM semantic_endpoints WHERE operation_id = ?1")?;
    let mut insert =
        conn.prepare("INSERT INTO semantic_endpoints (operation_id, embedding) VALUES (?1, ?2)")?;

    // Sequential, not parallelized: keeps peak memory bounded regardless of
    // how many operations a spec declares, at the cost of total wall time —
    // an acceptable trade for a one-time setup script.
    let count = rows.len();
    for row in rows {
        let operation_id = row.operation_id.clone();
        let text = [
            Some(row.method),
            Some(row.path),
            row.summary,
            row.description,
        ]
        .into_iter()
        .flatten()
        .collect::<Vec<_>>()
        .join(" ");
        let vector = embed(&text).map_err(|err| {
            anyhow::anyhow!(
                "failed to compute embedding for '{operation_id}' in '{}': {err}",
                path.display()
            )
        })?;
        delete.execute(rusqlite::params![row.operation_id])?;
        insert.execute(rusqlite::params![
            row.operation_id,
            vector_to_le_bytes(&vector)
        ])?;
    }

    // Dropping and recreating `semantic_endpoints` above (to change its
    // embedding dimension) frees its old pages, but SQLite doesn't shrink
    // the on-disk file or reuse those pages for anything smaller without
    // an explicit VACUUM — without this, the raw `.db` (and therefore its
    // recompressed `.db.zst`) would stay exactly as large as when it held
    // the wider 768-dim vectors, defeating the whole point of switching to
    // a narrower embedding model.
    conn.execute("VACUUM", [])
        .with_context(|| format!("failed to VACUUM '{}'", path.display()))?;

    Ok(count)
}

/// Verifies `semantic_endpoints` row count equals `endpoints` row count for
/// the store at `path` — a store can end up with `endpoints` rows but no
/// matching `semantic_endpoints` rows if embedding generation/insertion was
/// silently skipped or partially failed for some rows, and that must not be
/// allowed to pass as "populated". Returns the operation IDs present in
/// `endpoints` but missing from `semantic_endpoints` when counts diverge.
fn missing_operation_ids(path: &Path) -> anyhow::Result<Vec<String>> {
    let conn = open_store_read_write(path)?;
    let mut select = conn.prepare(
        "SELECT e.operation_id FROM endpoints e \
         LEFT JOIN semantic_endpoints s ON s.operation_id = e.operation_id \
         WHERE s.operation_id IS NULL",
    )?;
    let missing: Vec<String> = select
        .query_map([], |row| row.get(0))?
        .collect::<Result<_, _>>()?;
    Ok(missing)
}

/// Which store file(s) to populate: bare invocation (and `--all`) both walk
/// every version this project's ledger knows about (`VERSION_STORE_FILES`)
/// — defaulting to just the default version's store left every other
/// version's `semantic_endpoints` table at 0 rows unless `--all` was passed
/// explicitly, so the safe default is "populate everything". An explicit
/// path argument still targets exactly that one file, for a targeted
/// re-run.
fn targets_from(mut args: impl Iterator<Item = String>) -> Vec<PathBuf> {
    match args.next().as_deref() {
        Some("--all") | None => VERSION_STORE_FILES
            .iter()
            .map(|(_, file)| PathBuf::from(file))
            .collect(),
        Some(path) => vec![PathBuf::from(path)],
    }
}

fn targets() -> Vec<PathBuf> {
    targets_from(std::env::args().skip(1))
}

fn main() -> anyhow::Result<()> {
    let mut had_mismatch = false;
    for path in targets() {
        // SQLite needs a real uncompressed file to write into — if only
        // the `.db.zst` sibling is present (the normal committed state),
        // decompress it into place first.
        ensure_raw_db(&path)?;

        let count = populate_one(&path)?;
        println!(
            "populated embeddings for {count} operation(s) in '{}'",
            path.display()
        );

        let missing = missing_operation_ids(&path)?;
        if !missing.is_empty() {
            had_mismatch = true;
            eprintln!(
                "ERROR: '{}' has {} endpoint(s) missing from semantic_endpoints (row-count \
                 parity check failed): {}",
                path.display(),
                missing.len(),
                missing.join(", ")
            );
        }

        // On failure above, `?` already propagated and left the raw `.db`
        // on disk (not recompressed) so it's available for inspection —
        // only a fully processed path gets folded back into the `.db.zst`
        // this binary must always end by leaving behind.
        recompress_and_remove_raw(&path)?;
    }
    if had_mismatch {
        anyhow::bail!(
            "embedding population incomplete: one or more stores have semantic_endpoints \
             row counts below their endpoints row counts (see errors above)"
        );
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn target_selection_covers_defaults_all_and_explicit_paths_without_mutating_stores() {
        let expected = VERSION_STORE_FILES
            .iter()
            .map(|(_, file)| PathBuf::from(file))
            .collect::<Vec<_>>();
        assert_eq!(targets_from(std::iter::empty()), expected);
        assert_eq!(targets_from(["--all".to_string()].into_iter()), expected);
        assert_eq!(
            targets_from(["copy.db".to_string()].into_iter()),
            vec![PathBuf::from("copy.db")]
        );
    }
}