github_mcp/data/
store.rs

1// GitHub v3 REST API MCP server — generated by mcpify. Do not hand-edit.
2//
3// Single data-access layer against mcp_store.db — both the relational
4// `endpoints` table and the `semantic_endpoints` vec0 virtual table
5// (REQ-2.4.1's single-store consolidation). Mirrors the shape of
6// mcpify's own src/db/open.rs almost directly, since both the generator
7// and the generated project use the identical rusqlite+sqlite-vec crate
8// pair. Unlike mcpify's own src/db/schema.rs, this module never creates
9// tables — mcpify's shared pipeline (Story 5/6) already wrote
10// mcp_store.db's schema before any generated code runs.
11
12use std::collections::HashMap;
13use std::path::{Path, PathBuf};
14use std::sync::atomic::{AtomicU64, Ordering};
15use std::sync::{Mutex, Once, OnceLock};
16use std::time::Duration;
17
18use anyhow::{Context, Result};
19use rusqlite::backup::Backup;
20use rusqlite::{Connection, OpenFlags, Row};
21use serde::Serialize;
22
23static REGISTER_VEC_EXTENSION: Once = Once::new();
24
25// mcpify:versions:begin
26pub const VERSION_STORE_FILES: &[(&str, &str)] = &[
27    ("gh-2026-03-10", "mcp_store.db"),
28    ("ghec-2026-03-10", "mcp_store_vghec-2026-03-10.db"),
29    ("ghes-3.21", "mcp_store_vghes-3.21.db"),
30    ("ghes-3.20", "mcp_store_vghes-3.20.db"),
31    ("ghes-3.19", "mcp_store_vghes-3.19.db"),
32];
33
34// Each entry is zstd-compressed (level 19) — crates.io enforces a hard
35// 10MiB package-upload limit, and the raw uncompressed stores alone total
36// well over that once every API version carries real embeddings.
37// `resolve_store_path` decompresses these bytes back to a real on-disk
38// `.db` file before `rusqlite::Connection::open` ever sees them.
39const VERSION_STORE_BYTES: &[(&str, &[u8])] = &[
40    ("gh-2026-03-10", include_bytes!("../../mcp_store.db.zst")),
41    (
42        "ghec-2026-03-10",
43        include_bytes!("../../mcp_store_vghec-2026-03-10.db.zst"),
44    ),
45    (
46        "ghes-3.21",
47        include_bytes!("../../mcp_store_vghes-3.21.db.zst"),
48    ),
49    (
50        "ghes-3.20",
51        include_bytes!("../../mcp_store_vghes-3.20.db.zst"),
52    ),
53    (
54        "ghes-3.19",
55        include_bytes!("../../mcp_store_vghes-3.19.db.zst"),
56    ),
57];
58// mcpify:versions:end
59
60/// Resolves the active `api_version` (from the config cascade) to its
61/// store file. Every `.db` this crate supports is embedded into the
62/// compiled binary via `include_bytes!` (`VERSION_STORE_BYTES`) exactly
63/// like `validator.rs` embeds each version's schema — there is no
64/// filesystem fallback chain to reason about, and this crate never
65/// depends on any particular `.db` file existing anywhere on disk after
66/// `cargo install`. The one difference from a schema lookup: SQLite
67/// needs a real file to open a `Connection` against (unlike a `&[u8]`
68/// JSON schema, read directly from memory), so this extracts the
69/// embedded bytes to a fixed path in the OS temp dir on every call —
70/// cheap, since it only runs once per process start, and it means a
71/// rebuilt binary with different embedded bytes (a `populate_embeddings`
72/// re-run, an `add-version` update) can never be shadowed by a stale
73/// leftover from a previous install at that same path.
74pub fn resolve_store_path(api_version: &str) -> Result<PathBuf> {
75    let file = VERSION_STORE_FILES
76        .iter()
77        .find(|(label, _)| *label == api_version)
78        .map(|(_, file)| *file)
79        .with_context(|| format!("unknown api_version '{api_version}' — run the 'versions' command to see what's available"))?;
80    let bytes = VERSION_STORE_BYTES
81        .iter()
82        .find(|(label, _)| *label == api_version)
83        .map(|(_, bytes)| *bytes)
84        .with_context(|| format!("no embedded store data for api_version '{api_version}'"))?;
85
86    let mut dir = std::env::temp_dir();
87    dir.push(concat!(env!("CARGO_PKG_NAME"), "-store"));
88    std::fs::create_dir_all(&dir)
89        .with_context(|| format!("failed to create temp dir '{}'", dir.display()))?;
90
91    let path = dir.join(file);
92    // Always (re)writes rather than skipping when the path already
93    // exists: the extraction path doesn't vary by build, so a stale copy
94    // from a previous install (older embedded bytes, e.g. before a
95    // `populate_embeddings` re-run or an `add-version` update) would
96    // otherwise linger forever, silently serving outdated data. The
97    // write is cheap — this runs once per process start, not per query.
98    //
99    // Writes to a uniquely-named sibling file first, then `rename`s it
100    // into place, rather than writing `path` directly: `cached_store_connection`
101    // calls this on every tool invocation (not just once at startup), and
102    // tool calls run concurrently (each MCP request is its own tokio
103    // task) — a direct write would let one task's `open_store` observe
104    // another task's in-progress truncate, reading a momentarily-empty
105    // file and failing with "no such table: endpoints". `rename` within
106    // the same directory is atomic on both POSIX and Windows, so every
107    // reader sees either the complete previous copy or the complete new
108    // one, never a partial write.
109    // `bytes` is the zstd-compressed `.db.zst` payload (see
110    // `VERSION_STORE_BYTES`), not a valid SQLite file itself — it must be
111    // decompressed before `rusqlite::Connection::open` can read it.
112    let decompressed = zstd::stream::decode_all(bytes).with_context(|| {
113        format!("failed to decompress embedded store data for api_version '{api_version}'")
114    })?;
115    static UNIQUE: AtomicU64 = AtomicU64::new(0);
116    let tmp_path = dir.join(format!(
117        "{file}.{}.{}.tmp",
118        std::process::id(),
119        UNIQUE.fetch_add(1, Ordering::Relaxed)
120    ));
121    std::fs::write(&tmp_path, decompressed).with_context(|| {
122        format!(
123            "failed to extract embedded store data to '{}'",
124            tmp_path.display()
125        )
126    })?;
127    std::fs::rename(&tmp_path, &path).with_context(|| {
128        format!(
129            "failed to move extracted store data into place at '{}'",
130            path.display()
131        )
132    })?;
133
134    Ok(path)
135}
136
137const ENDPOINT_COLUMNS: &str = "operation_id, path, method, summary, description, input_schema, output_schema, auth_scheme_ref";
138
139/// Registers the `sqlite-vec` extension once per process, via
140/// `sqlite3_auto_extension` — matching the pattern the `sqlite-vec` crate
141/// itself documents for `rusqlite`.
142fn register_vec_extension() {
143    REGISTER_VEC_EXTENSION.call_once(|| unsafe {
144        #[allow(clippy::missing_transmute_annotations)]
145        rusqlite::ffi::sqlite3_auto_extension(Some(std::mem::transmute(
146            sqlite_vec::sqlite3_vec_init as *const (),
147        )));
148    });
149}
150
151/// Opens `mcp_store.db` read-only: this crate's binaries only ever read
152/// it, except `github-mcp-populate-embeddings` (a separate `[[bin]]`), which uses
153/// `open_store_read_write` instead.
154pub fn open_store(path: &Path) -> Result<Connection> {
155    register_vec_extension();
156    Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY)
157        .with_context(|| format!("failed to open '{}'", path.display()))
158}
159
160/// Read-write counterpart to `open_store` — for `bin/populate_embeddings.rs`,
161/// the one binary that actually writes to `mcp_store.db` (backfilling
162/// `semantic_endpoints`, whose table every other caller only ever reads
163/// from).
164pub fn open_store_read_write(path: &Path) -> Result<Connection> {
165    register_vec_extension();
166    Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_WRITE)
167        .with_context(|| format!("failed to open '{}'", path.display()))
168}
169
170/// Returns a process-wide, in-memory-backed connection for `api_version`:
171/// on first access, opens the on-disk file read-only via `open_store`,
172/// copies its entire contents into a fresh `:memory:` connection (via
173/// SQLite's backup API), then drops the on-disk connection immediately —
174/// so its file handle/lock is held only for that brief copy, not for the
175/// process's lifetime. This means an external process (a fresh
176/// `github-mcp-populate-embeddings` run, a deployment replacing the file) can update
177/// `mcp_store.db` without hitting "database is locked" against a live
178/// server, at the cost of the running process only picking up such an
179/// update on its next restart.
180///
181/// Returns the connection behind a `Mutex` rather than handing it out
182/// directly: callers must finish with the guard (and drop it) *before*
183/// any `.await` — `rusqlite::Connection` isn't `Sync`, so holding the
184/// guard across an await point would make the enclosing future
185/// non-`Send`, the same constraint `core/mcp_server.rs`'s tool handlers
186/// already respect for a plain `Connection`.
187pub fn cached_store_connection(api_version: &str) -> Result<&'static Mutex<Connection>> {
188    let path = resolve_store_path(api_version)?;
189    cached_in_memory_connection(api_version, &path)
190}
191
192/// Does the actual work for `cached_store_connection`, taking an explicit
193/// path (rather than resolving one from `api_version` itself) so it's
194/// testable against an arbitrary tempdir path — `cache_key` and `path`
195/// are separate parameters because two different `api_version`s should
196/// never collide in the process-wide cache even if (hypothetically) they
197/// resolved to the same file.
198fn cached_in_memory_connection(cache_key: &str, path: &Path) -> Result<&'static Mutex<Connection>> {
199    static CACHE: OnceLock<Mutex<HashMap<String, &'static Mutex<Connection>>>> = OnceLock::new();
200    let cache = CACHE.get_or_init(|| Mutex::new(HashMap::new()));
201
202    if let Some(conn) = cache.lock().unwrap().get(cache_key) {
203        return Ok(conn);
204    }
205
206    let disk_conn = open_store(path)?;
207    let mut mem_conn =
208        Connection::open_in_memory().context("failed to open an in-memory SQLite connection")?;
209    Backup::new(&disk_conn, &mut mem_conn)
210        .context("failed to start SQLite backup into memory")?
211        .run_to_completion(i32::MAX, Duration::from_millis(0), None)
212        .with_context(|| format!("failed to back up '{}' into memory", path.display()))?;
213    drop(disk_conn);
214
215    let leaked: &'static Mutex<Connection> = Box::leak(Box::new(Mutex::new(mem_conn)));
216    let mut cache = cache.lock().unwrap();
217    Ok(*cache.entry(cache_key.to_string()).or_insert(leaked))
218}
219
220#[derive(Debug, Clone, Serialize)]
221pub struct EndpointRecord {
222    pub operation_id: String,
223    pub path: String,
224    pub method: String,
225    pub summary: Option<String>,
226    pub description: Option<String>,
227    pub input_schema: serde_json::Value,
228    pub output_schema: serde_json::Value,
229    pub auth_scheme_ref: Option<String>,
230}
231
232fn row_to_endpoint(row: &Row) -> rusqlite::Result<EndpointRecord> {
233    let input_schema: String = row.get(5)?;
234    let output_schema: String = row.get(6)?;
235    Ok(EndpointRecord {
236        operation_id: row.get(0)?,
237        path: row.get(1)?,
238        method: row.get(2)?,
239        summary: row.get(3)?,
240        description: row.get(4)?,
241        input_schema: serde_json::from_str(&input_schema).unwrap_or(serde_json::Value::Null),
242        output_schema: serde_json::from_str(&output_schema).unwrap_or(serde_json::Value::Null),
243        auth_scheme_ref: row.get(7)?,
244    })
245}
246
247pub fn get_endpoint(conn: &Connection, operation_id: &str) -> Result<Option<EndpointRecord>> {
248    let mut stmt = conn.prepare(&format!(
249        "SELECT {ENDPOINT_COLUMNS} FROM endpoints WHERE operation_id = ?1"
250    ))?;
251    match stmt.query_row([operation_id], row_to_endpoint) {
252        Ok(record) => Ok(Some(record)),
253        Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
254        Err(err) => Err(err.into()),
255    }
256}
257
258pub fn list_endpoints(conn: &Connection) -> Result<Vec<EndpointRecord>> {
259    let mut stmt = conn.prepare(&format!("SELECT {ENDPOINT_COLUMNS} FROM endpoints"))?;
260    let rows = stmt.query_map([], row_to_endpoint)?;
261    Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
262}
263
264#[derive(Debug, Clone, Serialize)]
265pub struct SearchResult {
266    pub operation_id: String,
267    pub summary: Option<String>,
268    pub similarity: f64,
269}
270
271/// k-nearest-neighbor search over `semantic_endpoints` (sqlite-vec's
272/// documented `MATCH ... AND k = ...` query form). `query_embedding` must
273/// come from the same model as the vectors it's compared against — see
274/// `services::embedding_service`. Bound as a raw little-endian `f32` blob,
275/// the same wire format `bin/populate_embeddings.rs` writes (sqlite-vec
276/// accepts either that or a JSON array per-call, independent of how any
277/// given row was originally inserted, so this is a consistency choice, not
278/// a correctness requirement).
279///
280/// `similarity` is a simple `1 - distance` transform of sqlite-vec's L2
281/// distance, not a true cosine-similarity computation; since embeddings
282/// are normalized (unit vectors), L2-distance ordering already matches
283/// cosine-similarity ordering, so result *ranking* is correct even though
284/// the displayed score is an approximation.
285pub fn search_endpoints(
286    conn: &Connection,
287    query_embedding: &[f32],
288    limit: usize,
289) -> Result<Vec<SearchResult>> {
290    let blob: Vec<u8> = query_embedding
291        .iter()
292        .flat_map(|value| value.to_le_bytes())
293        .collect();
294
295    let mut stmt = conn.prepare(
296        "SELECT e.operation_id, e.summary, s.distance
297         FROM semantic_endpoints s
298         JOIN endpoints e ON e.operation_id = s.operation_id
299         WHERE s.embedding MATCH ?1 AND k = ?2
300         ORDER BY s.distance",
301    )?;
302    let rows = stmt.query_map(rusqlite::params![blob, limit], |row| {
303        let distance: f64 = row.get(2)?;
304        Ok(SearchResult {
305            operation_id: row.get(0)?,
306            summary: row.get(1)?,
307            similarity: 1.0 - distance,
308        })
309    })?;
310    Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
311}
312
313#[cfg(test)]
314mod tests {
315    use super::*;
316
317    /// Guards against `VERSION_STORE_FILES` and `VERSION_STORE_BYTES`
318    /// silently drifting apart — every `api_version` this crate lists must
319    /// resolve to embedded bytes, or `resolve_store_path` fails at runtime
320    /// for exactly that version and nothing else, which is easy to miss in
321    /// review since the two arrays are edited in different places.
322    #[test]
323    fn every_version_store_file_has_embedded_bytes() {
324        let file_labels: std::collections::HashSet<_> = VERSION_STORE_FILES
325            .iter()
326            .map(|(label, _)| *label)
327            .collect();
328        let byte_labels: std::collections::HashSet<_> = VERSION_STORE_BYTES
329            .iter()
330            .map(|(label, _)| *label)
331            .collect();
332        assert_eq!(file_labels, byte_labels);
333    }
334
335    /// Regression test: `resolve_store_path` used to `std::fs::write` the
336    /// shared extraction path directly, which let one thread's `open_store`
337    /// race another thread's in-progress truncate — exactly what happens
338    /// when multiple MCP tool calls run concurrently against the same
339    /// `api_version` (each hits this path via `cached_store_connection`).
340    /// The rename-into-place fix must make every one of these opens see a
341    /// complete file rather than an intermittently empty one.
342    #[test]
343    fn resolve_store_path_survives_concurrent_calls() {
344        let api_version = VERSION_STORE_FILES[0].0.to_string();
345        let handles: Vec<_> = (0..16)
346            .map(|_| {
347                let api_version = api_version.clone();
348                std::thread::spawn(move || {
349                    let path = resolve_store_path(&api_version).unwrap();
350                    open_store(&path).unwrap();
351                })
352            })
353            .collect();
354        for handle in handles {
355            handle.join().unwrap();
356        }
357    }
358
359    /// Builds a read-write connection with the same schema mcpify's shared
360    /// pipeline writes, and seeds it with one row — real usage never
361    /// creates this schema (mcpify already wrote it before any generated
362    /// code runs), so this setup is test-only fixture, not production
363    /// code these tests exercise.
364    fn seeded_store(path: &Path) -> Connection {
365        unsafe {
366            #[allow(clippy::missing_transmute_annotations)]
367            rusqlite::ffi::sqlite3_auto_extension(Some(std::mem::transmute(
368                sqlite_vec::sqlite3_vec_init as *const (),
369            )));
370        }
371        let conn = Connection::open(path).unwrap();
372        conn.execute(
373            "CREATE TABLE endpoints (
374                operation_id TEXT PRIMARY KEY,
375                path TEXT NOT NULL,
376                method TEXT NOT NULL,
377                summary TEXT,
378                description TEXT,
379                input_schema TEXT NOT NULL,
380                output_schema TEXT NOT NULL,
381                auth_scheme_ref TEXT
382            )",
383            [],
384        )
385        .unwrap();
386        conn.execute(
387            "CREATE VIRTUAL TABLE semantic_endpoints USING vec0(
388                operation_id TEXT PRIMARY KEY,
389                embedding FLOAT[4]
390            )",
391            [],
392        )
393        .unwrap();
394        conn.execute(
395            "INSERT INTO endpoints (operation_id, path, method, summary, description, input_schema, output_schema, auth_scheme_ref)
396             VALUES ('listWidgets', '/widgets', 'GET', 'List widgets', NULL, '{}', '[]', NULL)",
397            [],
398        )
399        .unwrap();
400        let embedding: Vec<u8> = [1.0f32, 0.0, 0.0, 0.0]
401            .iter()
402            .flat_map(|v| v.to_le_bytes())
403            .collect();
404        conn.execute(
405            "INSERT INTO semantic_endpoints (operation_id, embedding) VALUES ('listWidgets', ?1)",
406            rusqlite::params![embedding],
407        )
408        .unwrap();
409        conn
410    }
411
412    #[test]
413    fn get_endpoint_returns_a_seeded_row() {
414        let dir = tempfile::tempdir().unwrap();
415        let path = dir.path().join("mcp_store.db");
416        let _conn = seeded_store(&path);
417
418        let store = open_store(&path).unwrap();
419        let endpoint = get_endpoint(&store, "listWidgets").unwrap().unwrap();
420        assert_eq!(endpoint.path, "/widgets");
421        assert_eq!(endpoint.method, "GET");
422        assert_eq!(endpoint.summary.as_deref(), Some("List widgets"));
423    }
424
425    #[test]
426    fn get_endpoint_returns_none_for_an_unknown_operation() {
427        let dir = tempfile::tempdir().unwrap();
428        let path = dir.path().join("mcp_store.db");
429        let _conn = seeded_store(&path);
430
431        let store = open_store(&path).unwrap();
432        assert!(get_endpoint(&store, "unknownOp").unwrap().is_none());
433    }
434
435    #[test]
436    fn list_endpoints_returns_every_row() {
437        let dir = tempfile::tempdir().unwrap();
438        let path = dir.path().join("mcp_store.db");
439        let _conn = seeded_store(&path);
440
441        let store = open_store(&path).unwrap();
442        let endpoints = list_endpoints(&store).unwrap();
443        assert_eq!(endpoints.len(), 1);
444        assert_eq!(endpoints[0].operation_id, "listWidgets");
445    }
446
447    #[test]
448    fn search_endpoints_finds_the_nearest_neighbor() {
449        let dir = tempfile::tempdir().unwrap();
450        let path = dir.path().join("mcp_store.db");
451        let _conn = seeded_store(&path);
452
453        let store = open_store(&path).unwrap();
454        let results = search_endpoints(&store, &[1.0, 0.0, 0.0, 0.0], 5).unwrap();
455        assert_eq!(results.len(), 1);
456        assert_eq!(results[0].operation_id, "listWidgets");
457        assert!(results[0].similarity > 0.99);
458    }
459
460    #[test]
461    fn cached_in_memory_connection_serves_seeded_data() {
462        let dir = tempfile::tempdir().unwrap();
463        let path = dir.path().join("mcp_store.db");
464        let _conn = seeded_store(&path);
465
466        let cached = cached_in_memory_connection("cached-serves-seeded-data", &path).unwrap();
467        let endpoint = get_endpoint(&cached.lock().unwrap(), "listWidgets")
468            .unwrap()
469            .unwrap();
470        assert_eq!(endpoint.path, "/widgets");
471
472        // vec0 search must also work against the in-memory copy —
473        // `register_vec_extension`'s `sqlite3_auto_extension` applies
474        // process-wide, so it isn't specific to file-backed connections.
475        let results = search_endpoints(&cached.lock().unwrap(), &[1.0, 0.0, 0.0, 0.0], 5).unwrap();
476        assert_eq!(results.len(), 1);
477        assert_eq!(results[0].operation_id, "listWidgets");
478    }
479
480    #[test]
481    fn cached_in_memory_connection_holds_no_lingering_lock_on_the_disk_file() {
482        let dir = tempfile::tempdir().unwrap();
483        let path = dir.path().join("mcp_store.db");
484        let _conn = seeded_store(&path);
485
486        let cached = cached_in_memory_connection("cached-releases-disk-handle", &path).unwrap();
487
488        // If the disk connection were still open, removing the file out
489        // from under it would be the exact failure mode this fix exists
490        // to avoid.
491        std::fs::remove_file(&path).unwrap();
492
493        let endpoint = get_endpoint(&cached.lock().unwrap(), "listWidgets")
494            .unwrap()
495            .unwrap();
496        assert_eq!(endpoint.path, "/widgets");
497    }
498
499    #[test]
500    fn cached_in_memory_connection_reuses_the_same_connection_for_the_same_key() {
501        let dir = tempfile::tempdir().unwrap();
502        let path = dir.path().join("mcp_store.db");
503        let _conn = seeded_store(&path);
504
505        let first = cached_in_memory_connection("cached-reuses-same-key", &path).unwrap()
506            as *const Mutex<Connection>;
507        let second = cached_in_memory_connection("cached-reuses-same-key", &path).unwrap()
508            as *const Mutex<Connection>;
509        assert_eq!(
510            first, second,
511            "expected the same cached connection, not a fresh backup"
512        );
513    }
514}
github_mcp/data/store.rs

github_mcp/data/
store.rs
