github_mcp/data/
store.rs

1// GitHub v3 REST API MCP server — generated by mcpify. Do not hand-edit.
2//
3// Single data-access layer against mcp_store.db — both the relational
4// `endpoints` table and the `semantic_endpoints` vec0 virtual table
5// (REQ-2.4.1's single-store consolidation). Mirrors the shape of
6// mcpify's own src/db/open.rs almost directly, since both the generator
7// and the generated project use the identical rusqlite+sqlite-vec crate
8// pair. Unlike mcpify's own src/db/schema.rs, this module never creates
9// tables — mcpify's shared pipeline (Story 5/6) already wrote
10// mcp_store.db's schema before any generated code runs.
11
12use std::collections::HashMap;
13use std::path::{Path, PathBuf};
14use std::sync::atomic::{AtomicU64, Ordering};
15use std::sync::{Mutex, Once, OnceLock};
16use std::time::Duration;
17
18use anyhow::{Context, Result};
19use rusqlite::backup::Backup;
20use rusqlite::{Connection, OpenFlags, Row};
21use serde::Serialize;
22
23static REGISTER_VEC_EXTENSION: Once = Once::new();
24
25// mcpify:versions:begin
26pub const VERSION_STORE_FILES: &[(&str, &str)] = &[
27    ("gh-2026-03-10", "mcp_store.db"),
28    ("ghec-2026-03-10", "mcp_store_vghec-2026-03-10.db"),
29    ("ghes-3.21", "mcp_store_vghes-3.21.db"),
30    ("ghes-3.20", "mcp_store_vghes-3.20.db"),
31    ("ghes-3.19", "mcp_store_vghes-3.19.db"),
32];
33
34const VERSION_STORE_BYTES: &[(&str, &[u8])] = &[
35    ("gh-2026-03-10", include_bytes!("../../mcp_store.db.zst")),
36    (
37        "ghec-2026-03-10",
38        include_bytes!("../../mcp_store_vghec-2026-03-10.db.zst"),
39    ),
40    (
41        "ghes-3.21",
42        include_bytes!("../../mcp_store_vghes-3.21.db.zst"),
43    ),
44    (
45        "ghes-3.20",
46        include_bytes!("../../mcp_store_vghes-3.20.db.zst"),
47    ),
48    (
49        "ghes-3.19",
50        include_bytes!("../../mcp_store_vghes-3.19.db.zst"),
51    ),
52];
53// mcpify:versions:end
54
55/// Resolves the active `api_version` (from the config cascade) to its
56/// store file. Every `.db` this crate supports is embedded into the
57/// compiled binary via `include_bytes!` (`VERSION_STORE_BYTES`) exactly
58/// like `validator.rs` embeds each version's schema — there is no
59/// filesystem fallback chain to reason about, and this crate never
60/// depends on any particular `.db` file existing anywhere on disk after
61/// `cargo install`. The one difference from a schema lookup: SQLite
62/// needs a real file to open a `Connection` against (unlike a `&[u8]`
63/// JSON schema, read directly from memory), so this extracts the
64/// embedded bytes to a fixed path in the OS temp dir the first time a
65/// given `api_version` is requested in this process, then reuses that
66/// same path on every later call (see `EXTRACTED` below) — a rebuilt
67/// binary with different embedded bytes (a `populate_embeddings` re-run,
68/// an `add-version` update) still can never be shadowed by a stale
69/// leftover from a previous install, since a fresh process always
70/// extracts fresh.
71pub fn resolve_store_path(api_version: &str) -> Result<PathBuf> {
72    // `cached_store_connection` calls this on every tool invocation, and
73    // tool calls run concurrently (each MCP request is its own tokio
74    // task) — caching the resolved path per `api_version`, guarded by
75    // this same mutex, means the actual extract-and-rename-into-place
76    // below only ever runs once per `api_version` per process: every
77    // concurrent first caller blocks here rather than racing each other
78    // to write the same destination file. That race used to be real: on
79    // Windows, `rename`-ing over a destination another thread already has
80    // open via `open_store` can fail outright ("failed to move extracted
81    // store data into place") rather than just being non-atomic, unlike
82    // POSIX where the same rename silently succeeds.
83    static EXTRACTED: OnceLock<Mutex<HashMap<String, PathBuf>>> = OnceLock::new();
84    let extracted = EXTRACTED.get_or_init(|| Mutex::new(HashMap::new()));
85    let mut extracted = extracted.lock().unwrap();
86    if let Some(path) = extracted.get(api_version) {
87        return Ok(path.clone());
88    }
89
90    let file = VERSION_STORE_FILES
91        .iter()
92        .find(|(label, _)| *label == api_version)
93        .map(|(_, file)| *file)
94        .with_context(|| format!("unknown api_version '{api_version}' — run the 'versions' command to see what's available"))?;
95    let bytes = VERSION_STORE_BYTES
96        .iter()
97        .find(|(label, _)| *label == api_version)
98        .map(|(_, bytes)| *bytes)
99        .with_context(|| format!("no embedded store data for api_version '{api_version}'"))?;
100
101    let mut dir = std::env::temp_dir();
102    dir.push(concat!(env!("CARGO_PKG_NAME"), "-store"));
103    std::fs::create_dir_all(&dir)
104        .with_context(|| format!("failed to create temp dir '{}'", dir.display()))?;
105
106    let path = dir.join(file);
107    // Writes to a uniquely-named sibling file first, then `rename`s it
108    // into place, rather than writing `path` directly, so a *different*
109    // process sharing this same temp dir (a fresh `populate_embeddings`
110    // run, a second server instance starting up concurrently) never
111    // observes a momentarily-empty or partially-written file and fails
112    // with "no such table: endpoints". `rename` within the same
113    // directory is atomic on both POSIX and Windows with respect to
114    // content — a reader always sees either the complete previous copy
115    // or the complete new one, never a partial write. The per-`api_version`
116    // cache above prevents *this* process from ever racing itself, but a
117    // genuinely different OS process can still have `path` open at the
118    // exact moment of the rename below — see the retry loop there for why
119    // that's handled with a backoff instead of a lock.
120    //
121    // `bytes` is the zstd-compressed `.db.zst` payload (see
122    // `VERSION_STORE_BYTES`), not a valid SQLite file itself — it must be
123    // decompressed before `rusqlite::Connection::open` can read it.
124    let decompressed = zstd::stream::decode_all(bytes).with_context(|| {
125        format!("failed to decompress embedded store data for api_version '{api_version}'")
126    })?;
127    static UNIQUE: AtomicU64 = AtomicU64::new(0);
128    let tmp_path = dir.join(format!(
129        "{file}.{}.{}.tmp",
130        std::process::id(),
131        UNIQUE.fetch_add(1, Ordering::Relaxed)
132    ));
133    std::fs::write(&tmp_path, decompressed).with_context(|| {
134        format!(
135            "failed to extract embedded store data to '{}'",
136            tmp_path.display()
137        )
138    })?;
139    // The per-`api_version` cache above only prevents *this* process from
140    // racing itself; it can't stop a genuinely different OS process (a
141    // concurrently-starting second server instance, a `populate_embeddings`
142    // run) from having `path` open via `open_store` at the exact moment
143    // this rename happens. On Windows that can make the rename itself fail
144    // outright rather than just being non-atomic, unlike POSIX — but the
145    // other process's use of the file is always brief (open, backup into
146    // memory or write, close), so a few retries with a short backoff
147    // resolve it without needing any cross-process locking.
148    let mut rename_attempts = 0u32;
149    loop {
150        match std::fs::rename(&tmp_path, &path) {
151            Ok(()) => break,
152            Err(_) if rename_attempts < 5 => {
153                rename_attempts += 1;
154                std::thread::sleep(Duration::from_millis(50 * u64::from(rename_attempts)));
155            }
156            Err(err) => {
157                return Err(err).with_context(|| {
158                    format!(
159                        "failed to move extracted store data into place at '{}'",
160                        path.display()
161                    )
162                });
163            }
164        }
165    }
166
167    extracted.insert(api_version.to_string(), path.clone());
168    Ok(path)
169}
170
171const ENDPOINT_COLUMNS: &str = "operation_id, path, method, summary, description, input_schema, output_schema, auth_scheme_ref";
172
173/// Registers the `sqlite-vec` extension once per process, via
174/// `sqlite3_auto_extension` — matching the pattern the `sqlite-vec` crate
175/// itself documents for `rusqlite`.
176fn register_vec_extension() {
177    REGISTER_VEC_EXTENSION.call_once(|| unsafe {
178        #[allow(clippy::missing_transmute_annotations)]
179        rusqlite::ffi::sqlite3_auto_extension(Some(std::mem::transmute(
180            sqlite_vec::sqlite3_vec_init as *const (),
181        )));
182    });
183}
184
185/// Opens `mcp_store.db` read-only: this crate's binaries only ever read
186/// it, except `github-mcp-populate-embeddings` (a separate `[[bin]]`), which uses
187/// `open_store_read_write` instead.
188pub fn open_store(path: &Path) -> Result<Connection> {
189    register_vec_extension();
190    Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY)
191        .with_context(|| format!("failed to open '{}'", path.display()))
192}
193
194/// Read-write counterpart to `open_store` — for `bin/populate_embeddings.rs`,
195/// the one binary that actually writes to `mcp_store.db` (backfilling
196/// `semantic_endpoints`, whose table every other caller only ever reads
197/// from).
198pub fn open_store_read_write(path: &Path) -> Result<Connection> {
199    register_vec_extension();
200    Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_WRITE)
201        .with_context(|| format!("failed to open '{}'", path.display()))
202}
203
204/// Returns a process-wide, in-memory-backed connection for `api_version`:
205/// on first access, opens the on-disk file read-only via `open_store`,
206/// copies its entire contents into a fresh `:memory:` connection (via
207/// SQLite's backup API), then drops the on-disk connection immediately —
208/// so its file handle/lock is held only for that brief copy, not for the
209/// process's lifetime. This means an external process (a fresh
210/// `github-mcp-populate-embeddings` run, a deployment replacing the file) can update
211/// `mcp_store.db` without hitting "database is locked" against a live
212/// server, at the cost of the running process only picking up such an
213/// update on its next restart.
214///
215/// Returns the connection behind a `Mutex` rather than handing it out
216/// directly: callers must finish with the guard (and drop it) *before*
217/// any `.await` — `rusqlite::Connection` isn't `Sync`, so holding the
218/// guard across an await point would make the enclosing future
219/// non-`Send`, the same constraint `core/mcp_server.rs`'s tool handlers
220/// already respect for a plain `Connection`.
221pub fn cached_store_connection(api_version: &str) -> Result<&'static Mutex<Connection>> {
222    let path = resolve_store_path(api_version)?;
223    cached_in_memory_connection(api_version, &path)
224}
225
226/// Does the actual work for `cached_store_connection`, taking an explicit
227/// path (rather than resolving one from `api_version` itself) so it's
228/// testable against an arbitrary tempdir path — `cache_key` and `path`
229/// are separate parameters because two different `api_version`s should
230/// never collide in the process-wide cache even if (hypothetically) they
231/// resolved to the same file.
232fn cached_in_memory_connection(cache_key: &str, path: &Path) -> Result<&'static Mutex<Connection>> {
233    static CACHE: OnceLock<Mutex<HashMap<String, &'static Mutex<Connection>>>> = OnceLock::new();
234    let cache = CACHE.get_or_init(|| Mutex::new(HashMap::new()));
235
236    if let Some(conn) = cache.lock().unwrap().get(cache_key) {
237        return Ok(conn);
238    }
239
240    let disk_conn = open_store(path)?;
241    let mut mem_conn =
242        Connection::open_in_memory().context("failed to open an in-memory SQLite connection")?;
243    Backup::new(&disk_conn, &mut mem_conn)
244        .context("failed to start SQLite backup into memory")?
245        .run_to_completion(i32::MAX, Duration::from_millis(0), None)
246        .with_context(|| format!("failed to back up '{}' into memory", path.display()))?;
247    drop(disk_conn);
248
249    let leaked: &'static Mutex<Connection> = Box::leak(Box::new(Mutex::new(mem_conn)));
250    let mut cache = cache.lock().unwrap();
251    Ok(*cache.entry(cache_key.to_string()).or_insert(leaked))
252}
253
254#[derive(Debug, Clone, Serialize)]
255pub struct EndpointRecord {
256    pub operation_id: String,
257    pub path: String,
258    pub method: String,
259    pub summary: Option<String>,
260    pub description: Option<String>,
261    pub input_schema: serde_json::Value,
262    pub output_schema: serde_json::Value,
263    pub auth_scheme_ref: Option<String>,
264}
265
266fn row_to_endpoint(row: &Row) -> rusqlite::Result<EndpointRecord> {
267    let input_schema: String = row.get(5)?;
268    let output_schema: String = row.get(6)?;
269    Ok(EndpointRecord {
270        operation_id: row.get(0)?,
271        path: row.get(1)?,
272        method: row.get(2)?,
273        summary: row.get(3)?,
274        description: row.get(4)?,
275        input_schema: serde_json::from_str(&input_schema).unwrap_or(serde_json::Value::Null),
276        output_schema: serde_json::from_str(&output_schema).unwrap_or(serde_json::Value::Null),
277        auth_scheme_ref: row.get(7)?,
278    })
279}
280
281pub fn get_endpoint(conn: &Connection, operation_id: &str) -> Result<Option<EndpointRecord>> {
282    let mut stmt = conn.prepare(&format!(
283        "SELECT {ENDPOINT_COLUMNS} FROM endpoints WHERE operation_id = ?1"
284    ))?;
285    match stmt.query_row([operation_id], row_to_endpoint) {
286        Ok(record) => Ok(Some(record)),
287        Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
288        Err(err) => Err(err.into()),
289    }
290}
291
292pub fn list_endpoints(conn: &Connection) -> Result<Vec<EndpointRecord>> {
293    let mut stmt = conn.prepare(&format!("SELECT {ENDPOINT_COLUMNS} FROM endpoints"))?;
294    let rows = stmt.query_map([], row_to_endpoint)?;
295    Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
296}
297
298#[derive(Debug, Clone, Serialize)]
299pub struct SearchResult {
300    pub operation_id: String,
301    pub summary: Option<String>,
302    pub similarity: f64,
303}
304
305/// k-nearest-neighbor search over `semantic_endpoints` (sqlite-vec's
306/// documented `MATCH ... AND k = ...` query form). `query_embedding` must
307/// come from the same model as the vectors it's compared against — see
308/// `services::embedding_service`. Bound as a raw little-endian `f32` blob,
309/// the same wire format `bin/populate_embeddings.rs` writes (sqlite-vec
310/// accepts either that or a JSON array per-call, independent of how any
311/// given row was originally inserted, so this is a consistency choice, not
312/// a correctness requirement).
313///
314/// `similarity` is a simple `1 - distance` transform of sqlite-vec's L2
315/// distance, not a true cosine-similarity computation; since embeddings
316/// are normalized (unit vectors), L2-distance ordering already matches
317/// cosine-similarity ordering, so result *ranking* is correct even though
318/// the displayed score is an approximation.
319pub fn search_endpoints(
320    conn: &Connection,
321    query_embedding: &[f32],
322    limit: usize,
323) -> Result<Vec<SearchResult>> {
324    let blob: Vec<u8> = query_embedding
325        .iter()
326        .flat_map(|value| value.to_le_bytes())
327        .collect();
328
329    let mut stmt = conn.prepare(
330        "SELECT e.operation_id, e.summary, s.distance
331         FROM semantic_endpoints s
332         JOIN endpoints e ON e.operation_id = s.operation_id
333         WHERE s.embedding MATCH ?1 AND k = ?2
334         ORDER BY s.distance",
335    )?;
336    let rows = stmt.query_map(rusqlite::params![blob, limit], |row| {
337        let distance: f64 = row.get(2)?;
338        Ok(SearchResult {
339            operation_id: row.get(0)?,
340            summary: row.get(1)?,
341            similarity: 1.0 - distance,
342        })
343    })?;
344    Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
345}
346
347#[cfg(test)]
348mod tests {
349    use super::*;
350
351    /// Guards against `VERSION_STORE_FILES` and `VERSION_STORE_BYTES`
352    /// silently drifting apart — every `api_version` this crate lists must
353    /// resolve to embedded bytes, or `resolve_store_path` fails at runtime
354    /// for exactly that version and nothing else, which is easy to miss in
355    /// review since the two arrays are edited in different places.
356    #[test]
357    fn every_version_store_file_has_embedded_bytes() {
358        let file_labels: std::collections::HashSet<_> = VERSION_STORE_FILES
359            .iter()
360            .map(|(label, _)| *label)
361            .collect();
362        let byte_labels: std::collections::HashSet<_> = VERSION_STORE_BYTES
363            .iter()
364            .map(|(label, _)| *label)
365            .collect();
366        assert_eq!(file_labels, byte_labels);
367    }
368
369    /// Regression test: `resolve_store_path` used to `std::fs::write` the
370    /// shared extraction path directly, which let one thread's `open_store`
371    /// race another thread's in-progress truncate — exactly what happens
372    /// when multiple MCP tool calls run concurrently against the same
373    /// `api_version` (each hits this path via `cached_store_connection`).
374    /// The rename-into-place fix must make every one of these opens see a
375    /// complete file rather than an intermittently empty one.
376    #[test]
377    fn resolve_store_path_survives_concurrent_calls() {
378        let api_version = VERSION_STORE_FILES[0].0.to_string();
379        let handles: Vec<_> = (0..16)
380            .map(|_| {
381                let api_version = api_version.clone();
382                std::thread::spawn(move || {
383                    let path = resolve_store_path(&api_version).unwrap();
384                    open_store(&path).unwrap();
385                })
386            })
387            .collect();
388        for handle in handles {
389            handle.join().unwrap();
390        }
391    }
392
393    /// Builds a read-write connection with the same schema mcpify's shared
394    /// pipeline writes, and seeds it with one row — real usage never
395    /// creates this schema (mcpify already wrote it before any generated
396    /// code runs), so this setup is test-only fixture, not production
397    /// code these tests exercise.
398    fn seeded_store(path: &Path) -> Connection {
399        unsafe {
400            #[allow(clippy::missing_transmute_annotations)]
401            rusqlite::ffi::sqlite3_auto_extension(Some(std::mem::transmute(
402                sqlite_vec::sqlite3_vec_init as *const (),
403            )));
404        }
405        let conn = Connection::open(path).unwrap();
406        conn.execute(
407            "CREATE TABLE endpoints (
408                operation_id TEXT PRIMARY KEY,
409                path TEXT NOT NULL,
410                method TEXT NOT NULL,
411                summary TEXT,
412                description TEXT,
413                input_schema TEXT NOT NULL,
414                output_schema TEXT NOT NULL,
415                auth_scheme_ref TEXT
416            )",
417            [],
418        )
419        .unwrap();
420        conn.execute(
421            "CREATE VIRTUAL TABLE semantic_endpoints USING vec0(
422                operation_id TEXT PRIMARY KEY,
423                embedding FLOAT[4]
424            )",
425            [],
426        )
427        .unwrap();
428        conn.execute(
429            "INSERT INTO endpoints (operation_id, path, method, summary, description, input_schema, output_schema, auth_scheme_ref)
430             VALUES ('listWidgets', '/widgets', 'GET', 'List widgets', NULL, '{}', '[]', NULL)",
431            [],
432        )
433        .unwrap();
434        let embedding: Vec<u8> = [1.0f32, 0.0, 0.0, 0.0]
435            .iter()
436            .flat_map(|v| v.to_le_bytes())
437            .collect();
438        conn.execute(
439            "INSERT INTO semantic_endpoints (operation_id, embedding) VALUES ('listWidgets', ?1)",
440            rusqlite::params![embedding],
441        )
442        .unwrap();
443        conn
444    }
445
446    #[test]
447    fn get_endpoint_returns_a_seeded_row() {
448        let dir = tempfile::tempdir().unwrap();
449        let path = dir.path().join("mcp_store.db");
450        let _conn = seeded_store(&path);
451
452        let store = open_store(&path).unwrap();
453        let endpoint = get_endpoint(&store, "listWidgets").unwrap().unwrap();
454        assert_eq!(endpoint.path, "/widgets");
455        assert_eq!(endpoint.method, "GET");
456        assert_eq!(endpoint.summary.as_deref(), Some("List widgets"));
457    }
458
459    #[test]
460    fn get_endpoint_returns_none_for_an_unknown_operation() {
461        let dir = tempfile::tempdir().unwrap();
462        let path = dir.path().join("mcp_store.db");
463        let _conn = seeded_store(&path);
464
465        let store = open_store(&path).unwrap();
466        assert!(get_endpoint(&store, "unknownOp").unwrap().is_none());
467    }
468
469    #[test]
470    fn list_endpoints_returns_every_row() {
471        let dir = tempfile::tempdir().unwrap();
472        let path = dir.path().join("mcp_store.db");
473        let _conn = seeded_store(&path);
474
475        let store = open_store(&path).unwrap();
476        let endpoints = list_endpoints(&store).unwrap();
477        assert_eq!(endpoints.len(), 1);
478        assert_eq!(endpoints[0].operation_id, "listWidgets");
479    }
480
481    #[test]
482    fn search_endpoints_finds_the_nearest_neighbor() {
483        let dir = tempfile::tempdir().unwrap();
484        let path = dir.path().join("mcp_store.db");
485        let _conn = seeded_store(&path);
486
487        let store = open_store(&path).unwrap();
488        let results = search_endpoints(&store, &[1.0, 0.0, 0.0, 0.0], 5).unwrap();
489        assert_eq!(results.len(), 1);
490        assert_eq!(results[0].operation_id, "listWidgets");
491        assert!(results[0].similarity > 0.99);
492    }
493
494    #[test]
495    fn cached_in_memory_connection_serves_seeded_data() {
496        let dir = tempfile::tempdir().unwrap();
497        let path = dir.path().join("mcp_store.db");
498        let _conn = seeded_store(&path);
499
500        let cached = cached_in_memory_connection("cached-serves-seeded-data", &path).unwrap();
501        let endpoint = get_endpoint(&cached.lock().unwrap(), "listWidgets")
502            .unwrap()
503            .unwrap();
504        assert_eq!(endpoint.path, "/widgets");
505
506        // vec0 search must also work against the in-memory copy —
507        // `register_vec_extension`'s `sqlite3_auto_extension` applies
508        // process-wide, so it isn't specific to file-backed connections.
509        let results = search_endpoints(&cached.lock().unwrap(), &[1.0, 0.0, 0.0, 0.0], 5).unwrap();
510        assert_eq!(results.len(), 1);
511        assert_eq!(results[0].operation_id, "listWidgets");
512    }
513
514    #[test]
515    fn cached_in_memory_connection_holds_no_lingering_lock_on_the_disk_file() {
516        let dir = tempfile::tempdir().unwrap();
517        let path = dir.path().join("mcp_store.db");
518        // Dropped immediately after seeding: this is the *fixture's* own
519        // handle, not the one under test, and an open handle here would
520        // itself block the `remove_file` below on Windows (which — unlike
521        // POSIX — refuses to delete a file that's still open anywhere),
522        // masking whether `cached_in_memory_connection` released its own.
523        drop(seeded_store(&path));
524
525        let cached = cached_in_memory_connection("cached-releases-disk-handle", &path).unwrap();
526
527        // If the disk connection were still open, removing the file out
528        // from under it would be the exact failure mode this fix exists
529        // to avoid.
530        std::fs::remove_file(&path).unwrap();
531
532        let endpoint = get_endpoint(&cached.lock().unwrap(), "listWidgets")
533            .unwrap()
534            .unwrap();
535        assert_eq!(endpoint.path, "/widgets");
536    }
537
538    #[test]
539    fn cached_in_memory_connection_reuses_the_same_connection_for_the_same_key() {
540        let dir = tempfile::tempdir().unwrap();
541        let path = dir.path().join("mcp_store.db");
542        let _conn = seeded_store(&path);
543
544        let first = cached_in_memory_connection("cached-reuses-same-key", &path).unwrap()
545            as *const Mutex<Connection>;
546        let second = cached_in_memory_connection("cached-reuses-same-key", &path).unwrap()
547            as *const Mutex<Connection>;
548        assert_eq!(
549            first, second,
550            "expected the same cached connection, not a fresh backup"
551        );
552    }
553}
github_mcp/data/store.rs

github_mcp/data/
store.rs
