mrapids 0.1.31

// Index Store - SQLite-backed storage for operation cards and embeddings
// Supports keyword search and vector similarity search (cosine similarity in Rust)

#![allow(dead_code)]

use anyhow::{Context, Result};
use rusqlite::{params, Connection};
use std::path::Path;

use crate::core::cards::OperationCard;

/// Index store backed by SQLite
pub struct IndexStore {
    conn: Connection,
}

/// Compute cosine similarity between two embedding vectors
fn cosine_similarity(a: &[f32], b: &[f32]) -> f64 {
    let (mut dot, mut na, mut nb) = (0.0f64, 0.0f64, 0.0f64);
    for (x, y) in a.iter().zip(b) {
        let (x, y) = (*x as f64, *y as f64);
        dot += x * y;
        na += x * x;
        nb += y * y;
    }
    let d = na.sqrt() * nb.sqrt();
    if d == 0.0 {
        0.0
    } else {
        dot / d
    }
}

/// Deserialize a BLOB into a Vec<f32> (little-endian)
fn blob_to_embedding(blob: &[u8]) -> Vec<f32> {
    blob.chunks_exact(4)
        .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
        .collect()
}

impl IndexStore {
    /// Open or create an index store
    pub fn open(db_path: &Path) -> Result<Self> {
        let conn = Connection::open(db_path).context("Failed to open SQLite database")?;

        let store = Self { conn };

        store.ensure_schema()?;
        Ok(store)
    }

    /// Open an in-memory index store (for testing)
    pub fn open_in_memory() -> Result<Self> {
        let conn = Connection::open_in_memory().context("Failed to create in-memory SQLite")?;

        let store = Self { conn };

        store.ensure_schema()?;
        Ok(store)
    }

    /// Ensure the schema exists
    fn ensure_schema(&self) -> Result<()> {
        // Specs table - tracks indexed specs
        self.conn.execute(
            r#"
            CREATE TABLE IF NOT EXISTS specs (
                spec_id TEXT PRIMARY KEY,
                spec_path TEXT,
                title TEXT,
                version TEXT,
                base_url TEXT,
                operation_count INTEGER,
                indexed_at TEXT,
                spec_hash TEXT
            )
            "#,
            [],
        )?;

        // Operation cards table
        self.conn.execute(
            r#"
            CREATE TABLE IF NOT EXISTS operation_cards (
                id TEXT PRIMARY KEY,
                spec_id TEXT NOT NULL,
                operation_id TEXT NOT NULL,
                content_hash TEXT NOT NULL,
                method TEXT NOT NULL,
                path TEXT NOT NULL,
                summary TEXT,
                description TEXT,
                parameters_json TEXT,
                request_body_json TEXT,
                auth_required INTEGER,
                auth_type TEXT,
                auth_scopes TEXT,
                risk_level TEXT,
                tags TEXT,
                alias TEXT,
                embedding_text TEXT,
                indexed_at TEXT,
                FOREIGN KEY (spec_id) REFERENCES specs(spec_id)
            )
            "#,
            [],
        )?;

        // Create indexes for fast lookup
        self.conn.execute(
            "CREATE INDEX IF NOT EXISTS idx_cards_spec ON operation_cards(spec_id)",
            [],
        )?;
        self.conn.execute(
            "CREATE INDEX IF NOT EXISTS idx_cards_alias ON operation_cards(alias)",
            [],
        )?;
        self.conn.execute(
            "CREATE INDEX IF NOT EXISTS idx_cards_operation ON operation_cards(operation_id)",
            [],
        )?;
        self.conn.execute(
            "CREATE INDEX IF NOT EXISTS idx_cards_method ON operation_cards(method)",
            [],
        )?;
        self.conn.execute(
            "CREATE INDEX IF NOT EXISTS idx_cards_risk ON operation_cards(risk_level)",
            [],
        )?;

        // Embeddings table (separate for flexibility) — embedding stored as BLOB
        self.conn.execute(
            r#"
            CREATE TABLE IF NOT EXISTS embeddings (
                card_id TEXT PRIMARY KEY,
                dimensions INTEGER,
                embedding BLOB,
                provider TEXT,
                created_at TEXT,
                FOREIGN KEY (card_id) REFERENCES operation_cards(id)
            )
            "#,
            [],
        )?;

        // Spec vocabulary table - extracted terms for intelligent search suggestions
        self.conn.execute(
            r#"
            CREATE TABLE IF NOT EXISTS spec_vocabulary (
                spec_id TEXT NOT NULL,
                term TEXT NOT NULL,
                weight REAL DEFAULT 1.0,
                provenance TEXT NOT NULL,
                operation_ids TEXT,
                PRIMARY KEY (spec_id, term)
            )
            "#,
            [],
        )?;

        Ok(())
    }

    /// Add or update a spec in the index
    pub fn upsert_spec(
        &self,
        spec_id: &str,
        spec_path: &str,
        title: &str,
        version: &str,
        base_url: &str,
        operation_count: usize,
        spec_hash: &str,
    ) -> Result<()> {
        let now = chrono::Utc::now().to_rfc3339();
        self.conn.execute(
            r#"
            INSERT INTO specs (spec_id, spec_path, title, version, base_url, operation_count, spec_hash, indexed_at)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
            ON CONFLICT (spec_id) DO UPDATE SET
                spec_path = excluded.spec_path,
                title = excluded.title,
                version = excluded.version,
                base_url = excluded.base_url,
                operation_count = excluded.operation_count,
                spec_hash = excluded.spec_hash,
                indexed_at = excluded.indexed_at
            "#,
            params![spec_id, spec_path, title, version, base_url, operation_count as i32, spec_hash, now],
        )?;

        Ok(())
    }

    /// Insert or update an operation card
    pub fn upsert_card(&self, card: &OperationCard) -> Result<()> {
        let card_id = format!("{}:{}", card.spec_id, card.operation_id);
        let parameters_json = serde_json::to_string(&card.parameters)?;
        let request_body_json = card
            .request_body
            .as_ref()
            .map(|rb| serde_json::to_string(rb))
            .transpose()?;
        let auth_scopes = card.auth_scopes.join(",");
        let tags = card.tags.join(",");
        let now = chrono::Utc::now().to_rfc3339();

        self.conn.execute(
            r#"
            INSERT INTO operation_cards (
                id, spec_id, operation_id, content_hash, method, path,
                summary, description, parameters_json, request_body_json,
                auth_required, auth_type, auth_scopes, risk_level, tags,
                alias, embedding_text, indexed_at
            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            ON CONFLICT (id) DO UPDATE SET
                content_hash = excluded.content_hash,
                method = excluded.method,
                path = excluded.path,
                summary = excluded.summary,
                description = excluded.description,
                parameters_json = excluded.parameters_json,
                request_body_json = excluded.request_body_json,
                auth_required = excluded.auth_required,
                auth_type = excluded.auth_type,
                auth_scopes = excluded.auth_scopes,
                risk_level = excluded.risk_level,
                tags = excluded.tags,
                alias = excluded.alias,
                embedding_text = excluded.embedding_text,
                indexed_at = excluded.indexed_at
            "#,
            params![
                card_id,
                card.spec_id,
                card.operation_id,
                card.content_hash,
                card.method,
                card.path,
                card.summary,
                card.description,
                parameters_json,
                request_body_json,
                card.auth_required,
                card.auth_type,
                auth_scopes,
                card.risk_level,
                tags,
                card.alias,
                card.embedding_text,
                now
            ],
        )?;

        Ok(())
    }

    /// Insert embedding for a card (stored as BLOB of little-endian f32s)
    pub fn upsert_embedding(&self, card_id: &str, embedding: &[f32], provider: &str) -> Result<()> {
        let embedding_blob: Vec<u8> = embedding.iter().flat_map(|f| f.to_le_bytes()).collect();
        let now = chrono::Utc::now().to_rfc3339();

        self.conn.execute(
            r#"
            INSERT INTO embeddings (card_id, dimensions, embedding, provider, created_at)
            VALUES (?, ?, ?, ?, ?)
            ON CONFLICT (card_id) DO UPDATE SET
                dimensions = excluded.dimensions,
                embedding = excluded.embedding,
                provider = excluded.provider,
                created_at = excluded.created_at
            "#,
            params![
                card_id,
                embedding.len() as i32,
                embedding_blob,
                provider,
                now
            ],
        )?;

        Ok(())
    }

    /// Keyword search across cards
    ///
    /// Uses both full-phrase LIKE matching and per-token LIKE matching.
    /// Tokens are split from the query using identifier_splitter, so
    /// "find pets" matches operations containing "find" OR "pets" individually.
    pub fn keyword_search(
        &self,
        query: &str,
        limit: usize,
        spec_id: Option<&str>,
        method: Option<&str>,
        risk_level: Option<&str>,
    ) -> Result<Vec<SearchResult>> {
        use crate::core::identifier_splitter::split_identifier;

        let query_lower = query.to_lowercase();

        // Stop words: common verbs and articles that match too many operations.
        // Filtering these from SQL prevents result flooding.
        const STOP_WORDS: &[&str] = &[
            "get", "list", "show", "find", "fetch", "read", "view", "display", "create", "add",
            "new", "make", "post", "insert", "update", "edit", "change", "modify", "set", "patch",
            "put", "delete", "remove", "destroy", "drop", "the", "a", "an", "my", "all", "this",
            "that", "for", "from", "to", "me", "i", "we", "it", "is", "are", "was", "do", "does",
            "what", "how", "where", "which", "can",
        ];

        // Split query into tokens and filter out stop words
        let tokens = split_identifier(query);
        let meaningful_tokens: Vec<&String> = tokens
            .iter()
            .filter(|t| t.len() > 1 && !STOP_WORDS.contains(&t.as_str()))
            .collect();

        // Build the full-phrase query string with stop words removed
        let meaningful_phrase: String = query_lower
            .split_whitespace()
            .filter(|w| w.len() > 1 && !STOP_WORDS.contains(w))
            .collect::<Vec<_>>()
            .join(" ");

        // Build WHERE clause: full-phrase OR per-token matches
        let mut where_clauses = Vec::new();
        let mut params_vec: Vec<Box<dyn rusqlite::types::ToSql>> = Vec::new();

        // Full-phrase match (with stop words stripped)
        if !meaningful_phrase.is_empty() {
            let query_pattern = format!("%{}%", meaningful_phrase);
            where_clauses.push(
                "(LOWER(operation_id) LIKE ? OR LOWER(alias) LIKE ? OR LOWER(summary) LIKE ? OR LOWER(description) LIKE ? OR LOWER(path) LIKE ? OR LOWER(embedding_text) LIKE ?)".to_string()
            );
            for _ in 0..6 {
                params_vec.push(Box::new(query_pattern.clone()));
            }
        }

        // Per-token matches (only meaningful tokens, including alias)
        // Fall back to all tokens if everything was a stop word
        let search_tokens = if meaningful_tokens.is_empty() {
            tokens.iter().collect()
        } else {
            meaningful_tokens
        };
        for token in &search_tokens {
            if token.len() > 1 {
                let token_pattern = format!("%{}%", token);
                where_clauses.push(
                    "(LOWER(operation_id) LIKE ? OR LOWER(alias) LIKE ? OR LOWER(embedding_text) LIKE ?)".to_string()
                );
                params_vec.push(Box::new(token_pattern.clone()));
                params_vec.push(Box::new(token_pattern.clone()));
                params_vec.push(Box::new(token_pattern));
            }
        }

        // If no clauses at all (everything was stop words and < 2 chars), use original query
        if where_clauses.is_empty() {
            let query_pattern = format!("%{}%", query_lower);
            where_clauses.push(
                "(LOWER(operation_id) LIKE ? OR LOWER(summary) LIKE ? OR LOWER(path) LIKE ?)"
                    .to_string(),
            );
            for _ in 0..3 {
                params_vec.push(Box::new(query_pattern.clone()));
            }
        }

        let where_clause = where_clauses.join(" OR ");

        let mut sql = format!(
            r#"
            SELECT
                id, spec_id, operation_id, method, path, summary, description,
                auth_required, auth_type, risk_level, tags, alias, embedding_text
            FROM operation_cards
            WHERE ({})
            "#,
            where_clause
        );

        if let Some(sid) = spec_id {
            sql.push_str(" AND spec_id = ?");
            params_vec.push(Box::new(sid.to_string()));
        }
        if let Some(m) = method {
            sql.push_str(" AND UPPER(method) = UPPER(?)");
            params_vec.push(Box::new(m.to_string()));
        }
        if let Some(r) = risk_level {
            sql.push_str(" AND risk_level = ?");
            params_vec.push(Box::new(r.to_string()));
        }

        sql.push_str(&format!(" LIMIT {}", limit));

        let mut stmt = self.conn.prepare(&sql)?;

        // Build params slice from params_vec
        let param_refs: Vec<&dyn rusqlite::types::ToSql> =
            params_vec.iter().map(|b| b.as_ref()).collect();

        let rows = stmt.query_map(param_refs.as_slice(), |row| {
            Ok(SearchResult {
                id: row.get(0)?,
                spec_id: row.get(1)?,
                operation_id: row.get(2)?,
                method: row.get(3)?,
                path: row.get(4)?,
                summary: row.get(5)?,
                description: row.get(6)?,
                auth_required: row.get(7)?,
                auth_type: row.get(8)?,
                risk_level: row.get(9)?,
                tags: row
                    .get::<_, String>(10)?
                    .split(',')
                    .map(|s| s.to_string())
                    .collect(),
                alias: row.get(11)?,
                score: 1.0, // Keyword search doesn't have score
            })
        })?;

        let mut results = Vec::new();
        for row in rows {
            results.push(row?);
        }

        Ok(results)
    }

    /// Vector similarity search (cosine similarity computed in Rust)
    pub fn vector_search(
        &self,
        query_embedding: &[f32],
        limit: usize,
        spec_id: Option<&str>,
    ) -> Result<Vec<SearchResult>> {
        // Fetch all card+embedding pairs (filtered by spec_id if provided)
        let mut sql = String::from(
            r#"
            SELECT
                c.id, c.spec_id, c.operation_id, c.method, c.path,
                c.summary, c.description, c.auth_required, c.auth_type,
                c.risk_level, c.tags, c.alias,
                e.embedding
            FROM operation_cards c
            JOIN embeddings e ON c.id = e.card_id
            "#,
        );

        let mut params_vec: Vec<Box<dyn rusqlite::types::ToSql>> = Vec::new();
        if let Some(sid) = spec_id {
            sql.push_str(" WHERE c.spec_id = ?");
            params_vec.push(Box::new(sid.to_string()));
        }

        let mut stmt = self.conn.prepare(&sql)?;
        let param_refs: Vec<&dyn rusqlite::types::ToSql> =
            params_vec.iter().map(|b| b.as_ref()).collect();

        let rows = stmt.query_map(param_refs.as_slice(), |row| {
            let embedding_blob: Vec<u8> = row.get(12)?;
            Ok((
                SearchResult {
                    id: row.get(0)?,
                    spec_id: row.get(1)?,
                    operation_id: row.get(2)?,
                    method: row.get(3)?,
                    path: row.get(4)?,
                    summary: row.get(5)?,
                    description: row.get(6)?,
                    auth_required: row.get(7)?,
                    auth_type: row.get(8)?,
                    risk_level: row.get(9)?,
                    tags: row
                        .get::<_, String>(10)?
                        .split(',')
                        .map(|s| s.to_string())
                        .collect(),
                    alias: row.get(11)?,
                    score: 0.0, // Will be set below
                },
                embedding_blob,
            ))
        })?;

        // Compute cosine similarity in Rust, sort, and truncate
        let mut scored: Vec<SearchResult> = Vec::new();
        for row in rows {
            let (mut result, blob) = row?;
            let embedding = blob_to_embedding(&blob);
            result.score = cosine_similarity(query_embedding, &embedding);
            scored.push(result);
        }

        scored.sort_by(|a, b| {
            b.score
                .partial_cmp(&a.score)
                .unwrap_or(std::cmp::Ordering::Equal)
        });
        scored.truncate(limit);

        Ok(scored)
    }

    /// List all indexed specs
    pub fn list_specs(&self) -> Result<Vec<SpecInfo>> {
        let mut stmt = self.conn.prepare(
            r#"
            SELECT spec_id, spec_path, title, version, base_url, operation_count, indexed_at, spec_hash
            FROM specs
            ORDER BY indexed_at DESC
            "#,
        )?;

        let rows = stmt.query_map([], |row| {
            Ok(SpecInfo {
                spec_id: row.get(0)?,
                spec_path: row.get(1)?,
                title: row.get(2)?,
                version: row.get(3)?,
                base_url: row.get(4)?,
                operation_count: row.get(5)?,
                indexed_at: row.get(6)?,
                spec_hash: row.get(7)?,
            })
        })?;

        let mut specs = Vec::new();
        for row in rows {
            specs.push(row?);
        }

        Ok(specs)
    }

    /// Get index status
    pub fn get_status(&self) -> Result<IndexStatus> {
        let spec_count: i32 = self
            .conn
            .query_row("SELECT COUNT(*) FROM specs", [], |row| row.get(0))?;

        let card_count: i32 =
            self.conn
                .query_row("SELECT COUNT(*) FROM operation_cards", [], |row| row.get(0))?;

        let embedding_count: i32 =
            self.conn
                .query_row("SELECT COUNT(*) FROM embeddings", [], |row| row.get(0))?;

        Ok(IndexStatus {
            spec_count: spec_count as usize,
            card_count: card_count as usize,
            embedding_count: embedding_count as usize,
            has_embeddings: embedding_count > 0,
        })
    }

    /// Remove a spec and all its cards
    pub fn remove_spec(&self, spec_id: &str) -> Result<usize> {
        // Delete embeddings first (due to FK)
        self.conn.execute(
            "DELETE FROM embeddings WHERE card_id IN (SELECT id FROM operation_cards WHERE spec_id = ?)",
            params![spec_id],
        )?;

        // Delete vocabulary
        self.remove_vocabulary(spec_id)?;

        // Delete cards
        let deleted: usize = self.conn.execute(
            "DELETE FROM operation_cards WHERE spec_id = ?",
            params![spec_id],
        )?;

        // Delete spec
        self.conn
            .execute("DELETE FROM specs WHERE spec_id = ?", params![spec_id])?;

        Ok(deleted)
    }

    /// Clear all data
    pub fn clear(&self) -> Result<()> {
        self.conn.execute("DELETE FROM embeddings", [])?;
        self.conn.execute("DELETE FROM spec_vocabulary", [])?;
        self.conn.execute("DELETE FROM operation_cards", [])?;
        self.conn.execute("DELETE FROM specs", [])?;
        Ok(())
    }

    /// Build spec vocabulary from operation cards for intelligent search suggestions
    ///
    /// Extracts terms from operation_ids, paths, parameter names, summaries, and tags.
    /// Deduplicates, weights by provenance, caps at 500 terms per spec.
    pub fn build_vocabulary(&self, spec_id: &str, cards: &[OperationCard]) -> Result<usize> {
        use crate::core::identifier_splitter::split_identifier;
        use std::collections::HashMap;

        // Clear existing vocabulary for this spec
        self.remove_vocabulary(spec_id)?;

        // term -> (weight, provenance, operation_ids)
        let mut vocab: HashMap<String, (f64, String, Vec<String>)> = HashMap::new();

        let mut insert_term = |term: String, weight: f64, provenance: &str, op_id: &str| {
            let entry = vocab
                .entry(term)
                .or_insert_with(|| (0.0, String::new(), Vec::new()));
            if weight > entry.0 {
                entry.0 = weight;
                entry.1 = provenance.to_string();
            }
            if !entry.2.contains(&op_id.to_string()) {
                entry.2.push(op_id.to_string());
            }
        };

        for card in cards {
            let op_id = &card.operation_id;

            // operation_id tokens (weight 1.0)
            for token in split_identifier(op_id) {
                if token.len() > 1 {
                    insert_term(token, 1.0, "operation_id", op_id);
                }
            }

            // path tokens (weight 1.1)
            for token in split_identifier(&card.path) {
                if token.len() > 1 {
                    insert_term(token, 1.1, "path", op_id);
                }
            }

            // parameter name tokens (weight 1.0)
            for param in &card.parameters {
                for token in split_identifier(&param.name) {
                    if token.len() > 1 {
                        insert_term(token, 1.0, "param", op_id);
                    }
                }
            }

            // summary words (weight 0.8)
            if let Some(summary) = &card.summary {
                for word in summary.split_whitespace() {
                    let clean = word
                        .to_lowercase()
                        .chars()
                        .filter(|c| c.is_alphanumeric())
                        .collect::<String>();
                    if clean.len() > 2 {
                        insert_term(clean, 0.8, "summary", op_id);
                    }
                }
            }

            // tag tokens (weight 1.0)
            for tag in &card.tags {
                for token in split_identifier(tag) {
                    if token.len() > 1 {
                        insert_term(token, 1.0, "tag", op_id);
                    }
                }
            }
        }

        // Sort by weight desc and cap at 500
        let mut entries: Vec<_> = vocab.into_iter().collect();
        entries.sort_by(|a, b| {
            b.1 .0
                .partial_cmp(&a.1 .0)
                .unwrap_or(std::cmp::Ordering::Equal)
        });
        entries.truncate(500);

        // Insert into spec_vocabulary table
        let count = entries.len();
        for (term, (weight, provenance, op_ids)) in &entries {
            let op_ids_str = op_ids.join(",");
            self.conn.execute(
                r#"
                INSERT INTO spec_vocabulary (spec_id, term, weight, provenance, operation_ids)
                VALUES (?, ?, ?, ?, ?)
                ON CONFLICT (spec_id, term) DO UPDATE SET
                    weight = excluded.weight,
                    provenance = excluded.provenance,
                    operation_ids = excluded.operation_ids
                "#,
                params![spec_id, term, *weight, provenance, op_ids_str],
            )?;
        }

        Ok(count)
    }

    /// Lookup vocabulary terms matching the given tokens
    ///
    /// For each token, finds exact and fuzzy matches in spec_vocabulary.
    pub fn lookup_vocabulary(
        &self,
        spec_id: Option<&str>,
        tokens: &[String],
        limit: usize,
    ) -> Result<Vec<VocabMatch>> {
        use crate::core::fuzzy_matcher::is_fuzzy_match;

        let mut results = Vec::new();

        for token in tokens {
            if token.len() <= 1 {
                continue;
            }
            let token_lower = token.to_lowercase();
            let like_pattern = format!("%{}%", token_lower);

            let mut sql = String::from(
                "SELECT term, weight, provenance, operation_ids FROM spec_vocabulary WHERE LOWER(term) LIKE ?"
            );
            let mut params_vec: Vec<Box<dyn rusqlite::types::ToSql>> = vec![Box::new(like_pattern)];

            if let Some(sid) = spec_id {
                sql.push_str(" AND spec_id = ?");
                params_vec.push(Box::new(sid.to_string()));
            }
            sql.push_str(" ORDER BY weight DESC LIMIT 50");

            let mut stmt = self.conn.prepare(&sql)?;
            let param_refs: Vec<&dyn rusqlite::types::ToSql> =
                params_vec.iter().map(|b| b.as_ref()).collect();

            let rows = stmt.query_map(param_refs.as_slice(), |row| {
                let term: String = row.get(0)?;
                let weight: f64 = row.get(1)?;
                let provenance: String = row.get(2)?;
                let op_ids_str: String = row.get(3)?;
                Ok((term, weight, provenance, op_ids_str))
            })?;

            for row in rows {
                let (term, weight, provenance, op_ids_str) = row?;
                let op_ids: Vec<String> = op_ids_str
                    .split(',')
                    .filter(|s| !s.is_empty())
                    .map(|s| s.to_string())
                    .collect();
                results.push(VocabMatch {
                    term,
                    weight,
                    provenance,
                    operation_ids: op_ids,
                });
            }

            // Also do fuzzy matching against all terms for this spec
            let mut fuzzy_sql =
                String::from("SELECT term, weight, provenance, operation_ids FROM spec_vocabulary");
            let mut fuzzy_params: Vec<Box<dyn rusqlite::types::ToSql>> = Vec::new();
            if let Some(sid) = spec_id {
                fuzzy_sql.push_str(" WHERE spec_id = ?");
                fuzzy_params.push(Box::new(sid.to_string()));
            }

            let mut fuzzy_stmt = self.conn.prepare(&fuzzy_sql)?;
            let fuzzy_refs: Vec<&dyn rusqlite::types::ToSql> =
                fuzzy_params.iter().map(|b| b.as_ref()).collect();

            let fuzzy_rows = fuzzy_stmt.query_map(fuzzy_refs.as_slice(), |row| {
                let term: String = row.get(0)?;
                let weight: f64 = row.get(1)?;
                let provenance: String = row.get(2)?;
                let op_ids_str: String = row.get(3)?;
                Ok((term, weight, provenance, op_ids_str))
            })?;

            for row in fuzzy_rows {
                let (term, weight, provenance, op_ids_str) = row?;
                if is_fuzzy_match(&token_lower, &term)
                    && !results.iter().any(|r: &VocabMatch| r.term == term)
                {
                    let op_ids: Vec<String> = op_ids_str
                        .split(',')
                        .filter(|s| !s.is_empty())
                        .map(|s| s.to_string())
                        .collect();
                    results.push(VocabMatch {
                        term,
                        weight,
                        provenance,
                        operation_ids: op_ids,
                    });
                }
            }
        }

        // Deduplicate and sort by weight
        results.sort_by(|a, b| {
            b.weight
                .partial_cmp(&a.weight)
                .unwrap_or(std::cmp::Ordering::Equal)
        });
        results.dedup_by(|a, b| a.term == b.term);
        results.truncate(limit);

        Ok(results)
    }

    /// Remove vocabulary for a spec (called before rebuild)
    pub fn remove_vocabulary(&self, spec_id: &str) -> Result<()> {
        self.conn.execute(
            "DELETE FROM spec_vocabulary WHERE spec_id = ?",
            params![spec_id],
        )?;
        Ok(())
    }

    /// Get a card by ID
    pub fn get_card(&self, card_id: &str) -> Result<Option<OperationCard>> {
        let mut stmt = self.conn.prepare(
            r#"
            SELECT spec_id, operation_id, content_hash, method, path,
                   summary, description, parameters_json, request_body_json,
                   auth_required, auth_type, auth_scopes, risk_level, tags,
                   alias, embedding_text
            FROM operation_cards
            WHERE id = ?
            "#,
        )?;

        let result = stmt.query_row(params![card_id], |row| {
            let parameters_json: String = row.get(7)?;
            let request_body_json: Option<String> = row.get(8)?;
            let auth_scopes: String = row.get(11)?;
            let tags: String = row.get(13)?;

            Ok(OperationCard {
                spec_id: row.get(0)?,
                operation_id: row.get(1)?,
                content_hash: row.get(2)?,
                method: row.get(3)?,
                path: row.get(4)?,
                summary: row.get(5)?,
                description: row.get(6)?,
                parameters: serde_json::from_str(&parameters_json).unwrap_or_default(),
                request_body: request_body_json.and_then(|s| serde_json::from_str(&s).ok()),
                auth_required: row.get(9)?,
                auth_type: row.get(10)?,
                auth_scopes: auth_scopes
                    .split(',')
                    .filter(|s| !s.is_empty())
                    .map(|s| s.to_string())
                    .collect(),
                risk_level: row.get(12)?,
                tags: tags
                    .split(',')
                    .filter(|s| !s.is_empty())
                    .map(|s| s.to_string())
                    .collect(),
                alias: row.get::<_, Option<String>>(14)?.unwrap_or_default(),
                embedding_text: row.get(15)?,
                embedding: None,
            })
        });

        match result {
            Ok(card) => Ok(Some(card)),
            Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
            Err(e) => Err(e.into()),
        }
    }

    /// Resolve an alias to an operation_id. Returns None if no match.
    pub fn resolve_alias(&self, alias: &str) -> Result<Option<String>> {
        let mut stmt = self.conn.prepare(
            "SELECT operation_id FROM operation_cards WHERE LOWER(alias) = LOWER(?1) LIMIT 1",
        )?;
        let result = stmt.query_row(params![alias], |row| row.get::<_, String>(0));
        match result {
            Ok(op_id) => Ok(Some(op_id)),
            Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
            Err(e) => Err(e.into()),
        }
    }
}

/// Search result
#[derive(Debug, Clone)]
pub struct SearchResult {
    pub id: String,
    pub spec_id: String,
    pub operation_id: String,
    pub method: String,
    pub path: String,
    pub summary: Option<String>,
    pub description: Option<String>,
    pub auth_required: bool,
    pub auth_type: Option<String>,
    pub risk_level: String,
    pub tags: Vec<String>,
    pub alias: Option<String>,
    pub score: f64,
}

impl SearchResult {
    /// Format as display string
    pub fn display(&self) -> String {
        let summary = self.summary.as_deref().unwrap_or("");
        format!(
            "{} {} {} - {}",
            self.method, self.path, self.operation_id, summary
        )
    }

    /// Format as JSON
    pub fn to_json(&self) -> serde_json::Value {
        serde_json::json!({
            "id": self.id,
            "spec_id": self.spec_id,
            "operation_id": self.operation_id,
            "alias": self.alias,
            "method": self.method,
            "path": self.path,
            "summary": self.summary,
            "auth_required": self.auth_required,
            "risk_level": self.risk_level,
            "score": self.score,
        })
    }
}

/// Spec info
#[derive(Debug, Clone)]
pub struct SpecInfo {
    pub spec_id: String,
    pub spec_path: String,
    pub title: String,
    pub version: String,
    pub base_url: String,
    pub operation_count: i32,
    pub indexed_at: String,
    pub spec_hash: String,
}

/// Index status
#[derive(Debug, Clone)]
pub struct IndexStatus {
    pub spec_count: usize,
    pub card_count: usize,
    pub embedding_count: usize,
    pub has_embeddings: bool,
}

/// Vocabulary match from spec_vocabulary lookup
#[derive(Debug, Clone)]
pub struct VocabMatch {
    pub term: String,
    pub weight: f64,
    pub provenance: String,
    pub operation_ids: Vec<String>,
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_create_index_store() {
        let store = IndexStore::open_in_memory().unwrap();
        let status = store.get_status().unwrap();
        assert_eq!(status.spec_count, 0);
        assert_eq!(status.card_count, 0);
    }

    #[test]
    fn test_upsert_spec() {
        let store = IndexStore::open_in_memory().unwrap();
        store
            .upsert_spec(
                "test-api",
                "/path/to/spec.yaml",
                "Test API",
                "1.0.0",
                "https://api.test.com",
                10,
                "abc123",
            )
            .unwrap();

        let specs = store.list_specs().unwrap();
        assert_eq!(specs.len(), 1);
        assert_eq!(specs[0].spec_id, "test-api");
    }

    #[test]
    fn test_cosine_similarity() {
        // Identical vectors → 1.0
        let a = vec![1.0f32, 0.0, 0.0];
        let b = vec![1.0f32, 0.0, 0.0];
        assert!((cosine_similarity(&a, &b) - 1.0).abs() < 1e-6);

        // Orthogonal vectors → 0.0
        let a = vec![1.0f32, 0.0, 0.0];
        let b = vec![0.0f32, 1.0, 0.0];
        assert!(cosine_similarity(&a, &b).abs() < 1e-6);

        // Known angle
        let a = vec![1.0f32, 1.0];
        let b = vec![1.0f32, 0.0];
        let expected = 1.0 / 2.0f64.sqrt();
        assert!((cosine_similarity(&a, &b) - expected).abs() < 1e-6);
    }

    #[test]
    fn test_embedding_roundtrip() {
        let original: Vec<f32> = vec![0.1, -0.5, 3.15, 0.0, -1.0];
        let blob: Vec<u8> = original.iter().flat_map(|f| f.to_le_bytes()).collect();
        let recovered = blob_to_embedding(&blob);
        assert_eq!(original, recovered);
    }

    #[test]
    fn test_blob_to_embedding_empty() {
        let empty: Vec<u8> = vec![];
        let result = blob_to_embedding(&empty);
        assert!(result.is_empty());
    }

    // ========================================================================
    // Sprint 2: Vocabulary tests
    // ========================================================================

    fn make_test_card(
        op_id: &str,
        path: &str,
        summary: Option<&str>,
        tags: Vec<&str>,
    ) -> OperationCard {
        use crate::core::cards::CardParameter;
        OperationCard {
            spec_id: "test-api".to_string(),
            operation_id: op_id.to_string(),
            content_hash: "hash".to_string(),
            method: "GET".to_string(),
            path: path.to_string(),
            summary: summary.map(|s| s.to_string()),
            description: None,
            parameters: vec![CardParameter {
                name: "petId".to_string(),
                location: "path".to_string(),
                required: true,
                param_type: "integer".to_string(),
                format: None,
                description: None,
                enum_values: None,
            }],
            request_body: None,
            auth_required: false,
            auth_type: None,
            auth_scopes: vec![],
            risk_level: "read".to_string(),
            tags: tags.into_iter().map(|s| s.to_string()).collect(),
            alias: summary
                .map(|s| s.to_lowercase().replace(' ', ""))
                .unwrap_or_default(),
            embedding_text: String::new(),
            embedding: None,
        }
    }

    #[test]
    fn test_build_vocabulary_extracts_tokens() {
        let store = IndexStore::open_in_memory().unwrap();
        let cards = vec![make_test_card(
            "findPetsByStatus",
            "/pets/{petId}",
            Some("Find pets by status"),
            vec!["pets"],
        )];
        let count = store.build_vocabulary("test-api", &cards).unwrap();
        assert!(count > 0);

        // Check that key tokens were extracted
        let matches = store
            .lookup_vocabulary(Some("test-api"), &["find".to_string()], 20)
            .unwrap();
        assert!(!matches.is_empty(), "Should find 'find' in vocabulary");

        let matches = store
            .lookup_vocabulary(Some("test-api"), &["pets".to_string()], 20)
            .unwrap();
        assert!(!matches.is_empty(), "Should find 'pets' in vocabulary");

        let matches = store
            .lookup_vocabulary(Some("test-api"), &["status".to_string()], 20)
            .unwrap();
        assert!(!matches.is_empty(), "Should find 'status' in vocabulary");
    }

    #[test]
    fn test_vocab_provenance_tracked() {
        let store = IndexStore::open_in_memory().unwrap();
        let cards = vec![make_test_card("listUsers", "/users", None, vec!["users"])];
        store.build_vocabulary("test-api", &cards).unwrap();

        let matches = store
            .lookup_vocabulary(Some("test-api"), &["users".to_string()], 20)
            .unwrap();
        assert!(!matches.is_empty());
        // "users" appears in path and tags — provenance should reflect highest-weight source
        let users_match = matches.iter().find(|m| m.term == "users").unwrap();
        assert_eq!(
            users_match.provenance, "path",
            "path has weight 1.1, highest"
        );
    }

    #[test]
    fn test_vocab_weight_ordering() {
        let store = IndexStore::open_in_memory().unwrap();
        let cards = vec![make_test_card(
            "findPetsByStatus",
            "/pets/{petId}",
            Some("Find all available pets nearby"),
            vec!["pets"],
        )];
        store.build_vocabulary("test-api", &cards).unwrap();

        // "pets" from path (weight 1.1) should rank higher than "available" from summary (weight 0.8)
        let matches = store
            .lookup_vocabulary(
                Some("test-api"),
                &["pets".to_string(), "available".to_string()],
                20,
            )
            .unwrap();
        if matches.len() >= 2 {
            let pets_match = matches.iter().find(|m| m.term == "pets").unwrap();
            let avail_match = matches.iter().find(|m| m.term == "available").unwrap();
            assert!(
                pets_match.weight > avail_match.weight,
                "path tokens (1.1) should weigh more than summary words (0.8)"
            );
        }
    }

    #[test]
    fn test_vocab_lookup_finds_terms() {
        let store = IndexStore::open_in_memory().unwrap();
        let cards = vec![
            make_test_card(
                "getPetById",
                "/pets/{petId}",
                Some("Get a pet by its ID"),
                vec!["pets"],
            ),
            make_test_card(
                "listOrders",
                "/orders",
                Some("List all orders"),
                vec!["orders"],
            ),
        ];
        store.build_vocabulary("test-api", &cards).unwrap();

        // Lookup "pet" should find "pet" or "pets" via LIKE match
        let matches = store
            .lookup_vocabulary(Some("test-api"), &["pet".to_string()], 20)
            .unwrap();
        assert!(!matches.is_empty(), "Should find terms matching 'pet'");

        // Lookup "order" should find "orders"
        let matches = store
            .lookup_vocabulary(Some("test-api"), &["order".to_string()], 20)
            .unwrap();
        assert!(!matches.is_empty(), "Should find terms matching 'order'");
    }

    #[test]
    fn test_vocab_capped_at_limit() {
        let store = IndexStore::open_in_memory().unwrap();
        // Create many cards to generate lots of vocab
        let mut cards = Vec::new();
        for i in 0..50 {
            cards.push(make_test_card(
                &format!("operation{}", i),
                &format!("/resource{}/sub{}", i, i),
                Some(&format!("Summary word{} extra{} bonus{}", i, i, i)),
                vec![],
            ));
        }
        store.build_vocabulary("test-api", &cards).unwrap();

        // Lookup with a broad pattern, limit to 3
        let matches = store
            .lookup_vocabulary(Some("test-api"), &["resource".to_string()], 3)
            .unwrap();
        assert!(matches.len() <= 3, "Should respect limit parameter");
    }

    #[test]
    fn test_vocab_deduplicates() {
        let store = IndexStore::open_in_memory().unwrap();
        // "pets" appears in operation_id, path, and tags — should be deduplicated
        let cards = vec![
            make_test_card("listPets", "/pets", None, vec!["pets"]),
            make_test_card("getPets", "/pets/{id}", None, vec!["pets"]),
        ];
        store.build_vocabulary("test-api", &cards).unwrap();

        let matches = store
            .lookup_vocabulary(Some("test-api"), &["pets".to_string()], 20)
            .unwrap();
        let pets_entries: Vec<_> = matches.iter().filter(|m| m.term == "pets").collect();
        assert_eq!(pets_entries.len(), 1, "Same term should appear only once");
        // Should track both operation_ids
        assert!(
            pets_entries[0].operation_ids.len() >= 2,
            "Should track multiple operation_ids for same term"
        );
    }
}