semantic-memory 0.5.9

Local-first hybrid semantic search (SQLite + FTS5 + usearch 2.25) with bitemporal truth and typed receipts
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92

//! poly-kv bridge for semantic-memory integration.
//!
//! Gated behind the `poly-kv-codec` feature.

use crate::error::MemoryError;

/// Build a SharedKVPool from (key, embedding) pairs.
pub fn build_pool_from_embeddings(
    embeddings: &[(String, Vec<f32>)],
    embedding_dim: usize,
    seed: u64,
) -> Result<(poly_kv::SharedKVPool, poly_kv::PoolBuildReceipt), MemoryError> {
    let shape = poly_kv::KvTensorShape {
        attention_type: poly_kv::AttentionType::MHA,
        num_layers: 1,
        num_heads: 1,
        num_kv_heads: 1,
        head_dim: embedding_dim,
        hidden_size: embedding_dim,
    };

    let (pool, receipt) = poly_kv::SharedKVPool::build(embeddings, &shape, seed)
        .map_err(|e| MemoryError::Other(format!("poly-kv pool build failed: {e}")))?;

    Ok((pool, receipt))
}

/// Convert a PoolBuildReceipt into a ProveKvPoolGenerationV1.
pub fn receipt_to_generation(
    receipt: &poly_kv::PoolBuildReceipt,
    snapshot_digest: &str,
) -> crate::types::ProveKvPoolGenerationV1 {
    crate::types::ProveKvPoolGenerationV1 {
        schema_version: "provekv_pool_generation_v1".into(),
        generation_id: receipt.pool_digest.hex().to_string(),
        embedding_snapshot_digest: snapshot_digest.to_string(),
        source_digest: receipt.pool_digest.hex().to_string(),
        pool_manifest_digest: String::new(),
        codec_family: "fib_quant".to_string(),
        codec_profile: "k4_n32_paper_default".to_string(),
        vector_dim: 0,
        item_count: receipt.total_tokens as usize,
        payload_bytes: receipt.pool_size_bytes,
        created_at: chrono::Utc::now(),
    }
}

/// Serialize a SharedKVPool to JSON bytes for DB storage.
pub fn serialize_pool(pool: &poly_kv::SharedKVPool) -> Result<Vec<u8>, MemoryError> {
    // SharedKVPool doesn't derive Serialize, but its fields do.
    // Use a manual envelope.
    #[derive(serde::Serialize)]
    struct Envelope<'a> {
        manifest: &'a poly_kv::PoolManifest,
        layers: &'a [poly_kv::PoolLayer],
        policy: &'a poly_kv::CompressionPolicy,
    }
    let env = Envelope {
        manifest: &pool.manifest,
        layers: &pool.layers,
        policy: &pool.policy,
    };
    serde_json::to_vec(&env).map_err(|e| MemoryError::Other(format!("pool serialize: {e}")))
}

/// Deserialize a SharedKVPool from JSON bytes.
pub fn deserialize_pool(data: &[u8]) -> Result<poly_kv::SharedKVPool, MemoryError> {
    #[derive(serde::Deserialize)]
    struct Envelope {
        manifest: poly_kv::PoolManifest,
        layers: Vec<poly_kv::PoolLayer>,
        policy: poly_kv::CompressionPolicy,
    }
    let env: Envelope = serde_json::from_slice(data)
        .map_err(|e| MemoryError::Other(format!("pool deserialize: {e}")))?;
    Ok(poly_kv::SharedKVPool {
        manifest: env.manifest,
        layers: env.layers,
        policy: env.policy,
    })
}

/// Search a pool layer for tokens similar to a query.
pub fn search_pool_layer(
    pool: &poly_kv::SharedKVPool,
    layer_idx: usize,
    query: &[f32],
    top_k: usize,
) -> Result<Vec<(usize, f32)>, MemoryError> {
    pool.search_similar_tokens(layer_idx, query, top_k)
        .map_err(|e| MemoryError::Other(format!("pool search failed: {e}")))
}