memstead-base 0.8.0

Engine internals for Memstead — store, parser, validators, filesystem-mem engine. Internal library surface consumed by the memstead binaries — pre-1.0, experimental, no API stability promise.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100

//! Per-mem tantivy search indexes.
//!
//! The engine holds `search_indexes: HashMap<String, MemIndex>` for
//! its writable mems, each with a long-lived `IndexWriter`, lazily
//! built on first search and rebuilt from the store after any mutation
//! or reload invalidates the map (whole-map invalidation, not
//! incremental upkeep). [`execute_on_mem`] serves queries against the
//! built indexes.

pub mod query;
pub mod schema;
pub mod snippets;
pub mod tokenizer;
pub mod writer;

pub use query::execute_on_mem;

/// Build the text indexed into the schema-agnostic `metadata` field:
/// one "key value" line per metadata entry, keys included so "which
/// entities carry this field at all" is answerable by the same
/// free-text path. THE single builder — the index writer and the
/// matched-terms reporter both call it, so what is indexed and what is
/// reported can never drift.
pub fn metadata_index_text(entity: &crate::entity::Entity) -> String {
    let mut out = String::new();
    for (key, value) in &entity.metadata {
        out.push_str(key);
        out.push(' ');
        out.push_str(&value.to_frontmatter_string());
        out.push('\n');
    }
    out
}
pub use schema::IndexFields;
pub use snippets::{compute_matched_terms, compute_score_breakdown};
pub use writer::MemIndex;

use std::collections::HashMap;
use std::sync::Arc;

use memstead_schema::Schema;

use crate::store::Store;

/// Build per-mem indexes for every writable mem. Each mem's index
/// uses its own pinned schema from `writable_schemas`.
pub fn build_all(
    store: &Store,
    writable_schemas: &HashMap<String, Arc<Schema>>,
) -> HashMap<String, MemIndex> {
    let mut indexes: HashMap<String, MemIndex> = HashMap::new();

    // Write-mems keep their writers alive for the engine lifetime.
    for (name, schema) in writable_schemas {
        match MemIndex::build_in_ram(name.clone(), Some(schema)) {
            Ok(idx) => {
                indexes.insert(name.clone(), idx);
            }
            Err(e) => {
                tracing::warn!(
                    mem = name.as_str(),
                    error = %e,
                    "failed to create search index for writable mem; skipping"
                );
            }
        }
    }

    // Populate every index from the store, then commit. Warn-log on error
    // so a broken index doesn't take the engine down — callers can reload
    // to recover.
    for entity in store.all_entities() {
        if entity.stub {
            continue;
        }
        let Some(idx) = indexes.get_mut(&entity.mem) else {
            continue;
        };
        if let Err(e) = idx.index_entity(entity) {
            tracing::warn!(
                mem = entity.mem.as_str(),
                id = entity.id.as_ref(),
                error = %e,
                "failed to index entity during bulk build"
            );
        }
    }

    for idx in indexes.values_mut() {
        if let Err(e) = idx.commit() {
            tracing::warn!(
                mem = idx.mem.as_str(),
                error = %e,
                "failed to commit mem index after bulk build"
            );
        }
    }

    indexes
}