hippmem_engine/
lib.rs

1//! HIPPMEM · Native Association Memory Engine — unified external Rust API orchestration layer.
2//!
3//! Integrates [`hippmem_core`], [`hippmem_store`], [`hippmem_model`],
4//! [`hippmem_write`], [`hippmem_retrieval`], [`hippmem_consolidation`],
5//! providing seven core APIs: `write/retrieve/explain/consolidate/inspect/feedback`.
6//!
7//! Corresponds to 05-api-contract, 09-engine-assembly.
8
9pub mod consolidate_api;
10pub mod dump_api;
11pub mod explain_api;
12pub mod feedback_api;
13pub mod inspect_api;
14pub mod list_api;
15pub mod retrieve_api;
16pub mod runtime;
17pub mod traverse_api;
18pub mod write_api;
19
20use hippmem_core::config::{AlgoParams, EmbedderConfig};
21use hippmem_core::ids::MemoryId;
22use hippmem_core::model::links::AssociationLink;
23use hippmem_core::model::understanding::MemoryUnderstanding;
24use hippmem_core::model::unit::{MemoryStage, WriteContext};
25use hippmem_model::registry::{build_embedder, BackendSelection};
26use hippmem_model::traits::Embedder;
27use hippmem_store::fulltext::FulltextIndex;
28use hippmem_store::semantic::binary::BinaryCodeIndex;
29use hippmem_store::semantic::hnsw::FlatVectorIndex;
30use hippmem_store::store::{RedbStore, Store};
31use parking_lot::RwLock;
32use serde::Serialize;
33use std::path::PathBuf;
34use std::sync::Arc;
35
36// ── EngineError ──
37
38/// Engine external error type: converts lower-layer errors into a unified external error code.
39///
40/// Corresponds to 05 §7. MUST NOT expose underlying library types (constitution C2).
41#[derive(Debug, thiserror::Error)]
42pub enum EngineError {
43    /// Underlying storage error.
44    #[error("store: {0}")]
45    Store(String),
46
47    /// Memory not found.
48    #[error("not found: {0:?}")]
49    NotFound(MemoryId),
50
51    /// Invalid input parameter.
52    #[error("invalid input: {0}")]
53    InvalidInput(String),
54
55    /// Incompatible schema version.
56    #[error("schema too new: {0}")]
57    SchemaTooNew(u16),
58
59    /// Model invocation failed (non-fatal).
60    #[error("model: {0}")]
61    Model(String),
62
63    /// Backend unavailable (API key missing or network error).
64    #[error("backend unavailable: {0}")]
65    BackendUnavailable(String),
66
67    /// Internal error.
68    #[error("internal: {0}")]
69    Internal(String),
70}
71
72/// Engine-layer Result alias.
73pub type EngineResult<T> = Result<T, EngineError>;
74
75// ── Write types (05 §1) ──
76
77/// Input for writing a memory.
78pub struct WriteMemoryInput {
79    pub content: String,
80    pub content_type: Option<hippmem_core::model::enums::ContentType>,
81    pub context: WriteContext,
82    pub importance_hint: Option<f32>,
83    pub source_refs: Vec<hippmem_core::model::unit::SourceRef>,
84}
85
86/// Output of writing a memory.
87pub struct WriteMemoryOutput {
88    pub memory_id: MemoryId,
89    pub stage_reached: MemoryStage,
90    pub created_links: Vec<AssociationLink>,
91    pub understanding: MemoryUnderstanding,
92    pub warnings: Vec<WriteWarning>,
93}
94
95// ── Retrieval types (05 §2) ──
96
97use hippmem_core::model::links::{RecallChannel, RetrievalResult};
98
99/// Retrieval context.
100#[derive(Debug, Clone, Default)]
101pub struct RetrieveContext {
102    pub conversation_id: Option<u64>,
103    pub session_id: Option<u64>,
104    pub project_id: Option<u64>,
105    pub task_id: Option<u64>,
106    pub user_id: Option<u64>,
107    pub recent_memory_ids: Vec<MemoryId>,
108}
109
110/// Retrieval input.
111pub struct RetrieveInput {
112    pub query: String,
113    pub context: RetrieveContext,
114    pub top_k: usize,
115    pub max_hops: Option<usize>,
116    pub retrieval_mode: hippmem_core::model::links::RetrievalMode,
117}
118
119/// Retrieval output.
120pub struct RetrieveOutput {
121    pub results: Vec<RetrievalResult>,
122    pub trace: RetrievalTrace,
123    pub diagnostics: RetrievalDiagnostics,
124}
125
126/// Retrieval trace.
127pub struct RetrievalTrace {
128    pub seeds: Vec<SeedRecord>,
129    pub steps: Vec<hippmem_core::model::links::ActivationStep>,
130    pub hops_used: u8,
131    pub merged_count: usize,
132}
133
134/// Seed record.
135pub struct SeedRecord {
136    pub id: MemoryId,
137    pub channel: RecallChannel,
138    pub initial_energy: f32,
139    /// V9: in-channel rank (0 = best), for RRF diagnostics
140    pub rank_in_channel: Option<usize>,
141}
142
143/// Retrieval diagnostics.
144pub struct RetrievalDiagnostics {
145    pub channel_contributions: Vec<(RecallChannel, u32)>,
146    pub reranked: bool,
147    pub pruned_branches: u32,
148    pub backend_used: BackendUsage,
149    pub latency_ms: u32,
150}
151
152/// Backend usage info.
153pub struct BackendUsage {
154    pub embedder: String,
155    pub reranker: Option<String>,
156}
157
158// ── Feedback types (05 §6) ──
159
160/// Usage feedback input.
161pub struct FeedbackInput {
162    pub retrieval_id: u64,
163    pub used_memory_ids: Vec<MemoryId>,
164    pub signal: UsageSignal,
165}
166
167// ── Consolidation types (05 §5) ──
168
169/// Consolidation scope.
170#[derive(Debug, Clone)]
171pub enum ConsolidationScope {
172    Full,
173    Incremental,
174    ByMemoryType(hippmem_core::model::enums::ContentType),
175    ByTimeRange {
176        from: hippmem_core::time::Timestamp,
177        to: hippmem_core::time::Timestamp,
178    },
179    Reindex,
180    EdgesOnly,
181}
182
183// ── Explain/diagnostics types (05 §4 §6) ──
184
185use hippmem_core::model::links::LinkType;
186use hippmem_core::model::unit::{MemoryLifecycle, MemoryUnit};
187
188/// Explain output.
189pub struct Explanation {
190    pub memory_id: MemoryId,
191    pub content_summary: String,
192    pub current_importance: f32,
193    pub linked: Vec<LinkSummary>,
194    pub corrections: Vec<MemoryId>,
195    pub contradictions: Vec<MemoryId>,
196    pub recent_activations: u32,
197}
198
199pub struct LinkSummary {
200    pub target: MemoryId,
201    pub link_type: LinkType,
202    pub strength: f32,
203}
204
205pub enum InspectQuery {
206    Memory(MemoryId),
207    Edges(MemoryId),
208    Channel(RecallChannel),
209    StoreStats,
210    QueueStatus,
211    StrongestEdges { limit: usize },
212    Contradictions { limit: usize },
213}
214
215pub enum InspectReport {
216    Memory(Box<MemoryInspect>),
217    StoreStats(StoreStats),
218    QueueStatus(QueueStatus),
219}
220
221pub struct MemoryInspect {
222    pub unit: MemoryUnit,
223    pub out_edges: Vec<EdgeView>,
224    pub in_edges: Vec<EdgeView>,
225    pub stage: MemoryStage,
226    pub lifecycle: MemoryLifecycle,
227}
228
229#[derive(Debug, Clone)]
230pub struct EdgeView {
231    pub from: MemoryId,
232    pub to: MemoryId,
233    pub link_type: LinkType,
234    pub strength: f32,
235    pub confidence: f32,
236    pub activation_count: u32,
237    pub evidence: String,
238}
239
240pub struct StoreStats {
241    pub memory_count: u64,
242    pub edge_count: u64,
243    pub observing_edge_count: u64,
244    pub per_index_size: Vec<(RecallChannel, u64)>,
245    pub queue_backlog: u64,
246    pub store_bytes: u64,
247}
248
249pub struct QueueStatus {
250    pub pending_enrich: u64,
251    pub pending_consolidate: u64,
252    pub in_flight: u64,
253    pub oldest_pending_age_ms: u64,
254}
255
256/// Consolidation report.
257pub struct ConsolidationReport {
258    pub memories_processed: u64,
259    pub edges_decayed: u64,
260    pub edges_archived: u64,
261    pub edges_merged: u64,
262    pub observation_promoted: u64,
263    pub summaries_created: u64,
264    pub contradictions_found: u64,
265    pub reindexed: bool,
266    pub elapsed_ms: u64,
267}
268
269/// Usage signal.
270#[derive(Debug, Clone, Copy, PartialEq, Eq)]
271pub enum UsageSignal {
272    Referenced,
273    UserConfirmedCorrect,
274    TaskSucceeded,
275    UserRejected,
276}
277
278/// Write warning.
279#[derive(Debug, Clone, PartialEq)]
280pub enum WriteWarning {
281    /// Degraded extractor was used
282    ExtractorDegraded,
283    /// Dense vector generation deferred
284    EmbeddingDeferred,
285    /// Strong semantic dimensions deferred to enrich
286    StrongDimsDeferred,
287    /// Model invocation failed and was degraded
288    ModelError { detail: String },
289}
290
291// ── List API types ──
292
293/// Input parameters for paginated memory listing.
294#[derive(Debug, Clone)]
295pub struct ListInput {
296    /// Page size, default 20, max 100.
297    pub limit: usize,
298    /// Cursor: pass the MemoryId (u128 value) of the last item on the previous page to get the next page.
299    pub cursor: Option<u128>,
300    /// Filter by ContentType; None = no filter.
301    pub content_type: Option<hippmem_core::model::enums::ContentType>,
302}
303
304impl Default for ListInput {
305    fn default() -> Self {
306        Self {
307            limit: 20,
308            cursor: None,
309            content_type: None,
310        }
311    }
312}
313
314/// Output of paginated memory listing.
315#[derive(Debug, Clone, Serialize)]
316pub struct ListOutput {
317    pub items: Vec<ListItem>,
318    /// Cursor for the next page; None means this is the last page.
319    pub next_cursor: Option<u128>,
320    /// Total memory count (approximate).
321    pub total: u64,
322}
323
324/// Summary of a single memory in the list.
325#[derive(Debug, Clone, Serialize)]
326pub struct ListItem {
327    pub id: hippmem_core::ids::MemoryId,
328    /// Content preview: first 100 chars of raw.
329    pub content_preview: String,
330    pub content_type: hippmem_core::model::enums::ContentType,
331    pub created_at: hippmem_core::time::Timestamp,
332    pub importance: f32,
333    pub stage: hippmem_core::model::unit::MemoryStage,
334    pub lifecycle: hippmem_core::model::unit::MemoryLifecycle,
335    /// Number of outgoing edges of this memory.
336    pub edge_count: usize,
337}
338
339// ── Dump API types ──
340
341/// Full export input parameters.
342#[derive(Debug, Clone, Default)]
343pub struct DumpInput {
344    /// Output file path; None = return a JSON string.
345    pub output_path: Option<std::path::PathBuf>,
346}
347
348/// Full export output.
349#[derive(Debug, Clone, Serialize)]
350pub struct DumpOutput {
351    pub count: u64,
352    /// Path echo when written to a file.
353    pub written_to: Option<std::path::PathBuf>,
354    /// JSONL string returned when output_path is None.
355    pub json: Option<String>,
356}
357
358// ── Traverse API types ──
359
360/// Graph traversal input parameters.
361#[derive(Debug, Clone)]
362pub struct TraverseInput {
363    /// Start memory ID.
364    pub start_id: hippmem_core::ids::MemoryId,
365    /// BFS max depth, default 2, max 5.
366    pub max_depth: u8,
367    /// Traversal direction.
368    pub direction: TraverseDirection,
369    /// Filter edges by LinkType; None = no filter.
370    pub link_types: Option<Vec<hippmem_core::model::links::LinkType>>,
371}
372
373impl TraverseInput {
374    /// Creates default traversal params from the specified ID (depth=2, outgoing, no filter).
375    pub fn new(start_id: hippmem_core::ids::MemoryId) -> Self {
376        Self {
377            start_id,
378            max_depth: 2,
379            direction: TraverseDirection::Outgoing,
380            link_types: None,
381        }
382    }
383}
384
385impl Default for TraverseInput {
386    fn default() -> Self {
387        Self {
388            start_id: hippmem_core::ids::MemoryId(0),
389            max_depth: 2,
390            direction: TraverseDirection::Outgoing,
391            link_types: None,
392        }
393    }
394}
395
396/// Traversal direction.
397#[derive(Debug, Clone, Copy, PartialEq, Eq)]
398pub enum TraverseDirection {
399    /// Only outgoing edges.
400    Outgoing,
401    /// Only incoming edges.
402    Incoming,
403    /// Both directions.
404    Both,
405}
406
407/// Graph traversal output.
408#[derive(Debug, Clone)]
409pub struct TraverseOutput {
410    /// Nodes visited by BFS (excluding the start node).
411    pub nodes: Vec<TraverseNode>,
412    /// Edges traversed.
413    pub edges: Vec<EdgeView>,
414}
415
416/// Node in BFS traversal.
417#[derive(Debug, Clone)]
418pub struct TraverseNode {
419    pub id: hippmem_core::ids::MemoryId,
420    /// BFS depth: 1 = direct neighbor, 2 = neighbor of neighbor...
421    pub depth: u8,
422    pub content_preview: String,
423    pub content_type: hippmem_core::model::enums::ContentType,
424    pub importance: f32,
425}
426
427// ── Conversion from lower-layer errors ──
428
429impl From<hippmem_store::store::StoreError> for EngineError {
430    fn from(e: hippmem_store::store::StoreError) -> Self {
431        EngineError::Store(e.to_string())
432    }
433}
434
435// ── EngineConfig ──
436
437/// Background worker configuration.
438#[derive(Debug, Clone)]
439pub struct BackgroundConfig {
440    /// Strong-semantic enrich concurrency, default 2.
441    pub enrich_workers: usize,
442    /// Consolidation concurrency, default 1.
443    pub consolidate_workers: usize,
444    /// Background queue capacity (bounded), default 4096.
445    pub queue_capacity: usize,
446    /// Periodic consolidation trigger interval (ms), default 3_600_000 (1h).
447    pub consolidate_interval_ms: u64,
448    /// Whether to enable enrich, default true.
449    pub enrich_enabled: bool,
450}
451
452impl Default for BackgroundConfig {
453    fn default() -> Self {
454        Self {
455            enrich_workers: 2,
456            consolidate_workers: 1,
457            queue_capacity: 4096,
458            consolidate_interval_ms: 3_600_000,
459            enrich_enabled: true,
460        }
461    }
462}
463
464/// Engine construction configuration.
465///
466/// Corresponds to 05 §0. Configures persistence path, algorithm params,
467/// model backend selection, and background workers.
468#[derive(Debug, Clone)]
469pub struct EngineConfig {
470    /// Storage directory (the redb file will be created under this directory).
471    pub store_dir: PathBuf,
472    /// Algorithm params, defaults to `AlgoParams::default()`.
473    pub algo: AlgoParams,
474    /// Embedder backend config, defaults to deterministic 256d SimHash (matches V3 behavior).
475    pub embedder: EmbedderConfig,
476    /// Backend selection (extractor/reranker/summarizer), all default to `Auto`.
477    pub backend: BackendSelection,
478    /// Background worker configuration.
479    pub background: BackgroundConfig,
480}
481
482impl Default for EngineConfig {
483    fn default() -> Self {
484        Self {
485            store_dir: PathBuf::from("./hippmem_data"),
486            algo: AlgoParams::default(),
487            embedder: EmbedderConfig::default(),
488            backend: BackendSelection::default(),
489            background: BackgroundConfig::default(),
490        }
491    }
492}
493
494// ── Engine ──
495
496/// HIPPMEM unified orchestration facade.
497///
498/// Holds the persistent storage, model registry, and algorithm params,
499/// and exposes seven core APIs externally.
500///
501/// Corresponds to 05 §0, 09 §1.
502pub struct Engine {
503    /// Persistent storage (redb).
504    store: Arc<RedbStore>,
505    /// Algorithm params (hot-swappable).
506    #[allow(dead_code)]
507    params: Arc<RwLock<AlgoParams>>,
508    /// Embedder backend (config-driven, default deterministic 256d SimHash).
509    embedder: Arc<dyn Embedder>,
510    /// Backend config (extractor/reranker/summarizer; embedder migrated to `self.embedder`).
511    #[allow(dead_code)]
512    backend: BackendSelection,
513    /// Tantivy fulltext index (fulltext/ subdirectory of store dir; internal Mutex supports &self writes).
514    fulltext_index: parking_lot::Mutex<FulltextIndex>,
515    /// Tantivy fulltext index directory path (used for Reindex rebuild).
516    fulltext_dir: PathBuf,
517    /// Binary code index (in-memory Hamming distance recall, 03 §4.5 SemanticBinary channel).
518    binary_code_index: parking_lot::Mutex<BinaryCodeIndex>,
519    /// Dense vector index (in-memory brute-force L2 KNN, 03 §4.5 SemanticDense channel).
520    dense_vector_index: parking_lot::Mutex<FlatVectorIndex>,
521}
522
523impl Engine {
524    /// Opens/creates a HIPPMEM memory store.
525    ///
526    /// Corresponds to 05 §0 `Engine::open`.
527    /// - Automatically creates the `store_dir` parent directory.
528    /// - If a redb file already exists at the specified path, opens the existing store.
529    /// - Builds the Embedder backend from `config.embedder` (default deterministic 256d, constitution C5).
530    pub fn open(config: EngineConfig) -> EngineResult<Self> {
531        // Automatically create parent directory
532        if let Some(parent) = config.store_dir.parent() {
533            std::fs::create_dir_all(parent).map_err(|e| {
534                EngineError::Store(format!("cannot create storage directory: {}", e))
535            })?;
536        }
537
538        // Build Embedder backend (config-driven)
539        let embedder =
540            build_embedder(&config.embedder).map_err(|e| EngineError::Model(e.to_string()))?;
541
542        // Open/create redb storage
543        let store = RedbStore::open(&config.store_dir)?;
544
545        // Create/open Tantivy fulltext index (same dir as redb, fulltext/ subdirectory)
546        let fulltext_dir = config
547            .store_dir
548            .parent()
549            .map(|p| p.join("fulltext"))
550            .unwrap_or_else(|| PathBuf::from("hippmem_data").join("fulltext"));
551        let fulltext_index = FulltextIndex::open(&fulltext_dir)
552            .or_else(|_| FulltextIndex::create(&fulltext_dir))
553            .map_err(|e| {
554                EngineError::Store(format!("Tantivy index initialization failed: {}", e))
555            })?;
556
557        Ok(Self {
558            store: Arc::new(store),
559            params: Arc::new(RwLock::new(config.algo)),
560            embedder,
561            backend: config.backend,
562            fulltext_index: parking_lot::Mutex::new(fulltext_index),
563            fulltext_dir,
564            binary_code_index: parking_lot::Mutex::new(BinaryCodeIndex::new()),
565            dense_vector_index: parking_lot::Mutex::new(FlatVectorIndex::new()),
566        })
567    }
568
569    /// Graceful shutdown.
570    ///
571    /// Corresponds to 05 §0 `Engine::close`.
572    /// Currently only drops the store (redb auto-flushes); in the future it will wait for background workers to exit.
573    pub fn close(self) -> EngineResult<()> {
574        // Tantivy: commit unwritten documents and close
575        if let Err(e) = self.fulltext_index.lock().flush() {
576            // Non-fatal; tracing warn, does not block close
577            eprintln!("Tantivy flush failed: {}", e);
578        }
579        // store is dropped; redb auto-flushes and closes
580        drop(self.store);
581        Ok(())
582    }
583
584    /// Sets the fulltext index batch commit interval (auto commit every N entries).
585    /// Only used in batch write scenarios; production defaults to per-entry commit.
586    pub fn set_fulltext_commit_every(&self, n: usize) {
587        self.fulltext_index.lock().set_commit_every(n);
588    }
589
590    /// Force-commits all unwritten documents in the fulltext index.
591    pub fn flush_fulltext(&self) {
592        let _ = self.fulltext_index.lock().flush();
593    }
594}
hippmem_engine/lib.rs

hippmem_engine/
lib.rs
