hippmem_engine/
lib.rs

1//! HIPPMEM · Native Association Memory Engine — unified external Rust API orchestration layer.
2//!
3//! Integrates [`hippmem_core`], [`hippmem_store`], [`hippmem_model`],
4//! [`hippmem_write`], [`hippmem_retrieval`], [`hippmem_consolidation`],
5//! providing seven core APIs: `write/retrieve/explain/consolidate/inspect/feedback`.
6//!
7//! Corresponds to 05-api-contract, 09-engine-assembly.
8
9pub mod consolidate_api;
10pub mod dump_api;
11pub mod explain_api;
12pub mod feedback_api;
13pub mod inspect_api;
14pub mod list_api;
15pub mod retrieve_api;
16pub mod runtime;
17pub mod traverse_api;
18pub mod write_api;
19
20use hippmem_core::config::{AlgoParams, EmbedderConfig};
21use hippmem_core::ids::MemoryId;
22use hippmem_core::model::links::AssociationLink;
23use hippmem_core::model::understanding::MemoryUnderstanding;
24use hippmem_core::model::unit::{MemoryStage, WriteContext};
25use hippmem_model::registry::{build_embedder, BackendSelection};
26use hippmem_model::traits::Embedder;
27use hippmem_store::fulltext::FulltextIndex;
28use hippmem_store::semantic::binary::BinaryCodeIndex;
29use hippmem_store::semantic::hnsw::FlatVectorIndex;
30use hippmem_store::store::{RedbStore, Store};
31use parking_lot::RwLock;
32use serde::Serialize;
33use std::path::PathBuf;
34use std::sync::Arc;
35
36// ── EngineError ──
37
38/// Engine external error type: converts lower-layer errors into a unified external error code.
39///
40/// Corresponds to 05 §7. MUST NOT expose underlying library types (constitution C2).
41#[derive(Debug, thiserror::Error)]
42pub enum EngineError {
43    /// Underlying storage error.
44    #[error("store: {0}")]
45    Store(String),
46
47    /// Memory not found.
48    #[error("not found: {0:?}")]
49    NotFound(MemoryId),
50
51    /// Invalid input parameter.
52    #[error("invalid input: {0}")]
53    InvalidInput(String),
54
55    /// Incompatible schema version.
56    #[error("schema too new: {0}")]
57    SchemaTooNew(u16),
58
59    /// Model invocation failed (non-fatal).
60    #[error("model: {0}")]
61    Model(String),
62
63    /// Backend unavailable (API key missing or network error).
64    #[error("backend unavailable: {0}")]
65    BackendUnavailable(String),
66
67    /// Internal error.
68    #[error("internal: {0}")]
69    Internal(String),
70}
71
72/// Engine-layer Result alias.
73pub type EngineResult<T> = Result<T, EngineError>;
74
75// ── Write types (05 §1) ──
76
77/// Input for writing a memory.
78pub struct WriteMemoryInput {
79    pub content: String,
80    pub content_type: Option<hippmem_core::model::enums::ContentType>,
81    pub context: WriteContext,
82    pub importance_hint: Option<f32>,
83    pub source_refs: Vec<hippmem_core::model::unit::SourceRef>,
84}
85
86/// Output of writing a memory.
87pub struct WriteMemoryOutput {
88    pub memory_id: MemoryId,
89    pub stage_reached: MemoryStage,
90    pub created_links: Vec<AssociationLink>,
91    pub understanding: MemoryUnderstanding,
92    pub warnings: Vec<WriteWarning>,
93}
94
95// ── Retrieval types (05 §2) ──
96
97use hippmem_core::model::links::{RecallChannel, RetrievalResult};
98
99/// Retrieval context.
100#[derive(Debug, Clone, Default)]
101pub struct RetrieveContext {
102    pub conversation_id: Option<u64>,
103    pub session_id: Option<u64>,
104    pub project_id: Option<u64>,
105    pub task_id: Option<u64>,
106    pub user_id: Option<u64>,
107    pub recent_memory_ids: Vec<MemoryId>,
108}
109
110/// Retrieval input.
111pub struct RetrieveInput {
112    pub query: String,
113    pub context: RetrieveContext,
114    pub top_k: usize,
115    pub max_hops: Option<usize>,
116    pub retrieval_mode: hippmem_core::model::links::RetrievalMode,
117}
118
119/// Retrieval output.
120pub struct RetrieveOutput {
121    /// Identifier for this retrieval, used for feedback (see `Engine::feedback`).
122    pub retrieval_id: u64,
123    pub results: Vec<RetrievalResult>,
124    pub trace: RetrievalTrace,
125    pub diagnostics: RetrievalDiagnostics,
126}
127
128/// Retrieval trace.
129pub struct RetrievalTrace {
130    pub seeds: Vec<SeedRecord>,
131    pub steps: Vec<hippmem_core::model::links::ActivationStep>,
132    pub hops_used: u8,
133    pub merged_count: usize,
134}
135
136/// Seed record.
137pub struct SeedRecord {
138    pub id: MemoryId,
139    pub channel: RecallChannel,
140    pub initial_energy: f32,
141    /// V9: in-channel rank (0 = best), for RRF diagnostics
142    pub rank_in_channel: Option<usize>,
143}
144
145/// Retrieval diagnostics.
146pub struct RetrievalDiagnostics {
147    pub channel_contributions: Vec<(RecallChannel, u32)>,
148    pub reranked: bool,
149    pub pruned_branches: u32,
150    pub backend_used: BackendUsage,
151    pub latency_ms: u32,
152}
153
154/// Backend usage info.
155pub struct BackendUsage {
156    pub embedder: String,
157    pub reranker: Option<String>,
158}
159
160// ── Feedback types (05 §6) ──
161
162/// Usage feedback input.
163pub struct FeedbackInput {
164    pub retrieval_id: u64,
165    pub used_memory_ids: Vec<MemoryId>,
166    pub signal: UsageSignal,
167}
168
169// ── Consolidation types (05 §5) ──
170
171/// Consolidation scope.
172#[derive(Debug, Clone)]
173pub enum ConsolidationScope {
174    Full,
175    Incremental,
176    ByMemoryType(hippmem_core::model::enums::ContentType),
177    ByTimeRange {
178        from: hippmem_core::time::Timestamp,
179        to: hippmem_core::time::Timestamp,
180    },
181    Reindex,
182    EdgesOnly,
183}
184
185// ── Explain/diagnostics types (05 §4 §6) ──
186
187use hippmem_core::model::links::LinkType;
188use hippmem_core::model::unit::{MemoryLifecycle, MemoryUnit};
189
190/// Explain output.
191pub struct Explanation {
192    pub memory_id: MemoryId,
193    pub content_summary: String,
194    pub current_importance: f32,
195    pub linked: Vec<LinkSummary>,
196    pub corrections: Vec<MemoryId>,
197    pub contradictions: Vec<MemoryId>,
198    pub recent_activations: u32,
199}
200
201pub struct LinkSummary {
202    pub target: MemoryId,
203    pub link_type: LinkType,
204    pub strength: f32,
205}
206
207pub enum InspectQuery {
208    Memory(MemoryId),
209    Edges(MemoryId),
210    Channel(RecallChannel),
211    StoreStats,
212    QueueStatus,
213    StrongestEdges { limit: usize },
214    Contradictions { limit: usize },
215}
216
217pub enum InspectReport {
218    Memory(Box<MemoryInspect>),
219    StoreStats(StoreStats),
220    QueueStatus(QueueStatus),
221}
222
223pub struct MemoryInspect {
224    pub unit: MemoryUnit,
225    pub out_edges: Vec<EdgeView>,
226    pub in_edges: Vec<EdgeView>,
227    pub stage: MemoryStage,
228    pub lifecycle: MemoryLifecycle,
229}
230
231#[derive(Debug, Clone)]
232pub struct EdgeView {
233    pub from: MemoryId,
234    pub to: MemoryId,
235    pub link_type: LinkType,
236    pub strength: f32,
237    pub confidence: f32,
238    pub activation_count: u32,
239    pub evidence: String,
240}
241
242pub struct StoreStats {
243    pub memory_count: u64,
244    pub edge_count: u64,
245    pub observing_edge_count: u64,
246    pub per_index_size: Vec<(RecallChannel, u64)>,
247    pub queue_backlog: u64,
248    pub store_bytes: u64,
249}
250
251pub struct QueueStatus {
252    pub pending_enrich: u64,
253    pub pending_consolidate: u64,
254    pub in_flight: u64,
255    pub oldest_pending_age_ms: u64,
256}
257
258/// Consolidation report.
259pub struct ConsolidationReport {
260    pub memories_processed: u64,
261    pub edges_decayed: u64,
262    pub edges_archived: u64,
263    pub edges_merged: u64,
264    pub observation_promoted: u64,
265    pub summaries_created: u64,
266    pub contradictions_found: u64,
267    pub reindexed: bool,
268    pub elapsed_ms: u64,
269}
270
271/// Usage signal.
272#[derive(Debug, Clone, Copy, PartialEq, Eq)]
273pub enum UsageSignal {
274    Referenced,
275    UserConfirmedCorrect,
276    TaskSucceeded,
277    UserRejected,
278}
279
280/// Write warning.
281#[derive(Debug, Clone, PartialEq)]
282pub enum WriteWarning {
283    /// Degraded extractor was used
284    ExtractorDegraded,
285    /// Dense vector generation deferred
286    EmbeddingDeferred,
287    /// Strong semantic dimensions deferred to enrich
288    StrongDimsDeferred,
289    /// Model invocation failed and was degraded
290    ModelError { detail: String },
291}
292
293// ── List API types ──
294
295/// Input parameters for paginated memory listing.
296#[derive(Debug, Clone)]
297pub struct ListInput {
298    /// Page size, default 20, max 100.
299    pub limit: usize,
300    /// Cursor: pass the MemoryId (u128 value) of the last item on the previous page to get the next page.
301    pub cursor: Option<u128>,
302    /// Filter by ContentType; None = no filter.
303    pub content_type: Option<hippmem_core::model::enums::ContentType>,
304}
305
306impl Default for ListInput {
307    fn default() -> Self {
308        Self {
309            limit: 20,
310            cursor: None,
311            content_type: None,
312        }
313    }
314}
315
316/// Output of paginated memory listing.
317#[derive(Debug, Clone, Serialize)]
318pub struct ListOutput {
319    pub items: Vec<ListItem>,
320    /// Cursor for the next page; None means this is the last page.
321    pub next_cursor: Option<u128>,
322    /// Total memory count (approximate).
323    pub total: u64,
324}
325
326/// Summary of a single memory in the list.
327#[derive(Debug, Clone, Serialize)]
328pub struct ListItem {
329    pub id: hippmem_core::ids::MemoryId,
330    /// Content preview: first 100 chars of raw.
331    pub content_preview: String,
332    pub content_type: hippmem_core::model::enums::ContentType,
333    pub created_at: hippmem_core::time::Timestamp,
334    pub importance: f32,
335    pub stage: hippmem_core::model::unit::MemoryStage,
336    pub lifecycle: hippmem_core::model::unit::MemoryLifecycle,
337    /// Number of outgoing edges of this memory.
338    pub edge_count: usize,
339}
340
341// ── Dump API types ──
342
343/// Full export input parameters.
344#[derive(Debug, Clone, Default)]
345pub struct DumpInput {
346    /// Output file path; None = return a JSON string.
347    pub output_path: Option<std::path::PathBuf>,
348}
349
350/// Full export output.
351#[derive(Debug, Clone, Serialize)]
352pub struct DumpOutput {
353    pub count: u64,
354    /// Path echo when written to a file.
355    pub written_to: Option<std::path::PathBuf>,
356    /// JSONL string returned when output_path is None.
357    pub json: Option<String>,
358}
359
360// ── Traverse API types ──
361
362/// Graph traversal input parameters.
363#[derive(Debug, Clone)]
364pub struct TraverseInput {
365    /// Start memory ID.
366    pub start_id: hippmem_core::ids::MemoryId,
367    /// BFS max depth, default 2, max 5.
368    pub max_depth: u8,
369    /// Traversal direction.
370    pub direction: TraverseDirection,
371    /// Filter edges by LinkType; None = no filter.
372    pub link_types: Option<Vec<hippmem_core::model::links::LinkType>>,
373}
374
375impl TraverseInput {
376    /// Creates default traversal params from the specified ID (depth=2, outgoing, no filter).
377    pub fn new(start_id: hippmem_core::ids::MemoryId) -> Self {
378        Self {
379            start_id,
380            max_depth: 2,
381            direction: TraverseDirection::Outgoing,
382            link_types: None,
383        }
384    }
385}
386
387impl Default for TraverseInput {
388    fn default() -> Self {
389        Self {
390            start_id: hippmem_core::ids::MemoryId(0),
391            max_depth: 2,
392            direction: TraverseDirection::Outgoing,
393            link_types: None,
394        }
395    }
396}
397
398/// Traversal direction.
399#[derive(Debug, Clone, Copy, PartialEq, Eq)]
400pub enum TraverseDirection {
401    /// Only outgoing edges.
402    Outgoing,
403    /// Only incoming edges.
404    Incoming,
405    /// Both directions.
406    Both,
407}
408
409/// Graph traversal output.
410#[derive(Debug, Clone)]
411pub struct TraverseOutput {
412    /// Nodes visited by BFS (excluding the start node).
413    pub nodes: Vec<TraverseNode>,
414    /// Edges traversed.
415    pub edges: Vec<EdgeView>,
416}
417
418/// Node in BFS traversal.
419#[derive(Debug, Clone)]
420pub struct TraverseNode {
421    pub id: hippmem_core::ids::MemoryId,
422    /// BFS depth: 1 = direct neighbor, 2 = neighbor of neighbor...
423    pub depth: u8,
424    pub content_preview: String,
425    pub content_type: hippmem_core::model::enums::ContentType,
426    pub importance: f32,
427}
428
429// ── Conversion from lower-layer errors ──
430
431impl From<hippmem_store::store::StoreError> for EngineError {
432    fn from(e: hippmem_store::store::StoreError) -> Self {
433        EngineError::Store(e.to_string())
434    }
435}
436
437// ── EngineConfig ──
438
439/// Background worker configuration.
440#[derive(Debug, Clone)]
441pub struct BackgroundConfig {
442    /// Strong-semantic enrich concurrency, default 2.
443    pub enrich_workers: usize,
444    /// Consolidation concurrency, default 1.
445    pub consolidate_workers: usize,
446    /// Background queue capacity (bounded), default 4096.
447    pub queue_capacity: usize,
448    /// Periodic consolidation trigger interval (ms), default 3_600_000 (1h).
449    pub consolidate_interval_ms: u64,
450    /// Whether to enable enrich, default true.
451    pub enrich_enabled: bool,
452}
453
454impl Default for BackgroundConfig {
455    fn default() -> Self {
456        Self {
457            enrich_workers: 2,
458            consolidate_workers: 1,
459            queue_capacity: 4096,
460            consolidate_interval_ms: 3_600_000,
461            enrich_enabled: true,
462        }
463    }
464}
465
466/// Engine construction configuration.
467///
468/// Corresponds to 05 §0. Configures persistence path, algorithm params,
469/// model backend selection, and background workers.
470#[derive(Debug, Clone)]
471pub struct EngineConfig {
472    /// Storage directory (the redb file will be created under this directory).
473    pub store_dir: PathBuf,
474    /// Algorithm params, defaults to `AlgoParams::default()`.
475    pub algo: AlgoParams,
476    /// Embedder backend config, defaults to deterministic 256d SimHash (matches V3 behavior).
477    pub embedder: EmbedderConfig,
478    /// Backend selection (extractor/reranker/summarizer), all default to `Auto`.
479    pub backend: BackendSelection,
480    /// Background worker configuration.
481    pub background: BackgroundConfig,
482}
483
484impl Default for EngineConfig {
485    fn default() -> Self {
486        Self {
487            store_dir: PathBuf::from("./hippmem_data"),
488            algo: AlgoParams::default(),
489            embedder: EmbedderConfig::default(),
490            backend: BackendSelection::default(),
491            background: BackgroundConfig::default(),
492        }
493    }
494}
495
496// ── Engine ──
497
498/// HIPPMEM unified orchestration facade.
499///
500/// Holds the persistent storage, model registry, and algorithm params,
501/// and exposes seven core APIs externally.
502///
503/// Corresponds to 05 §0, 09 §1.
504pub struct Engine {
505    /// Persistent storage (redb).
506    store: Arc<RedbStore>,
507    /// Algorithm params (hot-swappable).
508    #[allow(dead_code)]
509    params: Arc<RwLock<AlgoParams>>,
510    /// Embedder backend (config-driven, default deterministic 256d SimHash).
511    embedder: Arc<dyn Embedder>,
512    /// Backend config (extractor/reranker/summarizer; embedder migrated to `self.embedder`).
513    #[allow(dead_code)]
514    backend: BackendSelection,
515    /// Tantivy fulltext index (fulltext/ subdirectory of store dir; internal Mutex supports &self writes).
516    fulltext_index: parking_lot::Mutex<FulltextIndex>,
517    /// Tantivy fulltext index directory path (used for Reindex rebuild).
518    fulltext_dir: PathBuf,
519    /// Binary code index (in-memory Hamming distance recall, 03 §4.5 SemanticBinary channel).
520    binary_code_index: parking_lot::Mutex<BinaryCodeIndex>,
521    /// Dense vector index (in-memory brute-force L2 KNN, 03 §4.5 SemanticDense channel).
522    dense_vector_index: parking_lot::Mutex<FlatVectorIndex>,
523}
524
525impl Engine {
526    /// Opens/creates a HIPPMEM memory store.
527    ///
528    /// Corresponds to 05 §0 `Engine::open`.
529    /// - Automatically creates the `store_dir` parent directory.
530    /// - If a redb file already exists at the specified path, opens the existing store.
531    /// - Builds the Embedder backend from `config.embedder` (default deterministic 256d, constitution C5).
532    pub fn open(config: EngineConfig) -> EngineResult<Self> {
533        // Automatically create parent directory
534        if let Some(parent) = config.store_dir.parent() {
535            std::fs::create_dir_all(parent).map_err(|e| {
536                EngineError::Store(format!("cannot create storage directory: {}", e))
537            })?;
538        }
539
540        // Build Embedder backend (config-driven)
541        let embedder =
542            build_embedder(&config.embedder).map_err(|e| EngineError::Model(e.to_string()))?;
543
544        // Open/create redb storage
545        let store = RedbStore::open(&config.store_dir)?;
546
547        // Create/open Tantivy fulltext index (same dir as redb, fulltext/ subdirectory)
548        let fulltext_dir = config
549            .store_dir
550            .parent()
551            .map(|p| p.join("fulltext"))
552            .unwrap_or_else(|| PathBuf::from("hippmem_data").join("fulltext"));
553        let fulltext_index = FulltextIndex::open(&fulltext_dir)
554            .or_else(|_| FulltextIndex::create(&fulltext_dir))
555            .map_err(|e| {
556                EngineError::Store(format!("Tantivy index initialization failed: {}", e))
557            })?;
558
559        Ok(Self {
560            store: Arc::new(store),
561            params: Arc::new(RwLock::new(config.algo)),
562            embedder,
563            backend: config.backend,
564            fulltext_index: parking_lot::Mutex::new(fulltext_index),
565            fulltext_dir,
566            binary_code_index: parking_lot::Mutex::new(BinaryCodeIndex::new()),
567            dense_vector_index: parking_lot::Mutex::new(FlatVectorIndex::new()),
568        })
569    }
570
571    /// Graceful shutdown.
572    ///
573    /// Corresponds to 05 §0 `Engine::close`.
574    /// Currently only drops the store (redb auto-flushes); in the future it will wait for background workers to exit.
575    pub fn close(self) -> EngineResult<()> {
576        // Tantivy: commit unwritten documents and close
577        if let Err(e) = self.fulltext_index.lock().flush() {
578            // Non-fatal; tracing warn, does not block close
579            eprintln!("Tantivy flush failed: {}", e);
580        }
581        // store is dropped; redb auto-flushes and closes
582        drop(self.store);
583        Ok(())
584    }
585
586    /// Sets the fulltext index batch commit interval (auto commit every N entries).
587    /// Only used in batch write scenarios; production defaults to per-entry commit.
588    pub fn set_fulltext_commit_every(&self, n: usize) {
589        self.fulltext_index.lock().set_commit_every(n);
590    }
591
592    /// Force-commits all unwritten documents in the fulltext index.
593    pub fn flush_fulltext(&self) {
594        let _ = self.fulltext_index.lock().flush();
595    }
596}
hippmem_engine/lib.rs

hippmem_engine/
lib.rs
