hippmem_engine/
lib.rs

1//! HIPPMEM · Native Association Memory Engine — unified external Rust API orchestration layer.
2//!
3//! Integrates [`hippmem_core`], [`hippmem_store`], [`hippmem_model`],
4//! [`hippmem_write`], [`hippmem_retrieval`], [`hippmem_consolidation`],
5//! providing seven core APIs: `write/retrieve/explain/consolidate/inspect/feedback`.
6//!
7//! Corresponds to 05-api-contract, 09-engine-assembly.
8
9pub mod consolidate_api;
10pub mod dump_api;
11pub mod explain_api;
12pub mod feedback_api;
13pub mod inspect_api;
14pub mod list_api;
15pub mod retrieve_api;
16pub mod runtime;
17mod signals;
18pub mod traverse_api;
19pub mod write_api;
20
21use hippmem_core::config::{AlgoParams, EmbedderConfig};
22use hippmem_core::ids::MemoryId;
23use hippmem_core::model::links::AssociationLink;
24use hippmem_core::model::understanding::MemoryUnderstanding;
25use hippmem_core::model::unit::{MemoryStage, WriteContext};
26use hippmem_model::registry::{build_embedder, BackendSelection};
27use hippmem_model::traits::Embedder;
28use hippmem_store::fulltext::FulltextIndex;
29use hippmem_store::semantic::binary::BinaryCodeIndex;
30use hippmem_store::semantic::hnsw::FlatVectorIndex;
31use hippmem_store::store::{RedbStore, Store};
32use parking_lot::RwLock;
33use serde::Serialize;
34use std::path::PathBuf;
35use std::sync::Arc;
36
37// ── EngineError ──
38
39/// Engine external error type: converts lower-layer errors into a unified external error code.
40///
41/// Corresponds to 05 §7. MUST NOT expose underlying library types (constitution C2).
42#[derive(Debug, thiserror::Error)]
43pub enum EngineError {
44    /// Underlying storage error.
45    #[error("store: {0}")]
46    Store(String),
47
48    /// Memory not found.
49    #[error("not found: {0:?}")]
50    NotFound(MemoryId),
51
52    /// Invalid input parameter.
53    #[error("invalid input: {0}")]
54    InvalidInput(String),
55
56    /// Incompatible schema version.
57    #[error("schema too new: {0}")]
58    SchemaTooNew(u16),
59
60    /// Model invocation failed (non-fatal).
61    #[error("model: {0}")]
62    Model(String),
63
64    /// Backend unavailable (API key missing or network error).
65    #[error("backend unavailable: {0}")]
66    BackendUnavailable(String),
67
68    /// Internal error.
69    #[error("internal: {0}")]
70    Internal(String),
71}
72
73/// Engine-layer Result alias.
74pub type EngineResult<T> = Result<T, EngineError>;
75
76// ── Write types (05 §1) ──
77
78/// Input for writing a memory.
79pub struct WriteMemoryInput {
80    pub content: String,
81    pub content_type: Option<hippmem_core::model::enums::ContentType>,
82    pub context: WriteContext,
83    pub importance_hint: Option<f32>,
84    pub source_refs: Vec<hippmem_core::model::unit::SourceRef>,
85}
86
87/// Output of writing a memory.
88pub struct WriteMemoryOutput {
89    pub memory_id: MemoryId,
90    pub stage_reached: MemoryStage,
91    pub created_links: Vec<AssociationLink>,
92    pub understanding: MemoryUnderstanding,
93    pub warnings: Vec<WriteWarning>,
94}
95
96// ── Retrieval types (05 §2) ──
97
98use hippmem_core::model::links::{RecallChannel, RetrievalResult};
99
100/// Retrieval context.
101#[derive(Debug, Clone, Default)]
102pub struct RetrieveContext {
103    pub conversation_id: Option<u64>,
104    pub session_id: Option<u64>,
105    pub project_id: Option<u64>,
106    pub task_id: Option<u64>,
107    pub user_id: Option<u64>,
108    pub recent_memory_ids: Vec<MemoryId>,
109}
110
111/// Retrieval input.
112pub struct RetrieveInput {
113    pub query: String,
114    pub context: RetrieveContext,
115    pub top_k: usize,
116    pub max_hops: Option<usize>,
117    pub retrieval_mode: hippmem_core::model::links::RetrievalMode,
118}
119
120/// Retrieval output.
121pub struct RetrieveOutput {
122    /// Identifier for this retrieval, used for feedback (see `Engine::feedback`).
123    pub retrieval_id: u64,
124    pub results: Vec<RetrievalResult>,
125    pub trace: RetrievalTrace,
126    pub diagnostics: RetrievalDiagnostics,
127}
128
129/// Retrieval trace.
130pub struct RetrievalTrace {
131    pub seeds: Vec<SeedRecord>,
132    pub steps: Vec<hippmem_core::model::links::ActivationStep>,
133    pub hops_used: u8,
134    pub merged_count: usize,
135}
136
137/// Seed record.
138pub struct SeedRecord {
139    pub id: MemoryId,
140    pub channel: RecallChannel,
141    pub initial_energy: f32,
142    /// V9: in-channel rank (0 = best), for RRF diagnostics
143    pub rank_in_channel: Option<usize>,
144}
145
146/// Retrieval diagnostics.
147pub struct RetrievalDiagnostics {
148    pub channel_contributions: Vec<(RecallChannel, u32)>,
149    pub reranked: bool,
150    pub pruned_branches: u32,
151    pub backend_used: BackendUsage,
152    pub latency_ms: u32,
153}
154
155/// Backend usage info.
156pub struct BackendUsage {
157    pub embedder: String,
158    pub reranker: Option<String>,
159}
160
161// ── Feedback types (05 §6) ──
162
163/// Usage feedback input.
164pub struct FeedbackInput {
165    pub retrieval_id: u64,
166    pub used_memory_ids: Vec<MemoryId>,
167    pub signal: UsageSignal,
168}
169
170// ── Consolidation types (05 §5) ──
171
172/// Consolidation scope.
173#[derive(Debug, Clone)]
174pub enum ConsolidationScope {
175    Full,
176    Incremental,
177    ByMemoryType(hippmem_core::model::enums::ContentType),
178    ByTimeRange {
179        from: hippmem_core::time::Timestamp,
180        to: hippmem_core::time::Timestamp,
181    },
182    Reindex,
183    EdgesOnly,
184}
185
186// ── Explain/diagnostics types (05 §4 §6) ──
187
188use hippmem_core::model::links::LinkType;
189use hippmem_core::model::unit::{MemoryLifecycle, MemoryUnit};
190
191/// Explain output.
192pub struct Explanation {
193    pub memory_id: MemoryId,
194    pub content_summary: String,
195    pub current_importance: f32,
196    pub linked: Vec<LinkSummary>,
197    pub corrections: Vec<MemoryId>,
198    pub contradictions: Vec<MemoryId>,
199    pub recent_activations: u32,
200}
201
202pub struct LinkSummary {
203    pub target: MemoryId,
204    pub link_type: LinkType,
205    pub strength: f32,
206}
207
208pub enum InspectQuery {
209    Memory(MemoryId),
210    Edges(MemoryId),
211    Channel(RecallChannel),
212    StoreStats,
213    QueueStatus,
214    StrongestEdges { limit: usize },
215    Contradictions { limit: usize },
216}
217
218pub enum InspectReport {
219    Memory(Box<MemoryInspect>),
220    StoreStats(StoreStats),
221    QueueStatus(QueueStatus),
222}
223
224pub struct MemoryInspect {
225    pub unit: MemoryUnit,
226    pub out_edges: Vec<EdgeView>,
227    pub in_edges: Vec<EdgeView>,
228    pub stage: MemoryStage,
229    pub lifecycle: MemoryLifecycle,
230}
231
232#[derive(Debug, Clone)]
233pub struct EdgeView {
234    pub from: MemoryId,
235    pub to: MemoryId,
236    pub link_type: LinkType,
237    pub strength: f32,
238    pub confidence: f32,
239    pub activation_count: u32,
240    pub evidence: String,
241}
242
243pub struct StoreStats {
244    pub memory_count: u64,
245    pub edge_count: u64,
246    pub observing_edge_count: u64,
247    pub per_index_size: Vec<(RecallChannel, u64)>,
248    pub queue_backlog: u64,
249    pub store_bytes: u64,
250}
251
252pub struct QueueStatus {
253    pub pending_enrich: u64,
254    pub pending_consolidate: u64,
255    pub in_flight: u64,
256    pub oldest_pending_age_ms: u64,
257}
258
259/// Consolidation report.
260pub struct ConsolidationReport {
261    pub memories_processed: u64,
262    pub edges_decayed: u64,
263    pub edges_archived: u64,
264    pub edges_merged: u64,
265    pub observation_promoted: u64,
266    pub summaries_created: u64,
267    pub contradictions_found: u64,
268    pub reindexed: bool,
269    pub elapsed_ms: u64,
270}
271
272/// Usage signal.
273#[derive(Debug, Clone, Copy, PartialEq, Eq)]
274pub enum UsageSignal {
275    Referenced,
276    UserConfirmedCorrect,
277    TaskSucceeded,
278    UserRejected,
279}
280
281/// Write warning.
282#[derive(Debug, Clone, PartialEq)]
283pub enum WriteWarning {
284    /// Degraded extractor was used
285    ExtractorDegraded,
286    /// Dense vector generation deferred
287    EmbeddingDeferred,
288    /// Strong semantic dimensions deferred to enrich
289    StrongDimsDeferred,
290    /// Model invocation failed and was degraded
291    ModelError { detail: String },
292}
293
294// ── List API types ──
295
296/// Input parameters for paginated memory listing.
297#[derive(Debug, Clone)]
298pub struct ListInput {
299    /// Page size, default 20, max 100.
300    pub limit: usize,
301    /// Cursor: pass the MemoryId (u128 value) of the last item on the previous page to get the next page.
302    pub cursor: Option<u128>,
303    /// Filter by ContentType; None = no filter.
304    pub content_type: Option<hippmem_core::model::enums::ContentType>,
305}
306
307impl Default for ListInput {
308    fn default() -> Self {
309        Self {
310            limit: 20,
311            cursor: None,
312            content_type: None,
313        }
314    }
315}
316
317/// Output of paginated memory listing.
318#[derive(Debug, Clone, Serialize)]
319pub struct ListOutput {
320    pub items: Vec<ListItem>,
321    /// Cursor for the next page; None means this is the last page.
322    pub next_cursor: Option<u128>,
323    /// Total memory count (approximate).
324    pub total: u64,
325}
326
327/// Summary of a single memory in the list.
328#[derive(Debug, Clone, Serialize)]
329pub struct ListItem {
330    pub id: hippmem_core::ids::MemoryId,
331    /// Content preview: first 100 chars of raw.
332    pub content_preview: String,
333    pub content_type: hippmem_core::model::enums::ContentType,
334    pub created_at: hippmem_core::time::Timestamp,
335    pub importance: f32,
336    pub stage: hippmem_core::model::unit::MemoryStage,
337    pub lifecycle: hippmem_core::model::unit::MemoryLifecycle,
338    /// Number of outgoing edges of this memory.
339    pub edge_count: usize,
340}
341
342// ── Dump API types ──
343
344/// Full export input parameters.
345#[derive(Debug, Clone, Default)]
346pub struct DumpInput {
347    /// Output file path; None = return a JSON string.
348    pub output_path: Option<std::path::PathBuf>,
349}
350
351/// Full export output.
352#[derive(Debug, Clone, Serialize)]
353pub struct DumpOutput {
354    pub count: u64,
355    /// Path echo when written to a file.
356    pub written_to: Option<std::path::PathBuf>,
357    /// JSONL string returned when output_path is None.
358    pub json: Option<String>,
359}
360
361// ── Traverse API types ──
362
363/// Graph traversal input parameters.
364#[derive(Debug, Clone)]
365pub struct TraverseInput {
366    /// Start memory ID.
367    pub start_id: hippmem_core::ids::MemoryId,
368    /// BFS max depth, default 2, max 5.
369    pub max_depth: u8,
370    /// Traversal direction.
371    pub direction: TraverseDirection,
372    /// Filter edges by LinkType; None = no filter.
373    pub link_types: Option<Vec<hippmem_core::model::links::LinkType>>,
374}
375
376impl TraverseInput {
377    /// Creates default traversal params from the specified ID (depth=2, outgoing, no filter).
378    pub fn new(start_id: hippmem_core::ids::MemoryId) -> Self {
379        Self {
380            start_id,
381            max_depth: 2,
382            direction: TraverseDirection::Outgoing,
383            link_types: None,
384        }
385    }
386}
387
388impl Default for TraverseInput {
389    fn default() -> Self {
390        Self {
391            start_id: hippmem_core::ids::MemoryId(0),
392            max_depth: 2,
393            direction: TraverseDirection::Outgoing,
394            link_types: None,
395        }
396    }
397}
398
399/// Traversal direction.
400#[derive(Debug, Clone, Copy, PartialEq, Eq)]
401pub enum TraverseDirection {
402    /// Only outgoing edges.
403    Outgoing,
404    /// Only incoming edges.
405    Incoming,
406    /// Both directions.
407    Both,
408}
409
410/// Graph traversal output.
411#[derive(Debug, Clone)]
412pub struct TraverseOutput {
413    /// Nodes visited by BFS (excluding the start node).
414    pub nodes: Vec<TraverseNode>,
415    /// Edges traversed.
416    pub edges: Vec<EdgeView>,
417}
418
419/// Node in BFS traversal.
420#[derive(Debug, Clone)]
421pub struct TraverseNode {
422    pub id: hippmem_core::ids::MemoryId,
423    /// BFS depth: 1 = direct neighbor, 2 = neighbor of neighbor...
424    pub depth: u8,
425    pub content_preview: String,
426    pub content_type: hippmem_core::model::enums::ContentType,
427    pub importance: f32,
428}
429
430// ── Conversion from lower-layer errors ──
431
432impl From<hippmem_store::store::StoreError> for EngineError {
433    fn from(e: hippmem_store::store::StoreError) -> Self {
434        EngineError::Store(e.to_string())
435    }
436}
437
438// ── EngineConfig ──
439
440/// Background worker configuration.
441#[derive(Debug, Clone)]
442pub struct BackgroundConfig {
443    /// Strong-semantic enrich concurrency, default 2.
444    pub enrich_workers: usize,
445    /// Consolidation concurrency, default 1.
446    pub consolidate_workers: usize,
447    /// Background queue capacity (bounded), default 4096.
448    pub queue_capacity: usize,
449    /// Periodic consolidation trigger interval (ms), default 3_600_000 (1h).
450    pub consolidate_interval_ms: u64,
451    /// Whether to enable enrich, default true.
452    pub enrich_enabled: bool,
453}
454
455impl Default for BackgroundConfig {
456    fn default() -> Self {
457        Self {
458            enrich_workers: 2,
459            consolidate_workers: 1,
460            queue_capacity: 4096,
461            consolidate_interval_ms: 3_600_000,
462            enrich_enabled: true,
463        }
464    }
465}
466
467/// Engine construction configuration.
468///
469/// Corresponds to 05 §0. Configures persistence path, algorithm params,
470/// model backend selection, and background workers.
471#[derive(Debug, Clone)]
472pub struct EngineConfig {
473    /// Storage directory (the redb file will be created under this directory).
474    pub store_dir: PathBuf,
475    /// Algorithm params, defaults to `AlgoParams::default()`.
476    pub algo: AlgoParams,
477    /// Embedder backend config, defaults to deterministic 256d SimHash (matches V3 behavior).
478    pub embedder: EmbedderConfig,
479    /// Backend selection (extractor/reranker/summarizer), all default to `Auto`.
480    pub backend: BackendSelection,
481    /// Background worker configuration.
482    pub background: BackgroundConfig,
483}
484
485impl Default for EngineConfig {
486    fn default() -> Self {
487        Self {
488            store_dir: PathBuf::from("./hippmem_data"),
489            algo: AlgoParams::default(),
490            embedder: EmbedderConfig::default(),
491            backend: BackendSelection::default(),
492            background: BackgroundConfig::default(),
493        }
494    }
495}
496
497// ── Engine ──
498
499/// HIPPMEM unified orchestration facade.
500///
501/// Holds the persistent storage, model registry, and algorithm params,
502/// and exposes seven core APIs externally.
503///
504/// Corresponds to 05 §0, 09 §1.
505pub struct Engine {
506    /// Persistent storage (redb).
507    store: Arc<RedbStore>,
508    /// Algorithm params (hot-swappable).
509    #[allow(dead_code)]
510    params: Arc<RwLock<AlgoParams>>,
511    /// Embedder backend (config-driven, default deterministic 256d SimHash).
512    embedder: Arc<dyn Embedder>,
513    /// Backend config (extractor/reranker/summarizer; embedder migrated to `self.embedder`).
514    #[allow(dead_code)]
515    backend: BackendSelection,
516    /// Tantivy fulltext index (fulltext/ subdirectory of store dir; internal Mutex supports &self writes).
517    fulltext_index: parking_lot::Mutex<FulltextIndex>,
518    /// Tantivy fulltext index directory path (used for Reindex rebuild).
519    fulltext_dir: PathBuf,
520    /// Binary code index (in-memory Hamming distance recall, 03 §4.5 SemanticBinary channel).
521    binary_code_index: parking_lot::Mutex<BinaryCodeIndex>,
522    /// Dense vector index (in-memory brute-force L2 KNN, 03 §4.5 SemanticDense channel).
523    dense_vector_index: parking_lot::Mutex<FlatVectorIndex>,
524}
525
526impl Engine {
527    /// Opens/creates a HIPPMEM memory store.
528    ///
529    /// Corresponds to 05 §0 `Engine::open`.
530    /// - Automatically creates the `store_dir` parent directory.
531    /// - If a redb file already exists at the specified path, opens the existing store.
532    /// - Builds the Embedder backend from `config.embedder` (default deterministic 256d, constitution C5).
533    pub fn open(config: EngineConfig) -> EngineResult<Self> {
534        // Automatically create parent directory
535        if let Some(parent) = config.store_dir.parent() {
536            std::fs::create_dir_all(parent).map_err(|e| {
537                EngineError::Store(format!("cannot create storage directory: {}", e))
538            })?;
539        }
540
541        // Build Embedder backend (config-driven)
542        let embedder =
543            build_embedder(&config.embedder).map_err(|e| EngineError::Model(e.to_string()))?;
544
545        // Open/create redb storage
546        let store = RedbStore::open(&config.store_dir)?;
547
548        // Create/open Tantivy fulltext index (same dir as redb, fulltext/ subdirectory)
549        let fulltext_dir = config
550            .store_dir
551            .parent()
552            .map(|p| p.join("fulltext"))
553            .unwrap_or_else(|| PathBuf::from("hippmem_data").join("fulltext"));
554        let fulltext_index = FulltextIndex::open(&fulltext_dir)
555            .or_else(|_| FulltextIndex::create(&fulltext_dir))
556            .map_err(|e| {
557                EngineError::Store(format!("Tantivy index initialization failed: {}", e))
558            })?;
559
560        Ok(Self {
561            store: Arc::new(store),
562            params: Arc::new(RwLock::new(config.algo)),
563            embedder,
564            backend: config.backend,
565            fulltext_index: parking_lot::Mutex::new(fulltext_index),
566            fulltext_dir,
567            binary_code_index: parking_lot::Mutex::new(BinaryCodeIndex::new()),
568            dense_vector_index: parking_lot::Mutex::new(FlatVectorIndex::new()),
569        })
570    }
571
572    /// Graceful shutdown.
573    ///
574    /// Corresponds to 05 §0 `Engine::close`.
575    /// Currently only drops the store (redb auto-flushes); in the future it will wait for background workers to exit.
576    pub fn close(self) -> EngineResult<()> {
577        // Tantivy: commit unwritten documents and close
578        if let Err(e) = self.fulltext_index.lock().flush() {
579            // Non-fatal; tracing warn, does not block close
580            eprintln!("Tantivy flush failed: {}", e);
581        }
582        // store is dropped; redb auto-flushes and closes
583        drop(self.store);
584        Ok(())
585    }
586
587    /// Sets the fulltext index batch commit interval (auto commit every N entries).
588    /// Only used in batch write scenarios; production defaults to per-entry commit.
589    pub fn set_fulltext_commit_every(&self, n: usize) {
590        self.fulltext_index.lock().set_commit_every(n);
591    }
592
593    /// Force-commits all unwritten documents in the fulltext index.
594    pub fn flush_fulltext(&self) {
595        let _ = self.fulltext_index.lock().flush();
596    }
597}
hippmem_engine/lib.rs

hippmem_engine/
lib.rs
