zeph_memory/
lib.rs

1// SPDX-FileCopyrightText: 2026 Andrei G <bug-ops>
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4//! Semantic memory layer for the Zeph agent.
5//!
6//! `zeph-memory` implements a two-backend hybrid memory system:
7//!
8//! - **[`store::SqliteStore`]** — relational persistence for messages, summaries,
9//!   persona facts, trajectory entries, and session metadata.
10//! - **[`embedding_store::EmbeddingStore`]** — Qdrant-backed vector index for semantic recall.
11//!   Falls back gracefully to [`db_vector_store::DbVectorStore`] when Qdrant is unavailable.
12//!
13//! The high-level entry point is [`semantic::SemanticMemory`], which combines both backends
14//! and exposes `remember` / `recall` / `summarize` operations consumed by `zeph-core`.
15//!
16//! # Architecture overview
17//!
18//! ```text
19//! SemanticMemory
20//! ├── SqliteStore  ── messages, summaries, corrections, persona, trajectory …
21//! └── EmbeddingStore ── Qdrant (primary) / DbVectorStore (fallback)
22//!         └── QdrantOps  ── thin gRPC wrapper over qdrant-client
23//! ```
24//!
25//! # Memory tiers
26//!
27//! Messages are classified into four tiers (see [`types::MemoryTier`]):
28//!
29//! | Tier | Description |
30//! |------|-------------|
31//! | `Working` | Current context window; never persisted. |
32//! | `Episodic` | Per-session messages stored in `SQLite`. |
33//! | `Semantic` | Cross-session distilled facts promoted from episodic. |
34//! | `Persona` | Long-lived user attributes (preferences, domain knowledge). |
35//!
36//! # Admission control
37//!
38//! Each `remember()` call is gated by [`admission::AdmissionControl`] (A-MAC, #2317), which
39//! evaluates five factors (future utility, factual confidence, semantic novelty, temporal
40//! recency, content-type prior) and rejects low-value messages before they reach the DB.
41//!
42//! # Memory routing
43//!
44//! [`router::HybridRouter`] classifies each recall query and dispatches to the appropriate
45//! backend: keyword (`SQLite` FTS5), semantic (Qdrant), graph (BFS traversal), episodic
46//! (timestamp-filtered FTS5), or hybrid (reciprocal-rank fusion of keyword + semantic).
47//!
48//! # Background loops
49//!
50//! Several background tasks maintain memory health:
51//!
52//! - [`eviction::start_eviction_loop`] — Ebbinghaus-curve eviction.
53//! - [`forgetting::start_forgetting_loop`] — `SleepGate` importance downscaling.
54//! - [`consolidation::start_consolidation_loop`] — cross-session fact merging.
55//! - [`tiers::start_tier_promotion_loop`] — Episodic → Semantic promotion.
56//! - [`semantic::start_tree_consolidation_loop`] — hierarchical note consolidation.
57//! - [`hebbian_consolidation::spawn_consolidation_loop`] — HL-F3/F4 cluster distillation.
58//! - [`episodic_consolidation::start_episodic_consolidation_loop`] — episodic → semantic fact extraction.
59//!
60//! # Feature flags
61//!
62//! | Feature | Description |
63//! |---------|-------------|
64//! | `sqlite` (default) | Enable SQLite persistence via `zeph-db`. |
65//! | `pdf` | Enable `PdfLoader` for PDF ingestion. |
66//! | `postgres` | Enable PostgreSQL support via `zeph-db`. |
67
68/// Upper bound clamped onto every caller-supplied `limit`/`top_k` search parameter
69/// inside this crate (issue #6553).
70///
71/// [`embedding_store::EmbeddingStore::search`], [`embedding_store::EmbeddingStore::search_collection`],
72/// [`embedding_registry::EmbeddingRegistry::search_raw`], and
73/// [`reasoning::ReasoningMemory::retrieve_by_embedding`] all enforce this bound directly, so the
74/// safety guarantee does not depend on every external caller (MCP tools, plugins, future
75/// call sites) remembering to clamp before forwarding a value here.
76///
77/// `100` bounds the Qdrant result set Zeph will allocate/deserialize in a single call. Note
78/// this **can** clamp a legitimate config-driven candidate pool: `memory.retrieval.depth` and
79/// `memory.session.recall_limit` have no upper-bound validation today, and `retrieval.depth`'s
80/// own doc actively recommends raising it ("higher for better MMR diversity") with no ceiling
81/// — an operator who does so past `100` gets a silently smaller ANN pool than configured. Each
82/// clamping call site logs a `tracing::warn!` the first time this happens so the degradation is
83/// observable rather than silent; there is deliberately no config knob to raise this ceiling,
84/// since doing so would reopen the oversized-result-set `DoS` this constant exists to close.
85///
86/// # Examples
87///
88/// ```
89/// use zeph_memory::MAX_SEARCH_LIMIT;
90///
91/// let requested = 5_000_usize;
92/// assert_eq!(requested.clamp(1, MAX_SEARCH_LIMIT), MAX_SEARCH_LIMIT);
93/// ```
94pub const MAX_SEARCH_LIMIT: usize = 100;
95
96/// Log a one-shot `tracing::warn!` the first time a caller's requested search limit is
97/// actually reduced by [`MAX_SEARCH_LIMIT`] (issue #6553 follow-up).
98///
99/// `warned` is a call-site-local flag (a `static AtomicBool`) so each of the four clamping
100/// functions warns independently and at most once per process — a config-driven candidate
101/// pool that regularly exceeds the ceiling would otherwise log on every search call.
102pub(crate) fn warn_if_search_limit_clamped(
103    site: &'static str,
104    requested: usize,
105    warned: &std::sync::atomic::AtomicBool,
106) {
107    if requested > MAX_SEARCH_LIMIT && !warned.swap(true, std::sync::atomic::Ordering::Relaxed) {
108        tracing::warn!(
109            site,
110            requested,
111            max = MAX_SEARCH_LIMIT,
112            "requested search limit exceeds MAX_SEARCH_LIMIT and was clamped — a config-driven \
113             candidate pool (e.g. memory.retrieval.depth) larger than this will silently return \
114             a smaller ANN pool than configured"
115        );
116    }
117}
118
119pub mod admission;
120pub mod anchored_summary;
121pub mod compaction_probe;
122pub mod compression;
123pub mod compression_guidelines;
124pub mod consolidation;
125pub mod document;
126pub mod episodic_consolidation;
127pub mod episodic_graph;
128pub mod facade;
129pub mod five_signal;
130pub mod forgetting;
131pub mod hebbian_consolidation;
132pub mod optical_forgetting;
133pub mod reasoning;
134pub mod recall_view;
135pub mod retrieval_failure_logger;
136pub mod scenes;
137mod sweep_helpers;
138pub mod tiered_retrieval;
139pub mod tiers;
140
141pub mod db_vector_store;
142pub mod embed_probe;
143pub mod embedding_registry;
144pub mod embedding_store;
145pub mod error;
146pub mod eviction;
147pub mod graph;
148pub mod in_memory_store;
149mod llm_judge;
150pub mod qdrant_ops;
151pub mod quality_gate;
152pub mod response_cache;
153pub mod router;
154pub mod semantic;
155pub mod shadow;
156pub mod snapshot;
157mod sqlite_time;
158pub mod store;
159#[cfg(any(test, feature = "testing"))]
160pub mod testing;
161pub mod token_counter;
162pub mod types;
163pub mod vector_store;
164
165pub use admission::{
166    AdmissionControl, AdmissionDecision, AdmissionFactors, AdmissionRejected, AdmissionWeights,
167    GoalGateConfig, compute_content_type_prior, compute_factual_confidence, log_admission_decision,
168};
169pub use anchored_summary::AnchoredSummary;
170pub use compaction_probe::{
171    CategoryScore, CompactionProbeResult, ProbeQuestion, ProbeVerdict, answer_probe_questions,
172    generate_probe_questions, score_answers, validate_compaction,
173};
174pub use compression::{
175    CompressionLevel, RetrievalPolicy,
176    promotion::{
177        PromotionCandidate, PromotionConfig, PromotionEngine, PromotionInput, SkillWriter,
178    },
179};
180pub use compression_guidelines::{
181    build_guidelines_update_prompt, sanitize_guidelines, start_guidelines_updater,
182    truncate_to_token_budget, update_guidelines_once,
183};
184pub use consolidation::{
185    ConsolidationConfig, ConsolidationResult, TopologyOp, run_consolidation_sweep,
186    start_consolidation_loop,
187};
188#[cfg(feature = "pdf")]
189pub use document::PdfLoader;
190pub use document::{
191    Chunk, Document, DocumentError, DocumentLoader, DocumentMetadata, IngestionPipeline,
192    SplitterConfig, TextLoader, TextSplitter,
193};
194pub use embed_probe::{ProbeError, probe_vector_size};
195pub use embedding_registry::{
196    EmbedFuture, Embeddable, EmbeddingRegistry, EmbeddingRegistryError, SyncStats,
197};
198pub use embedding_store::ensure_qdrant_collection;
199pub use episodic_consolidation::{
200    EpisodicConsolidationConfig, EpisodicConsolidationResult, run_episodic_consolidation_sweep,
201    start_episodic_consolidation_loop,
202};
203pub use episodic_graph::{
204    CausalLink, EmGraphConfig, EpisodicEvent, extract_events, fetch_recent_events, link_events,
205    recall_episodic_causal, store_events, store_links,
206};
207pub use error::MemoryError;
208pub use eviction::{EbbinghausPolicy, EvictionPolicy, start_eviction_loop};
209pub use facade::{
210    CompactionContext, CompactionResult, InMemoryFacade, MemoryEntry, MemoryFacade, MemoryMatch,
211    MemorySource,
212};
213pub use forgetting::{ForgettingConfig, ForgettingResult, start_forgetting_loop};
214pub use graph::EntityLockManager;
215pub use graph::experience::{EvolutionSweepStats, ExperienceStore};
216pub use graph::ingest::{
217    BatchIdResolution, ClaudeCodeJsonl, CodexJsonl, HubDegree, ImportBatchId, IngestDocument,
218    IngestFailure, IngestLedger, IngestProgress, IngestReport, IngestSourceAdapter,
219    IngestSourceKind, LedgerEntry, SubagentJsonl, TranscriptEntry,
220};
221pub use graph::{
222    BeliefMemConfig, BeliefRevisionConfig, BeliefStore, Community, Edge, EdgeType, Entity,
223    EntityType, GraphFact, GraphOrigin, GraphProvenance, GraphStore, PendingBelief, RpeRouter,
224    RpeSignal, extract_candidate_entities, noisy_or, time_decayed_prob,
225};
226pub use hebbian_consolidation::{
227    GraphRule, HebbianConsolidationCandidate, HebbianConsolidationOutcome,
228    run_consolidation_sweep as run_hebbian_consolidation_sweep,
229    spawn_consolidation_loop as spawn_hebbian_consolidation_loop,
230};
231pub use optical_forgetting::{
232    ContentFidelity, OpticalForgettingConfig, OpticalForgettingResult,
233    run_optical_forgetting_sweep, start_optical_forgetting_loop,
234};
235pub use qdrant_ops::QdrantOps;
236pub use reasoning::{
237    Outcome, ProcessTurnConfig, ReasoningMemory, ReasoningStrategy, SelfJudgeOutcome,
238    distill_strategy, process_turn as process_reasoning_turn, run_self_judge,
239};
240pub use recall_view::{RecallView, RecalledFact};
241pub use response_cache::ResponseCache;
242pub use retrieval_failure_logger::RetrievalFailureLogger;
243pub use router::{
244    AsyncMemoryRouter, HeuristicRouter, HybridRouter, LlmRouter, MemoryRoute, MemoryRouter,
245    RoutingDecision, TemporalRange, classify_graph_subgraph, parse_route_str,
246    strip_temporal_keywords,
247};
248pub use scenes::{
249    MemScene, SceneConfig, consolidate_scenes, list_scenes, start_scene_consolidation_loop,
250};
251pub use semantic::{
252    EmbedContext, ExtractionResult, ExtractionStats, GraphExtractionConfig, HebbianReinforcement,
253    HelaSpreadRuntime, ImportanceScoring, IngestBatchConfig, LinkingStats, MmrReranking,
254    NoteLinkingConfig, PersonaExtractionConfig, QueryBiasCorrection, RecalledMessage,
255    SharedPostExtractValidator, StructuredSummary, TemporalDecay, TrajectoryEntry,
256    TrajectoryExtractionConfig, TreeConsolidationConfig, TreeConsolidationResult,
257    build_summarization_prompt, contains_self_referential_language, extract_and_store,
258    extract_persona_facts, extract_trajectory_entries, link_memory_notes,
259    run_tree_consolidation_sweep, start_tree_consolidation_loop,
260};
261pub use snapshot::{ImportStats, MemorySnapshot, export_snapshot, import_snapshot};
262pub use store::agent_sessions::{AgentSessionRow, SessionChannel, SessionKind, SessionStatus};
263pub use store::compression_guidelines::CompressionFailurePair;
264pub use store::corrections::UserCorrectionRow;
265pub use store::experiments::{ExperimentResultRow, NewExperimentResult, SessionSummaryRow};
266pub use store::memory_tree::MemoryTreeRow;
267pub use store::persona::PersonaFactRow;
268pub use store::retrieval_failures::{RetrievalFailureRecord, RetrievalFailureType};
269pub use store::session_digest::SessionDigest;
270pub use store::trajectory::{NewTrajectoryEntry, TrajectoryEntryRow};
271pub use tiered_retrieval::{
272    IntentClass, TieredRetrievalConfig, TieredRetrievalResult, recall_tiered,
273};
274pub use tiers::{TierPromotionConfig, start_tier_promotion_loop};
275pub use token_counter::TokenCounter;
276pub use tokio_util::sync::CancellationToken;
277pub use types::{
278    ConversationId, EntityId, ExperienceId, MemSceneId, MemoryTier, MessageId, UsageRecord,
279    UsageSource,
280};
281pub use vector_store::{
282    FieldCondition, FieldValue, ScoredVectorPoint, VectorFilter, VectorPoint, VectorStore,
283    VectorStoreError,
284};
285pub use zeph_common::config::memory::HebbianConsolidationConfig;
286pub use zeph_common::memory::{FunctionalType, TokenCounting};
287pub use zeph_config::memory::CompressionGuidelinesConfig;
288pub use zeph_config::memory::EvictionConfig;
289pub use zeph_config::memory::TypeAwareComposeConfig;
290pub use zeph_config::memory::{CompactionProbeConfig, ProbeCategory};
zeph_memory/lib.rs

zeph_memory/
lib.rs
