1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
//! Metadata store module
//!
//! Handles chunk metadata queries with tenant isolation.
//! Payloads are NOT stored here - only in segment files.
pub mod pool;
pub mod sqlite;
pub use pool::{PooledConnection, SqliteConnectionPool};
pub use sqlite::SqliteMetadataStore;
use crate::error::Result;
use std::collections::HashMap;
pub type ChunkTypeCounts = (
HashMap<String, usize>,
HashMap<String, usize>,
HashMap<String, usize>,
);
use crate::store::StoreHealthSnapshot;
use crate::types::{ChunkId, ChunkStatus, ChunkType, LifecycleDelta, LifecycleMetadata, TenantId};
/// Index lifecycle state for a chunk.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum IndexState {
Pending,
Indexed,
Failed,
}
impl IndexState {
pub fn as_str(self) -> &'static str {
match self {
Self::Pending => "pending",
Self::Indexed => "indexed",
Self::Failed => "failed",
}
}
}
/// Metadata record for a chunk (no payload)
#[derive(Debug, Clone)]
pub struct ChunkMetadata {
pub chunk_id: ChunkId,
pub tenant_id: TenantId,
pub project_id: Option<String>,
pub segment_id: u64,
pub ordinal: u32,
pub chunk_type: ChunkType,
pub status: ChunkStatus,
pub timestamp_created: i64,
pub hash: String,
pub source_uri: Option<String>,
/// Lifecycle overlay (tier, supersession edges, retention window).
///
/// Added in A3. For chunks inserted before the lifecycle columns
/// existed, this falls back to `LifecycleMetadata::default()` at
/// read time (long-term tier, no supersession, no expiry).
pub lifecycle: LifecycleMetadata,
/// Optional canonical text for writer-driven digests / supersession
/// by content identity. Orthogonal to the chunk payload; empty for
/// regular chunks.
pub canonical_text: Option<String>,
/// Track E: write-time label declaring whether the chunk came from
/// a `conversation` (rapidly-evolving session memory) or a
/// `document` (curated, durable) ingestion. Defaults to Document
/// for legacy rows whose row was inserted before E1 shipped.
pub ingestion_mode: crate::types::IngestionMode,
}
/// Metadata store trait
pub trait MetadataStore: Send + Sync {
/// Insert metadata for a new chunk
fn insert(&self, metadata: &ChunkMetadata) -> Result<()>;
/// Insert multiple metadata rows atomically.
///
/// Implementations should treat this as all-or-nothing.
fn insert_many(&self, metadata: &[ChunkMetadata]) -> Result<()>;
/// Get metadata by chunk_id (tenant_id required for isolation)
fn get(&self, tenant_id: &TenantId, chunk_id: &ChunkId) -> Result<Option<ChunkMetadata>>;
/// List chunks for a tenant (non-deleted only)
fn list(&self, tenant_id: &TenantId, limit: usize, offset: usize)
-> Result<Vec<ChunkMetadata>>;
/// List chunks for a tenant, optionally scoped to a project (non-deleted only).
fn list_for_project(
&self,
tenant_id: &TenantId,
project_id: Option<&str>,
limit: usize,
offset: usize,
) -> Result<Vec<ChunkMetadata>>;
/// Mark chunk as deleted (soft delete)
fn mark_deleted(&self, tenant_id: &TenantId, chunk_id: &ChunkId) -> Result<bool>;
/// Get all chunk metadata for a tenant's segment (for tombstone
/// sync and compaction audits). Scoped by `tenant_id` because
/// `segment_id` is only unique within a tenant — the chunks
/// UNIQUE constraint is `(tenant_id, segment_id, ordinal)` and
/// `PersistentStore::next_segment_id()` allocates per-tenant, so
/// segment_id=1 legitimately exists in every tenant independently
/// (Item 2).
fn get_by_segment(&self, tenant_id: &TenantId, segment_id: u64) -> Result<Vec<ChunkMetadata>>;
/// Count chunks by status for a tenant
fn count_by_status(&self, tenant_id: &TenantId) -> Result<(usize, usize, usize)>;
/// Count chunk types via aggregate SQL, returning active/deleted/all maps.
fn count_chunk_types_by_status(
&self,
tenant_id: &TenantId,
project_id: Option<&str>,
) -> Result<ChunkTypeCounts>;
/// Compute read-only health aggregates for a tenant/project scope.
fn health_snapshot(
&self,
tenant_id: &TenantId,
project_id: Option<&str>,
duplicate_limit: usize,
) -> Result<StoreHealthSnapshot>;
/// Get all deleted chunk IDs for a tenant (for compaction)
fn get_deleted_chunk_ids(&self, tenant_id: &TenantId) -> Result<Vec<ChunkId>>;
/// Mark chunks as pending indexing.
fn mark_index_pending(
&self,
tenant_id: &TenantId,
chunk_ids: &[ChunkId],
now_ms: i64,
) -> Result<()>;
/// Mark chunks as successfully indexed.
fn mark_indexed(&self, tenant_id: &TenantId, chunk_ids: &[ChunkId], now_ms: i64) -> Result<()>;
/// Mark one chunk as failed indexing and increment attempt count.
fn mark_index_failed(
&self,
tenant_id: &TenantId,
chunk_id: &ChunkId,
error: &str,
now_ms: i64,
) -> Result<()>;
/// List pending index chunk IDs for one tenant.
fn list_pending_index_chunk_ids(
&self,
tenant_id: &TenantId,
limit: usize,
) -> Result<Vec<ChunkId>>;
/// Count pending/indexed/failed chunks for one tenant.
fn count_by_index_state(&self, tenant_id: &TenantId) -> Result<(usize, usize, usize)>;
/// Apply a lifecycle delta to one chunk row.
///
/// Triple-state semantics match `LifecycleDelta`:
/// - `None` leaves the field unchanged.
/// - `Some(value)` sets (for simple fields) or sets-or-clears (for
/// nested `Option<Option<T>>` fields such as `expires_at_ms`).
fn update_lifecycle(
&self,
tenant_id: &TenantId,
chunk_id: &ChunkId,
delta: &LifecycleDelta,
) -> Result<()>;
/// Atomically link a supersession pair inside one SQL transaction.
///
/// Marks `old_id` as `Superseded` with `superseded_by = new_id`, and
/// sets `supersedes = old_id` on `new_id`. Both rows receive the
/// same `lifecycle_updated_at_ms = now_ms`.
fn atomic_supersede(
&self,
tenant_id: &TenantId,
old_id: &ChunkId,
new_id: &ChunkId,
now_ms: i64,
) -> Result<()>;
/// Set the optional canonical text used by writer-driven digest /
/// supersession-by-content-identity flows.
fn set_canonical_text(
&self,
tenant_id: &TenantId,
chunk_id: &ChunkId,
canonical: &str,
) -> Result<()>;
/// List chunk IDs whose retention window has elapsed before `now_ms`.
///
/// Skips rows already in terminal lifecycle states (`deleted`,
/// `expired`, `superseded`, `error`) so sweeps do not retouch rows
/// that another writer has already moved out from under us. Uses
/// `expires_at_ms <= now_ms` to match `VisibilityPolicy::is_visible_at`.
fn list_expired_before(&self, tenant_id: &TenantId, now_ms: i64) -> Result<Vec<ChunkId>>;
/// Atomically promote one chunk to `status='expired'` with a
/// guard against concurrent status *and* retention-window
/// changes. The backend-level implementation re-checks two
/// predicates at UPDATE time so a row that was eligible when the
/// sweep SELECTed it is no longer promoted if either drifted:
///
/// 1. `status = 'final'` — any other status (deleted,
/// superseded, expired, error) is treated as "raced out from
/// under the sweep".
/// 2. `expires_at_ms IS NOT NULL AND expires_at_ms <= now_ms` —
/// if a concurrent writer cleared the expiry or pushed it
/// forward while leaving status=Final, the row is no longer
/// eligible and is skipped.
///
/// Stamps `lifecycle_updated_at_ms = now_ms` on a successful
/// promotion. Returns `true` when exactly one row was updated,
/// `false` when either guard rejected the update (or the row is
/// not present).
///
/// Default implementation reads the current row and then calls
/// `update_lifecycle`; this is NOT race-free. Backends with real
/// SQL should override with a single guarded `UPDATE ... WHERE
/// status = 'final' AND expires_at_ms IS NOT NULL AND
/// expires_at_ms <= :now_ms` statement.
fn mark_expired_if_final(
&self,
tenant_id: &TenantId,
chunk_id: &ChunkId,
now_ms: i64,
) -> Result<bool> {
let current = match self.get(tenant_id, chunk_id)? {
Some(m) => m,
None => return Ok(false),
};
use crate::types::ChunkStatus;
if current.status != ChunkStatus::Final {
return Ok(false);
}
self.update_lifecycle(
tenant_id,
chunk_id,
&LifecycleDelta {
status: Some(ChunkStatus::Expired),
lifecycle_updated_at_ms: Some(now_ms),
..Default::default()
},
)?;
Ok(true)
}
/// List superseded/expired chunk IDs older than the given cutoff
/// that are not yet demoted to the history tier. Feeds history
/// promotion (C4).
fn list_stale_superseded(
&self,
tenant_id: &TenantId,
older_than_ms: i64,
) -> Result<Vec<ChunkId>>;
/// Atomically promote one chunk to `tier='history'` with guards
/// against concurrent overlay refreshes. Only updates when the
/// row's current state at UPDATE time still satisfies all of:
/// * status IN ('superseded', 'expired')
/// * tier != 'history'
/// * lifecycle_updated_at_ms < `older_than_ms`
///
/// Stamps `lifecycle_updated_at_ms = now_ms` on a successful
/// promotion. Returns `true` when exactly one row was updated,
/// `false` when any guard rejected the update (row no longer
/// matches, was just refreshed, or is absent).
///
/// Default implementation reads then calls `update_lifecycle`; it
/// is NOT race-free. Backends with real SQL should override with
/// a single guarded `UPDATE` that embeds all three predicates.
fn promote_to_history_if_stale(
&self,
tenant_id: &TenantId,
chunk_id: &ChunkId,
older_than_ms: i64,
now_ms: i64,
) -> Result<bool> {
let current = match self.get(tenant_id, chunk_id)? {
Some(m) => m,
None => return Ok(false),
};
use crate::types::{ChunkStatus, MemoryTier};
let stale_status = matches!(
current.status,
ChunkStatus::Superseded | ChunkStatus::Expired
);
if !stale_status
|| current.lifecycle.tier == MemoryTier::History
|| current.lifecycle.lifecycle_updated_at_ms >= older_than_ms
{
return Ok(false);
}
self.update_lifecycle(
tenant_id,
chunk_id,
&LifecycleDelta {
tier: Some(MemoryTier::History),
lifecycle_updated_at_ms: Some(now_ms),
..Default::default()
},
)?;
Ok(true)
}
/// List chunk IDs hidden by lifecycle (superseded, expired, or
/// history-tier). Used by compaction (B2) to extend the HNSW
/// excluded-ID set beyond soft-deleted rows.
fn list_lifecycle_hidden(&self, tenant_id: &TenantId) -> Result<Vec<ChunkId>>;
/// List chunks whose `canonical_text` equals `canonical`, optionally
/// scoped to `project_id`. Feeds conflict-aware ingest (D3).
fn list_by_canonical_text(
&self,
tenant_id: &TenantId,
project_id: Option<&str>,
canonical: &str,
) -> Result<Vec<ChunkMetadata>>;
/// List live, visible chunks matching an exact content hash within
/// one tenant/project/chunk-kind scope. Unlike helpers where
/// `project_id = None` means "any project", this treats `None` as
/// `project_id IS NULL` so default write-time dedupe cannot cross a
/// project boundary accidentally.
fn list_live_by_content_hash(
&self,
tenant_id: &TenantId,
project_id: Option<&str>,
chunk_type: ChunkType,
hash: &str,
limit: usize,
) -> Result<Vec<ChunkMetadata>>;
/// List the most recently created chunks for a tenant, optionally
/// scoped to a project. `Option<&str>::None` means "any project"
/// (no filter), matching the historical semantics. For "rows
/// whose project_id IS NULL", call `list_recent_with_null_project`
/// instead — these two semantics are distinct for D3/D4 fuzzy
/// dedup with `scope: "project"` on an unscoped chunk.
fn list_recent_for_project(
&self,
tenant_id: &TenantId,
project_id: Option<&str>,
limit: usize,
) -> Result<Vec<ChunkMetadata>>;
/// List the most recently created chunks for a tenant whose
/// `project_id IS NULL`. Distinct from `list_recent_for_project`
/// with `project_id = None` because the latter widens to all
/// projects before applying `LIMIT`. D3/D4 fuzzy dedup with
/// `scope: "project"` on an unscoped incoming chunk needs this
/// pre-LIMIT NULL filter so a valid older NULL-project candidate
/// is not evicted by recent project-scoped traffic.
fn list_recent_with_null_project(
&self,
tenant_id: &TenantId,
limit: usize,
) -> Result<Vec<ChunkMetadata>>;
/// List chunks for OMF export (F2).
///
/// Excludes `candidate`, `deleted`, and `error` rows unconditionally; includes
/// `superseded` and `expired` so callers can decide whether to emit
/// them. Excludes `history`-tier rows unless `include_history` is
/// set. Ordered by creation time ascending for stable export.
fn list_for_export(
&self,
tenant_id: &TenantId,
project_id: Option<&str>,
include_history: bool,
) -> Result<Vec<ChunkMetadata>>;
}