use std::collections::HashSet;
use anyhow::Result;
use chrono::{DateTime, Utc};
use rusqlite::{OptionalExtension, params};
use serde::{Deserialize, Serialize};
use uuid::Uuid;
use crate::cold::{stub_lead_sql, stub_matches};
use crate::index::{EMBEDDING_DIM, Indexes};
use crate::schema::distillable_source;
use crate::sqlite_store::{SqliteStore, body_of, embedding_to_blob, parse_uuid, sync_dedup};
use crate::store::StoreAdmin;
const HOT: &str = "tier = 'hot' AND superseded_by IS NULL";
const DEDUP_CANDIDATE: &str = "tier = 'hot' AND superseded_by IS NULL AND source <> 'markdown'";
const DANGLING_RELATIONS: &str = "from_id NOT IN (SELECT id FROM memories) \
OR (relation_type = 'mentions' AND to_id NOT IN (SELECT id FROM entities)) \
OR (relation_type = 'derived_from' AND to_id NOT IN (SELECT id FROM conversations)) \
OR (relation_type NOT IN ('mentions', 'derived_from') \
AND to_id NOT IN (SELECT id FROM memories))";
const DEAD_VEC_KEYS: &str = "SELECT COUNT(*) FROM vec_keys vk \
LEFT JOIN memories m ON m.id = vk.memory_id \
WHERE m.id IS NULL OR m.tier <> 'hot' OR m.superseded_by IS NOT NULL \
OR m.embedding IS NULL";
const RECLAIMABLE_SYNC_LOG: &str = "SELECT s.local_seq FROM sync_log s \
WHERE ((s.node_type = 'memory' AND s.op IN ('create', 'update') \
AND EXISTS (SELECT 1 FROM sync_log d \
WHERE d.node_type = 'memory' AND d.node_id = s.node_id \
AND d.op = 'delete' AND d.local_seq > s.local_seq)) \
OR (s.node_type = 'relation' AND s.op = 'create' \
AND (substr(s.node_id, 1, 36) IN (SELECT node_id FROM tombstones) \
OR substr(s.node_id, 38, 36) IN (SELECT node_id FROM tombstones))))";
const ORPHANED_MARKERS: &str = "SELECT dl.memory_id FROM distillation_log dl \
JOIN memories m ON m.id = dl.memory_id \
WHERE m.tier = 'hot' AND m.superseded_by IS NULL \
AND dl.memory_id <> dl.distilled_id \
AND NOT EXISTS (SELECT 1 FROM memories d WHERE d.id = dl.distilled_id) \
AND dl.distilled_id IN (SELECT node_id FROM tombstones)";
macro_rules! inheriting_keeper {
() => {
concat!(
"k.tier = 'hot' AND k.superseded_by IS NULL AND ",
distillable_source!("k.source"),
" AND k.id NOT IN (SELECT memory_id FROM distillation_log) \
AND (dl.memory_id = dl.distilled_id \
OR EXISTS (SELECT 1 FROM memories d WHERE d.id = dl.distilled_id))"
)
};
}
const INHERITABLE_MARKERS: &str = concat!(
"SELECT k.id, dl.distilled_id, dl.model, \
dl.memory_id = dl.distilled_id AS skip \
FROM memories k \
JOIN memories t ON t.content_prefix = k.content_prefix AND t.content = k.content \
AND t.id <> k.id \
JOIN distillation_log dl ON dl.memory_id = t.id \
WHERE ",
inheriting_keeper!(),
" GROUP BY k.id"
);
fn inheritable_cold_markers() -> String {
format!(
"SELECT k.id, dl.distilled_id, dl.model, dl.memory_id = dl.distilled_id AS skip, \
CAST(k.content AS BLOB), t.content \
FROM distillation_log dl \
JOIN memories t ON t.id = dl.memory_id AND t.archived_segment IS NOT NULL \
JOIN memories k ON k.content_prefix = {lead} AND k.id <> t.id \
WHERE {keeper}",
lead = stub_lead_sql("t.content"),
keeper = inheriting_keeper!(),
)
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct IntegrityReport {
pub memories_total: u64,
pub hot: u64,
pub archived: u64,
pub superseded: u64,
pub distilled_live: u64,
pub raw_pending: u64,
pub raw_retired: u64,
pub raw_retire_pending: u64,
pub missing_embeddings: u64,
pub duplicate_hot_groups: u64,
pub orphaned_distillation_markers: u64,
pub dead_vec_keys: u64,
pub dangling_relations: u64,
pub sync_log_rows: u64,
pub sync_log_reclaimable_rows: u64,
pub tombstones: u64,
pub stale_peer_cursors: u64,
pub outbox_pending: u64,
pub index_vectors: Option<u64>,
pub index_docs: Option<u64>,
pub index_consistent: Option<bool>,
pub db_bytes: u64,
pub freelist_bytes: u64,
}
impl IntegrityReport {
pub fn drift(&self) -> Vec<(&'static str, u64)> {
let mut out = Vec::new();
let mut push = |name: &'static str, value: u64| {
if value > 0 {
out.push((name, value));
}
};
push("raw_retire_pending", self.raw_retire_pending);
push("duplicate_hot_groups", self.duplicate_hot_groups);
push(
"orphaned_distillation_markers",
self.orphaned_distillation_markers,
);
push("dead_vec_keys", self.dead_vec_keys);
push("dangling_relations", self.dangling_relations);
push("sync_log_reclaimable_rows", self.sync_log_reclaimable_rows);
push("stale_peer_cursors", self.stale_peer_cursors);
push("missing_embeddings", self.missing_embeddings);
if self.index_consistent == Some(false) {
push("index_inconsistent", 1);
}
out
}
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct RepairStats {
pub dry_run: bool,
pub duplicates_superseded: u64,
pub distillation_markers_inherited: u64,
pub orphaned_markers_dropped: u64,
pub raws_retired: u64,
pub relations_pruned: u64,
pub vec_keys_deleted: u64,
pub vec_keys_gc_skipped: bool,
pub sync_log_rows_deleted: u64,
pub stale_cursors_dropped: u64,
pub embeddings_backfilled: u64,
pub embedding_failures: u64,
pub index_rebuilt: bool,
}
impl RepairStats {
pub fn is_clean(&self) -> bool {
self.raws_retired == 0
&& self.duplicates_superseded == 0
&& self.distillation_markers_inherited == 0
&& self.orphaned_markers_dropped == 0
&& self.relations_pruned == 0
&& self.vec_keys_deleted == 0
&& !self.vec_keys_gc_skipped
&& self.sync_log_rows_deleted == 0
&& self.stale_cursors_dropped == 0
&& self.embeddings_backfilled == 0
&& self.embedding_failures == 0
&& !self.index_rebuilt
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MaintenanceSummary {
pub ran_at: DateTime<Utc>,
pub stats: RepairStats,
pub report_after: IntegrityReport,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct LiveCounts {
pub stored_total: u64,
pub live: u64,
pub archived: u64,
pub superseded: u64,
pub by_type: std::collections::BTreeMap<String, u64>,
}
pub type EmbedFn<'a> = &'a dyn Fn(&str) -> Result<Vec<f32>>;
pub type EmbedBatchFn<'a> = &'a dyn Fn(&[&str]) -> Result<Vec<Vec<f32>>>;
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct ChunkBackfill {
pub memories: u64,
pub chunks: u64,
pub failed: u64,
pub pending: u64,
}
pub(crate) fn is_stale_cursor(peer_id: &str) -> bool {
let bare = peer_id.strip_prefix("push:").unwrap_or(peer_id);
Uuid::parse_str(bare).is_err()
}
pub(crate) fn live_counts_on(conn: &rusqlite::Connection) -> Result<LiveCounts> {
let mut stmt = conn.prepare(
"SELECT CASE WHEN tier = 'archived' THEN 'archived' \
WHEN superseded_by IS NOT NULL THEN 'superseded' \
ELSE 'live' END, memory_type, COUNT(*) \
FROM memories GROUP BY 1, 2",
)?;
let rows = stmt.query_map([], |r| {
Ok((
r.get::<_, String>(0)?,
r.get::<_, String>(1)?,
r.get::<_, i64>(2)? as u64,
))
})?;
let mut counts = LiveCounts::default();
for row in rows {
let (state, memory_type, n) = row?;
counts.stored_total += n;
match state.as_str() {
"archived" => counts.archived += n,
"superseded" => counts.superseded += n,
_ => {
counts.live += n;
*counts.by_type.entry(memory_type).or_default() += n;
}
}
}
Ok(counts)
}
impl SqliteStore {
pub fn live_counts(&self) -> Result<LiveCounts> {
live_counts_on(&self.reader())
}
pub fn outbox_pending(&self) -> Result<u64> {
let watermark = self.index_watermark()?;
let n: i64 = self.reader().query_row(
"SELECT COUNT(*) FROM index_outbox WHERE seq > ?1",
params![watermark],
|r| r.get(0),
)?;
Ok(n as u64)
}
pub fn integrity_report(&self, indexes: Option<&Indexes>) -> Result<IntegrityReport> {
let consolidation = self.consolidation_stats()?;
let stale_peer_cursors = self
.get_all_sync_states()?
.iter()
.filter(|s| is_stale_cursor(&s.peer_id))
.count() as u64;
let (index_vectors, index_docs, index_consistent) = match indexes {
Some(idx) => (
Some(idx.vector_len() as u64),
Some(idx.lexical_len()),
Some(idx.consistent_with(self)?),
),
None => (None, None, None),
};
let conn = self.reader();
let count = |sql: &str| -> Result<u64> {
let n: i64 = conn.query_row(sql, [], |r| r.get(0))?;
Ok(n as u64)
};
let page_size = crate::sqlite_store::page_size(&conn)? as u64;
Ok(IntegrityReport {
memories_total: count("SELECT COUNT(*) FROM memories")?,
hot: count(&format!("SELECT COUNT(*) FROM memories WHERE {HOT}"))?,
archived: count("SELECT COUNT(*) FROM memories WHERE tier = 'archived'")?,
superseded: count("SELECT COUNT(*) FROM memories WHERE superseded_by IS NOT NULL")?,
distilled_live: consolidation.distilled_live,
raw_pending: consolidation.raw_pending,
raw_retired: consolidation.raw_retired,
raw_retire_pending: consolidation.raw_retire_pending,
missing_embeddings: count(&format!(
"SELECT COUNT(*) FROM memories WHERE {HOT} AND embedding IS NULL"
))?,
duplicate_hot_groups: count(&format!(
"SELECT COUNT(*) FROM (SELECT 1 FROM memories WHERE {DEDUP_CANDIDATE} \
AND content_prefix IN (SELECT content_prefix FROM memories \
WHERE {DEDUP_CANDIDATE} \
GROUP BY content_prefix HAVING COUNT(*) > 1) \
GROUP BY content HAVING COUNT(*) > 1)"
))?,
orphaned_distillation_markers: count(&format!(
"SELECT COUNT(*) FROM ({ORPHANED_MARKERS})"
))?,
dead_vec_keys: count(DEAD_VEC_KEYS)?,
dangling_relations: count(&format!(
"SELECT COUNT(*) FROM relations WHERE {DANGLING_RELATIONS}"
))?,
sync_log_rows: count("SELECT COUNT(*) FROM sync_log")?,
sync_log_reclaimable_rows: count(&format!(
"SELECT COUNT(*) FROM ({RECLAIMABLE_SYNC_LOG})"
))?,
tombstones: count("SELECT COUNT(*) FROM tombstones")?,
stale_peer_cursors,
outbox_pending: count("SELECT COUNT(*) FROM index_outbox")?,
index_vectors,
index_docs,
index_consistent,
db_bytes: count("PRAGMA page_count")? * page_size,
freelist_bytes: count("PRAGMA freelist_count")? * page_size,
})
}
pub fn repair(
&self,
mut indexes: Option<&mut Indexes>,
embed: Option<EmbedFn<'_>>,
dry_run: bool,
) -> Result<RepairStats> {
let mut stats = RepairStats {
dry_run,
..Default::default()
};
stats.duplicates_superseded = self.dedup_hot_memories(dry_run)?;
stats.distillation_markers_inherited = self.inherit_distillation_markers(dry_run)?;
stats.orphaned_markers_dropped = self.drop_orphaned_markers(dry_run)?;
stats.raws_retired = if dry_run {
self.consolidation_stats()?.raw_retire_pending
} else {
self.retire_consolidated_raws()? as u64
};
stats.relations_pruned = if dry_run {
self.count_dangling_relations()?
} else {
self.prune_dangling_relations()?
};
if let Some(idx) = indexes.as_deref_mut()
&& !dry_run
{
idx.drain(self)?;
}
match self.gc_vec_keys(dry_run)? {
Some(n) => stats.vec_keys_deleted = n,
None => stats.vec_keys_gc_skipped = true,
}
stats.sync_log_rows_deleted = self.gc_sync_log(dry_run)?;
stats.stale_cursors_dropped = self.drop_stale_peer_cursors(dry_run)?;
if let Some(embed) = embed {
let (done, failed) = self.backfill_embeddings(embed, dry_run)?;
stats.embeddings_backfilled = done;
stats.embedding_failures = failed;
}
if let Some(idx) = indexes {
if !dry_run {
idx.drain(self)?;
}
if !idx.consistent_with(self)? {
stats.index_rebuilt = true;
if !dry_run {
idx.rebuild(self)?;
}
}
}
Ok(stats)
}
fn dedup_hot_memories(&self, dry_run: bool) -> Result<u64> {
const BATCH: usize = 500;
let mut rows: Vec<(String, DateTime<Utc>, Uuid)> = {
let conn = self.reader();
let mut stmt = conn.prepare(&format!(
"SELECT id, content, created_at FROM memories WHERE {DEDUP_CANDIDATE} \
AND content_prefix IN (SELECT content_prefix FROM memories \
WHERE {DEDUP_CANDIDATE} \
GROUP BY content_prefix HAVING COUNT(*) > 1)"
))?;
let rows = stmt.query_map([], |r| {
let created = DateTime::parse_from_rfc3339(&r.get::<_, String>(2)?)
.map(|d| d.with_timezone(&Utc))
.unwrap_or(DateTime::UNIX_EPOCH);
Ok((
r.get::<_, String>(1)?,
created,
parse_uuid(&r.get::<_, String>(0)?)?,
))
})?;
rows.collect::<rusqlite::Result<Vec<_>>>()?
};
rows.sort();
let mut pairs: Vec<(Uuid, Uuid)> = Vec::new();
let mut keeper: Option<(&Uuid, &str)> = None;
for (content, _, id) in &rows {
match keeper {
Some((keep_id, keep_content)) if keep_content == content => {
pairs.push((*id, *keep_id));
}
_ => keeper = Some((id, content)),
}
}
if dry_run {
return Ok(pairs.len() as u64);
}
let mut superseded = 0u64;
for chunk in pairs.chunks(BATCH) {
superseded += self.with_txn(|tx| {
let mut n = 0u64;
for (loser, winner) in chunk {
if self.supersede_on_tx(tx, *loser, *winner)? {
n += 1;
}
}
Ok(n)
})?;
}
Ok(superseded)
}
fn inherit_distillation_markers(&self, dry_run: bool) -> Result<u64> {
let rows: Vec<(Uuid, Uuid, Option<String>, bool)> = {
let conn = self.reader();
let mut stmt = conn.prepare(INHERITABLE_MARKERS)?;
let rows = stmt.query_map([], |r| {
Ok((
parse_uuid(&r.get::<_, String>(0)?)?,
parse_uuid(&r.get::<_, String>(1)?)?,
r.get::<_, Option<String>>(2)?,
r.get::<_, bool>(3)?,
))
})?;
rows.collect::<rusqlite::Result<Vec<_>>>()?
};
let mut rows = rows;
let mut keepers: HashSet<Uuid> = rows.iter().map(|r| r.0).collect();
for (keeper, distilled_id, model, skip) in self.cold_twin_markers()? {
if keepers.insert(keeper) {
rows.push((keeper, distilled_id, model, skip));
}
}
if dry_run {
return Ok(rows.len() as u64);
}
for (keeper, distilled_id, model, skip) in &rows {
let target = if *skip { *keeper } else { *distilled_id };
self.mark_consolidated(*keeper, target, model.as_deref().unwrap_or("inherited"))?;
}
Ok(rows.len() as u64)
}
fn cold_twin_markers(&self) -> Result<Vec<(Uuid, Uuid, Option<String>, bool)>> {
let conn = self.reader();
let mut stmt = conn.prepare(&inheritable_cold_markers())?;
let rows = stmt.query_map([], |r| {
Ok((
parse_uuid(&r.get::<_, String>(0)?)?,
parse_uuid(&r.get::<_, String>(1)?)?,
r.get::<_, Option<String>>(2)?,
r.get::<_, bool>(3)?,
r.get::<_, Vec<u8>>(4)?,
r.get::<_, String>(5)?,
))
})?;
let mut out = Vec::new();
for row in rows {
let (keeper, distilled_id, model, skip, content, stub) = row?;
if String::from_utf8(content).is_ok_and(|c| stub_matches(&stub, &c)) {
out.push((keeper, distilled_id, model, skip));
}
}
Ok(out)
}
fn drop_orphaned_markers(&self, dry_run: bool) -> Result<u64> {
let ids: Vec<Uuid> = {
let conn = self.reader();
let mut stmt = conn.prepare(ORPHANED_MARKERS)?;
let rows = stmt.query_map([], |r| parse_uuid(&r.get::<_, String>(0)?))?;
rows.collect::<rusqlite::Result<Vec<_>>>()?
};
if dry_run {
return Ok(ids.len() as u64);
}
let mut dropped = 0u64;
for id in &ids {
dropped += self.with_txn(|tx| Ok(u64::from(self.drop_marker_on_tx(tx, *id)?)))?;
}
Ok(dropped)
}
fn count_dangling_relations(&self) -> Result<u64> {
let conn = self.reader();
let n: i64 = conn.query_row(
&format!("SELECT COUNT(*) FROM relations WHERE {DANGLING_RELATIONS}"),
[],
|r| r.get(0),
)?;
Ok(n as u64)
}
fn gc_vec_keys(&self, dry_run: bool) -> Result<Option<u64>> {
let watermark = self.index_watermark()?;
let pending: i64 = {
let conn = self.reader();
conn.query_row(
"SELECT COUNT(*) FROM index_outbox WHERE seq > ?1",
params![watermark],
|r| r.get(0),
)?
};
if pending > 0 {
return Ok(None);
}
if dry_run {
let conn = self.reader();
let n: i64 = conn.query_row(DEAD_VEC_KEYS, [], |r| r.get(0))?;
return Ok(Some(n as u64));
}
let conn = self.conn();
let n = conn.execute(
"DELETE FROM vec_keys WHERE key IN (SELECT vk.key FROM vec_keys vk \
LEFT JOIN memories m ON m.id = vk.memory_id \
WHERE m.id IS NULL OR m.tier <> 'hot' OR m.superseded_by IS NOT NULL \
OR m.embedding IS NULL)",
[],
)?;
Ok(Some(n as u64))
}
fn gc_sync_log(&self, dry_run: bool) -> Result<u64> {
let mut conn = self.conn();
conn.execute_batch("PRAGMA cache_size = -262144;")?;
conn.execute_batch("DROP TABLE IF EXISTS gc_victims;")?;
conn.execute(
&format!("CREATE TEMP TABLE gc_victims AS {RECLAIMABLE_SYNC_LOG}"),
[],
)?;
conn.execute_batch("CREATE INDEX idx_gc_victims_seq ON gc_victims(local_seq);")?;
let victims: i64 = conn.query_row("SELECT COUNT(*) FROM gc_victims", [], |r| r.get(0))?;
let mut deleted = 0u64;
if !dry_run {
let mut cursor: i64 = 0;
loop {
let chunk: Vec<i64> = {
let mut stmt = conn.prepare(
"SELECT local_seq FROM gc_victims WHERE local_seq > ?1 \
ORDER BY local_seq LIMIT 20000",
)?;
let rows = stmt.query_map(params![cursor], |r| r.get::<_, i64>(0))?;
rows.collect::<rusqlite::Result<Vec<_>>>()?
};
let Some(last) = chunk.last().copied() else {
break;
};
cursor = last;
let placeholders = chunk.iter().map(|_| "?").collect::<Vec<_>>().join(",");
let tx = conn.transaction()?;
sync_dedup::keep_keys_of(&tx, &chunk)?;
deleted += tx.execute(
&format!("DELETE FROM sync_log WHERE local_seq IN ({placeholders})"),
rusqlite::params_from_iter(chunk.iter()),
)? as u64;
tx.commit()?;
}
}
conn.execute_batch("DROP TABLE IF EXISTS gc_victims;")?;
Ok(if dry_run { victims as u64 } else { deleted })
}
fn drop_stale_peer_cursors(&self, dry_run: bool) -> Result<u64> {
let stale: Vec<String> = self
.get_all_sync_states()?
.into_iter()
.filter(|s| is_stale_cursor(&s.peer_id))
.map(|s| s.peer_id)
.collect();
if dry_run {
return Ok(stale.len() as u64);
}
let conn = self.conn();
let mut dropped = 0u64;
for peer_id in &stale {
dropped += conn.execute(
"DELETE FROM sync_state WHERE peer_id = ?1",
params![peer_id],
)? as u64;
}
Ok(dropped)
}
pub fn backfill_embeddings(&self, embed: EmbedFn<'_>, dry_run: bool) -> Result<(u64, u64)> {
const BATCH: usize = 64;
let rows: Vec<(Uuid, Option<String>, String)> = {
let conn = self.reader();
let mut stmt = conn.prepare(&format!(
"SELECT id, context, content FROM memories WHERE {HOT} AND embedding IS NULL \
ORDER BY created_at"
))?;
let rows = stmt.query_map([], |r| {
Ok((
parse_uuid(&r.get::<_, String>(0)?)?,
r.get::<_, Option<String>>(1)?,
r.get::<_, String>(2)?,
))
})?;
rows.collect::<rusqlite::Result<Vec<_>>>()?
};
if dry_run {
return Ok((rows.len() as u64, 0));
}
let mut done = 0u64;
let mut failed = 0u64;
for chunk in rows.chunks(BATCH) {
let mut embedded: Vec<(Uuid, Vec<f32>)> = Vec::with_capacity(chunk.len());
for (id, context, content) in chunk {
let body = body_of(context.clone(), content.clone());
match embed(&body) {
Ok(v) if v.len() == EMBEDDING_DIM => embedded.push((*id, v)),
Ok(v) => {
failed += 1;
tracing::warn!(
memory_id = %id,
dim = v.len(),
"embedding backfill produced the wrong dimension"
);
}
Err(e) => {
failed += 1;
tracing::warn!(memory_id = %id, "embedding backfill failed: {e:#}");
}
}
}
done += self.with_txn(|tx| {
let mut n = 0u64;
for (id, vector) in &embedded {
let changed = tx.execute(
"UPDATE memories SET embedding = ?2, updated_at = ?3 \
WHERE id = ?1 AND embedding IS NULL",
params![
id.to_string(),
embedding_to_blob(vector),
Utc::now().to_rfc3339()
],
)?;
if changed == 0 {
continue;
}
Self::append_outbox(tx, *id, "upsert")?;
if let Some(memory) = Self::memory_on_tx(tx, *id)? {
self.log_memory_update(tx, &memory)?;
}
n += 1;
}
Ok(n)
})?;
}
Ok((done, failed))
}
pub fn backfill_chunk_embeddings(
&self,
embed: EmbedBatchFn<'_>,
max_memories: usize,
budget: std::time::Duration,
) -> Result<ChunkBackfill> {
const BATCH: usize = 16;
let started = std::time::Instant::now();
let (work, pending) = self.scan_chunk_work(max_memories)?;
let mut out = ChunkBackfill::default();
let mut rest = work.as_slice();
while !rest.is_empty() && (out.memories + out.failed == 0 || started.elapsed() < budget) {
let mut take = 1;
let mut texts = rest[0].chunks.len();
while take < rest.len() && texts + rest[take].chunks.len() <= BATCH {
texts += rest[take].chunks.len();
take += 1;
}
let (batch, tail) = rest.split_at(take);
rest = tail;
let refs: Vec<&str> = batch
.iter()
.flat_map(|w| w.chunks.iter().map(String::as_str))
.collect();
let vectors = match embed(&refs) {
Ok(v) if v.len() == refs.len() => v,
Ok(v) => {
out.failed += batch.len() as u64;
tracing::warn!(
expected = refs.len(),
got = v.len(),
"chunk embedding returned the wrong number of vectors"
);
continue;
}
Err(e) => {
out.failed += batch.len() as u64;
tracing::warn!("chunk embedding failed: {e:#}");
continue;
}
};
let mut vectors = vectors.into_iter();
for w in batch {
let set: Vec<(usize, String, Vec<f32>)> = w
.chunks
.iter()
.cloned()
.enumerate()
.zip(vectors.by_ref())
.map(|((ord, text), v)| (ord, text, v))
.collect();
if set.iter().any(|(_, _, v)| v.len() != EMBEDDING_DIM) {
out.failed += 1;
tracing::warn!(memory_id = %w.memory_id, "chunk embedding had the wrong dimension");
continue;
}
if self.store_chunks(w.memory_id, &w.content_hash, &set)? {
out.memories += 1;
out.chunks += set.len() as u64;
}
}
}
out.pending = (pending as u64).saturating_sub(out.memories);
self.set_chunks_pending(out.pending as usize)?;
Ok(out)
}
pub fn sync_log_row_exists(&self, local_seq: u64) -> Result<bool> {
let conn = self.reader();
let hit: Option<i64> = conn
.query_row(
"SELECT 1 FROM sync_log WHERE local_seq = ?1",
params![local_seq as i64],
|r| r.get(0),
)
.optional()?;
Ok(hit.is_some())
}
}