use std::collections::HashMap;
use std::fs;
use std::num::{NonZeroU64, NonZeroUsize};
use std::path::Path;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use anyhow::{anyhow, bail, Context, Result};
use chrono::{DateTime, TimeZone, Utc};
use fuzzy_matcher::skim::SkimMatcherV2;
use fuzzy_matcher::FuzzyMatcher;
use nucleo_matcher::pattern::{AtomKind, CaseMatching, Normalization, Pattern};
use nucleo_matcher::{Config as NucleoConfig, Matcher as NucleoMatcher, Utf32Str};
use rayon::prelude::*;
use rusqlite::{params, Connection, ErrorCode, OptionalExtension};
use crate::hashing::FramedSha256;
use crate::message_search::selected_message_field_parts;
use crate::models::{
AnalysisRequest, EditOp, FileCrossRef, FileEdit, FileEditSummary, FileQuery,
MessageClassificationMatch, MessageFilters, MessageHit, MessageSearchMode, ParsedSession,
ParserHealth, PlanningCount, Provider, ProviderParserHealth, Role, SearchExplain, SearchField,
SearchFilters, SearchHit, SessionRecord, SessionTimeProfile, SessionWithTranscript,
};
use crate::runtime::ExecutionRuntime;
use crate::util::{
fold_caseless, snippet_from_match, snippet_from_match_while, UnicodeLowerNeedle, SIGMAS,
};
pub const SCHEMA_VERSION: i64 = 5;
const PARSER_SCHEMA_VERSION: i64 = 5;
const PRE_FTS_PARSER_SCHEMA_VERSION: i64 = 3;
pub const MIN_READABLE_SCHEMA_VERSION: i64 = 2;
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum SchemaState {
Missing,
Current,
Older {
current: i64,
required: i64,
},
Newer {
current: i64,
supported: i64,
},
UnknownProviders {
providers: Vec<String>,
},
RepairableLayout {
reason: String,
},
RecoveryRequired {
reason: String,
},
}
impl SchemaState {
pub(crate) fn from_version(current: i64) -> Self {
match current.cmp(&SCHEMA_VERSION) {
std::cmp::Ordering::Less => Self::Older {
current,
required: SCHEMA_VERSION,
},
std::cmp::Ordering::Equal => Self::Current,
std::cmp::Ordering::Greater => Self::Newer {
current,
supported: SCHEMA_VERSION,
},
}
}
pub(crate) fn unknown_providers_message(path: &Path, providers: &[String]) -> String {
format!(
"index {} contains sessions from provider(s) {} that this aise {} does not recognize, \
so a newer aise wrote it; upgrade aise (`aise package update`) before opening it",
path.display(),
providers.join(", "),
env!("CARGO_PKG_VERSION")
)
}
}
pub(crate) fn unknown_index_providers(conn: &Connection) -> Result<Vec<String>> {
let table_exists: bool = conn.query_row(
"select exists(select 1 from sqlite_schema where type = 'table' and name = 'sessions')",
[],
|row| row.get(0),
)?;
if !table_exists {
return Ok(Vec::new());
}
let mut statement = conn.prepare("select distinct provider from sessions order by provider")?;
let providers = statement
.query_map([], |row| row.get::<_, String>(0))?
.collect::<rusqlite::Result<Vec<_>>>()?;
Ok(providers
.into_iter()
.filter(|provider| Provider::from_db_str(provider).is_err())
.collect())
}
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
pub enum MessageOrder {
OldestFirst,
NewestFirst,
}
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
pub(crate) enum MessageBatchControl {
Continue,
Stop,
}
#[derive(Debug)]
pub(crate) struct MessageBatchVisitOutcome {
pub(crate) rows_visited: usize,
pub(crate) exhausted: bool,
pub(crate) explain: Option<SearchExplain>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum SourceFileRefreshState {
Current,
ChangedSameParser,
ChangedParser,
New,
}
struct PreparedMessageQuery {
sql: String,
args: Vec<rusqlite::types::Value>,
explain: Option<SearchExplain>,
}
const TRIGRAM_PREFILTER_MIN_CORPUS: i64 = 50_000;
const TRIGRAM_BASE_REBUILD_DELTA: i64 = TRIGRAM_PREFILTER_MIN_CORPUS;
const FUZZY_SCORE_BATCH_SIZE: usize = 512;
const SESSION_SCORE_BATCH_SIZE: usize = 128;
const SESSION_SCORE_BATCH_BYTES: usize = 8 * 1024 * 1024;
const FUZZY_BATCH_BYTES: usize = 8 * 1024 * 1024;
const fn fuzzy_batch_is_full(rows: usize, bytes: usize) -> bool {
rows >= FUZZY_SCORE_BATCH_SIZE || bytes >= FUZZY_BATCH_BYTES
}
pub const DEFAULT_BUSY_TIMEOUT_MS: u64 = 5_000;
pub const DEFAULT_AUTO_REINDEX_BUSY_TIMEOUT_MS: u64 = 10_000;
pub const DEFAULT_AUTO_REINDEX_INTERVAL_MS: u64 = 1_500;
const SQLITE_PAGE_CACHE_KIB: i64 = 64 * 1_024;
const SQLITE_MMAP_BYTES: i64 = 256 * 1_024 * 1_024;
pub(crate) const MESSAGE_KIND_INDEX: &str = "idx_messages_excluded_kinds";
const MESSAGE_KIND_INDEX_COLUMNS: &str = "(kind, role, provider, session_id, ts, tool_name)";
fn excluded_kinds_predicate() -> String {
let classes = crate::models::MessageKind::EXCLUDABLE_BY_DEFAULT
.iter()
.map(|kind| format!("'{}'", kind.as_str()))
.collect::<Vec<_>>()
.join(", ");
format!("kind in ({classes})")
}
type ProgressReporter = Box<dyn Fn(&str) + Send + Sync>;
const SESSIONS_FTS_RETIRED_KEY: &str = "sessions_fts_retired";
const AUTO_REINDEX_COMPLETED_MS_KEY: &str = "auto_reindex_completed_ms";
const AUTO_REINDEX_COMPLETED_NS_KEY: &str = "auto_reindex_completed_ns";
const AUTO_REINDEX_PARSER_CONTRACT_KEYS: [&str; 4] = [
"auto_reindex_parser_contract_0",
"auto_reindex_parser_contract_1",
"auto_reindex_parser_contract_2",
"auto_reindex_parser_contract_3",
];
const AUTO_REINDEX_FRESHNESS_SQL: &str =
"select completed.value, word0.value, word1.value, word2.value, word3.value
from index_metadata completed
join index_metadata word0 on word0.key = ?2
join index_metadata word1 on word1.key = ?3
join index_metadata word2 on word2.key = ?4
join index_metadata word3 on word3.key = ?5
where completed.key = ?1";
const NANOSECONDS_PER_MILLISECOND: i64 = 1_000_000;
const NANOSECONDS_PER_SECOND: i64 = 1_000_000_000;
const CORRECTION_CANDIDATES_SQL: &str =
"select m.id, m.session_id, m.seq, m.provider, m.ts, m.content, m.authorship
from messages m
where m.record_relation = 'original'
and (
m.authorship = 'human'
or (
m.authorship = 'mixed'
and exists (
select 1
from message_content_parts p
where p.message_id = m.id and p.authorship = 'human'
)
)
)";
const CORRECTION_HUMAN_PARTS_SQL: &str = "select m.id, p.start_char, p.end_char
from messages m
join message_content_parts p on p.message_id = m.id
where m.authorship = 'mixed'
and m.record_relation = 'original'
and p.authorship = 'human'";
struct TrigramRebuild {
base_max: i64,
rebuilt: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct StorageAllocation {
pub total_bytes: u64,
pub reclaimable_bytes: u64,
}
fn elapsed_ms(now_ms: i64, earlier_ms: i64) -> u64 {
now_ms.saturating_sub(earlier_ms).max(0) as u64
}
pub struct Db {
conn: Connection,
runtime: Arc<ExecutionRuntime>,
access_scope: crate::search_scope::EffectiveAccessScope,
prefilter_min_corpus: i64,
trigram_rebuild_delta: i64,
implicit_index_maintenance: bool,
progress: Option<ProgressReporter>,
}
pub(crate) struct AnalysisTraversalReceipt {
pub(crate) selected_sessions: u64,
pub(crate) messages_in_selected_sessions: u64,
pub(crate) analyzed_user_messages: u64,
pub(crate) has_more: bool,
pub(crate) last_selected_session_id: Option<String>,
pub(crate) max_selected_session_updated_at: Option<DateTime<Utc>>,
pub(crate) corpus_digest: String,
}
const QUERY_PROGRESS_HANDLER_OPCODES: i32 = 10_000;
struct ProgressHandlerReset<'connection>(&'connection Connection);
impl Drop for ProgressHandlerReset<'_> {
fn drop(&mut self) {
let cleared = self.0.progress_handler(0, None::<fn() -> bool>);
debug_assert!(
cleared.is_ok(),
"clearing the SQLite progress handler failed: {cleared:?}"
);
}
}
struct ReadSnapshotRollback<'connection> {
connection: &'connection Connection,
active: bool,
}
#[derive(Debug)]
pub(crate) struct ReadSnapshotCleanupError {
operation: Option<String>,
cleanup: String,
}
impl ReadSnapshotCleanupError {
pub(crate) fn new(operation: Option<anyhow::Error>, cleanup: anyhow::Error) -> Self {
Self {
operation: operation.map(|error| format!("{error:#}")),
cleanup: format!("{cleanup:#}"),
}
}
}
impl std::fmt::Display for ReadSnapshotCleanupError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self.operation.as_deref() {
Some(operation) => write!(
formatter,
"message search failed ({operation}) and its read snapshot also failed to release: {}",
self.cleanup
),
None => write!(
formatter,
"failed to release message-search read snapshot: {}",
self.cleanup
),
}
}
}
impl std::error::Error for ReadSnapshotCleanupError {}
impl<'connection> ReadSnapshotRollback<'connection> {
fn begin(connection: &'connection Connection) -> Result<Self> {
anyhow::ensure!(
connection.is_autocommit(),
"cannot start a read snapshot while another transaction is active"
);
connection.execute_batch("begin deferred")?;
Ok(Self {
connection,
active: true,
})
}
fn rollback(mut self) -> Result<()> {
self.connection.execute_batch("rollback")?;
self.active = false;
Ok(())
}
}
impl Drop for ReadSnapshotRollback<'_> {
fn drop(&mut self) {
if self.active {
let _ = self.connection.execute_batch("rollback");
}
}
}
#[derive(Debug)]
pub(crate) struct QueryCancelled;
impl std::fmt::Display for QueryCancelled {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str("query cancelled")
}
}
impl std::error::Error for QueryCancelled {}
pub(crate) struct QueryCancellation {
cancelled: Arc<AtomicBool>,
interrupt: Mutex<Option<rusqlite::InterruptHandle>>,
}
impl QueryCancellation {
pub(crate) fn new() -> Self {
Self {
cancelled: Arc::new(AtomicBool::new(false)),
interrupt: Mutex::new(None),
}
}
pub(crate) fn cancel(&self) {
self.cancelled.store(true, Ordering::Release);
let interrupt = self
.interrupt
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(interrupt) = interrupt.as_ref() {
interrupt.interrupt();
}
}
pub(crate) fn is_cancelled(&self) -> bool {
self.cancelled.load(Ordering::Acquire)
}
pub(crate) fn ensure_active(&self) -> Result<()> {
if self.is_cancelled() {
return Err(QueryCancelled.into());
}
Ok(())
}
pub(crate) fn flag(&self) -> &AtomicBool {
self.cancelled.as_ref()
}
pub(crate) fn flag_arc(&self) -> Arc<AtomicBool> {
Arc::clone(&self.cancelled)
}
pub(crate) fn register(&self, connection: &Connection) {
let mut interrupt = self
.interrupt
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*interrupt = Some(connection.get_interrupt_handle());
if self.is_cancelled() {
interrupt
.as_ref()
.expect("interrupt handle was installed above")
.interrupt();
}
}
}
#[cfg(test)]
pub(crate) fn with_sqlite_query_timeout<T>(
connection: &Connection,
timeout_ms: Option<NonZeroU64>,
operation: &str,
recovery: &str,
run: impl FnOnce() -> Result<T>,
) -> Result<T> {
with_sqlite_query_control(connection, timeout_ms, None, operation, recovery, run)
}
pub(crate) fn with_sqlite_query_control<T>(
connection: &Connection,
timeout_ms: Option<NonZeroU64>,
cancellation: Option<Arc<AtomicBool>>,
operation: &str,
recovery: &str,
run: impl FnOnce() -> Result<T>,
) -> Result<T> {
if timeout_ms.is_none() && cancellation.is_none() {
return run();
}
let deadline = timeout_ms
.map(|timeout_ms| {
Instant::now()
.checked_add(Duration::from_millis(timeout_ms.get()))
.ok_or_else(|| anyhow!("{operation} timeout_ms is too large for this platform"))
})
.transpose()?;
let progress_cancellation = cancellation.clone();
connection
.progress_handler(
QUERY_PROGRESS_HANDLER_OPCODES,
Some(move || {
progress_cancellation
.as_ref()
.is_some_and(|flag| flag.load(Ordering::Acquire))
|| deadline.is_some_and(|deadline| Instant::now() >= deadline)
}),
)
.with_context(|| format!("{operation} could not arm its query stop conditions"))?;
let reset = ProgressHandlerReset(connection);
let result = run();
drop(reset);
result.map_err(|error| {
if error.chain().any(|cause| {
matches!(
cause.downcast_ref::<rusqlite::Error>(),
Some(rusqlite::Error::SqliteFailure(inner, _))
if inner.code == rusqlite::ErrorCode::OperationInterrupted
)
}) {
if cancellation
.as_ref()
.is_some_and(|flag| flag.load(Ordering::Acquire))
{
anyhow!("{operation} was cancelled")
} else if let Some(timeout_ms) = timeout_ms {
anyhow!(
"{operation} timed out after {} ms; {recovery}",
timeout_ms.get()
)
} else {
error
}
} else {
error
}
})
}
macro_rules! session_record_columns {
() => {
"s.id, s.provider, s.provider_session_id, s.title, s.summary, s.cwd, s.repo_root, \
s.created_at, s.updated_at, s.last_message_at, s.preview_text, s.source_path, \
s.message_count, s.parse_version, s.raw_metadata_json, s.parse_warning, s.discovery_source, \
s.parent_session_id, s.agent_label"
};
}
impl Db {
pub fn open(path: &Path) -> Result<Self> {
Self::open_with_busy_timeout(path, DEFAULT_BUSY_TIMEOUT_MS)
}
pub fn open_with_busy_timeout(path: &Path, busy_timeout_ms: u64) -> Result<Self> {
let worker_threads = std::thread::available_parallelism().unwrap_or(NonZeroUsize::MIN);
Self::open_with_threads(path, busy_timeout_ms, worker_threads)
}
pub fn open_with_threads(
path: &Path,
busy_timeout_ms: u64,
worker_threads: NonZeroUsize,
) -> Result<Self> {
if let Some(parent) = path.parent() {
fs::create_dir_all(parent)
.with_context(|| format!("failed to create {}", parent.display()))?;
}
let conn = Connection::open(path)?;
conn.busy_timeout(Duration::from_millis(busy_timeout_ms))?;
crate::sql_functions::register(&conn)?;
let db = Self {
conn,
runtime: Arc::new(ExecutionRuntime::new(worker_threads)),
access_scope: crate::search_scope::EffectiveAccessScope::All,
prefilter_min_corpus: TRIGRAM_PREFILTER_MIN_CORPUS,
trigram_rebuild_delta: TRIGRAM_BASE_REBUILD_DELTA,
implicit_index_maintenance: true,
progress: None,
};
db.init()?;
Ok(db)
}
pub(crate) fn open_existing_read_only_with_threads(
path: &Path,
busy_timeout_ms: u64,
worker_threads: NonZeroUsize,
) -> Result<Self> {
Self::open_existing_read_only_with_runtime(
path,
busy_timeout_ms,
Arc::new(ExecutionRuntime::new(worker_threads)),
)
}
pub(crate) fn open_existing_read_only_with_runtime(
path: &Path,
busy_timeout_ms: u64,
runtime: Arc<ExecutionRuntime>,
) -> Result<Self> {
let flags = rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY
| rusqlite::OpenFlags::SQLITE_OPEN_URI
| rusqlite::OpenFlags::SQLITE_OPEN_PRIVATE_CACHE;
let conn = Connection::open_with_flags(path, flags).with_context(|| {
format!(
"failed to open existing index {} read-only even though it was just verified to \
exist; check file permissions, or another process may have removed or \
relocated it in the meantime",
path.display()
)
})?;
conn.busy_timeout(Duration::from_millis(busy_timeout_ms))?;
crate::sql_functions::register(&conn)?;
conn.pragma_update(None, "query_only", true)?;
conn.pragma_update(None, "foreign_keys", true)?;
conn.pragma_update(None, "temp_store", 2_i64)?;
conn.pragma_update(None, "cache_size", -SQLITE_PAGE_CACHE_KIB)?;
conn.pragma_update(None, "mmap_size", SQLITE_MMAP_BYTES)?;
Ok(Self {
conn,
runtime,
access_scope: crate::search_scope::EffectiveAccessScope::All,
prefilter_min_corpus: TRIGRAM_PREFILTER_MIN_CORPUS,
trigram_rebuild_delta: TRIGRAM_BASE_REBUILD_DELTA,
implicit_index_maintenance: false,
progress: None,
})
}
pub(crate) fn execution_runtime(&self) -> Arc<ExecutionRuntime> {
Arc::clone(&self.runtime)
}
pub fn worker_threads(&self) -> usize {
self.runtime.worker_threads()
}
#[cfg(test)]
fn worker_pool_build_count(&self) -> usize {
self.runtime.pool_build_count()
}
pub(crate) fn interrupt_handle(&self) -> rusqlite::InterruptHandle {
self.conn.get_interrupt_handle()
}
pub(crate) fn interrupt_while(&self, cancellation: Arc<AtomicBool>) -> Result<()> {
self.conn
.progress_handler(
QUERY_PROGRESS_HANDLER_OPCODES,
Some(move || cancellation.load(Ordering::Acquire)),
)
.context("could not arm query cancellation on this connection")
}
pub(crate) fn install_query_cancellation(
&self,
cancellation: &QueryCancellation,
) -> Result<()> {
self.interrupt_while(cancellation.flag_arc())?;
cancellation.register(&self.conn);
Ok(())
}
pub(crate) fn with_read_snapshot<T>(&self, run: impl FnOnce() -> Result<T>) -> Result<T> {
if !self.conn.is_autocommit() {
return run();
}
let snapshot = ReadSnapshotRollback::begin(&self.conn)?;
let result = run();
let cleanup = snapshot.rollback();
match (result, cleanup) {
(Ok(value), Ok(())) => Ok(value),
(Err(error), Ok(())) => Err(error),
(Ok(_), Err(cleanup_error)) => {
Err(ReadSnapshotCleanupError::new(None, cleanup_error).into())
}
(Err(error), Err(cleanup_error)) => {
Err(ReadSnapshotCleanupError::new(Some(error), cleanup_error).into())
}
}
}
pub(crate) fn set_access_scope(
&mut self,
access_scope: crate::search_scope::EffectiveAccessScope,
) {
self.access_scope = access_scope;
}
pub(crate) const fn access_scope(&self) -> &crate::search_scope::EffectiveAccessScope {
&self.access_scope
}
fn validate_access_scope(&self) -> Result<()> {
self.access_scope.validate_stable()
}
pub fn set_busy_timeout_ms(&self, busy_timeout_ms: u64) -> Result<()> {
self.conn
.busy_timeout(Duration::from_millis(busy_timeout_ms))?;
Ok(())
}
pub fn busy_timeout_ms(&self) -> Result<u64> {
let timeout: i64 = self
.conn
.query_row("pragma busy_timeout", [], |row| row.get(0))?;
Ok(timeout.max(0) as u64)
}
pub(crate) fn storage_allocation(&self) -> Result<StorageAllocation> {
let (page_size, page_count, freelist_count): (i64, i64, i64) = self.conn.query_row(
"select (select page_size from pragma_page_size),
(select page_count from pragma_page_count),
(select freelist_count from pragma_freelist_count)",
[],
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
)?;
if page_size < 0 || page_count < 0 || freelist_count < 0 {
return Err(anyhow::anyhow!(
"SQLite returned negative allocation metadata: page_size={page_size}, page_count={page_count}, freelist_count={freelist_count}"
));
}
let page_size = page_size as u64;
let total_bytes = page_size.checked_mul(page_count as u64).ok_or_else(|| {
anyhow::anyhow!(
"SQLite allocation size overflow: page_size={page_size}, page_count={page_count}"
)
})?;
let reclaimable_bytes = page_size
.checked_mul(freelist_count as u64)
.ok_or_else(|| {
anyhow::anyhow!(
"SQLite reclaimable size overflow: page_size={page_size}, freelist_count={freelist_count}"
)
})?;
Ok(StorageAllocation {
total_bytes,
reclaimable_bytes,
})
}
pub fn with_busy_timeout_ms<T>(
&self,
busy_timeout_ms: u64,
f: impl FnOnce() -> Result<T>,
) -> Result<T> {
let original = self.busy_timeout_ms()?;
self.set_busy_timeout_ms(busy_timeout_ms)?;
let result = f();
let restore = self.set_busy_timeout_ms(original);
match (result, restore) {
(Ok(value), Ok(())) => Ok(value),
(Err(err), Ok(())) => Err(err),
(Ok(_), Err(err)) => Err(err),
(Err(err), Err(restore_err)) => Err(err.context(format!(
"also failed to restore busy_timeout to its prior value ({original}ms): {restore_err:#}"
))),
}
}
fn with_immediate_transaction<T>(&self, f: impl FnOnce() -> Result<T>) -> Result<T> {
self.conn.execute_batch("begin immediate")?;
let result = f();
match result {
Ok(value) => {
self.conn.execute_batch("commit")?;
Ok(value)
}
Err(err) => {
let _ = self.conn.execute_batch("rollback");
Err(err)
}
}
}
pub fn is_sqlite_busy_error(err: &anyhow::Error) -> bool {
err.chain().any(|source| {
source
.downcast_ref::<rusqlite::Error>()
.and_then(rusqlite::Error::sqlite_error_code)
.is_some_and(|code| {
matches!(code, ErrorCode::DatabaseBusy | ErrorCode::DatabaseLocked)
})
})
}
pub fn auto_reindex_is_fresh(&self, interval_ms: u64) -> Result<bool> {
self.auto_reindex_is_fresh_at(Utc::now().timestamp_millis(), interval_ms)
}
fn auto_reindex_is_fresh_at(&self, now_ms: i64, interval_ms: u64) -> Result<bool> {
let Some((completed_ms, parser_contract)) = self.auto_reindex_freshness_metadata()? else {
return Ok(false);
};
Ok(elapsed_ms(now_ms, completed_ms) < interval_ms
&& parser_contract == crate::source::provider_parse_contract_fingerprint())
}
pub fn mark_auto_reindex_complete(&self) -> Result<()> {
self.mark_auto_reindex_complete_at(Utc::now())
}
pub fn auto_reindex_completed_at(&self) -> Result<Option<DateTime<Utc>>> {
if let Some(value) = self.index_metadata_i64(AUTO_REINDEX_COMPLETED_NS_KEY)? {
return Ok(DateTime::from_timestamp(
value.div_euclid(NANOSECONDS_PER_SECOND),
value.rem_euclid(NANOSECONDS_PER_SECOND) as u32,
));
}
Ok(self
.index_metadata_i64(AUTO_REINDEX_COMPLETED_MS_KEY)?
.and_then(|value| Utc.timestamp_millis_opt(value).single()))
}
pub(crate) fn mark_auto_reindex_complete_at(&self, completed_at: DateTime<Utc>) -> Result<()> {
let completed_ms = completed_at.timestamp_millis();
let completed_ns = completed_at
.timestamp_nanos_opt()
.unwrap_or_else(|| completed_ms.saturating_mul(NANOSECONDS_PER_MILLISECOND));
let parser_contract = crate::source::provider_parse_contract_fingerprint();
self.conn.execute(
"insert into index_metadata (key, value) values
(?1, ?2), (?3, ?4), (?5, ?6), (?7, ?8), (?9, ?10), (?11, ?12)
on conflict(key) do update set value = excluded.value",
params![
AUTO_REINDEX_COMPLETED_MS_KEY,
completed_ms,
AUTO_REINDEX_COMPLETED_NS_KEY,
completed_ns,
AUTO_REINDEX_PARSER_CONTRACT_KEYS[0],
parser_contract[0],
AUTO_REINDEX_PARSER_CONTRACT_KEYS[1],
parser_contract[1],
AUTO_REINDEX_PARSER_CONTRACT_KEYS[2],
parser_contract[2],
AUTO_REINDEX_PARSER_CONTRACT_KEYS[3],
parser_contract[3],
],
)?;
Ok(())
}
fn auto_reindex_freshness_metadata(&self) -> Result<Option<(i64, [i64; 4])>> {
self.conn
.query_row(
AUTO_REINDEX_FRESHNESS_SQL,
params![
AUTO_REINDEX_COMPLETED_MS_KEY,
AUTO_REINDEX_PARSER_CONTRACT_KEYS[0],
AUTO_REINDEX_PARSER_CONTRACT_KEYS[1],
AUTO_REINDEX_PARSER_CONTRACT_KEYS[2],
AUTO_REINDEX_PARSER_CONTRACT_KEYS[3],
],
|row| {
Ok((
row.get(0)?,
[row.get(1)?, row.get(2)?, row.get(3)?, row.get(4)?],
))
},
)
.optional()
.map_err(Into::into)
}
fn index_metadata_i64(&self, key: &str) -> Result<Option<i64>> {
self.conn
.query_row(
"select value from index_metadata where key = ?1",
[key],
|row| row.get(0),
)
.optional()
.map_err(Into::into)
}
pub(crate) fn set_implicit_index_maintenance(&mut self, enabled: bool) {
self.implicit_index_maintenance = enabled;
}
pub fn set_progress_reporter(&mut self, reporter: impl Fn(&str) + Send + Sync + 'static) {
self.progress = Some(Box::new(reporter));
}
fn report_progress(&self, message: &str) {
if let Some(reporter) = &self.progress {
reporter(message);
}
}
fn init(&self) -> Result<()> {
self.conn.execute_batch(
"
pragma journal_mode = wal;
pragma foreign_keys = on;
-- Read-perf tuning for the analytics/search workload over a multi-GB index:
-- queries like `corrections` fetch thousands of message rows by rowid (the
-- role/ts index doesn't cover `content`), which is random I/O across the file.
-- A larger page cache + memory-mapped reads + in-memory temp store cut that
-- cost; synchronous=normal is the documented-safe durability level under WAL.
pragma synchronous = normal;
pragma temp_store = memory;
create table if not exists sessions (
id text primary key,
provider text not null,
provider_session_id text not null,
title text,
summary text,
cwd text,
repo_root text,
created_at text,
updated_at text,
last_message_at text,
preview_text text not null,
source_path text not null,
message_count integer,
parse_version text not null,
raw_metadata_json text,
parse_warning text,
discovery_source text not null,
parent_session_id text,
agent_label text
);
create table if not exists transcripts (
session_id text primary key references sessions(id) on delete cascade,
transcript_text text not null
);
create table if not exists files_seen (
provider text not null,
source_path text not null,
mtime_ns integer not null,
size_bytes integer not null,
last_indexed_at text not null,
content_hash text,
parse_version text not null default '',
-- Incremental tail-parse checkpoint (§7): byte offset (at a newline boundary)
-- up to which the file is parsed, and a fingerprint of the file's leading bytes
-- used to detect rewrite/rotation. NULL = no checkpoint → always a full parse.
tail_byte_offset integer,
prefix_fingerprint text,
primary key(provider, source_path)
);
create table if not exists index_metadata (
key text primary key,
value integer not null
);
create index if not exists idx_sessions_provider on sessions(provider);
create index if not exists idx_sessions_updated_at on sessions(updated_at desc);
create index if not exists idx_sessions_provider_id on sessions(provider_session_id);
create table if not exists messages (
id integer primary key,
session_id text not null references sessions(id) on delete cascade,
provider text not null,
seq integer not null,
role text not null,
ts text,
tool_name text,
kind text not null default 'unknown',
tool_call_id text,
is_compaction integer not null default 0,
content text not null,
authorship text not null default 'unknown',
correlation_authority text,
correlation_scope text,
correlation_id text,
record_relation text not null default 'unknown'
check (record_relation in ('original', 'mirror', 'unknown')),
check (
(correlation_authority is null and correlation_scope is null and correlation_id is null)
or
(correlation_authority is not null and correlation_scope is not null and correlation_id is not null)
)
);
-- Bare ts index for date-range message filters that span all roles; the
-- composites below lead with role/session_id and so cannot serve a bare
-- `ts between ? and ?` scan.
create index if not exists idx_messages_ts on messages(ts);
-- Composite indexes serve the hot filter + ORDER BY combinations straight from
-- the index (no temp B-tree sort) and, by leftmost-prefix, subsume a bare
-- (session_id) or (role) index: (session_id, seq) covers `where session_id=?`
-- [+ `order by seq`] (message search / get / context); (role, ts) covers
-- `where role=?` [+ `order by ts`] (corrections / planning / stats). Older
-- branch builds created standalone (session_id) and (role) indexes before these
-- composites existed — drop them so every index converges on this final shape;
-- they were pure write-amplification (an upstream index never had them).
drop index if exists idx_messages_session;
drop index if exists idx_messages_role;
create index if not exists idx_messages_session_seq on messages(session_id, seq);
create index if not exists idx_messages_provider_session_seq
on messages(provider, session_id, seq);
create index if not exists idx_messages_role_ts on messages(role, ts);
create index if not exists idx_messages_tool_name
on messages(tool_name) where tool_name is not null;
create table if not exists file_edits (
id integer primary key,
session_id text not null references sessions(id) on delete cascade,
provider text not null,
seq integer not null,
ts text,
tool text not null,
file_path text not null,
file_name text not null,
new_content text,
edits_json text
);
create index if not exists idx_file_edits_session on file_edits(session_id);
create index if not exists idx_file_edits_provider on file_edits(provider);
create index if not exists idx_file_edits_path on file_edits(file_path);
create index if not exists idx_file_edits_name on file_edits(file_name);
",
)?;
self.conn
.pragma_update(None, "cache_size", -SQLITE_PAGE_CACHE_KIB)?;
self.conn
.pragma_update(None, "mmap_size", SQLITE_MMAP_BYTES)?;
for (name, definition) in [
("kind", "kind text not null default 'unknown'"),
("tool_call_id", "tool_call_id text"),
("authorship", "authorship text not null default 'unknown'"),
("correlation_authority", "correlation_authority text"),
("correlation_scope", "correlation_scope text"),
("correlation_id", "correlation_id text"),
(
"record_relation",
"record_relation text not null default 'unknown'",
),
] {
let exists: bool = self.conn.query_row(
"select exists(select 1 from pragma_table_info('messages') where name = ?1)",
params![name],
|row| row.get(0),
)?;
if !exists {
self.conn
.execute_batch(&format!("alter table messages add column {definition}"))?;
}
}
self.conn.execute_batch(
"drop index if exists idx_messages_authorship_role_provider_ts;
create index if not exists idx_messages_tool_calls
on messages(session_id, seq) where kind = 'tool_call';
create index if not exists idx_messages_provider_session_seq
on messages(provider, session_id, seq);
create index if not exists idx_messages_session_authorship_relation_seq
on messages(session_id, authorship, record_relation, seq);
create index if not exists idx_messages_authorship_relation_ts
on messages(authorship, record_relation, ts desc, session_id, seq);
create index if not exists idx_messages_correlation
on messages(correlation_authority, correlation_scope, correlation_id)
where correlation_id is not null;
create table if not exists message_content_parts (
message_id integer not null references messages(id) on delete cascade,
ordinal integer not null check (ordinal >= 0),
start_char integer not null check (start_char >= 0),
end_char integer not null check (end_char > start_char),
authorship text not null,
origin text not null,
primary key(message_id, ordinal)
) without rowid;
create trigger if not exists messages_correlation_all_or_none_insert
before insert on messages
when ((new.correlation_authority is not null)
+ (new.correlation_scope is not null)
+ (new.correlation_id is not null)) not in (0, 3)
begin
select raise(abort, 'message correlation authority, scope, and id must be all null or all non-null');
end;
create trigger if not exists messages_correlation_all_or_none_update
before update of correlation_authority, correlation_scope, correlation_id on messages
when ((new.correlation_authority is not null)
+ (new.correlation_scope is not null)
+ (new.correlation_id is not null)) not in (0, 3)
begin
select raise(abort, 'message correlation authority, scope, and id must be all null or all non-null');
end;",
)?;
let retired: bool = self.conn.query_row(
"select exists(select 1 from index_metadata where key = ?1)",
params![SESSIONS_FTS_RETIRED_KEY],
|row| row.get(0),
)?;
if !retired {
self.conn
.execute_batch("drop table if exists sessions_fts")?;
self.conn.execute(
"insert into index_metadata (key, value) values (?1, 1)
on conflict(key) do update set value = excluded.value",
params![SESSIONS_FTS_RETIRED_KEY],
)?;
}
self.conn.execute_batch(
"create virtual table if not exists sessions_fts using fts5(
title, summary, preview_text, transcript_text
)",
)?;
crate::fts::install_released_message_word_index(&self.conn)?;
let messages_exist: bool =
self.conn
.query_row("select exists(select 1 from messages)", [], |row| {
row.get(0)
})?;
if !messages_exist {
self.ensure_message_kind_index()?;
}
if self.schema_version()? == 0 && !messages_exist {
let tx = self.conn.unchecked_transaction()?;
crate::fts::install_target_message_search_indexes(&tx)?;
tx.execute_batch(&format!("pragma user_version = {SCHEMA_VERSION}"))?;
tx.commit()?;
}
let indexed_messages_exist: bool = self.conn.query_row(
"select exists(select 1 from messages_fts_docsize)",
[],
|row| row.get(0),
)?;
if messages_exist && !indexed_messages_exist {
self.conn
.execute_batch("insert into messages_fts(messages_fts) values('rebuild')")?;
}
if self.schema_version()? < 4 {
crate::trigram_index::ensure_schema(&self.conn)?;
self.conn.execute_batch(
"drop trigger if exists messages_tri_ai;
drop trigger if exists messages_tri_ad;
drop trigger if exists messages_tri_au;
drop table if exists messages_trigram_terms;
drop table if exists messages_trigram_vocab;
drop table if exists messages_trigram;",
)?;
}
self.conn.execute_batch(
"create virtual table if not exists messages_vocab
using fts5vocab('messages_fts', 'row');",
)?;
let schema_version = self.schema_version()?;
let message_search_layout_problem = match schema_version {
4 => crate::indexer::message_search_layout_problem(&self.conn)?,
SCHEMA_VERSION => crate::indexer::current_schema_layout_problem(&self.conn)?,
_ => None,
};
if message_search_layout_problem.is_some() && crate::indexer::base_data_intact(&self.conn)?
{
self.migrate_message_search_schema_exclusive()?;
}
self.ensure_column("files_seen", "tail_byte_offset", "tail_byte_offset integer")?;
self.ensure_column(
"files_seen",
"prefix_fingerprint",
"prefix_fingerprint text",
)?;
let parse_version_added = self.ensure_column(
"files_seen",
"parse_version",
"parse_version text not null default ''",
)?;
if parse_version_added {
self.backfill_source_parse_versions()?;
}
self.ensure_column("sessions", "parent_session_id", "parent_session_id text")?;
self.ensure_column("sessions", "agent_label", "agent_label text")?;
Ok(())
}
fn ensure_column(&self, table: &str, column: &str, column_decl: &str) -> Result<bool> {
let present = self
.conn
.prepare(&format!("pragma table_info({table})"))?
.query_map([], |row| row.get::<_, String>(1))?
.filter_map(Result::ok)
.any(|name| name == column);
if !present {
self.conn
.execute_batch(&format!("alter table {table} add column {column_decl}"))?;
}
Ok(!present)
}
fn backfill_source_parse_versions(&self) -> Result<()> {
self.conn.execute_batch(
"with source_versions as (
select provider, source_path, min(parse_version) as parse_version
from sessions
group by provider, source_path
having count(distinct parse_version) = 1
)
update files_seen
set parse_version = coalesce((
select source_versions.parse_version
from source_versions
where source_versions.provider = files_seen.provider
and source_versions.source_path = files_seen.source_path
), '')
where parse_version = ''",
)?;
Ok(())
}
pub(crate) fn ensure_trigram_base(&self) -> Result<i64> {
anyhow::ensure!(
self.schema_version()? < 4,
"custom trigram maintenance is unavailable on schema v4; SQLite FTS5 maintains messages_trigram incrementally"
);
if !crate::trigram_index::schema_is_compatible(&self.conn)? {
anyhow::ensure!(
self.implicit_index_maintenance,
"substring/regex search index has an incompatible schema and automatic \
maintenance is disabled; rerun without --index-refresh existing-only to rebuild it"
);
return self
.rebuild_trigram_base_with_writer_lock()
.map(|rebuild| rebuild.base_max);
}
let base_max = crate::trigram_index::base_max_id(&self.conn)?;
if !self.implicit_index_maintenance {
return Ok(base_max);
}
let max_id: i64 =
self.conn
.query_row("select coalesce(max(id), 0) from messages", [], |row| {
row.get(0)
})?;
if (base_max == 0 && max_id > 0) || (max_id - base_max) > self.trigram_rebuild_delta {
return match self.rebuild_trigram_base_with_writer_lock() {
Ok(rebuild) => Ok(rebuild.base_max),
Err(err) if Self::is_sqlite_busy_error(&err) => {
self.report_progress(
"substring/regex search index is already being updated; scanning the unindexed delta directly",
);
Ok(base_max)
}
Err(err) => Err(err),
};
}
Ok(base_max)
}
fn rebuild_trigram_base_with_writer_lock(&self) -> Result<TrigramRebuild> {
self.with_immediate_transaction(|| {
let schema_compatible = crate::trigram_index::schema_is_compatible(&self.conn)?;
if !schema_compatible {
self.report_progress(
"substring/regex search index had an incompatible schema; rebuilding it from the message table",
);
self.conn.execute_batch(
"drop table if exists trigram_postings; drop table if exists trigram_meta;",
)?;
crate::trigram_index::ensure_schema(&self.conn)?;
}
let base_max = crate::trigram_index::base_max_id(&self.conn)?;
let max_id: i64 =
self.conn
.query_row("select coalesce(max(id), 0) from messages", [], |row| {
row.get(0)
})?;
if schema_compatible
&& !((base_max == 0 && max_id > 0)
|| (max_id - base_max) > self.trigram_rebuild_delta)
{
return Ok(TrigramRebuild {
base_max,
rebuilt: false,
});
}
let count: i64 = self
.conn
.query_row("select count(*) from messages", [], |row| row.get(0))?;
self.report_progress(&format!(
"building substring/regex search index in parallel (one-time over {count} messages)…"
));
let base_max = crate::trigram_index::build_in_current_transaction(
&self.conn,
&self.runtime,
)?;
Ok(TrigramRebuild {
base_max,
rebuilt: true,
})
})
.and_then(|rebuild| {
if rebuild.rebuilt {
self.checkpoint_truncate()?;
}
Ok(rebuild)
})
}
fn stage_candidates(
&self,
base_max: i64,
candidates: &std::collections::HashSet<i64>,
) -> Result<()> {
self.conn.execute_batch(
"create temp table if not exists _trigram_cand (id integer primary key);
delete from _trigram_cand;",
)?;
let tx = self.conn.unchecked_transaction()?;
{
let mut stmt = tx.prepare("insert or ignore into _trigram_cand(id) values (?1)")?;
for id in candidates {
stmt.execute([id])?;
}
}
tx.execute(
"insert or ignore into _trigram_cand(id) select id from messages where id > ?1",
[base_max],
)?;
tx.commit()?;
Ok(())
}
pub fn checkpoint_truncate(&self) -> Result<()> {
self.conn
.query_row("pragma wal_checkpoint(truncate)", [], |_| Ok(()))
.or_else(|err| match err {
rusqlite::Error::QueryReturnedNoRows => Ok(()),
other => Err(other),
})?;
Ok(())
}
#[cfg(test)]
pub(crate) fn execute_batch_for_test(&self, sql: &str) -> Result<()> {
self.conn.execute_batch(sql)?;
Ok(())
}
#[cfg(test)]
pub(crate) fn wal_checkpoint_busy_for_test(&self) -> Result<i64> {
self.conn
.query_row("pragma wal_checkpoint(passive)", [], |row| row.get(0))
.map_err(Into::into)
}
pub fn optimize_fts(&self) -> Result<()> {
self.conn
.execute_batch("insert into messages_fts(messages_fts) values('optimize');")?;
if self.schema_version()? >= 4 {
self.conn.execute_batch(
"insert into messages_trigram(messages_trigram) values('optimize');",
)?;
}
Ok(())
}
pub fn vacuum(&self) -> Result<()> {
self.conn.execute_batch("vacuum")?;
Ok(())
}
pub fn needs_backfill(&self) -> Result<bool> {
Ok(self.schema_version()? < PARSER_SCHEMA_VERSION)
}
pub fn schema_is_readable(&self) -> Result<bool> {
Ok(match self.schema_state()? {
SchemaState::Current => true,
SchemaState::Older { current, .. } => current >= MIN_READABLE_SCHEMA_VERSION,
SchemaState::Missing
| SchemaState::Newer { .. }
| SchemaState::UnknownProviders { .. }
| SchemaState::RepairableLayout { .. }
| SchemaState::RecoveryRequired { .. } => false,
})
}
pub(crate) fn schema_state(&self) -> Result<SchemaState> {
match SchemaState::from_version(self.schema_version()?) {
SchemaState::Current => {
let providers = unknown_index_providers(&self.conn)?;
Ok(if providers.is_empty() {
SchemaState::Current
} else {
SchemaState::UnknownProviders { providers }
})
}
other => Ok(other),
}
}
pub fn schema_version(&self) -> Result<i64> {
self.conn
.query_row("pragma user_version", [], |row| row.get(0))
.map_err(Into::into)
}
pub(crate) fn migrate_message_search_schema_exclusive(&self) -> Result<()> {
let schema_version = self.schema_version()?;
if schema_version == SCHEMA_VERSION
&& crate::indexer::current_schema_layout_problem(&self.conn)?.is_none()
{
return Ok(());
}
if schema_version == 4
&& crate::indexer::message_search_layout_problem(&self.conn)?.is_none()
{
return Ok(());
}
let count: i64 = self
.conn
.query_row("select count(*) from messages", [], |row| row.get(0))?;
self.report_progress(&format!(
"rebuilding message-search indexes in place (one-time over {count} messages)…"
));
let target_version = if schema_version == 4 {
4
} else {
SCHEMA_VERSION
};
crate::fts::migrate_message_search_schema_offline(&self.conn, target_version)
}
pub fn mark_schema_current(&self) -> Result<()> {
let target = if self.schema_version()? >= 4 {
PARSER_SCHEMA_VERSION
} else {
PRE_FTS_PARSER_SCHEMA_VERSION
};
self.conn
.execute_batch(&format!("pragma user_version = {target}"))?;
Ok(())
}
pub fn clear_all(&self) -> Result<()> {
self.with_immediate_transaction(|| {
self.conn.execute_batch(
"
delete from transcripts;
delete from messages;
delete from file_edits;
delete from sessions;
delete from files_seen;
",
)?;
Ok(())
})
}
pub(crate) fn clear_trigram_base(&self) -> Result<()> {
if self.schema_version()? >= 4 {
return Ok(());
}
self.conn.execute_batch(
"
delete from trigram_postings;
delete from trigram_meta;
",
)?;
Ok(())
}
pub fn is_file_current(
&self,
provider: Provider,
path: &str,
mtime_ns: i64,
size: i64,
parse_version: &str,
) -> Result<bool> {
Ok(
self.source_file_refresh_state(provider, path, mtime_ns, size, parse_version)?
== SourceFileRefreshState::Current,
)
}
pub(crate) fn source_file_refresh_state(
&self,
provider: Provider,
path: &str,
mtime_ns: i64,
size: i64,
parse_version: &str,
) -> Result<SourceFileRefreshState> {
Ok(match self.file_seen_state(provider, path)? {
None => SourceFileRefreshState::New,
Some((_, _, stored_version)) if stored_version != parse_version => {
SourceFileRefreshState::ChangedParser
}
Some((stored_mtime, stored_size, _))
if stored_mtime == mtime_ns && stored_size == size =>
{
SourceFileRefreshState::Current
}
Some(_) => SourceFileRefreshState::ChangedSameParser,
})
}
fn file_seen_state(
&self,
provider: Provider,
path: &str,
) -> Result<Option<(i64, i64, String)>> {
self.conn
.query_row(
"select mtime_ns, size_bytes, parse_version from files_seen
where provider = ?1 and source_path = ?2",
params![provider.as_str(), path],
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
)
.optional()
.map_err(Into::into)
}
pub(crate) fn indexed_source_identities(
&self,
) -> Result<Vec<(Provider, String, usize, String)>> {
let mut stmt = self.conn.prepare(
"select provider, source_path, count(*), min(id) from sessions
group by provider, source_path",
)?;
let rows = stmt
.query_map([], |row| {
Ok((
decode_db_enum(
row.get::<_, String>(0)?,
0,
"sessions.provider",
Provider::from_db_str,
)?,
row.get::<_, String>(1)?,
row.get::<_, i64>(2)? as usize,
row.get::<_, String>(3)?,
))
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
Ok(rows)
}
pub(crate) fn exact_session_source_path(&self, id: &str) -> Result<Option<String>> {
self.conn
.query_row(
"select source_path from sessions where id = ?1",
params![id],
|row| row.get(0),
)
.optional()
.map_err(Into::into)
}
pub fn upsert_session(
&self,
parsed: &ParsedSession,
mtime_ns: i64,
size_bytes: i64,
) -> Result<()> {
self.upsert_session_with_mode(parsed, mtime_ns, size_bytes, true, &[])
}
pub fn replace_session(
&self,
parsed: &ParsedSession,
mtime_ns: i64,
size_bytes: i64,
) -> Result<()> {
self.upsert_session_with_mode(parsed, mtime_ns, size_bytes, false, &[])
}
pub(crate) fn upsert_session_reconciling_sources(
&self,
parsed: &ParsedSession,
mtime_ns: i64,
size_bytes: i64,
source_aliases: &[String],
allow_append_optimization: bool,
) -> Result<()> {
self.upsert_session_with_mode(
parsed,
mtime_ns,
size_bytes,
allow_append_optimization,
source_aliases,
)
}
fn upsert_session_with_mode(
&self,
parsed: &ParsedSession,
mtime_ns: i64,
size_bytes: i64,
allow_append_optimization: bool,
source_aliases: &[String],
) -> Result<()> {
let tx = self.conn.unchecked_transaction()?;
let session = &parsed.session;
for alias in source_aliases
.iter()
.filter(|alias| alias.as_str() != session.source_path)
{
tx.execute(
"delete from sessions where provider = ?1 and source_path = ?2",
params![session.provider.as_str(), alias],
)?;
tx.execute(
"delete from files_seen where provider = ?1 and source_path = ?2",
params![session.provider.as_str(), alias],
)?;
}
tx.execute(
"delete from sessions
where provider = ?1 and source_path = ?2 and id != ?3",
params![session.provider.as_str(), session.source_path, session.id],
)?;
tx.execute(
"
insert into sessions (
id, provider, provider_session_id, title, summary, cwd, repo_root, created_at,
updated_at, last_message_at, preview_text, source_path, message_count, parse_version,
raw_metadata_json, parse_warning, discovery_source, parent_session_id, agent_label
) values (
?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8,
?9, ?10, ?11, ?12, ?13, ?14,
?15, ?16, ?17, ?18, ?19
)
on conflict(id) do update set
provider = excluded.provider,
provider_session_id = excluded.provider_session_id,
title = excluded.title,
summary = excluded.summary,
cwd = excluded.cwd,
repo_root = excluded.repo_root,
created_at = excluded.created_at,
updated_at = excluded.updated_at,
last_message_at = excluded.last_message_at,
preview_text = excluded.preview_text,
source_path = excluded.source_path,
message_count = excluded.message_count,
parse_version = excluded.parse_version,
raw_metadata_json = excluded.raw_metadata_json,
parse_warning = excluded.parse_warning,
discovery_source = excluded.discovery_source,
parent_session_id = excluded.parent_session_id,
agent_label = excluded.agent_label
",
params![
session.id,
session.provider.as_str(),
session.provider_session_id,
session.title,
session.summary,
session.cwd,
session.repo_root,
session.created_at.map(|value| value.to_rfc3339()),
session.updated_at.map(|value| value.to_rfc3339()),
session.last_message_at.map(|value| value.to_rfc3339()),
session.preview_text,
session.source_path,
session.message_count,
session.parse_version,
session.raw_metadata_json,
session.parse_warning,
session.discovery_source,
session.parent_session_id,
session.agent_label,
],
)?;
tx.execute(
"
insert into transcripts (session_id, transcript_text)
values (?1, ?2)
on conflict(session_id) do update set transcript_text = excluded.transcript_text
",
params![session.id, parsed.transcript_text],
)?;
tx.execute(
"
insert into files_seen (
provider, source_path, mtime_ns, size_bytes, last_indexed_at,
content_hash, parse_version
)
values (?1, ?2, ?3, ?4, ?5, null, ?6)
on conflict(provider, source_path) do update set
mtime_ns = excluded.mtime_ns,
size_bytes = excluded.size_bytes,
last_indexed_at = excluded.last_indexed_at,
parse_version = excluded.parse_version
",
params![
session.provider.as_str(),
session.source_path,
mtime_ns,
size_bytes,
Utc::now().to_rfc3339(),
session.parse_version,
],
)?;
let existing_count: i64 = tx.query_row(
"select count(*) from messages where session_id = ?1",
params![session.id],
|row| row.get(0),
)?;
let parsed_count = parsed.messages.len() as i64;
let append_from: Option<usize> =
if allow_append_optimization && existing_count > 0 && parsed_count > existing_count {
let boundary = &parsed.messages[(existing_count - 1) as usize];
let existing_boundary: Option<String> = tx
.query_row(
"select content from messages where session_id = ?1 and seq = ?2",
params![session.id, boundary.seq],
|row| row.get(0),
)
.optional()?;
(existing_boundary.as_deref() == Some(boundary.content.as_str()))
.then_some(existing_count as usize)
} else {
None
};
let new_messages = match append_from {
Some(start) => &parsed.messages[start..],
None => {
tx.execute(
"delete from messages where session_id = ?1",
params![session.id],
)?;
&parsed.messages[..]
}
};
insert_messages(&tx, session, new_messages.iter().map(|m| (m.seq, m)))?;
tx.execute(
"delete from file_edits where session_id = ?1",
params![session.id],
)?;
insert_file_edits(&tx, session, parsed.file_edits.iter().map(|e| (e.seq, e)))?;
tx.commit()?;
Ok(())
}
pub fn purge_injected_messages(&self) -> Result<usize> {
let deleted = self.conn.execute(
"delete from messages where role = 'user' and (\
content like '<local-command-stdout>%' \
or content like '<local-command-stderr>%' \
or content like '<local-command-caveat>%' \
or content like '<environment_context>%' \
or content like '<turn_aborted>%')",
[],
)?;
Ok(deleted)
}
pub fn file_checkpoint(
&self,
provider: Provider,
source_path: &str,
required_parse_version: &str,
) -> Result<Option<(i64, String)>> {
let row = self
.conn
.query_row(
"select tail_byte_offset, prefix_fingerprint from files_seen
where provider = ?1 and source_path = ?2 and parse_version = ?3",
params![provider.as_str(), source_path, required_parse_version],
|row| {
Ok((
row.get::<_, Option<i64>>(0)?,
row.get::<_, Option<String>>(1)?,
))
},
)
.optional()?;
Ok(match row {
Some((Some(offset), Some(fingerprint))) => Some((offset, fingerprint)),
_ => None,
})
}
pub fn set_file_checkpoint(
&self,
provider: Provider,
source_path: &str,
tail_byte_offset: i64,
prefix_fingerprint: &str,
) -> Result<()> {
self.conn.execute(
"update files_seen set tail_byte_offset = ?3, prefix_fingerprint = ?4
where provider = ?1 and source_path = ?2",
params![
provider.as_str(),
source_path,
tail_byte_offset,
prefix_fingerprint
],
)?;
Ok(())
}
pub fn append_tail(
&self,
tail: &crate::tail::TailParse,
mtime_ns: i64,
size_bytes: i64,
) -> Result<()> {
let session = &tail.session;
let tx = self.conn.unchecked_transaction()?;
let existing_count: i64 = tx.query_row(
"select count(*) from messages where session_id = ?1",
params![session.id],
|row| row.get(0),
)?;
insert_messages(
&tx,
session,
tail.new_messages
.iter()
.enumerate()
.map(|(i, m)| (existing_count + i as i64, m)),
)?;
let existing_edit_seq: i64 = tx.query_row(
"select coalesce(max(seq), -1) from file_edits where session_id = ?1",
params![session.id],
|row| row.get(0),
)?;
insert_file_edits(
&tx,
session,
tail.new_file_edits
.iter()
.enumerate()
.map(|(i, e)| (existing_edit_seq + 1 + i as i64, e)),
)?;
let new_count = existing_count + tail.new_messages.len() as i64;
tx.execute(
"update sessions set
updated_at = case when ?2 is not null and ?2 > coalesce(updated_at, '') then ?2
else updated_at end,
last_message_at = case when ?3 is not null and ?3 > coalesce(last_message_at, '') then ?3
else last_message_at end,
title = ?4,
preview_text = ?5,
cwd = coalesce(cwd, ?6),
message_count = ?7
where id = ?1",
params![
session.id,
session.updated_at.map(|value| value.to_rfc3339()),
session.last_message_at.map(|value| value.to_rfc3339()),
session.title,
session.preview_text,
session.cwd,
new_count,
],
)?;
if !tail.new_transcript.is_empty() {
tx.execute(
"update transcripts set transcript_text =
case when transcript_text = '' then ?2
else transcript_text || char(10) || char(10) || ?2 end
where session_id = ?1",
params![session.id, tail.new_transcript],
)?;
}
tx.execute(
"update files_seen set
mtime_ns = ?3, size_bytes = ?4, last_indexed_at = ?5,
tail_byte_offset = ?6, prefix_fingerprint = ?7, parse_version = ?8
where provider = ?1 and source_path = ?2",
params![
session.provider.as_str(),
session.source_path,
mtime_ns,
size_bytes,
Utc::now().to_rfc3339(),
tail.new_tail_offset,
tail.new_fingerprint,
session.parse_version,
],
)?;
tx.commit()?;
Ok(())
}
pub fn message_count(&self) -> Result<i64> {
Ok(self
.conn
.query_row("select count(*) from messages", [], |row| row.get(0))?)
}
pub fn has_sessions(&self) -> Result<bool> {
Ok(self
.conn
.query_row("select exists(select 1 from sessions limit 1)", [], |row| {
row.get(0)
})?)
}
pub fn messages_fts_count(&self) -> Result<i64> {
Ok(self
.conn
.query_row("select count(*) from messages_fts_docsize", [], |row| {
row.get(0)
})?)
}
pub fn message_role_counts(&self, filters: &MessageFilters) -> Result<Vec<(String, i64)>> {
use rusqlite::types::Value;
self.validate_access_scope()?;
let mut sql = String::from("select m.role, count(*) from messages m where 1 = 1");
let mut args: Vec<Value> = Vec::new();
append_message_filters(&mut sql, &mut args, filters, &self.access_scope);
sql.push_str(" group by m.role order by m.role");
let mut stmt = self.conn.prepare(&sql)?;
let rows = stmt
.query_map(rusqlite::params_from_iter(args.iter()), |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
Ok(rows)
}
pub fn vocabulary(
&self,
trigram: bool,
limit: usize,
prefix: Option<&str>,
) -> Result<Vec<(String, i64, i64)>> {
use rusqlite::types::Value;
let lim: i64 = if limit == 0 { -1 } else { limit as i64 };
let schema_version = self.schema_version()?;
let (source, term, doc, total) = if trigram && schema_version >= 4 {
("messages_trigram_terms", "term", "doc", "doc")
} else if trigram {
self.ensure_trigram_base()?;
("trigram_postings", "tg", "df", "df")
} else {
("messages_vocab", "term", "doc", "cnt")
};
let folded = prefix
.map(fold_caseless)
.filter(|prefix| !prefix.is_empty());
let mut args: Vec<Value> = vec![Value::Integer(lim)];
let mut filter = String::new();
if let Some(prefix) = folded {
filter = format!(" where {term} >= ?{}", args.len() + 1);
if let Some(upper) = prefix_upper_bound(&prefix) {
args.push(Value::Text(prefix));
args.push(Value::Text(upper));
filter.push_str(&format!(" and {term} < ?{}", args.len()));
} else {
args.push(Value::Text(prefix));
}
}
let sql = format!(
"select {term}, {doc}, {total} from {source}{filter} \
order by {total} desc, {term} limit ?1"
);
let mut stmt = self.conn.prepare(&sql)?;
let rows = stmt
.query_map(rusqlite::params_from_iter(args.iter()), |row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, i64>(1)?,
row.get::<_, i64>(2)?,
))
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
Ok(rows)
}
pub fn search_messages(
&self,
query: &str,
filters: &MessageFilters,
) -> Result<Vec<MessageHit>> {
Ok(self.search_messages_with_explain(query, filters, false)?.0)
}
pub(crate) fn search_message_plan(
&self,
plan: &crate::message_search::MessageRetrievalPlan,
include_explain: bool,
) -> Result<(Vec<MessageHit>, Option<SearchExplain>)> {
let (query, filters, order) = Self::message_plan_execution(plan)?;
self.search_messages_with_explain_order(query, &filters, include_explain, order)
}
pub(crate) fn visit_message_plan_batches(
&self,
plan: &crate::message_search::MessageRetrievalPlan,
include_explain: bool,
batch_rows: NonZeroUsize,
visitor: impl FnMut(Vec<MessageHit>) -> Result<MessageBatchControl>,
) -> Result<MessageBatchVisitOutcome> {
let (query, filters, order) = Self::message_plan_execution(plan)?;
self.visit_messages_in_batches(query, &filters, include_explain, order, batch_rows, visitor)
}
fn message_plan_execution(
plan: &crate::message_search::MessageRetrievalPlan,
) -> Result<(&str, MessageFilters, MessageOrder)> {
use crate::message_search::{MatchWindow, MessageQuery, ResolvedExtent};
let query = plan.query.text().unwrap_or("");
let match_mode = match &plan.query {
MessageQuery::All | MessageQuery::Literal(_) => MessageSearchMode::Literal,
MessageQuery::Regex(_) => MessageSearchMode::Regex,
MessageQuery::Fuzzy(_) => MessageSearchMode::Fuzzy,
};
let (limit, offset) = match plan.extent {
ResolvedExtent::Page { limit, offset } => (
limit
.get()
.checked_add(1)
.ok_or_else(|| anyhow!("message page limit plus look-ahead overflows"))?,
offset,
),
ResolvedExtent::AllResults { offset } => (0, offset),
};
let filters = MessageFilters {
role: plan.predicates.role,
kinds: plan.predicates.kinds.clone(),
field: Some(plan.target.field()),
argument_path: plan
.target
.argument_path()
.map(|pointer| pointer.as_str().to_string()),
providers: plan.predicates.providers.clone(),
session_id: plan.predicates.session_id.clone(),
workspace_path_prefix: plan.predicates.workspace_path_prefix.clone(),
transcript_path_prefix: plan.predicates.transcript_path_prefix.clone(),
exclude_workspace_path_prefixes: plan
.predicates
.exclude_workspace_path_prefixes
.clone(),
exclude_transcript_path_prefixes: plan
.predicates
.exclude_transcript_path_prefixes
.clone(),
exclude_session_ids: plan.predicates.exclude_session_ids.clone(),
since: plan.predicates.time.since(),
until: plan.predicates.time.until(),
seq_from: plan.predicates.sequence.and_then(|range| range.from()),
seq_to: plan.predicates.sequence.and_then(|range| range.to()),
match_mode,
tool: plan.predicates.tool_name_contains.clone(),
no_compaction: !plan.predicates.include_compaction,
limit,
offset,
..Default::default()
};
let order = if plan.match_window == Some(MatchWindow::Latest) {
MessageOrder::NewestFirst
} else {
MessageOrder::OldestFirst
};
Ok((query, filters, order))
}
pub(crate) fn search_messages_ordered(
&self,
query: &str,
filters: &MessageFilters,
order: MessageOrder,
) -> Result<Vec<MessageHit>> {
anyhow::ensure!(
order == MessageOrder::OldestFirst || filters.session_id.is_some(),
"newest-first message order requires session_id because sequence numbers are session-local"
);
Ok(self
.search_messages_with_explain_order(query, filters, false, order)?
.0)
}
pub fn read_session_messages(
&self,
filters: &MessageFilters,
order: MessageOrder,
) -> Result<Vec<MessageHit>> {
anyhow::ensure!(
filters.session_id.is_some(),
"read_session_messages requires filters.session_id because sequence numbers are session-local"
);
let mut hits = self.search_messages_ordered("", filters, order)?;
if order == MessageOrder::NewestFirst {
hits.reverse();
}
Ok(hits)
}
pub fn search_messages_with_explain(
&self,
query: &str,
filters: &MessageFilters,
include_explain: bool,
) -> Result<(Vec<MessageHit>, Option<SearchExplain>)> {
self.search_messages_with_explain_order(
query,
filters,
include_explain,
MessageOrder::OldestFirst,
)
}
fn prepare_non_fuzzy_message_query(
&self,
query: &str,
filters: &MessageFilters,
include_explain: bool,
order: MessageOrder,
) -> Result<PreparedMessageQuery> {
use rusqlite::types::Value;
debug_assert_ne!(filters.match_mode, MessageSearchMode::Fuzzy);
let field = filters.field.unwrap_or(SearchField::Content);
let mut sql = String::from(
"select m.session_id, m.provider, m.seq, m.role, m.ts, m.tool_name, m.kind, m.tool_call_id, m.content \
from messages m where 1 = 1",
);
let mut args = Vec::new();
append_message_filters(&mut sql, &mut args, filters, &self.access_scope);
let explain = if field == SearchField::Content {
let literal_query =
filters.match_mode == MessageSearchMode::Literal && !query.is_empty();
if literal_query {
sql.push_str(" and unicode_lower_contains(m.content, ?)");
args.push(Value::Text(fold_caseless(query)));
}
if filters.match_mode == MessageSearchMode::Regex {
regex::Regex::new(query).map_err(|error| anyhow!("invalid regex: {error}"))?;
sql.push_str(" and rust_regexp(?, m.content)");
args.push(Value::Text(query.to_string()));
}
let prefilter_pattern = if filters.match_mode == MessageSearchMode::Regex {
Some(query.to_string())
} else if literal_query {
Some(regex::escape(query))
} else {
None
};
if self.schema_version()? >= 4 {
let prefilter = prefilter_pattern
.as_deref()
.and_then(crate::trigram::trigram_prefilter);
if let Some(fts_query) = &prefilter {
sql.push_str(
" and m.id in (
select rowid from messages_trigram
where messages_trigram match ?
)",
);
args.push(Value::Text(fts_query.clone()));
}
if include_explain {
Some(SearchExplain {
prefilter: prefilter.clone(),
candidates: prefilter
.as_deref()
.map(|fts_query| self.fts5_candidate_count(filters, fts_query))
.transpose()?,
prefilter_skipped: prefilter
.is_none()
.then(|| "no required literal of at least three characters".into()),
corpus: self.filtered_corpus_count(filters)?,
})
} else {
None
}
} else {
let (use_trigram_candidates, explain) = self.prepare_content_prefilter(
prefilter_pattern.as_deref(),
filters,
include_explain,
)?;
if use_trigram_candidates {
sql.push_str(" and m.id in (select id from _trigram_cand)");
}
explain
}
} else {
let mut derived_explain = None;
match (field, filters.match_mode) {
(SearchField::ToolName, MessageSearchMode::Literal) => {
sql.push_str(" and unicode_lower_contains(m.tool_name, ?)");
args.push(Value::Text(fold_caseless(query)));
}
(SearchField::ToolName, MessageSearchMode::Regex) => {
regex::Regex::new(query).map_err(|error| anyhow!("invalid regex: {error}"))?;
sql.push_str(" and rust_regexp(?, m.tool_name)");
args.push(Value::Text(query.to_string()));
}
(SearchField::ToolArgument, mode) => {
if mode == MessageSearchMode::Regex {
regex::Regex::new(query)
.map_err(|error| anyhow!("invalid regex: {error}"))?;
}
let prefilter_pattern = match mode {
MessageSearchMode::Literal if !query.is_empty() => {
Some(regex::escape(query))
}
MessageSearchMode::Regex => Some(query.to_string()),
MessageSearchMode::Literal => None,
MessageSearchMode::Fuzzy => unreachable!("fuzzy query rejected above"),
};
let prefilter = (self.schema_version()? >= 4)
.then(|| {
prefilter_pattern
.as_deref()
.and_then(json_projection_prefilter_groups)
.map(|groups| crate::trigram::render_prefilter_groups(&groups))
})
.flatten();
if let Some(fts_query) = &prefilter {
sql.push_str(
" and m.id in (
select rowid from messages_trigram
where messages_trigram match ?
)",
);
args.push(Value::Text(fts_query.clone()));
}
sql.push_str(" and m.kind = 'tool_call'");
match mode {
MessageSearchMode::Literal => {
sql.push_str(
" and unicode_lower_contains(rust_json_pointer(?, m.content), ?)",
);
args.push(Value::Text(
filters.argument_path.clone().unwrap_or_default(),
));
args.push(Value::Text(fold_caseless(query)));
}
MessageSearchMode::Regex => {
sql.push_str(" and rust_regexp(?, rust_json_pointer(?, m.content))");
args.push(Value::Text(query.to_string()));
args.push(Value::Text(
filters.argument_path.clone().unwrap_or_default(),
));
}
MessageSearchMode::Fuzzy => unreachable!("fuzzy query rejected above"),
}
if include_explain {
let corpus =
self.corpus_count_with_base("m.kind = 'tool_call'", filters)?;
let candidates = prefilter
.as_deref()
.map(|fts_query| {
self.fts5_candidate_count_with_base(
"m.kind = 'tool_call'",
filters,
fts_query,
)
})
.transpose()?;
derived_explain = Some(SearchExplain {
prefilter: prefilter.clone(),
candidates,
prefilter_skipped: prefilter.is_none().then(|| {
"no JSON-stable required literal of at least three characters"
.to_string()
}),
corpus,
});
}
}
(_, MessageSearchMode::Fuzzy) => unreachable!("fuzzy query rejected above"),
(SearchField::Content, _) => unreachable!("content handled above"),
}
if include_explain {
match derived_explain {
Some(explain) => Some(explain),
None => Some(SearchExplain {
prefilter: None,
candidates: None,
prefilter_skipped: Some(
"tool name verified in SQLite without a prefilter".to_string(),
),
corpus: self.corpus_count_with_base("m.tool_name is not null", filters)?,
}),
}
} else {
None
}
};
sql.push_str(if order == MessageOrder::NewestFirst {
" order by m.session_id, m.seq desc"
} else {
" order by m.session_id, m.seq"
});
if filters.limit > 0 {
sql.push_str(" limit ?");
args.push(Value::Integer(filters.limit as i64));
if filters.offset > 0 {
sql.push_str(" offset ?");
args.push(Value::Integer(filters.offset as i64));
}
} else if filters.offset > 0 {
sql.push_str(" limit -1 offset ?");
args.push(Value::Integer(filters.offset as i64));
}
Ok(PreparedMessageQuery { sql, args, explain })
}
fn search_messages_with_explain_order(
&self,
query: &str,
filters: &MessageFilters,
include_explain: bool,
order: MessageOrder,
) -> Result<(Vec<MessageHit>, Option<SearchExplain>)> {
self.validate_access_scope()?;
filters.validate(query)?;
let field = filters.field.unwrap_or(SearchField::Content);
if filters.match_mode != MessageSearchMode::Fuzzy {
let prepared =
self.prepare_non_fuzzy_message_query(query, filters, include_explain, order)?;
let hits = self.query_message_hits(&prepared.sql, &prepared.args)?;
let explain = prepared.explain.clone();
return Ok((hits, explain));
}
self.with_read_snapshot(|| {
self.fuzzy_search_in_snapshot(query, filters, field, include_explain, order)
})
}
fn fuzzy_search_in_snapshot(
&self,
query: &str,
filters: &MessageFilters,
field: SearchField,
include_explain: bool,
order: MessageOrder,
) -> Result<(Vec<MessageHit>, Option<SearchExplain>)> {
use rusqlite::types::Value;
debug_assert!(
!self.conn.is_autocommit(),
"every statement of one fuzzy search must share one read snapshot"
);
if field != SearchField::Content {
return self.search_derived_message_field(
query,
filters,
field,
include_explain,
order,
);
}
let mut sql = String::from(
"select m.session_id, m.provider, m.seq, m.role, m.ts, m.tool_name, m.kind, m.tool_call_id, m.content \
from messages m where 1 = 1",
);
let mut args: Vec<Value> = Vec::new();
append_message_filters(&mut sql, &mut args, filters, &self.access_scope);
sql.push_str(if order == MessageOrder::NewestFirst {
" order by m.session_id, m.seq desc"
} else {
" order by m.session_id, m.seq"
});
let ranked_limit = match self.fuzzy_ranking_window("1 = 1", filters)? {
FuzzyRankingWindow::Rank { limit, .. } => limit,
FuzzyRankingWindow::PastCorpus { corpus } => {
return Ok((
Vec::new(),
include_explain.then(|| empty_past_corpus(corpus)),
))
}
};
let pattern = Pattern::new(
query,
CaseMatching::Ignore,
Normalization::Smart,
AtomKind::Fuzzy,
);
let query_lower = fold_caseless(query);
let mut stmt = self.conn.prepare(&sql)?;
let rows = stmt.query_map(rusqlite::params_from_iter(args.iter()), row_to_message_hit)?;
let mut batch = Vec::with_capacity(FUZZY_SCORE_BATCH_SIZE);
let mut batch_bytes = 0_usize;
let mut top = Vec::new();
let mut corpus = 0_i64;
let mut matched = 0_i64;
for row in rows {
let hit = row?;
corpus += 1;
batch_bytes = batch_bytes.saturating_add(hit.content.len());
batch.push(hit);
if fuzzy_batch_is_full(batch.len(), batch_bytes) {
batch_bytes = 0;
let scored = self.runtime.install(|| {
score_fuzzy_message_hits(&pattern, &query_lower, std::mem::take(&mut batch))
})?;
matched += scored.len() as i64;
top.extend(scored);
if top.len() >= top_k_compaction_threshold(ranked_limit) {
retain_top_fuzzy_hits(&mut top, ranked_limit);
}
}
}
if !batch.is_empty() {
let scored = self
.runtime
.install(|| score_fuzzy_message_hits(&pattern, &query_lower, batch))?;
matched += scored.len() as i64;
top.extend(scored);
}
let hits = finish_fuzzy_hits(top, ranked_limit, filters.offset);
let explain = include_explain.then(|| scored_whole_corpus(matched, corpus));
Ok((hits, explain))
}
pub(crate) fn visit_messages_in_batches(
&self,
query: &str,
filters: &MessageFilters,
include_explain: bool,
order: MessageOrder,
batch_rows: NonZeroUsize,
mut visitor: impl FnMut(Vec<MessageHit>) -> Result<MessageBatchControl>,
) -> Result<MessageBatchVisitOutcome> {
self.validate_access_scope()?;
filters.validate(query)?;
anyhow::ensure!(
filters.match_mode != MessageSearchMode::Fuzzy,
"batched exhaustive traversal supports literal, regex, and queryless searches; pass a finite limit to the bounded fuzzy search"
);
anyhow::ensure!(
order == MessageOrder::OldestFirst || filters.session_id.is_some(),
"newest-first message order requires session_id because sequence numbers are session-local"
);
let legacy_prepared = if self.schema_version()? < 4 {
anyhow::ensure!(
!include_explain,
"batched message-search receipts require database schema 4 so planner counts and results share one snapshot; run `aise reindex --full` before requesting a receipt"
);
Some(self.prepare_non_fuzzy_message_query(query, filters, false, order)?)
} else {
None
};
let owned_snapshot = self
.conn
.is_autocommit()
.then(|| ReadSnapshotRollback::begin(&self.conn))
.transpose()?;
let initial_prepared = match legacy_prepared {
Some(prepared) => prepared,
None => self.prepare_non_fuzzy_message_query(query, filters, include_explain, order)?,
};
let prepared = if order == MessageOrder::NewestFirst {
let boundary = {
let mut statement = self.conn.prepare(&initial_prepared.sql)?;
let mut rows = statement.query_map(
rusqlite::params_from_iter(initial_prepared.args.iter()),
row_to_message_hit,
)?;
let boundary = rows.next().transpose()?;
drop(rows);
drop(statement);
boundary
};
let Some(boundary) = boundary else {
let explain = initial_prepared.explain.clone();
if let Some(snapshot) = owned_snapshot {
snapshot.rollback()?;
}
return Ok(MessageBatchVisitOutcome {
rows_visited: 0,
exhausted: true,
explain,
});
};
let mut chronological_filters = filters.clone();
chronological_filters.limit = 0;
chronological_filters.offset = 0;
chronological_filters.seq_to = Some(
chronological_filters
.seq_to
.map_or(boundary.seq, |current| current.min(boundary.seq)),
);
let mut chronological = self.prepare_non_fuzzy_message_query(
query,
&chronological_filters,
false,
MessageOrder::OldestFirst,
)?;
chronological.explain = initial_prepared.explain;
chronological
} else {
initial_prepared
};
let mut statement = self.conn.prepare(&prepared.sql)?;
let mut rows = statement.query_map(
rusqlite::params_from_iter(prepared.args.iter()),
row_to_message_hit,
)?;
let mut batch = Vec::new();
batch.try_reserve_exact(batch_rows.get()).map_err(|error| {
anyhow!(
"batch_rows {} cannot be allocated: {error}; pass a smaller positive row count",
batch_rows
)
})?;
let mut rows_visited = 0_usize;
let mut exhausted = false;
loop {
match rows.next() {
Some(row) => {
batch.push(row?);
rows_visited = rows_visited
.checked_add(1)
.ok_or_else(|| anyhow!("batched message row count overflow"))?;
if batch.len() == batch_rows.get()
&& visitor(std::mem::take(&mut batch))? == MessageBatchControl::Stop
{
break;
}
if batch.capacity() < batch_rows.get() {
batch
.try_reserve_exact(batch_rows.get() - batch.capacity())
.map_err(|error| {
anyhow!(
"batch_rows {} cannot be allocated: {error}; pass a smaller positive row count",
batch_rows
)
})?;
}
}
None => {
exhausted = true;
if !batch.is_empty() {
let _ = visitor(batch)?;
}
break;
}
}
}
drop(rows);
drop(statement);
if let Some(snapshot) = owned_snapshot {
snapshot.rollback()?;
}
Ok(MessageBatchVisitOutcome {
rows_visited,
exhausted,
explain: exhausted.then(|| prepared.explain.clone()).flatten(),
})
}
fn fts5_candidate_count(&self, filters: &MessageFilters, fts_query: &str) -> Result<i64> {
self.fts5_candidate_count_with_base("1 = 1", filters, fts_query)
}
fn fts5_candidate_count_with_base(
&self,
base_predicate: &str,
filters: &MessageFilters,
fts_query: &str,
) -> Result<i64> {
use rusqlite::types::Value;
let mut sql = format!("select count(*) from messages m where {base_predicate}");
let mut args = Vec::new();
append_message_filters(&mut sql, &mut args, filters, &self.access_scope);
sql.push_str(
" and m.id in (
select rowid from messages_trigram where messages_trigram match ?
)",
);
args.push(Value::Text(fts_query.to_string()));
Ok(self
.conn
.query_row(&sql, rusqlite::params_from_iter(args.iter()), |row| {
row.get(0)
})?)
}
fn search_derived_message_field(
&self,
query: &str,
filters: &MessageFilters,
field: SearchField,
include_explain: bool,
order: MessageOrder,
) -> Result<(Vec<MessageHit>, Option<SearchExplain>)> {
if field == SearchField::ToolName
&& filters.match_mode == MessageSearchMode::Fuzzy
&& self.schema_version()? >= 4
{
return self.search_tool_name_fuzzy_indexed(query, filters, include_explain);
}
let mut sql = String::from(
"select m.session_id, m.provider, m.seq, m.role, m.ts, m.tool_name, m.kind, m.tool_call_id, m.content \
from messages m where 1 = 1",
);
let mut args = Vec::new();
append_message_filters(&mut sql, &mut args, filters, &self.access_scope);
debug_assert_eq!(filters.match_mode, MessageSearchMode::Fuzzy);
let base_predicate = fuzzy_derived_scan_predicate(field);
sql.push_str(" and ");
sql.push_str(base_predicate);
sql.push_str(if order == MessageOrder::NewestFirst {
" order by m.session_id, m.seq desc"
} else {
" order by m.session_id, m.seq"
});
let pattern = Pattern::new(
query,
CaseMatching::Ignore,
Normalization::Smart,
AtomKind::Fuzzy,
);
let query_lower = fold_caseless(query);
let ranked_limit = match self.fuzzy_ranking_window(base_predicate, filters)? {
FuzzyRankingWindow::Rank { limit, .. } => limit,
FuzzyRankingWindow::PastCorpus { corpus } => {
return Ok((
Vec::new(),
include_explain.then(|| empty_past_corpus(corpus)),
))
}
};
let mut top = Vec::new();
let mut corpus = 0_i64;
let mut matched = 0_i64;
let mut stmt = self.conn.prepare(&sql)?;
let rows = stmt.query_map(rusqlite::params_from_iter(args.iter()), row_to_message_hit)?;
let mut batch = Vec::with_capacity(FUZZY_SCORE_BATCH_SIZE);
let mut batch_bytes = 0_usize;
for row in rows {
let hit = row?;
corpus += 1;
batch_bytes = batch_bytes.saturating_add(hit.content.len());
batch.push(hit);
if fuzzy_batch_is_full(batch.len(), batch_bytes) {
let scored = self.runtime.install(|| {
score_fuzzy_derived_hits(
&pattern,
&query_lower,
field,
filters.argument_path.as_deref(),
std::mem::take(&mut batch),
)
})?;
batch_bytes = 0;
matched += scored.len() as i64;
top.extend(scored);
if top.len() >= top_k_compaction_threshold(ranked_limit) {
retain_top_fuzzy_hits(&mut top, ranked_limit);
}
}
}
if !batch.is_empty() {
let scored = self.runtime.install(|| {
score_fuzzy_derived_hits(
&pattern,
&query_lower,
field,
filters.argument_path.as_deref(),
batch,
)
})?;
matched += scored.len() as i64;
top.extend(scored);
}
let hits = finish_fuzzy_hits(top, ranked_limit, filters.offset);
let explain = include_explain.then(|| scored_whole_corpus(matched, corpus));
Ok((hits, explain))
}
fn global_tool_name_vocabulary(&self) -> Result<Vec<String>> {
let mut stmt = self.conn.prepare(
"with recursive names(name) as (
select min(tool_name) from messages where tool_name is not null
union all
select (select min(tool_name) from messages where tool_name > names.name)
from names where name is not null
)
select name from names where name is not null",
)?;
let rows = stmt.query_map([], |row| row.get::<_, String>(0))?;
rows.collect::<rusqlite::Result<Vec<_>>>()
.map_err(Into::into)
}
#[cfg(test)]
fn search_tool_name_fuzzy_direct_for_test(
&self,
query: &str,
filters: &MessageFilters,
) -> Result<Vec<MessageHit>> {
let mut sql = String::from(
"select m.session_id, m.provider, m.seq, m.role, m.ts, m.tool_name,
m.kind, m.tool_call_id, m.content
from messages m
where m.tool_name is not null",
);
let mut args = Vec::new();
append_message_filters(&mut sql, &mut args, filters, &self.access_scope);
sql.push_str(" order by m.session_id, m.seq");
let rows = self.query_message_hits(&sql, &args)?;
let pattern = Pattern::new(
query,
CaseMatching::Ignore,
Normalization::Smart,
AtomKind::Fuzzy,
);
let scored = score_fuzzy_derived_hits(
&pattern,
&fold_caseless(query),
SearchField::ToolName,
None,
rows,
);
Ok(finish_fuzzy_hits(
scored,
fuzzy_ranked_limit(filters)?,
filters.offset,
))
}
fn search_tool_name_fuzzy_indexed(
&self,
query: &str,
filters: &MessageFilters,
include_explain: bool,
) -> Result<(Vec<MessageHit>, Option<SearchExplain>)> {
use rusqlite::types::Value;
debug_assert!(
!self.conn.is_autocommit(),
"the fuzzy tool-name statements must share one read snapshot"
);
let (ranked_limit, counted_corpus) =
match self.fuzzy_ranking_window("m.tool_name is not null", filters)? {
FuzzyRankingWindow::Rank { limit, corpus } => (limit, corpus),
FuzzyRankingWindow::PastCorpus { corpus } => {
return Ok((
Vec::new(),
include_explain.then(|| empty_past_corpus(corpus)),
))
}
};
let names = self.global_tool_name_vocabulary()?;
let pattern = Pattern::new(
query,
CaseMatching::Ignore,
Normalization::Smart,
AtomKind::Fuzzy,
);
let query_lower = fold_caseless(query);
let query_needle = UnicodeLowerNeedle::from_lowered(&query_lower);
let mut matcher = NucleoMatcher::new(NucleoConfig::DEFAULT);
let mut utf32_buf = Vec::new();
let mut matching_names = HashMap::new();
for name in names {
utf32_buf.clear();
if let Some(score) = pattern.score(Utf32Str::new(&name, &mut utf32_buf), &mut matcher) {
let exact_phrase = query_needle.contains(&name);
matching_names.insert(name, (score, exact_phrase));
}
}
let explained_matching_rows = if include_explain && !matching_names.is_empty() {
let selected_names = serde_json::to_string(&matching_names.keys().collect::<Vec<_>>())?;
let mut sql = String::from(
"select count(*) from json_each(?) selected
join messages m on m.tool_name = selected.value
where m.tool_name is not null",
);
let mut args = vec![Value::Text(selected_names)];
append_message_filters(&mut sql, &mut args, filters, &self.access_scope);
Some(
self.conn
.query_row(&sql, rusqlite::params_from_iter(args.iter()), |row| {
row.get::<_, i64>(0)
})?,
)
} else {
None
};
let mut scored_names = matching_names.into_iter().collect::<Vec<_>>();
scored_names.sort_by(|(left_name, left), (right_name, right)| {
right
.0
.cmp(&left.0)
.then_with(|| right.1.cmp(&left.1))
.then_with(|| left_name.cmp(right_name))
});
let mut top = Vec::new();
let mut matched = 0_i64;
for tier in scored_names.chunk_by(|left, right| left.1 == right.1) {
if top.len() == ranked_limit {
break;
}
let selected_names =
serde_json::to_string(&tier.iter().map(|(name, _)| name).collect::<Vec<_>>())?;
let remaining = ranked_limit - top.len();
let mut sql = String::from(
"select m.session_id, m.provider, m.seq, m.role, m.ts, m.tool_name,
m.kind, m.tool_call_id, m.content
from json_each(?) selected
join messages m on m.tool_name = selected.value
where m.tool_name is not null",
);
let mut args = vec![Value::Text(selected_names)];
append_message_filters(&mut sql, &mut args, filters, &self.access_scope);
sql.push_str(" order by m.session_id, m.seq limit ?");
args.push(Value::Integer(remaining as i64));
let mut stmt = self.conn.prepare(&sql)?;
let rows =
stmt.query_map(rusqlite::params_from_iter(args.iter()), row_to_message_hit)?;
for row in rows {
let mut hit = row?;
hit.fuzzy_score = Some(tier[0].1 .0);
top.push((hit, tier[0].1 .1));
matched += 1;
}
}
let hits = top
.into_iter()
.skip(filters.offset)
.map(|(hit, _)| hit)
.collect();
let explain = include_explain
.then(|| {
let corpus = match counted_corpus {
Some(corpus) => corpus,
None => self.corpus_count_with_base("m.tool_name is not null", filters)?,
};
Ok::<_, anyhow::Error>(SearchExplain {
prefilter: None,
candidates: Some(explained_matching_rows.unwrap_or(matched)),
prefilter_skipped: Some(
"complete filtered tool-name vocabulary scored before matching rows are loaded"
.to_string(),
),
corpus,
})
})
.transpose()?;
Ok((hits, explain))
}
fn query_message_hits(
&self,
sql: &str,
args: &[rusqlite::types::Value],
) -> Result<Vec<MessageHit>> {
let mut stmt = self.conn.prepare(sql)?;
let rows = stmt.query_map(rusqlite::params_from_iter(args.iter()), row_to_message_hit)?;
rows.collect::<rusqlite::Result<Vec<_>>>()
.map_err(Into::into)
}
fn filtered_corpus_count(&self, filters: &MessageFilters) -> Result<i64> {
self.corpus_count_with_base("1 = 1", filters)
}
fn fuzzy_ranking_window(
&self,
base_predicate: &str,
filters: &MessageFilters,
) -> Result<FuzzyRankingWindow> {
let requested = fuzzy_ranked_limit(filters)?;
if requested <= FUZZY_SCORE_BATCH_SIZE {
return Ok(FuzzyRankingWindow::Rank {
limit: requested,
corpus: None,
});
}
let corpus = self.corpus_count_with_base(base_predicate, filters)?;
let corpus_rows = usize::try_from(corpus).unwrap_or(usize::MAX);
if filters.offset > 0 && filters.offset >= corpus_rows {
return Ok(FuzzyRankingWindow::PastCorpus { corpus });
}
Ok(FuzzyRankingWindow::Rank {
limit: requested.min(corpus_rows),
corpus: Some(corpus),
})
}
fn corpus_count_with_base(
&self,
base_predicate: &str,
filters: &MessageFilters,
) -> Result<i64> {
let mut corpus = 0_i64;
for (sql, args, sign) in self.corpus_count_statements(base_predicate, filters) {
let count: i64 =
self.conn
.query_row(&sql, rusqlite::params_from_iter(args.iter()), |row| {
row.get(0)
})?;
corpus += sign * count;
}
Ok(corpus)
}
fn corpus_count_statements(
&self,
base_predicate: &str,
filters: &MessageFilters,
) -> Vec<(String, Vec<rusqlite::types::Value>, i64)> {
let build = |class: ClassSelection<'_>| {
let mut sql = format!("select count(*) from messages m where {base_predicate}");
if matches!(class, ClassSelection::Only(_)) {
sql.push_str(&format!(" and m.{}", excluded_kinds_predicate()));
}
let mut args = Vec::new();
append_message_filters_with_class(
&mut sql,
&mut args,
filters,
&self.access_scope,
class,
);
if filters.tool.is_some() {
sql.push_str(" and m.tool_name is not null");
}
(sql, args)
};
let excludable = crate::models::MessageKind::EXCLUDABLE_BY_DEFAULT;
match filters.kind_predicate() {
crate::models::KindPredicate::AllExcept(excluded) => {
debug_assert!(excluded.iter().all(|kind| excludable.contains(kind)));
let (total_sql, total_args) = build(ClassSelection::Unrestricted);
let (excluded_sql, excluded_args) = build(ClassSelection::Only(&excluded));
vec![
(total_sql, total_args, 1),
(excluded_sql, excluded_args, -1),
]
}
crate::models::KindPredicate::Only(kinds)
if !kinds.is_empty() && kinds.iter().all(|kind| excludable.contains(kind)) =>
{
let (sql, args) = build(ClassSelection::Only(&kinds));
vec![(sql, args, 1)]
}
crate::models::KindPredicate::Only(_) => {
let (sql, args) = build(ClassSelection::FromFilters);
vec![(sql, args, 1)]
}
}
}
pub(crate) fn message_kind_index_exists(&self) -> Result<bool> {
Ok(self.conn.query_row(
"select exists(select 1 from sqlite_schema where type = 'index' and name = ?1)",
params![MESSAGE_KIND_INDEX],
|row| row.get(0),
)?)
}
pub(crate) fn ensure_message_kind_index(&self) -> Result<bool> {
if self.message_kind_index_exists()? {
return Ok(false);
}
self.report_progress(
"building the message-class index once (one scan of every indexed message) so \
receipts and class-filtered counts read the index instead of every message row",
);
let started = std::time::Instant::now();
self.conn.execute_batch(&format!(
"create index if not exists {MESSAGE_KIND_INDEX} on messages{MESSAGE_KIND_INDEX_COLUMNS} \
where {}",
excluded_kinds_predicate()
))?;
self.report_progress(&format!(
"built the message-class index in {:.1?}",
started.elapsed()
));
Ok(true)
}
fn filtered_corpus_reaches(&self, filters: &MessageFilters, threshold: i64) -> Result<bool> {
use rusqlite::types::Value;
anyhow::ensure!(threshold > 0, "corpus probe threshold must be positive");
let mut inner = String::from("select 1 from messages m where 1 = 1");
let mut args: Vec<Value> = Vec::new();
append_message_filters(&mut inner, &mut args, filters, &self.access_scope);
inner.push_str(" limit 1 offset ?");
args.push(Value::Integer(threshold - 1));
let sql = format!("select exists({inner})");
Ok(self
.conn
.query_row(&sql, rusqlite::params_from_iter(args.iter()), |row| {
row.get(0)
})?)
}
fn corpus_count(&self, filters: &MessageFilters, cached: Option<i64>) -> Result<i64> {
cached.map_or_else(|| self.filtered_corpus_count(filters), Ok)
}
fn staged_candidate_count(&self, filters: &MessageFilters) -> Result<i64> {
use rusqlite::types::Value;
let mut sql = String::from("select count(*) from messages m where 1 = 1");
let mut args: Vec<Value> = Vec::new();
append_message_filters(&mut sql, &mut args, filters, &self.access_scope);
sql.push_str(" and m.id in (select id from _trigram_cand)");
Ok(self
.conn
.query_row(&sql, rusqlite::params_from_iter(args.iter()), |row| {
row.get(0)
})?)
}
fn prepare_content_prefilter(
&self,
pattern: Option<&str>,
filters: &MessageFilters,
include_explain: bool,
) -> Result<(bool, Option<SearchExplain>)> {
let Some(pattern) = pattern else {
let explain = include_explain
.then(|| {
self.filtered_corpus_count(filters)
.map(|corpus| SearchExplain {
prefilter: None,
candidates: None,
prefilter_skipped: None,
corpus,
})
})
.transpose()?;
return Ok((false, explain));
};
let corpus = if include_explain {
Some(self.filtered_corpus_count(filters)?)
} else {
None
};
if pattern.contains(SIGMAS) {
let explain = if include_explain {
Some(SearchExplain {
prefilter: None,
candidates: None,
prefilter_skipped: Some(
"the pre-v4 compatibility index spells sigma by position, so this query scans directly"
.into(),
),
corpus: self.corpus_count(filters, corpus)?,
})
} else {
None
};
return Ok((false, explain));
}
let Some(groups) = crate::trigram::trigram_prefilter_groups(pattern) else {
let explain = if include_explain {
Some(SearchExplain {
prefilter: None,
candidates: None,
prefilter_skipped: None,
corpus: self.corpus_count(filters, corpus)?,
})
} else {
None
};
return Ok((false, explain));
};
let use_prefilter = !filters.narrows_corpus()
|| if let Some(corpus) = corpus {
corpus >= self.prefilter_min_corpus
} else {
self.filtered_corpus_reaches(filters, self.prefilter_min_corpus)?
};
let prefilter = include_explain.then(|| crate::trigram::render_prefilter_groups(&groups));
if !use_prefilter {
let explain = if include_explain {
Some(SearchExplain {
prefilter,
candidates: None,
prefilter_skipped: Some(format!(
"structured filters reduced the corpus below the pre-v4 compatibility prefilter threshold ({})",
self.prefilter_min_corpus
)),
corpus: self.corpus_count(filters, corpus)?,
})
} else {
None
};
return Ok((false, explain));
}
let base_max = self.ensure_trigram_base()?;
let candidates = crate::trigram_index::candidates(&self.conn, &groups)?;
self.stage_candidates(base_max, &candidates)?;
let explain = if include_explain {
Some(SearchExplain {
prefilter,
candidates: Some(self.staged_candidate_count(filters)?),
prefilter_skipped: None,
corpus: self.corpus_count(filters, corpus)?,
})
} else {
None
};
Ok((true, explain))
}
pub fn explain_message_search(
&self,
query: &str,
filters: &MessageFilters,
) -> Result<SearchExplain> {
let (_, explain) = self.search_messages_with_explain(query, filters, true)?;
explain.context("message search explanation was not produced")
}
pub fn message_context(
&self,
session_id: &str,
seq: i64,
before: i64,
after: i64,
) -> Result<Vec<MessageHit>> {
use rusqlite::types::Value;
self.validate_access_scope()?;
anyhow::ensure!(before >= 0, "context before must be non-negative");
anyhow::ensure!(after >= 0, "context after must be non-negative");
let mut sql = String::from(
"select session_id, provider, seq, role, ts, tool_name, kind, tool_call_id, content from messages
where session_id = ? and seq between ? and ?",
);
let mut args = vec![
Value::Text(session_id.to_string()),
Value::Integer(seq.saturating_sub(before)),
Value::Integer(seq.saturating_add(after)),
];
push_access_scope(&mut sql, &mut args, "session_id", &self.access_scope);
sql.push_str(" order by seq");
let mut stmt = self.conn.prepare(&sql)?;
let rows = stmt.query_map(rusqlite::params_from_iter(args.iter()), row_to_message_hit)?;
rows.collect::<rusqlite::Result<Vec<_>>>()
.map_err(Into::into)
}
pub(crate) fn message_context_windows(
&self,
anchors: &[(String, i64)],
before: i64,
after: i64,
) -> Result<Vec<Vec<MessageHit>>> {
use rusqlite::types::Value;
self.validate_access_scope()?;
anyhow::ensure!(before >= 0, "context before must be non-negative");
anyhow::ensure!(after >= 0, "context after must be non-negative");
let mut windows = vec![Vec::new(); anchors.len()];
if anchors.is_empty() {
return Ok(windows);
}
let bounds = anchors
.iter()
.map(|(session_id, seq)| {
(
session_id,
seq.saturating_sub(before),
seq.saturating_add(after),
)
})
.collect::<Vec<_>>();
let mut sql = String::from(
"with anchors(ord, session_id, lower_seq, upper_seq) as materialized (
select cast(key as integer),
json_extract(value, '$[0]'),
json_extract(value, '$[1]'),
json_extract(value, '$[2]')
from json_each(?)
)
select a.ord, m.session_id, m.provider, m.seq, m.role, m.ts, m.tool_name,
m.kind, m.tool_call_id, m.content
from anchors a
join messages m on m.session_id = a.session_id
and m.seq between a.lower_seq and a.upper_seq
where 1 = 1",
);
let mut args = vec![Value::Text(serde_json::to_string(&bounds)?)];
push_access_scope(&mut sql, &mut args, "m.session_id", &self.access_scope);
sql.push_str(" order by a.ord, m.seq");
let mut stmt = self.conn.prepare(&sql)?;
let rows = stmt.query_map(rusqlite::params_from_iter(args.iter()), |row| {
Ok((row.get::<_, i64>(0)?, row_to_message_hit_at(row, 1)?))
})?;
for row in rows {
let (ord, hit) = row?;
let window = usize::try_from(ord)
.ok()
.and_then(|ordinal| windows.get_mut(ordinal))
.with_context(|| {
format!(
"context window ordinal {ord} falls outside the {} anchors queried",
anchors.len()
)
})?;
window.push(hit);
}
Ok(windows)
}
pub fn session_metadata(
&self,
ids: &[String],
) -> Result<std::collections::HashMap<String, crate::models::SessionMeta>> {
use crate::models::SessionMeta;
use rusqlite::types::Value;
use std::collections::HashMap;
self.validate_access_scope()?;
let mut map = HashMap::new();
if ids.is_empty() {
return Ok(map);
}
let mut sql = String::from(
"select s.id, s.provider_session_id, s.cwd, s.repo_root, s.title, s.updated_at,
s.last_message_at, s.message_count, s.parse_warning, s.parent_session_id,
s.agent_label
from json_each(?) requested
join sessions s on s.id = requested.value
where 1 = 1",
);
let mut args = vec![Value::Text(serde_json::to_string(ids)?)];
push_access_scope(&mut sql, &mut args, "s.id", &self.access_scope);
let mut stmt = self.conn.prepare(&sql)?;
let rows = stmt.query_map(rusqlite::params_from_iter(args.iter()), |row| {
Ok((
row.get::<_, String>(0)?,
SessionMeta {
provider_session_id: row.get(1)?,
cwd: row.get(2)?,
repo_root: row.get(3)?,
title: row.get(4)?,
updated_at: row
.get::<_, Option<String>>(5)?
.as_deref()
.and_then(crate::util::parse_datetime),
last_message_at: row
.get::<_, Option<String>>(6)?
.as_deref()
.and_then(crate::util::parse_datetime),
message_count: row.get(7)?,
parse_warning: row.get(8)?,
parent_session_id: row.get(9)?,
agent_label: row.get(10)?,
},
))
})?;
for row in rows {
let (id, meta) = row?;
map.insert(id, meta);
}
Ok(map)
}
pub fn session_raw_provider_metadata(
&self,
ids: &[String],
) -> Result<std::collections::BTreeMap<String, Box<serde_json::value::RawValue>>> {
use rusqlite::types::Value;
use std::collections::BTreeMap;
self.validate_access_scope()?;
let mut map = BTreeMap::new();
if ids.is_empty() {
return Ok(map);
}
let mut sql = String::from(
"select s.id, s.raw_metadata_json
from json_each(?) requested
join sessions s on s.id = requested.value
where s.raw_metadata_json is not null",
);
let mut args = vec![Value::Text(serde_json::to_string(ids)?)];
push_access_scope(&mut sql, &mut args, "s.id", &self.access_scope);
let mut statement = self.conn.prepare(&sql)?;
let rows = statement.query_map(rusqlite::params_from_iter(args.iter()), |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
})?;
for row in rows {
let (session_id, raw_json) = row?;
let raw = serde_json::value::RawValue::from_string(raw_json).map_err(|error| {
anyhow::anyhow!(
"session {session_id} contains invalid raw provider metadata JSON: {error}"
)
})?;
map.insert(session_id, raw);
}
Ok(map)
}
pub(crate) fn search_attributable_human_messages(
&self,
query: &str,
filters: &MessageFilters,
) -> Result<Vec<MessageHit>> {
use rusqlite::types::Value;
self.validate_access_scope()?;
if filters.match_mode == MessageSearchMode::Fuzzy {
bail!("attributable human-message analytics do not support fuzzy matching");
}
if filters.field.unwrap_or(SearchField::Content) != SearchField::Content {
bail!("attributable human-message analytics search message content only");
}
if filters.match_mode == MessageSearchMode::Regex {
regex::Regex::new(query).map_err(|error| anyhow!("invalid regex: {error}"))?;
}
let mut inner = String::from(
"select m.session_id, m.provider, m.seq, m.role, m.ts, m.tool_name, m.kind,
m.tool_call_id,
case
when m.authorship = 'human' then m.content
else (
select group_concat(fragment, '')
from (
select substr(m.content, p.start_char + 1,
p.end_char - p.start_char) as fragment
from message_content_parts p
where p.message_id = m.id and p.authorship = 'human'
order by p.ordinal
)
)
end as content
from messages m
where m.record_relation = 'original'
and (
m.authorship = 'human'
or (
m.authorship = 'mixed'
and exists (
select 1 from message_content_parts p
where p.message_id = m.id and p.authorship = 'human'
)
)
)",
);
let mut args: Vec<Value> = Vec::new();
append_message_filters(&mut inner, &mut args, filters, &self.access_scope);
let mut sql = format!("select * from ({inner}) attributable where 1 = 1");
if !query.is_empty() {
match filters.match_mode {
MessageSearchMode::Literal => {
sql.push_str(" and unicode_lower_contains(attributable.content, ?)");
args.push(Value::Text(fold_caseless(query)));
}
MessageSearchMode::Regex => {
sql.push_str(" and rust_regexp(?, attributable.content)");
args.push(Value::Text(query.to_string()));
}
MessageSearchMode::Fuzzy => unreachable!("fuzzy rejected above"),
}
}
sql.push_str(" order by attributable.session_id, attributable.seq");
if filters.limit > 0 {
sql.push_str(" limit ? offset ?");
args.push(Value::Integer(filters.limit as i64));
args.push(Value::Integer(filters.offset as i64));
} else if filters.offset > 0 {
sql.push_str(" limit -1 offset ?");
args.push(Value::Integer(filters.offset as i64));
}
let mut stmt = self.conn.prepare(&sql)?;
let rows = stmt.query_map(rusqlite::params_from_iter(args.iter()), row_to_message_hit)?;
rows.collect::<rusqlite::Result<Vec<_>>>()
.map_err(Into::into)
}
pub(crate) fn find_corrections(
&self,
policies: &crate::corrections::ResolvedCorrectionPolicySet,
filters: &MessageFilters,
) -> Result<Vec<MessageClassificationMatch>> {
self.validate_access_scope()?;
if policies.is_empty() {
return Ok(Vec::new());
}
self.find_corrections_with_after_candidates(policies, filters, || {})
}
fn find_corrections_with_after_candidates(
&self,
policies: &crate::corrections::ResolvedCorrectionPolicySet,
filters: &MessageFilters,
after_candidates: impl FnOnce(),
) -> Result<Vec<MessageClassificationMatch>> {
use rusqlite::types::Value;
let mut filters = filters.clone();
filters.role = Some(Role::User);
filters
.session_kinds
.get_or_insert_with(|| vec![crate::models::SessionKind::User]);
struct CorrectionCandidate {
message_id: i64,
session_id: String,
message_seq: i64,
provider: Provider,
ts: Option<String>,
content: String,
human_parts: Option<Vec<(usize, usize, usize)>>,
}
fn resolve_human_part_bytes(
message_id: i64,
content: &str,
ranges: &[(usize, usize)],
) -> Result<Vec<(usize, usize, usize)>> {
let mut resolved = Vec::with_capacity(ranges.len());
let mut range_index = 0;
let mut start_byte = None;
for (char_index, byte_index) in content
.char_indices()
.map(|(byte_index, _)| byte_index)
.chain(std::iter::once(content.len()))
.enumerate()
{
while let Some(&(start_char, end_char)) = ranges.get(range_index) {
if start_byte.is_none() && char_index == start_char {
start_byte = Some(byte_index);
}
if char_index == end_char {
let start_byte = start_byte.take().ok_or_else(|| {
anyhow!(
"message {message_id} human content part starts beyond its content"
)
})?;
resolved.push((start_char, start_byte, byte_index));
range_index += 1;
continue;
}
break;
}
}
if range_index != ranges.len() {
bail!(
"message {message_id} human content part ends beyond its content; \
reindex the source transcript"
);
}
Ok(resolved)
}
let rows = self.with_read_snapshot(|| {
let mut sql = String::from(CORRECTION_CANDIDATES_SQL);
let mut args: Vec<Value> = Vec::new();
append_message_filters(&mut sql, &mut args, &filters, &self.access_scope);
sql.push_str(" order by m.ts desc, m.session_id asc, m.seq asc");
let mut stmt = self.conn.prepare(&sql)?;
let mut rows: Vec<CorrectionCandidate> = stmt
.query_map(rusqlite::params_from_iter(args.iter()), |row| {
let authorship = row.get::<_, String>(6)?;
Ok(CorrectionCandidate {
message_id: row.get(0)?,
session_id: row.get(1)?,
message_seq: row.get(2)?,
provider: decode_db_enum(
row.get::<_, String>(3)?,
3,
"messages.provider",
Provider::from_db_str,
)?,
ts: row.get(4)?,
content: row.get(5)?,
human_parts: (authorship == "mixed").then(Vec::new),
})
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
drop(stmt);
after_candidates();
if rows.iter().any(|row| row.human_parts.is_some()) {
let mut parts_sql = String::from(CORRECTION_HUMAN_PARTS_SQL);
let mut parts_args: Vec<Value> = Vec::new();
append_message_filters(
&mut parts_sql,
&mut parts_args,
&filters,
&self.access_scope,
);
parts_sql.push_str(" order by m.id, p.ordinal");
let mut human_parts_by_message: HashMap<i64, Vec<(usize, usize)>> = HashMap::new();
let mut parts_stmt = self.conn.prepare(&parts_sql)?;
let part_rows =
parts_stmt.query_map(rusqlite::params_from_iter(parts_args.iter()), |row| {
Ok((
row.get::<_, i64>(0)?,
row.get::<_, i64>(1)?,
row.get::<_, i64>(2)?,
))
})?;
for part in part_rows {
let (message_id, start_char, end_char) = part?;
let start_char = usize::try_from(start_char).with_context(|| {
format!(
"message {message_id} has a negative human content-part start coordinate"
)
})?;
let end_char = usize::try_from(end_char).with_context(|| {
format!(
"message {message_id} has a negative human content-part end coordinate"
)
})?;
human_parts_by_message
.entry(message_id)
.or_default()
.push((start_char, end_char));
}
for row in &mut rows {
if let Some(parts) = &mut row.human_parts {
let ranges = human_parts_by_message
.remove(&row.message_id)
.unwrap_or_default();
*parts =
resolve_human_part_bytes(row.message_id, &row.content, &ranges)?;
}
}
}
Ok(rows)
})?;
let classify = |row: CorrectionCandidate| -> Option<MessageClassificationMatch> {
let CorrectionCandidate {
session_id,
message_seq,
provider,
ts,
content,
human_parts,
..
} = row;
let (identity, hit) = match &human_parts {
Some(parts) => policies.classify_fragments(parts.iter().map(
|(start_char, start_byte, end_byte)| {
(*start_char, &content[*start_byte..*end_byte])
},
)),
None => policies.classify(&content),
}?;
let ts = ts.as_deref().and_then(|value| {
chrono::DateTime::parse_from_rfc3339(value)
.ok()
.map(|dt| dt.with_timezone(&Utc))
});
Some(MessageClassificationMatch {
session_id,
message_seq,
provider,
ts,
policy_name: identity.name.clone(),
category: hit.category.to_string(),
matched_text: hit.matched_text,
match_start_char: hit.match_start_char,
match_end_char_exclusive: hit.match_end_char_exclusive,
content,
})
};
use rayon::prelude::*;
let mut out: Vec<MessageClassificationMatch> = if self.runtime.worker_threads() <= 1 {
rows.into_iter().filter_map(classify).collect()
} else {
self.runtime
.install(|| rows.into_par_iter().filter_map(classify).collect())?
};
if filters.offset > 0 {
out.drain(..filters.offset.min(out.len()));
}
if filters.limit > 0 {
out.truncate(filters.limit);
}
Ok(out)
}
pub fn planning_usage(
&self,
filters: &MessageFilters,
command_filters: &[regex::Regex],
) -> Result<Vec<PlanningCount>> {
use rusqlite::types::Value;
use std::collections::{HashMap, HashSet};
self.validate_access_scope()?;
let mut sql = String::from(
"select m.session_id, s.repo_root, s.cwd, m.content from messages m \
join sessions s on s.id = m.session_id where 1 = 1",
);
let mut args: Vec<Value> = Vec::new();
let mut filters = filters.clone();
filters.role = Some(Role::Slash);
append_message_filters(&mut sql, &mut args, &filters, &self.access_scope);
let mut stmt = self.conn.prepare(&sql)?;
let raw = stmt.query_map(rusqlite::params_from_iter(args.iter()), |row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, Option<String>>(1)?,
row.get::<_, Option<String>>(2)?,
row.get::<_, String>(3)?,
))
})?;
let mut agg: HashMap<String, (i64, HashSet<String>, HashSet<String>)> = HashMap::new();
for row in raw {
let (session_id, repo_root, cwd, content) = row?;
if let Some(command) = crate::util::slash_command_token(&content) {
if !command_filters.is_empty()
&& !command_filters.iter().any(|re| re.is_match(&command))
{
continue;
}
let project = repo_root.or(cwd).unwrap_or_default();
let entry = agg.entry(command).or_default();
entry.0 += 1;
entry.1.insert(session_id);
if !project.is_empty() {
entry.2.insert(project);
}
}
}
let mut counts: Vec<PlanningCount> = agg
.into_iter()
.map(|(command, (count, sessions, projects))| PlanningCount {
command,
count,
unique_sessions: sessions.len() as i64,
unique_projects: projects.len() as i64,
})
.collect();
counts.sort_by(|a, b| {
b.count
.cmp(&a.count)
.then_with(|| a.command.cmp(&b.command))
});
if filters.limit > 0 {
counts.truncate(filters.limit);
}
Ok(counts)
}
pub fn file_search(&self, query: &FileQuery) -> Result<Vec<FileEditSummary>> {
use rusqlite::types::Value;
self.validate_access_scope()?;
let mut sql = String::from(
"select file_path, file_name, count(*) as edits, \
count(distinct session_id) as sessions, max(ts) as last_edited \
from file_edits where 1 = 1",
);
let mut args: Vec<Value> = Vec::new();
if let Some(pattern) = &query.pattern {
let (col, like) = glob_clause(pattern);
sql.push_str(&format!(" and {col} like ? escape '\\'"));
args.push(Value::Text(like));
}
push_file_filters(&mut sql, &mut args, query, &self.access_scope);
push_ts_window(&mut sql, &mut args, "ts", query.since, query.until);
sql.push_str(" group by file_path");
let mut having: Vec<&str> = Vec::new();
if let Some(min) = query.min_edits {
having.push("count(*) >= ?");
args.push(Value::Integer(min));
}
if let Some(max) = query.max_edits {
having.push("count(*) <= ?");
args.push(Value::Integer(max));
}
if !having.is_empty() {
sql.push_str(" having ");
sql.push_str(&having.join(" and "));
}
sql.push_str(" order by edits desc, last_edited desc");
if query.limit > 0 || query.offset > 0 {
sql.push_str(" limit ? offset ?");
args.push(Value::Integer(if query.limit == 0 {
-1
} else {
query.limit as i64
}));
args.push(Value::Integer(query.offset as i64));
}
let mut stmt = self.conn.prepare(&sql)?;
let rows = stmt
.query_map(rusqlite::params_from_iter(args.iter()), |row| {
Ok(FileEditSummary {
file_path: row.get(0)?,
file_name: row.get(1)?,
edits: row.get(2)?,
sessions: row.get(3)?,
last_edited: row
.get::<_, Option<String>>(4)?
.as_deref()
.and_then(crate::util::parse_datetime),
})
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
Ok(rows)
}
pub fn file_cross_ref(&self, query: &FileQuery) -> Result<Vec<FileCrossRef>> {
use rusqlite::types::Value;
self.validate_access_scope()?;
let mut sql = String::from(
"select file_path, session_id, provider, count(*) as edits \
from file_edits where 1 = 1",
);
let mut args: Vec<Value> = Vec::new();
if let Some(pattern) = &query.pattern {
let (col, like) = glob_clause(pattern);
sql.push_str(&format!(" and {col} like ? escape '\\'"));
args.push(Value::Text(like));
}
push_file_filters(&mut sql, &mut args, query, &self.access_scope);
push_ts_window(&mut sql, &mut args, "ts", query.since, query.until);
sql.push_str(" group by file_path, session_id order by file_path, edits desc");
if query.limit > 0 || query.offset > 0 {
sql.push_str(" limit ? offset ?");
args.push(Value::Integer(if query.limit == 0 {
-1
} else {
query.limit as i64
}));
args.push(Value::Integer(query.offset as i64));
}
let mut stmt = self.conn.prepare(&sql)?;
let rows = stmt
.query_map(rusqlite::params_from_iter(args.iter()), |row| {
let provider: String = row.get(2)?;
Ok(FileCrossRef {
file_path: row.get(0)?,
session_id: row.get(1)?,
provider: decode_db_enum(
provider,
2,
"file_edits.provider",
Provider::from_db_str,
)?,
edits: row.get(3)?,
})
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
Ok(rows)
}
pub fn file_edits_for(
&self,
file: &str,
session_id: Option<&str>,
) -> Result<Vec<(String, Provider, FileEdit)>> {
let session_id = session_id
.map(|id| self.resolve_session_record(id).map(|session| session.id))
.transpose()?;
self.file_edits_for_query(
file,
&FileQuery {
session_id,
..Default::default()
},
)
}
pub fn file_edits_for_query(
&self,
file: &str,
query: &FileQuery,
) -> Result<Vec<(String, Provider, FileEdit)>> {
self.file_edits_for_scoped(file, query)
}
pub fn file_edits_for_session_id(
&self,
file: &str,
session_id: &str,
) -> Result<Vec<(String, Provider, FileEdit)>> {
self.file_edits_for_scoped(
file,
&FileQuery {
session_id: Some(session_id.to_string()),
..Default::default()
},
)
}
fn file_edits_for_scoped(
&self,
file: &str,
query: &FileQuery,
) -> Result<Vec<(String, Provider, FileEdit)>> {
use rusqlite::types::Value;
self.validate_access_scope()?;
let mut sql = String::from(
"select session_id, provider, seq, ts, tool, file_path, file_name, new_content, edits_json \
from file_edits where (file_name = ? or file_path = ? or file_path like ?)",
);
let mut args: Vec<Value> = vec![
Value::Text(file.to_string()),
Value::Text(file.to_string()),
Value::Text(format!("%/{file}")),
];
push_file_filters(&mut sql, &mut args, query, &self.access_scope);
sql.push_str(" order by session_id, seq");
let mut stmt = self.conn.prepare(&sql)?;
let raw = stmt.query_map(rusqlite::params_from_iter(args.iter()), |row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, String>(1)?,
row.get::<_, i64>(2)?,
row.get::<_, Option<String>>(3)?,
row.get::<_, String>(4)?,
row.get::<_, String>(5)?,
row.get::<_, String>(6)?,
row.get::<_, Option<String>>(7)?,
row.get::<_, Option<String>>(8)?,
))
})?;
let mut out = Vec::new();
for row in raw {
let (
session_id,
provider,
seq,
ts,
tool,
file_path,
file_name,
new_content,
edits_json,
) = row?;
let edits: Vec<EditOp> = match edits_json.as_deref() {
Some(json) => serde_json::from_str(json).with_context(|| {
format!("corrupt edits_json for {file_path} in session {session_id}")
})?,
None => Vec::new(),
};
out.push((
session_id,
decode_db_enum(provider, 1, "file_edits.provider", Provider::from_db_str)?,
FileEdit {
seq,
ts: ts.as_deref().and_then(crate::util::parse_datetime),
tool,
file_path,
file_name,
new_content,
edits,
},
));
}
Ok(out)
}
pub fn file_edit_count(&self) -> Result<i64> {
Ok(self
.conn
.query_row("select count(*) from file_edits", [], |row| row.get(0))?)
}
pub fn list_recent(&self, filters: &SearchFilters) -> Result<Vec<SessionRecord>> {
self.list_recent_page(filters, 0)
}
pub fn list_recent_page(
&self,
filters: &SearchFilters,
offset: usize,
) -> Result<Vec<SessionRecord>> {
self.validate_access_scope()?;
let mut sql = format!(
"select {} from sessions s where 1 = 1",
session_record_columns!()
);
let mut params_vec = Vec::new();
push_session_filters(&mut sql, &mut params_vec, filters, &self.access_scope);
use std::fmt::Write as _;
sql.push_str(" order by s.updated_at desc, s.id asc");
if filters.limit != 0 {
write!(sql, " limit {}", filters.limit)?;
if offset != 0 {
write!(sql, " offset {offset}")?;
}
} else if offset != 0 {
write!(sql, " limit -1 offset {offset}")?;
}
let mut stmt = self.conn.prepare(&sql)?;
let rows = stmt.query_map(
rusqlite::params_from_iter(params_vec.iter()),
row_to_session_record,
)?;
let mut results = Vec::new();
for row in rows {
results.push(row?);
}
Ok(results)
}
pub(crate) fn analysis_documents(
&self,
filters: &SearchFilters,
cursor: Option<&crate::models::AnalysisCursor>,
) -> Result<crate::models::AnalysisDocumentPage> {
self.validate_access_scope()?;
if filters.limit == 0 {
return Err(anyhow!(
"analysis document page limit must be greater than zero"
));
}
let transaction = self.conn.unchecked_transaction()?;
analysis_document_page(
&transaction,
filters,
cursor,
filters.limit,
&self.access_scope,
)
}
pub(crate) fn visit_analysis_sessions(
&self,
request: &AnalysisRequest,
session_batch_size: std::num::NonZeroUsize,
mut visitor: impl FnMut(
crate::models::SessionRecord,
i64,
i64,
&mut dyn Iterator<Item = Result<(i64, String)>>,
) -> Result<()>,
) -> Result<AnalysisTraversalReceipt> {
self.validate_access_scope()?;
let transaction = self.conn.unchecked_transaction()?;
let mut count_stmt = transaction.prepare(ANALYSIS_MESSAGE_COUNTS_SQL)?;
let mut human_message_stmt =
transaction.prepare(ANALYSIS_ATTRIBUTABLE_HUMAN_MESSAGES_SQL)?;
let mut cursor = None;
let mut selected_sessions = 0_u64;
let mut messages_in_selected_sessions = 0_u64;
let mut analyzed_user_messages = 0_u64;
let mut has_more = false;
let mut last_selected_session_id = None;
let mut max_selected_session_updated_at = None;
let mut corpus_digest = FramedSha256::new(b"aise-analysis-corpus-v1");
loop {
let remaining = match request.selection().max_sessions() {
None => usize::MAX,
Some(max_sessions) => max_sessions
.get()
.saturating_sub(selected_sessions.try_into().unwrap_or(usize::MAX)),
};
if remaining == 0 {
break;
}
let limit = session_batch_size.get().min(remaining);
let (sessions, next_cursor) = analysis_session_page(
&transaction,
request.scope(),
cursor.as_ref(),
limit,
&self.access_scope,
)?;
if sessions.is_empty() {
if next_cursor.is_some() {
bail!(
"internal error: an analysis page returned a pagination cursor but no \
documents; this indicates a bug in aise's pagination logic — please \
file an issue at https://github.com/ahundt/ai-session-search/issues \
with the filters/query that triggered it, or send a pull request fixing \
analysis_session_page's cursor logic in db.rs"
);
}
break;
}
let page_len = sessions.len();
for session in sessions {
let (message_count, human_message_count) = count_stmt
.query_row([&session.id], |row| {
Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?))
})?;
corpus_digest.update_u8(0);
corpus_digest.update_json(&session)?;
corpus_digest.update_i64(message_count);
corpus_digest.update_i64(human_message_count);
messages_in_selected_sessions = messages_in_selected_sessions
.checked_add(u64::try_from(message_count)?)
.ok_or_else(|| anyhow!("analysis total message count overflow"))?;
analyzed_user_messages = analyzed_user_messages
.checked_add(u64::try_from(human_message_count)?)
.ok_or_else(|| anyhow!("analysis human-authored message count overflow"))?;
last_selected_session_id = Some(session.id.clone());
if let Some(updated_at) = session.updated_at.or(session.created_at) {
max_selected_session_updated_at = Some(
max_selected_session_updated_at
.map_or(updated_at, |current: DateTime<Utc>| current.max(updated_at)),
);
}
let session_id = session.id.clone();
let mut rows = human_message_stmt
.query_map([&session.id], |row| {
Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?))
})?
.map(|row| {
let (seq, content) = row.map_err(anyhow::Error::from)?;
corpus_digest.update_u8(1);
corpus_digest.update_bytes(session_id.as_bytes());
corpus_digest.update_i64(seq);
corpus_digest.update_bytes(content.as_bytes());
Ok((seq, content))
});
visitor(session, message_count, human_message_count, &mut rows)?;
for unread in rows {
unread?;
}
}
selected_sessions = selected_sessions
.checked_add(page_len as u64)
.ok_or_else(|| anyhow!("analysis document count overflow"))?;
if request
.selection()
.max_sessions()
.is_some_and(|maximum| selected_sessions >= maximum.get() as u64)
{
has_more = next_cursor.is_some();
break;
}
let Some(next_cursor) = next_cursor else {
break;
};
if cursor
.as_ref()
.is_some_and(|previous: &crate::models::AnalysisCursor| {
previous.as_str() >= next_cursor.as_str()
})
{
bail!(
"internal error: analysis pagination did not advance to a new cursor; this \
indicates a bug in aise's pagination logic — please file an issue at \
https://github.com/ahundt/ai-session-search/issues with the filters/query \
that triggered it, or send a pull request fixing analysis_session_page's \
cursor logic in db.rs"
);
}
cursor = Some(next_cursor);
}
Ok(AnalysisTraversalReceipt {
selected_sessions,
messages_in_selected_sessions,
analyzed_user_messages,
has_more,
last_selected_session_id,
max_selected_session_updated_at,
corpus_digest: corpus_digest.finish(),
})
}
pub fn search(
&self,
query: &str,
filters: &SearchFilters,
current_repo: Option<&str>,
scoring: &crate::config::ScoringConfig,
) -> Result<Vec<SearchHit>> {
self.search_with_cancellation(query, filters, current_repo, scoring, None)
}
pub(crate) fn search_cancellable(
&self,
query: &str,
filters: &SearchFilters,
current_repo: Option<&str>,
scoring: &crate::config::ScoringConfig,
cancellation: &QueryCancellation,
) -> Result<Vec<SearchHit>> {
self.search_with_cancellation(query, filters, current_repo, scoring, Some(cancellation))
}
fn search_with_cancellation(
&self,
query: &str,
filters: &SearchFilters,
current_repo: Option<&str>,
scoring: &crate::config::ScoringConfig,
cancellation: Option<&QueryCancellation>,
) -> Result<Vec<SearchHit>> {
self.validate_access_scope()?;
if let Some(cancellation) = cancellation {
cancellation.ensure_active()?;
}
let query_lower = fold_caseless(query);
let tokens: Vec<&str> = query_lower.split_whitespace().collect();
let query_needle = UnicodeLowerNeedle::from_lowered(&query_lower);
let token_needles = tokens
.iter()
.map(|token| UnicodeLowerNeedle::from_lowered(token))
.collect::<Vec<_>>();
let context = SessionScoreContext {
query,
query_needle: &query_needle,
tokens: &tokens,
token_needles: &token_needles,
current_repo,
scoring,
cancellation,
now: Utc::now(),
};
let mut hits = Vec::new();
let ranked_limit = filters.limit;
let mut sql = format!(
"select {}, coalesce(t.transcript_text, '') as transcript_text
from sessions s
left join transcripts t on t.session_id = s.id
where 1 = 1",
session_record_columns!()
);
let mut params_vec = Vec::new();
push_session_filters(&mut sql, &mut params_vec, filters, &self.access_scope);
sql.push_str(" order by s.id asc");
let mut stmt = self.conn.prepare(&sql)?;
let candidates = stmt.query_map(
rusqlite::params_from_iter(params_vec.iter()),
row_to_session_with_transcript,
)?;
let mut batch = Vec::with_capacity(SESSION_SCORE_BATCH_SIZE);
let mut batch_bytes = 0_usize;
for record in candidates {
if let Some(cancellation) = cancellation {
cancellation.ensure_active()?;
}
let record = record?;
batch_bytes = batch_bytes.saturating_add(session_search_record_bytes(&record));
batch.push(record);
if batch.len() == SESSION_SCORE_BATCH_SIZE || batch_bytes >= SESSION_SCORE_BATCH_BYTES {
let scored = self
.runtime
.install(|| score_session_records(std::mem::take(&mut batch), &context))??;
if let Some(cancellation) = cancellation {
cancellation.ensure_active()?;
}
batch_bytes = 0;
hits.extend(scored);
if ranked_limit > 0 && hits.len() >= top_k_compaction_threshold(ranked_limit) {
retain_top_session_hits(&mut hits, ranked_limit);
}
}
}
if !batch.is_empty() {
hits.extend(
self.runtime
.install(|| score_session_records(batch, &context))??,
);
}
if let Some(cancellation) = cancellation {
cancellation.ensure_active()?;
}
if ranked_limit > 0 {
retain_top_session_hits(&mut hits, ranked_limit);
}
if let Some(cancellation) = cancellation {
cancellation.ensure_active()?;
}
hits.sort_by(compare_session_hits);
if let Some(cancellation) = cancellation {
cancellation.ensure_active()?;
}
Ok(hits)
}
pub fn resolve_session(&self, value: &str) -> Result<SessionWithTranscript> {
let exact = self.session_resolution_matches(
RESOLVE_SESSION_EXACT_SQL,
value,
row_to_session_with_transcript,
)?;
if !exact.is_empty() {
return unique_session_match(value, exact, |session| &session.session.id);
}
let prefix = literal_like_prefix_pattern(value);
let matches = self.session_resolution_matches(
RESOLVE_SESSION_PREFIX_SQL,
&prefix,
row_to_session_with_transcript,
)?;
unique_session_match(value, matches, |session| &session.session.id)
}
pub fn resolve_session_record(&self, value: &str) -> Result<SessionRecord> {
let exact = self.session_resolution_matches(
RESOLVE_SESSION_RECORD_EXACT_SQL,
value,
row_to_session_record,
)?;
if !exact.is_empty() {
return unique_session_match(value, exact, |session| &session.id);
}
let prefix = literal_like_prefix_pattern(value);
let matches = self.session_resolution_matches(
RESOLVE_SESSION_RECORD_PREFIX_SQL,
&prefix,
row_to_session_record,
)?;
unique_session_match(value, matches, |session| &session.id)
}
pub(crate) fn inspect_session_transcript<T>(
&self,
value: &str,
cancellation: &QueryCancellation,
inspect: impl FnOnce(&SessionRecord, &str) -> Result<T>,
) -> Result<T> {
self.with_read_snapshot(|| {
let session = self.resolve_session_record(value)?;
cancellation.ensure_active()?;
let mut statement = self.conn.prepare(
"select coalesce(transcript_text, '') from transcripts where session_id = ?1",
)?;
let mut rows = statement.query([&session.id])?;
let transcript = match rows.next()? {
Some(row) => row.get_ref(0)?.as_str()?,
None => "",
};
cancellation.ensure_active()?;
let result = inspect(&session, transcript);
drop(rows);
drop(statement);
result
})
}
fn session_resolution_matches<T>(
&self,
base_sql: &str,
value: &str,
mapper: fn(&rusqlite::Row<'_>) -> rusqlite::Result<T>,
) -> Result<Vec<T>> {
self.access_scope.validate_stable()?;
let mut sql = base_sql.to_string();
let mut args = vec![rusqlite::types::Value::Text(value.to_string())];
push_access_scope(&mut sql, &mut args, "s.id", &self.access_scope);
let mut stmt = self.conn.prepare(&sql)?;
let rows = stmt.query_map(rusqlite::params_from_iter(args.iter()), mapper)?;
let mut matches = Vec::new();
for row in rows {
matches.push(row?);
}
Ok(matches)
}
pub fn count_parse_warnings(&self) -> Result<i64> {
self.conn
.query_row(
"select count(*) from sessions where parse_warning is not null and parse_warning != ''",
[],
|row| row.get(0),
)
.map_err(Into::into)
}
pub fn parser_health(&self) -> Result<ParserHealth> {
let schema_version: i64 = self
.conn
.query_row("pragma user_version", [], |row| row.get(0))?;
let mut stmt = self.conn.prepare(
"select provider, parse_version, count(*) from sessions
group by provider, parse_version",
)?;
let grouped = stmt
.query_map([], |row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, String>(1)?,
row.get::<_, i64>(2)?,
))
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
let providers = crate::source::PROVIDERS
.into_iter()
.map(|provider| {
let expected = crate::util::provider_parse_version(provider);
let mut indexed_sessions = 0;
let mut current_sessions = 0;
for (stored_provider, parse_version, count) in &grouped {
if stored_provider == provider.as_str() {
indexed_sessions += count;
if parse_version == expected {
current_sessions += count;
}
}
}
ProviderParserHealth {
provider,
expected_parse_version: expected.to_string(),
indexed_sessions,
current_sessions,
stale_sessions: indexed_sessions - current_sessions,
}
})
.collect::<Vec<_>>();
let indexed_sessions = providers.iter().map(|item| item.indexed_sessions).sum();
let current_sessions = providers.iter().map(|item| item.current_sessions).sum();
Ok(ParserHealth {
schema_version,
expected_schema_version: SCHEMA_VERSION,
schema_current: schema_version == SCHEMA_VERSION,
indexed_sessions,
current_sessions,
stale_sessions: indexed_sessions - current_sessions,
parse_warnings: self.count_parse_warnings()?,
providers,
})
}
pub(crate) fn stale_session_sources(&self) -> Result<Vec<(Provider, String)>> {
let mut stmt = self
.conn
.prepare("select provider, parse_version, source_path from sessions")?;
let rows = stmt
.query_map([], |row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, String>(1)?,
row.get::<_, String>(2)?,
))
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
rows.into_iter()
.filter_map(|(provider, parse_version, source_path)| {
let provider =
match decode_db_enum(provider, 0, "sessions.provider", Provider::from_db_str) {
Ok(provider) => provider,
Err(error) => return Some(Err(error.into())),
};
(parse_version != crate::util::provider_parse_version(provider))
.then_some(Ok((provider, source_path)))
})
.collect()
}
pub fn session_time_profile(&self, session_id: &str) -> Result<SessionTimeProfile> {
use rusqlite::types::Value;
self.validate_access_scope()?;
let mut sql = String::from(
"with ordered as (
select ts, kind, lag(ts) over (order by seq) as previous_ts
from messages where session_id = ?",
);
let mut args = vec![Value::Text(session_id.to_string())];
push_access_scope(&mut sql, &mut args, "session_id", &self.access_scope);
sql.push_str(
") select count(*), count(ts), min(ts), max(ts),
max(case when previous_ts is null or ts is null then null
else unixepoch(ts) - unixepoch(previous_ts) end),
coalesce(sum(kind = 'tool_call'), 0),
coalesce(sum(kind = 'tool_result'), 0)
from ordered",
);
let row = self
.conn
.query_row(&sql, rusqlite::params_from_iter(args.iter()), |row| {
Ok((
row.get::<_, i64>(0)?,
row.get::<_, i64>(1)?,
row.get::<_, Option<String>>(2)?,
row.get::<_, Option<String>>(3)?,
row.get::<_, Option<i64>>(4)?,
row.get::<_, i64>(5)?,
row.get::<_, i64>(6)?,
))
})?;
let parse_ts = |value: Option<String>| {
value.and_then(|value| {
chrono::DateTime::parse_from_rfc3339(&value)
.ok()
.map(|timestamp| timestamp.with_timezone(&Utc))
})
};
let first_timestamp = parse_ts(row.2);
let last_timestamp = parse_ts(row.3);
Ok(SessionTimeProfile {
messages: row.0,
timestamped_messages: row.1,
undated_messages: row.0 - row.1,
observed_span_seconds: first_timestamp
.zip(last_timestamp)
.map(|(first, last)| (last - first).num_seconds()),
first_timestamp,
last_timestamp,
max_message_gap_seconds: row.4,
tool_calls: row.5,
tool_results: row.6,
})
}
pub fn counts_by_provider(&self) -> Result<HashMap<String, i64>> {
let mut stmt = self
.conn
.prepare("select provider, count(*) from sessions group by provider")?;
let rows = stmt.query_map([], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
})?;
let mut out = HashMap::new();
for row in rows {
let (provider, count) = row?;
out.insert(provider, count);
}
Ok(out)
}
}
const ANALYSIS_MESSAGE_COUNTS_SQL: &str = "select count(*),
coalesce(sum(
case
when m.record_relation = 'original'
and (
m.authorship = 'human'
or (
m.authorship = 'mixed'
and exists (
select 1
from message_content_parts p
where p.message_id = m.id and p.authorship = 'human'
)
)
)
then 1
else 0
end
), 0)
from messages m
where m.session_id = ?1";
const ANALYSIS_ATTRIBUTABLE_HUMAN_MESSAGES_SQL: &str = "select seq, group_concat(fragment, '')
from (
select m.id as message_id, m.seq as seq, -1 as ordinal, m.content as fragment
from messages m
where m.session_id = ?1
and m.authorship = 'human'
and m.record_relation = 'original'
union all
select m.id as message_id, m.seq as seq, p.ordinal as ordinal,
substr(m.content, p.start_char + 1, p.end_char - p.start_char) as fragment
from messages m
join message_content_parts p on p.message_id = m.id
where m.session_id = ?1
and m.authorship = 'mixed'
and m.record_relation = 'original'
and p.authorship = 'human'
order by seq, message_id, ordinal
)
group by message_id, seq
order by seq";
fn analysis_session_page(
transaction: &rusqlite::Transaction<'_>,
filters: &SearchFilters,
cursor: Option<&crate::models::AnalysisCursor>,
page_limit: usize,
access: &crate::search_scope::EffectiveAccessScope,
) -> Result<(Vec<SessionRecord>, Option<crate::models::AnalysisCursor>)> {
use crate::models::AnalysisCursor;
use std::fmt::Write as _;
if page_limit == 0 {
bail!("analysis document page limit must be greater than zero");
}
let fetch_limit = page_limit
.checked_add(1)
.ok_or_else(|| anyhow!("analysis document page limit is too large"))?;
let mut sql = format!(
"select {} from sessions s where 1 = 1",
session_record_columns!()
);
let mut params_vec = Vec::new();
push_session_filters(&mut sql, &mut params_vec, filters, access);
if let Some(cursor) = cursor {
sql.push_str(" and s.id > ? ");
params_vec.push(cursor.as_str().to_string());
}
write!(sql, " order by s.id asc limit {fetch_limit}")?;
let mut session_stmt = transaction.prepare(&sql)?;
let session_rows = session_stmt.query_map(
rusqlite::params_from_iter(params_vec.iter()),
row_to_session_record,
)?;
let mut sessions = Vec::new();
for row in session_rows {
sessions.push(row?);
}
let has_more = sessions.len() > page_limit;
sessions.truncate(page_limit);
let next_cursor = has_more
.then(|| sessions.last().map(|session| session.id.clone()))
.flatten()
.map(AnalysisCursor::after);
Ok((sessions, next_cursor))
}
fn analysis_document_page(
transaction: &rusqlite::Transaction<'_>,
filters: &SearchFilters,
cursor: Option<&crate::models::AnalysisCursor>,
page_limit: usize,
access: &crate::search_scope::EffectiveAccessScope,
) -> Result<crate::models::AnalysisDocumentPage> {
use crate::models::{AnalysisDocument, AnalysisDocumentPage};
let (sessions, next_cursor) =
analysis_session_page(transaction, filters, cursor, page_limit, access)?;
let mut count_stmt = transaction.prepare(ANALYSIS_MESSAGE_COUNTS_SQL)?;
let mut human_message_stmt = transaction.prepare(ANALYSIS_ATTRIBUTABLE_HUMAN_MESSAGES_SQL)?;
let mut documents = Vec::with_capacity(sessions.len());
for session in sessions {
let (message_count, human_message_count) = count_stmt.query_row([&session.id], |row| {
Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?))
})?;
let rows = human_message_stmt.query_map([&session.id], |row| {
Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?))
})?;
let mut user_text = String::new();
let mut first_user_text = None;
for row in rows {
let (_seq, content) = row?;
if first_user_text.is_none() {
first_user_text = Some(content.clone());
}
if !user_text.is_empty() {
user_text.push(' ');
}
user_text.push_str(&content);
}
documents.push(AnalysisDocument {
session,
user_text,
first_user_text,
message_count,
user_message_count: human_message_count,
});
}
Ok(AnalysisDocumentPage {
documents,
next_cursor,
})
}
fn push_session_filters(
sql: &mut String,
params_vec: &mut Vec<String>,
filters: &SearchFilters,
access: &crate::search_scope::EffectiveAccessScope,
) {
if let Some(provider) = filters.provider {
sql.push_str(" and s.provider = ? ");
params_vec.push(provider.as_str().to_string());
}
if let Some(path_prefix) = &filters.path_prefix {
push_session_path_prefix(sql, params_vec, path_prefix);
}
push_session_access_scope(sql, params_vec, access);
push_session_exclusions(sql, params_vec, filters);
push_session_kinds(sql, filters);
if let Some(parent_session_id) = &filters.parent_session_id {
sql.push_str(" and s.parent_session_id = ? ");
params_vec.push(parent_session_id.clone());
}
push_session_time_window(sql, params_vec, filters.since, filters.until);
if filters.warnings_only {
sql.push_str(" and s.parse_warning is not null and s.parse_warning != '' ");
}
}
fn until_bound_text(until: chrono::DateTime<Utc>) -> String {
until.to_rfc3339_opts(chrono::SecondsFormat::Nanos, false)
}
fn prefix_upper_bound(prefix: &str) -> Option<String> {
let mut head = prefix.to_string();
while let Some(last) = head.pop() {
let next = match u32::from(last) + 1 {
0xD800 => 0xE000,
code => code,
};
if let Some(next) = char::from_u32(next) {
head.push(next);
return Some(head);
}
}
None
}
fn push_path_prefix(
sql: &mut String,
args: &mut Vec<rusqlite::types::Value>,
id_col: &str,
path_prefix: Option<&str>,
) {
use std::fmt::Write as _;
if let Some(prefix) = path_prefix {
let _ = write!(sql, " and {id_col} in (select id from sessions where ");
push_path_condition(sql, args, prefix);
sql.push(')');
}
}
fn push_path_condition(sql: &mut String, args: &mut Vec<rusqlite::types::Value>, prefix: &str) {
use rusqlite::types::Value;
sql.push_str(
"(coalesce(cwd, '') = ? or coalesce(cwd, '') like ? escape '\\' \
or coalesce(repo_root, '') = ? or coalesce(repo_root, '') like ? escape '\\' \
or coalesce(source_path, '') = ? or coalesce(source_path, '') like ? escape '\\')",
);
let (exact, child_pattern) = path_prefix_patterns(prefix);
args.push(Value::Text(exact.clone()));
args.push(Value::Text(child_pattern.clone()));
args.push(Value::Text(exact.clone()));
args.push(Value::Text(child_pattern.clone()));
args.push(Value::Text(exact));
args.push(Value::Text(child_pattern));
}
#[derive(Clone, Copy)]
enum SessionPathDomain {
Workspace,
Transcript,
}
fn push_domain_path_prefix(
sql: &mut String,
args: &mut Vec<rusqlite::types::Value>,
id_col: &str,
prefix: Option<&str>,
domain: SessionPathDomain,
) {
use std::fmt::Write as _;
if let Some(prefix) = prefix {
let _ = write!(sql, " and {id_col} in (select id from sessions where ");
push_domain_path_condition(sql, args, prefix, domain);
sql.push(')');
}
}
fn push_domain_path_condition(
sql: &mut String,
args: &mut Vec<rusqlite::types::Value>,
prefix: &str,
domain: SessionPathDomain,
) {
use rusqlite::types::Value;
let (exact, child_pattern) = path_prefix_patterns(prefix);
match domain {
SessionPathDomain::Workspace => {
sql.push_str(
"(coalesce(cwd, '') = ? or coalesce(cwd, '') like ? escape '\\' \
or coalesce(repo_root, '') = ? or coalesce(repo_root, '') like ? escape '\\')",
);
args.push(Value::Text(exact.clone()));
args.push(Value::Text(child_pattern.clone()));
args.push(Value::Text(exact));
args.push(Value::Text(child_pattern));
}
SessionPathDomain::Transcript => {
sql.push_str(
"(coalesce(source_path, '') = ? or coalesce(source_path, '') like ? escape '\\')",
);
args.push(Value::Text(exact));
args.push(Value::Text(child_pattern));
}
}
}
fn push_access_scope(
sql: &mut String,
args: &mut Vec<rusqlite::types::Value>,
id_col: &str,
access: &crate::search_scope::EffectiveAccessScope,
) {
use std::fmt::Write as _;
let prefixes: Vec<&str> = access.workspace_prefixes().collect();
if prefixes.is_empty() {
return;
}
let _ = write!(sql, " and {id_col} in (select id from sessions where ");
for (index, prefix) in prefixes.into_iter().enumerate() {
if index > 0 {
sql.push_str(" or ");
}
push_domain_path_condition(sql, args, prefix, SessionPathDomain::Workspace);
}
sql.push(')');
}
fn push_session_access_scope(
sql: &mut String,
args: &mut Vec<String>,
access: &crate::search_scope::EffectiveAccessScope,
) {
let prefixes: Vec<&str> = access.workspace_prefixes().collect();
if prefixes.is_empty() {
return;
}
sql.push_str(" and (");
for (index, prefix) in prefixes.into_iter().enumerate() {
if index > 0 {
sql.push_str(" or ");
}
let (exact, child_pattern) = path_prefix_patterns(prefix);
sql.push_str(
"(coalesce(s.cwd, '') = ? or coalesce(s.cwd, '') like ? escape '\\' \
or coalesce(s.repo_root, '') = ? or coalesce(s.repo_root, '') like ? escape '\\')",
);
args.push(exact.clone());
args.push(child_pattern.clone());
args.push(exact);
args.push(child_pattern);
}
sql.push(')');
}
fn push_exclude_domain_path_prefixes(
sql: &mut String,
args: &mut Vec<rusqlite::types::Value>,
id_col: &str,
prefixes: &[String],
domain: SessionPathDomain,
) {
use std::fmt::Write as _;
if prefixes.is_empty() {
return;
}
let _ = write!(sql, " and {id_col} not in (select id from sessions where ");
for (index, prefix) in prefixes.iter().enumerate() {
if index > 0 {
sql.push_str(" or ");
}
push_domain_path_condition(sql, args, prefix, domain);
}
sql.push(')');
}
fn push_exclude_path_prefixes(
sql: &mut String,
args: &mut Vec<rusqlite::types::Value>,
id_col: &str,
prefixes: &[String],
) {
use std::fmt::Write as _;
if prefixes.is_empty() {
return;
}
let _ = write!(sql, " and {id_col} not in (select id from sessions where ");
for (i, prefix) in prefixes.iter().enumerate() {
if i > 0 {
sql.push_str(" or ");
}
push_path_condition(sql, args, prefix);
}
sql.push(')');
}
#[derive(Debug, Clone, Copy)]
enum ClassSelection<'a> {
FromFilters,
Unrestricted,
Only(&'a [crate::models::MessageKind]),
}
fn append_message_filters(
sql: &mut String,
args: &mut Vec<rusqlite::types::Value>,
filters: &MessageFilters,
access: &crate::search_scope::EffectiveAccessScope,
) {
append_message_filters_with_class(sql, args, filters, access, ClassSelection::FromFilters);
}
fn append_message_filters_with_class(
sql: &mut String,
args: &mut Vec<rusqlite::types::Value>,
filters: &MessageFilters,
access: &crate::search_scope::EffectiveAccessScope,
class: ClassSelection<'_>,
) {
use rusqlite::types::Value;
if let Some(role) = filters.role {
sql.push_str(" and m.role = ?");
args.push(Value::Text(role.as_str().to_string()));
}
let predicate = match class {
ClassSelection::FromFilters => Some(filters.kind_predicate()),
ClassSelection::Unrestricted => None,
ClassSelection::Only(kinds) => Some(crate::models::KindPredicate::Only(kinds.to_vec())),
};
match predicate {
None => {}
Some(crate::models::KindPredicate::AllExcept(excluded)) => {
for kind in excluded {
sql.push_str(" and m.kind != ?");
args.push(Value::Text(kind.as_str().to_string()));
}
}
Some(crate::models::KindPredicate::Only(kinds)) if kinds.is_empty() => {
sql.push_str(" and 0");
}
Some(crate::models::KindPredicate::Only(kinds)) => {
sql.push_str(" and m.kind in (");
for (i, kind) in kinds.iter().enumerate() {
if i > 0 {
sql.push_str(", ");
}
sql.push('?');
args.push(Value::Text(kind.as_str().to_string()));
}
sql.push(')');
}
}
if let Some(providers) = &filters.providers {
if providers.is_empty() {
sql.push_str(" and 0");
} else {
sql.push_str(" and m.provider in (");
for (index, provider) in providers.iter().enumerate() {
if index > 0 {
sql.push_str(", ");
}
sql.push('?');
args.push(Value::Text(provider.as_str().to_string()));
}
sql.push(')');
}
}
if let Some(session_id) = &filters.session_id {
sql.push_str(" and m.session_id = ?");
args.push(Value::Text(session_id.clone()));
}
push_message_session_kinds(sql, "m.session_id", filters.session_kinds.as_ref());
push_access_scope(sql, args, "m.session_id", access);
push_path_prefix(sql, args, "m.session_id", filters.path_prefix.as_deref());
push_exclude_path_prefixes(sql, args, "m.session_id", &filters.exclude_path_prefixes);
push_domain_path_prefix(
sql,
args,
"m.session_id",
filters.workspace_path_prefix.as_deref(),
SessionPathDomain::Workspace,
);
push_domain_path_prefix(
sql,
args,
"m.session_id",
filters.transcript_path_prefix.as_deref(),
SessionPathDomain::Transcript,
);
push_exclude_domain_path_prefixes(
sql,
args,
"m.session_id",
&filters.exclude_workspace_path_prefixes,
SessionPathDomain::Workspace,
);
push_exclude_domain_path_prefixes(
sql,
args,
"m.session_id",
&filters.exclude_transcript_path_prefixes,
SessionPathDomain::Transcript,
);
for session_id in &filters.exclude_session_ids {
sql.push_str(" and m.session_id <> ?");
args.push(Value::Text(session_id.clone()));
}
if let Some(tool) = &filters.tool {
sql.push_str(" and unicode_lower_contains(m.tool_name, ?)");
args.push(Value::Text(fold_caseless(tool)));
}
if let Some(seq_from) = filters.seq_from {
sql.push_str(" and m.seq >= ?");
args.push(Value::Integer(seq_from));
}
if let Some(seq_to) = filters.seq_to {
sql.push_str(" and m.seq <= ?");
args.push(Value::Integer(seq_to));
}
push_ts_window(sql, args, "m.ts", filters.since, filters.until);
}
fn insert_messages<'a>(
tx: &rusqlite::Transaction<'_>,
session: &SessionRecord,
rows: impl Iterator<Item = (i64, &'a crate::models::Message)>,
) -> Result<()> {
let mut stmt = tx.prepare(
"insert into messages
(session_id, provider, seq, role, ts, tool_name, kind, tool_call_id, is_compaction,
content, authorship, correlation_authority, correlation_scope, correlation_id,
record_relation)
values (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)",
)?;
let mut part_stmt = tx.prepare(
"insert into message_content_parts
(message_id, ordinal, start_char, end_char, authorship, origin)
values (?1, ?2, ?3, ?4, ?5, ?6)",
)?;
for (seq, message) in rows {
if let Err(detail) = message.provenance.validate(&message.content) {
bail!(
"invalid provenance for message sequence {seq} in session '{}': {detail}",
session.id
);
}
let correlation = message.provenance.correlation_identity.as_ref();
stmt.execute(params![
session.id,
session.provider.as_str(),
seq,
message.role.as_str(),
message.ts.map(|ts| ts.to_rfc3339()),
message.tool_name,
message.kind.as_str(),
message.tool_call_id,
message.is_compaction as i64,
message.content,
message.provenance.authorship.as_str(),
correlation.map(|value| value.authority.as_str()),
correlation.map(|value| value.scope.as_str()),
correlation.map(|value| value.id.as_str()),
message.provenance.record_relation.as_str(),
])?;
let message_id = tx.last_insert_rowid();
for part in &message.provenance.content_parts {
part_stmt.execute(params![
message_id,
i64::from(part.ordinal),
i64::try_from(part.start_char).context("content-part start exceeds SQLite i64")?,
i64::try_from(part.end_char).context("content-part end exceeds SQLite i64")?,
part.authorship.as_str(),
part.origin.as_str(),
])?;
}
}
Ok(())
}
fn insert_file_edits<'a>(
tx: &rusqlite::Transaction<'_>,
session: &SessionRecord,
rows: impl Iterator<Item = (i64, &'a FileEdit)>,
) -> Result<()> {
let mut stmt = tx.prepare(
"insert into file_edits
(session_id, provider, seq, ts, tool, file_path, file_name, new_content, edits_json)
values (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
)?;
for (seq, edit) in rows {
let edits_json = if edit.edits.is_empty() {
None
} else {
Some(serde_json::to_string(&edit.edits)?)
};
stmt.execute(params![
session.id,
session.provider.as_str(),
seq,
edit.ts.map(|ts| ts.to_rfc3339()),
edit.tool,
edit.file_path,
edit.file_name,
edit.new_content,
edits_json,
])?;
}
Ok(())
}
fn push_ts_window(
sql: &mut String,
args: &mut Vec<rusqlite::types::Value>,
col: &str,
since: Option<chrono::DateTime<Utc>>,
until: Option<chrono::DateTime<Utc>>,
) {
use rusqlite::types::Value;
use std::fmt::Write as _;
if let Some(since) = since {
let _ = write!(sql, " and {col} >= ?");
args.push(Value::Text(since.to_rfc3339()));
}
if let Some(until) = until {
let _ = write!(sql, " and {col} <= ?");
args.push(Value::Text(until_bound_text(until)));
}
}
fn push_file_filters(
sql: &mut String,
args: &mut Vec<rusqlite::types::Value>,
query: &FileQuery,
access: &crate::search_scope::EffectiveAccessScope,
) {
use rusqlite::types::Value;
if let Some(provider) = query.provider {
sql.push_str(" and provider = ?");
args.push(Value::Text(provider.as_str().to_string()));
}
if let Some(session_id) = query.session_id.as_deref() {
sql.push_str(" and session_id = ?");
args.push(Value::Text(session_id.to_string()));
}
push_access_scope(sql, args, "session_id", access);
push_path_prefix(sql, args, "session_id", query.path_prefix.as_deref());
push_exclude_path_prefixes(sql, args, "session_id", &query.exclude_path_prefixes);
for session_id in &query.exclude_session_ids {
sql.push_str(" and session_id <> ?");
args.push(Value::Text(session_id.clone()));
}
}
fn path_prefix_parts(prefix: &str) -> (String, String) {
let bytes = prefix.as_bytes();
let windows_style = prefix.starts_with(r"\\")
|| matches!(bytes, [drive, b':', b'\\' | b'/', ..] if drive.is_ascii_alphabetic());
let separator = if windows_style { '\\' } else { '/' };
let exact = prefix.trim_end_matches(separator).to_string();
let child = format!("{exact}{separator}");
(exact, child)
}
fn path_prefix_patterns(prefix: &str) -> (String, String) {
let (exact, child) = path_prefix_parts(prefix);
(exact, literal_like_prefix_pattern(&child))
}
fn literal_like_prefix_pattern(value: &str) -> String {
let mut escaped = String::with_capacity(value.len() + 2);
for ch in value.chars() {
match ch {
'%' | '_' | '\\' => {
escaped.push('\\');
escaped.push(ch);
}
other => escaped.push(other),
}
}
escaped.push('%');
escaped
}
fn push_session_path_prefix(sql: &mut String, args: &mut Vec<String>, path_prefix: &str) {
let (exact, child_pattern) = path_prefix_patterns(path_prefix);
sql.push_str(
" and ((coalesce(s.cwd, '') = ? or coalesce(s.cwd, '') like ? escape '\\') \
or (coalesce(s.repo_root, '') = ? or coalesce(s.repo_root, '') like ? escape '\\') \
or (coalesce(s.source_path, '') = ? or coalesce(s.source_path, '') like ? escape '\\')) ",
);
args.push(exact.clone());
args.push(child_pattern.clone());
args.push(exact.clone());
args.push(child_pattern.clone());
args.push(exact);
args.push(child_pattern);
}
fn push_session_exclusions(sql: &mut String, args: &mut Vec<String>, filters: &SearchFilters) {
for session_id in &filters.exclude_session_ids {
sql.push_str(" and s.id <> ? ");
args.push(session_id.clone());
}
for prefix in &filters.exclude_path_prefixes {
let (exact, child_pattern) = path_prefix_patterns(prefix);
sql.push_str(
" and not ((coalesce(s.cwd, '') = ? or coalesce(s.cwd, '') like ? escape '\\') \
or (coalesce(s.repo_root, '') = ? or coalesce(s.repo_root, '') like ? escape '\\') \
or (coalesce(s.source_path, '') = ? or coalesce(s.source_path, '') like ? escape '\\')) ",
);
args.push(exact.clone());
args.push(child_pattern.clone());
args.push(exact.clone());
args.push(child_pattern.clone());
args.push(exact);
args.push(child_pattern);
}
}
fn session_kind_predicate(selected: &[crate::models::SessionKind], alias: &str) -> Option<String> {
if selected.is_empty() {
return Some("0".to_string());
}
if crate::models::SessionKind::all()
.iter()
.all(|kind| selected.contains(kind))
{
return None;
}
let mut predicate = String::from("(");
for (index, kind) in selected.iter().enumerate() {
if index > 0 {
predicate.push_str(" or ");
}
predicate.push_str(&kind.sql_predicate(alias));
}
predicate.push(')');
Some(predicate)
}
fn push_session_kinds(sql: &mut String, filters: &SearchFilters) {
if let Some(predicate) = session_kind_predicate(&filters.effective_session_kinds(), "s") {
sql.push_str(" and ");
sql.push_str(&predicate);
sql.push(' ');
}
}
fn push_message_session_kinds(
sql: &mut String,
id_col: &str,
session_kinds: Option<&Vec<crate::models::SessionKind>>,
) {
use std::fmt::Write as _;
let Some(selected) = session_kinds else {
return;
};
let Some(predicate) = session_kind_predicate(selected, "sessions") else {
return;
};
if predicate == "0" {
sql.push_str(" and 0");
return;
}
let _ = write!(
sql,
" and {id_col} in (select id from sessions where {predicate})"
);
}
fn push_session_time_window(
sql: &mut String,
args: &mut Vec<String>,
since: Option<chrono::DateTime<Utc>>,
until: Option<chrono::DateTime<Utc>>,
) {
if let Some(since) = since {
sql.push_str(" and coalesce(s.updated_at, s.created_at) >= ? ");
args.push(since.to_rfc3339());
}
if let Some(until) = until {
sql.push_str(" and coalesce(s.created_at, s.updated_at) <= ? ");
args.push(until_bound_text(until));
}
}
fn row_to_message_hit(row: &rusqlite::Row<'_>) -> rusqlite::Result<MessageHit> {
row_to_message_hit_at(row, 0)
}
fn decode_db_enum<T>(
value: String,
column_index: usize,
context: &'static str,
decode: impl FnOnce(&str) -> std::result::Result<T, String>,
) -> rusqlite::Result<T> {
decode(&value).map_err(|reason| {
rusqlite::Error::FromSqlConversionFailure(
column_index,
rusqlite::types::Type::Text,
std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!("{reason} (while reading {context})"),
)
.into(),
)
})
}
fn row_to_message_hit_at(row: &rusqlite::Row<'_>, offset: usize) -> rusqlite::Result<MessageHit> {
let ts: Option<String> = row.get(offset + 4)?;
Ok(MessageHit {
session_id: row.get(offset)?,
provider: decode_db_enum(
row.get::<_, String>(offset + 1)?,
offset + 1,
"messages.provider",
Provider::from_db_str,
)?,
seq: row.get(offset + 2)?,
role: decode_db_enum(
row.get::<_, String>(offset + 3)?,
offset + 3,
"messages.role",
Role::from_db_str,
)?,
ts: ts.and_then(|value| {
chrono::DateTime::parse_from_rfc3339(&value)
.ok()
.map(|dt| dt.with_timezone(&Utc))
}),
tool_name: row.get(offset + 5)?,
kind: crate::models::MessageKind::from_db_str(&row.get::<_, String>(offset + 6)?),
tool_call_id: row.get(offset + 7)?,
fuzzy_score: None,
content: row.get(offset + 8)?,
})
}
fn json_projection_prefilter_groups(pattern: &str) -> Option<Vec<Vec<String>>> {
let groups = crate::trigram::trigram_prefilter_groups(pattern)?;
let groups = groups
.into_iter()
.map(|group| {
group
.into_iter()
.filter(|trigram| {
trigram.chars().all(|character| {
!character.is_control() && !matches!(character, '"' | '\\')
})
})
.collect::<Vec<_>>()
})
.collect::<Vec<_>>();
(!groups.is_empty() && groups.iter().all(|group| !group.is_empty())).then_some(groups)
}
fn score_fuzzy_derived_hits(
pattern: &Pattern,
query_lower: &str,
field: SearchField,
argument_path: Option<&str>,
hits: Vec<MessageHit>,
) -> Vec<(MessageHit, bool)> {
let exact_phrase_needle = UnicodeLowerNeedle::from_lowered(query_lower);
hits.into_par_iter()
.map_init(
|| (NucleoMatcher::new(NucleoConfig::DEFAULT), Vec::new()),
|(matcher, utf32_buf), mut hit| {
let value = selected_message_field_parts(&hit, field, argument_path)?.into_owned();
utf32_buf.clear();
pattern
.score(Utf32Str::new(&value, utf32_buf), matcher)
.map(|score| {
hit.fuzzy_score = Some(score);
let exact_phrase = exact_phrase_needle.contains(&value);
(hit, exact_phrase)
})
},
)
.filter_map(std::convert::identity)
.collect()
}
fn score_fuzzy_message_hits(
pattern: &Pattern,
query_lower: &str,
hits: Vec<MessageHit>,
) -> Vec<(MessageHit, bool)> {
let exact_phrase_needle = UnicodeLowerNeedle::from_lowered(query_lower);
hits.into_par_iter()
.map_init(
|| (NucleoMatcher::new(NucleoConfig::DEFAULT), Vec::new()),
|(matcher, utf32_buf), mut hit| {
let score = {
let haystack = Utf32Str::new(&hit.content, utf32_buf);
pattern.score(haystack, matcher)
};
score.map(|score| {
hit.fuzzy_score = Some(score);
let exact_phrase = exact_phrase_needle.contains(&hit.content);
(hit, exact_phrase)
})
},
)
.filter_map(std::convert::identity)
.collect()
}
fn compare_fuzzy_hits(left: &(MessageHit, bool), right: &(MessageHit, bool)) -> std::cmp::Ordering {
right
.0
.fuzzy_score
.unwrap_or_default()
.cmp(&left.0.fuzzy_score.unwrap_or_default())
.then_with(|| right.1.cmp(&left.1))
.then_with(|| left.0.session_id.cmp(&right.0.session_id))
.then_with(|| left.0.seq.cmp(&right.0.seq))
}
fn scored_whole_corpus(matched: i64, corpus: i64) -> SearchExplain {
SearchExplain {
prefilter: None,
candidates: Some(matched),
prefilter_skipped: Some(
"complete filtered corpus scored, retaining at most min(offset + limit, corpus) ranked rows"
.to_string(),
),
corpus,
}
}
fn empty_past_corpus(corpus: i64) -> SearchExplain {
SearchExplain {
prefilter: None,
candidates: None,
prefilter_skipped: Some(
"offset starts at or past the last eligible row, so no row was scored".to_string(),
),
corpus,
}
}
const fn fuzzy_derived_scan_predicate(field: SearchField) -> &'static str {
match field {
SearchField::ToolArgument => "m.kind = 'tool_call'",
SearchField::ToolName | SearchField::Content => "1 = 1",
}
}
enum FuzzyRankingWindow {
Rank { limit: usize, corpus: Option<i64> },
PastCorpus { corpus: i64 },
}
fn fuzzy_ranked_limit(filters: &MessageFilters) -> Result<usize> {
filters
.offset
.checked_add(filters.limit)
.ok_or_else(|| anyhow!("fuzzy offset + limit exceeds the platform addressable range"))
}
fn top_k_compaction_threshold(limit: usize) -> usize {
limit.saturating_mul(2).max(FUZZY_SCORE_BATCH_SIZE)
}
fn retain_top_fuzzy_hits(scored: &mut Vec<(MessageHit, bool)>, limit: usize) {
if scored.len() > limit {
scored.select_nth_unstable_by(limit, compare_fuzzy_hits);
scored.truncate(limit);
}
}
fn finish_fuzzy_hits(
mut scored: Vec<(MessageHit, bool)>,
ranked_limit: usize,
offset: usize,
) -> Vec<MessageHit> {
retain_top_fuzzy_hits(&mut scored, ranked_limit);
scored.sort_by(compare_fuzzy_hits);
scored
.into_iter()
.skip(offset)
.map(|(hit, _)| hit)
.collect()
}
fn session_search_record_bytes(record: &SessionWithTranscript) -> usize {
record.transcript_text.len()
+ record.session.preview_text.len()
+ record.session.title.as_ref().map_or(0, String::len)
+ record.session.summary.as_ref().map_or(0, String::len)
+ record.session.cwd.as_ref().map_or(0, String::len)
+ record.session.repo_root.as_ref().map_or(0, String::len)
}
struct SessionScoreContext<'a> {
query: &'a str,
query_needle: &'a UnicodeLowerNeedle,
tokens: &'a [&'a str],
token_needles: &'a [UnicodeLowerNeedle],
current_repo: Option<&'a str>,
scoring: &'a crate::config::ScoringConfig,
cancellation: Option<&'a QueryCancellation>,
now: DateTime<Utc>,
}
fn session_contains(
needle: &UnicodeLowerNeedle,
value: &str,
cancellation: Option<&QueryCancellation>,
) -> Result<bool> {
needle
.contains_while(value, || {
cancellation.is_none_or(|token| !token.is_cancelled())
})
.ok_or_else(|| QueryCancelled.into())
}
fn session_snippet(
value: &str,
query: &str,
cancellation: Option<&QueryCancellation>,
) -> Result<String> {
match cancellation {
Some(token) => snippet_from_match_while(value, query, 160, || !token.is_cancelled())
.ok_or_else(|| QueryCancelled.into()),
None => Ok(snippet_from_match(value, query, 160)),
}
}
fn score_session_records(
records: Vec<SessionWithTranscript>,
context: &SessionScoreContext<'_>,
) -> Result<Vec<SearchHit>> {
let SessionScoreContext {
query,
query_needle,
tokens,
token_needles,
current_repo,
scoring,
cancellation,
now,
} = context;
records
.into_par_iter()
.map(|record| -> Result<Option<SearchHit>> {
if let Some(cancellation) = cancellation {
cancellation.ensure_active()?;
}
let matcher = SkimMatcherV2::default().smart_case();
let title = record.session.title.as_deref().unwrap_or_default();
let summary = record.session.summary.as_deref().unwrap_or_default();
let cwd = record.session.cwd.as_deref().unwrap_or_default();
let repo_root = record.session.repo_root.as_deref().unwrap_or_default();
let preview = record.session.preview_text.as_str();
let transcript = record.transcript_text.as_str();
let haystacks = [
("title", title),
("summary", summary),
("cwd", cwd),
("repo", repo_root),
("preview", preview),
("transcript", transcript),
];
let mut score = 0_i64;
let mut best_source = "fuzzy".to_string();
let mut best_source_score = i64::MIN;
let mut best_snippet = session_snippet(preview, query, *cancellation)?;
let mut term_coverage = vec![false; tokens.len()];
let mut matched = false;
for (source, value) in haystacks {
let mut source_score = 0_i64;
if session_contains(query_needle, value, *cancellation)? {
matched = true;
source_score += match source {
"title" => scoring.title_score,
"summary" => scoring.summary_score,
"cwd" | "repo" => scoring.path_score,
"preview" => scoring.preview_score,
_ => scoring.other_score,
};
}
for (index, token) in tokens.iter().enumerate() {
if !token.is_empty()
&& session_contains(&token_needles[index], value, *cancellation)?
{
matched = true;
source_score += scoring.token_bonus;
term_coverage[index] = true;
}
}
if matches!(source, "title" | "cwd" | "repo" | "preview") {
if let Some(fuzzy_score) = matcher.fuzzy_match(value, query) {
matched = true;
source_score += fuzzy_score;
}
}
score += source_score;
if source_score > best_source_score {
best_source_score = source_score;
best_source = source.to_string();
best_snippet = session_snippet(value, query, *cancellation)?;
}
}
if !matched {
return Ok(None);
}
if tokens.len() > 1 && term_coverage.iter().all(|matched| *matched) {
score += scoring.all_tokens_bonus;
}
if let Some(updated_at) = record.session.updated_at {
let age_days = (*now - updated_at)
.num_days()
.clamp(0, scoring.recency_max_days);
score += (scoring.recency_max_days - age_days) * scoring.recency_weight;
}
if let (Some(current_repo), Some(repo_root)) =
(current_repo, record.session.repo_root.as_deref())
{
if *current_repo == repo_root {
score += scoring.current_repo_bonus;
if best_source == "fuzzy" {
best_source = "repo".to_string();
best_snippet = snippet_from_match(repo_root, query, 160);
}
}
}
Ok(Some(SearchHit {
session: record.session,
score,
match_source: best_source,
match_snippet: best_snippet,
}))
})
.filter_map(|scored| scored.transpose())
.collect::<Result<Vec<_>>>()
}
fn compare_session_hits(left: &SearchHit, right: &SearchHit) -> std::cmp::Ordering {
right
.score
.cmp(&left.score)
.then_with(|| right.session.updated_at.cmp(&left.session.updated_at))
.then_with(|| left.session.id.cmp(&right.session.id))
}
fn retain_top_session_hits(hits: &mut Vec<SearchHit>, limit: usize) {
if hits.len() > limit {
hits.select_nth_unstable_by(limit, compare_session_hits);
hits.truncate(limit);
}
}
fn glob_clause(pattern: &str) -> (&'static str, String) {
let mut like = String::with_capacity(pattern.len() + 1);
for ch in pattern.chars() {
match ch {
'*' => like.push('%'),
'?' => like.push('_'),
'%' | '_' | '\\' => {
like.push('\\');
like.push(ch);
}
other => like.push(other),
}
}
if pattern.contains('/') {
("file_path", format!("%{like}"))
} else {
("file_name", like)
}
}
const RESOLVE_SESSION_EXACT_SQL: &str = concat!(
"select ",
session_record_columns!(),
", coalesce(t.transcript_text, '') as transcript_text \
from sessions s \
left join transcripts t on t.session_id = s.id \
where (s.id = ?1 or s.provider_session_id = ?1)"
);
const RESOLVE_SESSION_PREFIX_SQL: &str = concat!(
"select ",
session_record_columns!(),
", coalesce(t.transcript_text, '') as transcript_text \
from sessions s \
left join transcripts t on t.session_id = s.id \
where (s.id like ?1 escape '\\' or s.provider_session_id like ?1 escape '\\')"
);
const RESOLVE_SESSION_RECORD_EXACT_SQL: &str = concat!(
"select ",
session_record_columns!(),
" from sessions s where (s.id = ?1 or s.provider_session_id = ?1)"
);
const RESOLVE_SESSION_RECORD_PREFIX_SQL: &str = concat!(
"select ",
session_record_columns!(),
" from sessions s where (s.id like ?1 escape '\\' or s.provider_session_id like ?1 escape '\\')"
);
fn unique_session_match<T>(
value: &str,
mut matches: Vec<T>,
id_of: impl Fn(&T) -> &str,
) -> Result<T> {
if matches.len() > 1 {
let is_exact =
|id: &str| id == value || id.split_once(':').is_some_and(|(_, rest)| rest == value);
let mut exact = matches
.iter()
.enumerate()
.filter(|(_, m)| is_exact(id_of(m)));
if let (Some((index, _)), None) = (exact.next(), exact.next()) {
return Ok(matches.remove(index));
}
}
match matches.len() {
0 => Err(anyhow!(
"no session matches '{value}' — run `aise list` to see recent session \
ids, or `aise search <keywords>` to find one"
)),
1 => Ok(matches.remove(0)),
_ => {
let shown: Vec<&str> = matches.iter().take(8).map(id_of).collect();
let more = matches.len().saturating_sub(shown.len());
let suffix = if more > 0 {
format!(" (+{more} more)")
} else {
String::new()
};
Err(anyhow!(
"session prefix '{value}' is ambiguous — {} sessions match: {}{}. \
Pass a longer prefix or the full id.",
matches.len(),
shown.join(", "),
suffix
))
}
}
}
fn row_to_session_with_transcript(
row: &rusqlite::Row<'_>,
) -> rusqlite::Result<SessionWithTranscript> {
Ok(SessionWithTranscript {
session: row_to_session_record(row)?,
transcript_text: row.get("transcript_text")?,
})
}
fn row_to_session_record(row: &rusqlite::Row<'_>) -> rusqlite::Result<SessionRecord> {
let provider: String = row.get(1)?;
Ok(SessionRecord {
id: row.get(0)?,
provider: decode_db_enum(provider, 1, "sessions.provider", Provider::from_db_str)?,
provider_session_id: row.get(2)?,
title: row.get(3)?,
summary: row.get(4)?,
cwd: row.get(5)?,
repo_root: row.get(6)?,
created_at: row
.get::<_, Option<String>>(7)?
.as_deref()
.and_then(crate::util::parse_datetime),
updated_at: row
.get::<_, Option<String>>(8)?
.as_deref()
.and_then(crate::util::parse_datetime),
last_message_at: row
.get::<_, Option<String>>(9)?
.as_deref()
.and_then(crate::util::parse_datetime),
preview_text: row.get(10)?,
source_path: row.get(11)?,
message_count: row.get(12)?,
parse_version: row.get(13)?,
raw_metadata_json: row.get(14)?,
parse_warning: row.get(15)?,
discovery_source: row.get(16)?,
parent_session_id: row.get(17)?,
agent_label: row.get(18)?,
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::models::SessionKind;
const TEST_BUSY_TIMEOUT_MS: u64 = 250;
const TEST_NO_WAIT_BUSY_TIMEOUT_MS: u64 = 0;
#[test]
fn message_row_rejects_corrupt_provider_and_role_with_recovery_context() {
let connection = Connection::open_in_memory().unwrap();
for (provider, role, bad_value, column, advice, forbidden) in [
(
"not-a-provider",
"user",
"not-a-provider",
"messages.provider",
"upgrade aise",
"reindex --full",
),
(
"claude",
"not-a-role",
"not-a-role",
"messages.role",
"aise reindex --full",
"upgrade aise",
),
] {
let error = connection
.query_row(
"select 'session', ?1, 0, ?2, null, null, 'conversation', null, 'content'",
params![provider, role],
row_to_message_hit,
)
.unwrap_err();
let error = error.to_string();
assert!(error.contains(bad_value), "{error}");
assert!(error.contains(column), "{error}");
assert!(error.contains(advice), "{error}");
assert!(!error.contains(forbidden), "{error}");
}
}
fn test_policies(
categories: &[(&str, &[&str])],
) -> crate::corrections::ResolvedCorrectionPolicySet {
let spec = crate::corrections::CorrectionPolicySpec {
schema_version: crate::corrections::CORRECTION_POLICY_SCHEMA_VERSION,
name: "test-policy".to_string(),
version: "1.0.0".to_string(),
categories: categories
.iter()
.map(
|(name, patterns)| crate::corrections::CorrectionCategorySpec {
name: (*name).to_string(),
patterns: patterns.iter().map(|p| (*p).to_string()).collect(),
},
)
.collect(),
};
crate::corrections::ResolvedCorrectionPolicySet::from_policies(vec![spec
.compile_in_memory(crate::corrections::CorrectionPolicySource::Embedded)
.expect("test correction policy compiles")])
}
fn default_policies() -> crate::corrections::ResolvedCorrectionPolicySet {
crate::corrections::ResolvedCorrectionPolicySet::from_policies(vec![
crate::corrections::embedded_policy().expect("the built-in capability compiles"),
])
}
fn schema_fingerprint(conn: &Connection) -> Vec<(String, String, String, Option<String>)> {
conn.prepare(
"select type, name, tbl_name, sql
from sqlite_schema
where name not like 'sqlite_autoindex_%'
order by type, name, tbl_name",
)
.unwrap()
.query_map([], |row| {
Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?))
})
.unwrap()
.collect::<rusqlite::Result<_>>()
.unwrap()
}
fn enable_v3_custom_trigram_compatibility(db: &Db) {
db.conn
.execute_batch(
"drop trigger if exists messages_ai;
drop trigger if exists messages_ad;
drop trigger if exists messages_au;
drop table if exists messages_trigram_terms;
drop table if exists messages_trigram_vocab;
drop table if exists messages_trigram;",
)
.unwrap();
crate::fts::install_released_message_word_index(&db.conn).unwrap();
crate::trigram_index::ensure_schema(&db.conn).unwrap();
db.conn.pragma_update(None, "user_version", 3).unwrap();
}
#[derive(Debug, Clone, Copy)]
enum MessageContentMode {
Literal,
Regex,
Fuzzy,
}
impl MessageContentMode {
fn search(
self,
db: &Db,
pattern: &str,
mut filters: MessageFilters,
) -> Result<Vec<MessageHit>> {
filters.match_mode = match self {
Self::Literal => MessageSearchMode::Literal,
Self::Regex => MessageSearchMode::Regex,
Self::Fuzzy => MessageSearchMode::Fuzzy,
};
db.search_messages(pattern, &filters)
}
}
const MESSAGE_CONTENT_MODE_CASES: [(MessageContentMode, &str); 3] = [
(MessageContentMode::Literal, "shared needle"),
(MessageContentMode::Regex, r"shared\s+needle"),
(MessageContentMode::Fuzzy, "shared needle"),
];
fn hit_keys(hits: Vec<MessageHit>) -> Vec<(String, i64)> {
hits.into_iter()
.map(|hit| (hit.session_id, hit.seq))
.collect()
}
fn utc(value: &str) -> DateTime<Utc> {
DateTime::parse_from_rfc3339(value)
.unwrap()
.with_timezone(&Utc)
}
#[test]
fn glob_clause_maps_basename_and_path() {
assert_eq!(glob_clause("*.rs"), ("file_name", "%.rs".to_string()));
assert_eq!(glob_clause("db.rs"), ("file_name", "db.rs".to_string()));
assert_eq!(
glob_clause("src/*.rs"),
("file_path", "%src/%.rs".to_string())
);
assert_eq!(glob_clause("a_b%c"), ("file_name", "a\\_b\\%c".to_string()));
}
#[test]
fn a_spawner_resolves_by_its_own_id_even_though_its_runs_extend_it() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions (
id, provider, provider_session_id, updated_at, preview_text,
source_path, parse_version, discovery_source, parent_session_id
) values
('claude:s6', 'claude', 's6', '2026-03-01T00:00:00+00:00', '', '/p', 'v1',
'test', null),
('claude:s6/agent-a', 'claude', 's6/agent-a', '2026-03-02T00:00:00+00:00', '',
'/a', 'v1', 'test', 'claude:s6'),
('claude:s6/agent-b', 'claude', 's6/agent-b', '2026-03-03T00:00:00+00:00', '',
'/b', 'v1', 'test', 'claude:s6');",
)
.unwrap();
for spelling in ["s6", "claude:s6"] {
assert_eq!(
db.resolve_session_record(spelling).unwrap().id,
"claude:s6",
"the spawner's exact id must win over the runs that extend it: {spelling}"
);
}
assert_eq!(
db.resolve_session_record("s6/agent-a").unwrap().id,
"claude:s6/agent-a"
);
let error = db
.resolve_session_record("s6/agent-")
.expect_err("a genuine prefix collision must still be refused")
.to_string();
assert!(error.contains("ambiguous"), "{error}");
assert!(
error.contains("claude:s6/agent-a") && error.contains("claude:s6/agent-b"),
"the refusal must name the candidates: {error}"
);
}
#[test]
fn session_date_window_intersects_known_span_and_falls_back_for_null_endpoints() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions (
id, provider, provider_session_id, created_at, updated_at, preview_text,
source_path, parse_version, discovery_source
) values
('claude:long-running', 'claude', 'long-running', '2026-01-10T00:00:00+00:00',
'2026-03-10T00:00:00+00:00', '', '/long', 'v1', 'test'),
('claude:february', 'claude', 'february', '2026-02-05T00:00:00+00:00',
'2026-02-06T00:00:00+00:00', '', '/february', 'v1', 'test'),
('claude:never-updated', 'claude', 'never-updated', '2026-03-11T00:00:00+00:00',
null, '', '/never', 'v1', 'test'),
('claude:creation-missing', 'claude', 'creation-missing', null,
'2026-01-20T00:00:00+00:00', '', '/creation-missing', 'v1', 'test');",
)
.unwrap();
let ids = |since: Option<&str>, until: Option<&str>| {
let mut ids: Vec<String> = db
.list_recent(&SearchFilters {
since: since.and_then(crate::util::parse_datetime),
until: until.and_then(crate::util::parse_datetime),
..SearchFilters::default()
})
.unwrap()
.into_iter()
.map(|session| session.id)
.collect();
ids.sort();
ids
};
assert_eq!(
ids(
Some("2026-01-01T00:00:00+00:00"),
Some("2026-01-31T23:59:59+00:00")
),
vec!["claude:creation-missing", "claude:long-running"],
"January intersects the long-running span and both point fallbacks"
);
assert_eq!(
ids(
Some("2026-02-15T00:00:00+00:00"),
Some("2026-02-15T00:00:00+00:00")
),
vec!["claude:long-running"],
"an exact instant inside the known span is included"
);
assert_eq!(
ids(None, Some("2026-01-31T23:59:59+00:00")),
vec!["claude:creation-missing", "claude:long-running"],
"until compares the session start, not terminal activity"
);
assert_eq!(
ids(Some("2026-03-01T00:00:00+00:00"), None),
vec!["claude:long-running", "claude:never-updated"],
"since compares the session end"
);
assert!(ids(Some("2026-03-12T00:00:00+00:00"), None).is_empty());
}
#[test]
fn session_kinds_select_user_started_runs_or_both() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions (
id, provider, provider_session_id, updated_at, preview_text,
source_path, parse_version, discovery_source, parent_session_id, agent_label
) values
('claude:parent', 'claude', 'parent', '2026-03-01T00:00:00+00:00', '',
'/parent', 'v1', 'test', null, null),
('claude:parent/agent-a', 'claude', 'parent/agent-a', '2026-03-02T00:00:00+00:00',
'', '/a', 'v1', 'test', 'claude:parent', 'Explore'),
('claude:other/agent-b', 'claude', 'other/agent-b', '2026-03-03T00:00:00+00:00',
'', '/b', 'v1', 'test', 'claude:other', 'general-purpose');",
)
.unwrap();
let ids = |kinds: Option<Vec<SessionKind>>, parent: Option<&str>| {
let mut ids: Vec<String> = db
.list_recent(&SearchFilters {
session_kinds: kinds,
parent_session_id: parent.map(str::to_string),
..SearchFilters::default()
})
.unwrap()
.into_iter()
.map(|session| session.id)
.collect();
ids.sort();
ids
};
assert_eq!(
ids(None, None),
vec![
"claude:other/agent-b",
"claude:parent",
"claude:parent/agent-a"
],
"naming no class returns every class, so indexed work is never invisible"
);
assert_eq!(
ids(Some(vec![SessionKind::User]), None),
vec!["claude:parent"]
);
assert_eq!(
ids(Some(vec![SessionKind::User, SessionKind::User]), None),
vec!["claude:parent"],
"repeating one set member must not make it equivalent to the complete set"
);
assert_eq!(
ids(Some(vec![SessionKind::Subagent]), None),
vec!["claude:other/agent-b", "claude:parent/agent-a"]
);
assert_eq!(
ids(Some(vec![SessionKind::User, SessionKind::Subagent]), None),
ids(None, None),
"naming both classes is the default set spelled out"
);
assert!(
ids(Some(Vec::new()), None).is_empty(),
"deselecting every class matches nothing rather than silently matching all"
);
assert_eq!(
ids(None, Some("claude:parent")),
vec!["claude:parent/agent-a"]
);
assert!(ids(None, Some("claude:parent/agent-a")).is_empty());
assert_eq!(
ids(Some(vec![SessionKind::Subagent]), Some("claude:other")),
vec!["claude:other/agent-b"],
"a run whose parent is not indexed is still a session"
);
}
#[test]
fn list_recent_is_sql_bounded_and_does_not_read_transcripts() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions (
id, provider, provider_session_id, updated_at, preview_text,
source_path, parse_version, discovery_source
) values
('claude:old', 'claude', 'old', '2026-01-01T00:00:00+00:00', '', '/old', 'v1', 'test'),
('claude:middle', 'claude', 'middle', '2026-02-01T00:00:00+00:00', '', '/middle', 'v1', 'test'),
('claude:new', 'claude', 'new', '2026-03-01T00:00:00+00:00', '', '/new', 'v1', 'test');
drop table transcripts;",
)
.unwrap();
let mut filters = SearchFilters {
limit: 2,
..SearchFilters::default()
};
let sessions = db.list_recent(&filters).unwrap();
assert_eq!(
sessions
.into_iter()
.map(|session| session.id)
.collect::<Vec<_>>(),
["claude:new", "claude:middle"]
);
filters.limit = 0;
assert_eq!(
db.list_recent(&filters)
.unwrap()
.into_iter()
.map(|session| session.id)
.collect::<Vec<_>>(),
["claude:new", "claude:middle", "claude:old"]
);
}
#[test]
fn analysis_documents_keyset_pages_normalized_user_text_and_empty_sessions() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions (
id, provider, provider_session_id, preview_text,
source_path, parse_version, discovery_source
) values
('claude:a', 'claude', 'a', '', '/a', 'v1', 'test'),
('claude:b', 'claude', 'b', '', '/b', 'v1', 'test'),
('codex:c', 'codex', 'c', '', '/c', 'v1', 'test');
insert into messages (
session_id, provider, seq, role, content, authorship, record_relation
) values
('claude:a', 'claude', 0, 'user', 'first request', 'human', 'original'),
('claude:a', 'claude', 1, 'assistant', 'answer', 'agent', 'original'),
('claude:a', 'claude', 2, 'user', 'second request', 'human', 'original'),
('codex:c', 'codex', 0, 'user', 'other provider', 'human', 'original');
drop table transcripts;",
)
.unwrap();
let filters = SearchFilters {
provider: Some(Provider::Claude),
limit: 1,
..SearchFilters::default()
};
let first = db.analysis_documents(&filters, None).unwrap();
assert_eq!(first.documents.len(), 1);
assert_eq!(first.documents[0].session.id, "claude:a");
assert_eq!(first.documents[0].message_count, 3);
assert_eq!(first.documents[0].user_message_count, 2);
assert_eq!(first.documents[0].user_text, "first request second request");
assert_eq!(
first.documents[0].first_user_text.as_deref(),
Some("first request")
);
let second = db
.analysis_documents(&filters, first.next_cursor.as_ref())
.unwrap();
assert_eq!(second.documents.len(), 1);
assert_eq!(second.documents[0].session.id, "claude:b");
assert_eq!(second.documents[0].message_count, 0);
assert_eq!(second.documents[0].user_message_count, 0);
assert!(second.documents[0].user_text.is_empty());
assert!(second.documents[0].first_user_text.is_none());
assert!(second.next_cursor.is_none());
}
#[test]
fn analysis_selects_original_human_authorship_and_human_parts_of_mixed_messages() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions (
id, provider, provider_session_id, preview_text,
source_path, parse_version, discovery_source
) values ('claude:authorship', 'claude', 'authorship', '', '/authorship', 'v1', 'test');
insert into messages (
id, session_id, provider, seq, role, kind, content, authorship, record_relation
) values
(101, 'claude:authorship', 'claude', 0, 'user', 'conversation',
'human original', 'human', 'original'),
(102, 'claude:authorship', 'claude', 1, 'user', 'conversation',
'agent delegation', 'agent', 'original'),
(103, 'claude:authorship', 'claude', 2, 'user', 'conversation',
'human α\ngenerated β\nhuman γ', 'mixed', 'original'),
(104, 'claude:authorship', 'claude', 3, 'user', 'conversation',
'mirrored human', 'human', 'mirror'),
(105, 'claude:authorship', 'claude', 4, 'user', 'conversation',
'legacy uncertainty', 'unknown', 'unknown');
insert into message_content_parts (
message_id, ordinal, start_char, end_char, authorship, origin
) values
(103, 0, 0, 8, 'human', 'direct_input'),
(103, 1, 8, 20, 'generated', 'tool_payload'),
(103, 2, 20, 27, 'human', 'direct_input');",
)
.unwrap();
let filters = SearchFilters {
limit: 1,
..SearchFilters::default()
};
let page = db.analysis_documents(&filters, None).unwrap();
let document = &page.documents[0];
assert_eq!(document.message_count, 5);
assert_eq!(document.user_message_count, 2);
assert_eq!(
document.user_text, "human original human α\nhuman γ",
"analysis must exclude agent, generated, mirrored, and unknown-attribution text"
);
assert_eq!(document.first_user_text.as_deref(), Some("human original"));
let request = crate::models::AnalysisRequest::new(
SearchFilters::default(),
crate::models::AnalysisSessionSelection::AllEligible,
)
.unwrap();
let mut chunks = Vec::new();
let receipt = db
.visit_analysis_sessions(
&request,
std::num::NonZeroUsize::new(1).unwrap(),
|_session, _message_count, human_message_count, messages| {
assert_eq!(human_message_count, 2);
chunks.extend(messages.collect::<Result<Vec<_>>>()?);
Ok(())
},
)
.unwrap();
assert_eq!(
chunks,
[
(0, "human original".to_string()),
(2, "human α\nhuman γ".to_string())
]
);
assert_eq!(receipt.analyzed_user_messages, 2);
}
#[test]
fn analysis_visit_keeps_one_snapshot_across_bounded_pages() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("index.db");
let db = Db::open(&path).unwrap();
let writer = Db::open(&path).unwrap();
for (id, source) in [
("claude:a", "/fixture/a.jsonl"),
("claude:z", "/fixture/z.jsonl"),
] {
let mut parsed =
crate::util::minimal_record(Provider::Claude, Path::new(source), "test".into());
parsed.session.id = id.into();
parsed.session.provider_session_id = id.into();
parsed.messages.push(crate::models::Message {
seq: 0,
role: crate::models::Role::User,
ts: None,
tool_name: None,
kind: crate::models::MessageKind::Conversation,
tool_call_id: None,
is_compaction: false,
content: format!("content for {id}"),
provenance: crate::models::MessageProvenance::default(),
});
db.upsert_session(&parsed, 0, 0).unwrap();
}
let mut late = crate::util::minimal_record(
Provider::Claude,
Path::new("/fixture/m.jsonl"),
"test".into(),
);
late.session.id = "claude:m".into();
late.session.provider_session_id = "claude:m".into();
let request = crate::models::AnalysisRequest::new(
SearchFilters::default(),
crate::models::AnalysisSessionSelection::AllEligible,
)
.unwrap();
let mut seen = Vec::new();
let receipt = db
.visit_analysis_sessions(
&request,
std::num::NonZeroUsize::new(1).unwrap(),
|session, _message_count, _user_message_count, _chunks| {
seen.push(session.id);
if seen.len() == 1 {
writer.upsert_session(&late, 0, 0)?;
}
Ok(())
},
)
.unwrap();
assert_eq!(receipt.selected_sessions, 2);
assert!(!receipt.has_more);
assert_eq!(seen, ["claude:a", "claude:z"]);
assert_eq!(
receipt.last_selected_session_id.as_deref(),
Some("claude:z")
);
assert_eq!(
db.resolve_session_record("claude:m").unwrap().id,
"claude:m"
);
let original_population = crate::models::AnalysisRequest::new(
SearchFilters {
exclude_session_ids: vec!["claude:m".to_owned()],
..SearchFilters::default()
},
crate::models::AnalysisSessionSelection::AllEligible,
)
.unwrap();
let consumed = db
.visit_analysis_sessions(
&original_population,
std::num::NonZeroUsize::new(1).unwrap(),
|_session, _message_count, _user_message_count, chunks| {
for chunk in chunks {
chunk?;
}
Ok(())
},
)
.unwrap();
assert_eq!(
receipt.corpus_digest, consumed.corpus_digest,
"the owner must hash unread human-authored message rows after a successful visitor"
);
let later = db
.visit_analysis_sessions(
&request,
std::num::NonZeroUsize::new(1).unwrap(),
|_session, _message_count, _user_message_count, _chunks| Ok(()),
)
.unwrap();
assert_eq!(later.selected_sessions, 3);
assert_ne!(
receipt.corpus_digest, later.corpus_digest,
"the earlier receipt must describe only its original read snapshot"
);
}
#[test]
fn analysis_documents_rejects_unbounded_pages() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let error = db
.analysis_documents(
&SearchFilters {
limit: 0,
..SearchFilters::default()
},
None,
)
.unwrap_err();
assert!(error.to_string().contains("greater than zero"));
}
#[test]
fn planning_usage_optionally_filters_by_command_patterns() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values ('s1','claude','s1','','/p','1','test')",
[],
)
.unwrap();
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values ('s2','codex','s2','','/p2','1','test')",
[],
)
.unwrap();
let slash = |id: i64, session_id: &str, provider: &str, seq: i64, content: &str| {
db.conn
.execute(
"insert into messages (id, session_id, provider, seq, role, content) \
values (?1,?2,?3,?4,'slash',?5)",
params![id, session_id, provider, seq, content],
)
.unwrap();
};
slash(1, "s1", "claude", 0, "/cmd-a make a plan");
slash(2, "s1", "claude", 1, "/cmd-b");
slash(3, "s1", "claude", 2, "/cmd-a refine it");
slash(4, "s2", "codex", 0, "/cmd-c ship the fix");
let all = db.planning_usage(&MessageFilters::default(), &[]).unwrap();
assert_eq!(all.len(), 3, "all distinct commands counted");
let only = db
.planning_usage(
&MessageFilters::default(),
&[regex::Regex::new(r"^/cmd-a$").unwrap()],
)
.unwrap();
assert_eq!(only.len(), 1);
assert_eq!(only[0].command, "/cmd-a");
assert_eq!(only[0].count, 2);
let command_token_only = db
.planning_usage(
&MessageFilters::default(),
&[regex::Regex::new(r"^/cmd-b$").unwrap()],
)
.unwrap();
assert_eq!(
command_token_only
.iter()
.map(|row| row.command.as_str())
.collect::<Vec<_>>(),
vec!["/cmd-b"],
"planning filters command tokens, not arbitrary slash-message body text"
);
let codex = db
.planning_usage(
&MessageFilters {
providers: Some(vec![Provider::Codex]),
..Default::default()
},
&[],
)
.unwrap();
assert_eq!(codex.len(), 1);
assert_eq!(codex[0].command, "/cmd-c");
}
#[test]
fn search_messages_uses_exact_literal_substring_semantics() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values ('s1','claude','s1','','/p','1','test')",
[],
)
.unwrap();
let insert = |id: i64, seq: i64, content: &str| {
db.conn
.execute(
"insert into messages (id, session_id, provider, seq, role, content) \
values (?1,'s1','claude',?2,'user',?3)",
params![id, seq, content],
)
.unwrap();
};
insert(1, 0, "please handle the error");
insert(2, 1, "the handler crashed");
insert(3, 2, "we mishandled the input");
insert(4, 3, "error handling code here");
insert(5, 4, "use a => b arrow");
insert(6, 5, "literal /goal command");
insert(7, 6, "plain goal token");
insert(8, 7, "compile C++ today");
insert(9, 8, "flag --path passed");
insert(10, 9, "CAFÉ diagnostic");
insert(11, 10, "Ä°stanbul diagnostic");
let seqs = |query: &str| -> Vec<i64> {
let mut v: Vec<i64> = db
.search_messages(query, &MessageFilters::default())
.unwrap()
.into_iter()
.map(|h| h.seq)
.collect();
v.sort();
v
};
assert_eq!(seqs("handle"), vec![0, 1, 2]);
assert_eq!(seqs("handled"), vec![2]);
assert_eq!(seqs("error handling"), vec![3]);
assert_eq!(seqs("=>"), vec![4]);
assert_eq!(seqs("/goal"), vec![5]);
assert_eq!(seqs("goal"), vec![5, 6]);
assert_eq!(seqs("C++"), vec![7]);
assert_eq!(seqs("--path"), vec![8]);
assert_eq!(seqs("café"), vec![9]);
assert_eq!(seqs("Ä°S"), vec![10]);
assert_eq!(
seqs("Ä°ST"),
vec![10],
"the trigram candidate gate must remain a superset of Unicode lowercase matching"
);
assert_eq!(seqs("").len(), 11);
let re = db
.search_messages(
"h.ndler",
&MessageFilters {
match_mode: MessageSearchMode::Regex,
..Default::default()
},
)
.unwrap();
assert_eq!(re.into_iter().map(|h| h.seq).collect::<Vec<_>>(), vec![1]);
}
#[test]
fn fuzzy_message_search_ranks_approximate_matches_without_changing_literal_search() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values ('s1','claude','s1','','/p','1','test')",
[],
)
.unwrap();
for (seq, content) in [
(
0,
"please avoid magic values and keep settings configurable",
),
(1, "hard-coded timeout should move into config"),
(2, "unrelated transcript text"),
(3, "magic numbers should move into named values"),
] {
db.conn
.execute(
"insert into messages (session_id, provider, seq, role, content) \
values ('s1','claude',?1,'user',?2)",
params![seq, content],
)
.unwrap();
}
let literal = db
.search_messages("magic config", &MessageFilters::default())
.unwrap();
assert!(literal.is_empty(), "literal search remains exact");
let fuzzy = db
.search_messages(
"magic config",
&MessageFilters {
match_mode: MessageSearchMode::Fuzzy,
limit: 10,
..Default::default()
},
)
.unwrap();
assert_eq!(fuzzy.iter().map(|hit| hit.seq).collect::<Vec<_>>(), vec![0]);
assert!(fuzzy[0].fuzzy_score.is_some());
let fuzzy_phrase = db
.search_messages(
"magic values",
&MessageFilters {
match_mode: MessageSearchMode::Fuzzy,
limit: 10,
..Default::default()
},
)
.unwrap();
assert_eq!(fuzzy_phrase[0].seq, 0, "exact phrase wins fuzzy ties");
}
#[test]
fn schema_v4_fuzzy_search_filters_structurally_before_exhaustive_scoring() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("index.db");
{
let db = Db::open(&path).unwrap();
for (id, provider, seq, role, content) in [
(
"s1",
"claude",
0,
"user",
"incremental trigram acceleration",
),
("s1", "claude", 1, "assistant", "unrelated response"),
("s2", "codex", 0, "user", "incremental index design"),
] {
db.conn
.execute(
"insert or ignore into sessions
(id, provider, provider_session_id, preview_text, source_path,
parse_version, discovery_source)
values (?1, ?2, ?1, '', '/p', '1', 'test')",
params![id, provider],
)
.unwrap();
db.conn
.execute(
"insert into messages
(session_id, provider, seq, role, content)
values (?1, ?2, ?3, ?4, ?5)",
params![id, provider, seq, role, content],
)
.unwrap();
}
}
let db = Db::open(&path).unwrap();
let (hits, explain) = db
.search_messages_with_explain(
"incrmental trigram",
&MessageFilters {
providers: Some(vec![Provider::Claude]),
role: Some(Role::User),
match_mode: MessageSearchMode::Fuzzy,
limit: 10,
..Default::default()
},
true,
)
.unwrap();
assert_eq!(hits.iter().map(|hit| hit.seq).collect::<Vec<_>>(), vec![0]);
assert!(hits[0].fuzzy_score.is_some());
let explain = explain.unwrap();
assert_eq!(explain.prefilter.as_deref(), None);
assert_eq!(
explain.prefilter_skipped.as_deref(),
Some("complete filtered corpus scored, retaining at most min(offset + limit, corpus) ranked rows")
);
assert_eq!(explain.corpus, 1);
}
#[test]
fn exhaustive_fuzzy_ranking_keeps_a_best_hit_beyond_the_old_candidate_cap() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, preview_text,
source_path, parse_version, discovery_source)
values ('s1','claude','s1','','/p','1','test')",
[],
)
.unwrap();
let tx = db.conn.unchecked_transaction().unwrap();
{
let mut insert = tx
.prepare(
"insert into messages(session_id, provider, seq, role, content)
values ('s1','claude',?1,'user',?2)",
)
.unwrap();
for seq in 0..1_300_i64 {
insert
.execute(params![
seq,
format!("magic values configurable filler {seq}")
])
.unwrap();
}
insert.execute(params![1_300_i64, "magic config"]).unwrap();
}
tx.commit().unwrap();
let hits = db
.search_messages(
"magic config",
&MessageFilters {
match_mode: MessageSearchMode::Fuzzy,
limit: 1,
..Default::default()
},
)
.unwrap();
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].seq, 1_300);
let first_two = db
.search_messages(
"magic config",
&MessageFilters {
match_mode: MessageSearchMode::Fuzzy,
limit: 2,
..Default::default()
},
)
.unwrap();
let second = db
.search_messages(
"magic config",
&MessageFilters {
match_mode: MessageSearchMode::Fuzzy,
limit: 1,
offset: 1,
..Default::default()
},
)
.unwrap();
assert_eq!(second.len(), 1);
assert_eq!(second[0].session_id, first_two[1].session_id);
assert_eq!(second[0].seq, first_two[1].seq);
assert_eq!(second[0].fuzzy_score, first_two[1].fuzzy_score);
}
#[test]
fn search_messages_path_prefix_scopes_by_session_root_and_metadata_enriches() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let session = |id: &str, cwd: &str, repo: &str, title: &str| {
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, title, cwd, \
repo_root, preview_text, source_path, parse_version, discovery_source) \
values (?1,'claude',?1,?2,?3,?4,'','/p','1','test')",
params![id, title, cwd, repo],
)
.unwrap();
};
session("a", "/Users/x/proj-a", "/Users/x/proj-a", "Proj A");
session("b", "/Users/x/proj-b", "/Users/x/proj-b", "Proj B");
let msg = |id: i64, sid: &str| {
db.conn
.execute(
"insert into messages (id, session_id, provider, seq, role, content) \
values (?1,?2,'claude',0,'user','shared keyword here')",
params![id, sid],
)
.unwrap();
};
msg(1, "a");
msg(2, "b");
let scoped = db
.search_messages(
"keyword",
&MessageFilters {
path_prefix: Some("/Users/x/proj-a".into()),
..Default::default()
},
)
.unwrap();
assert_eq!(
scoped
.iter()
.map(|h| h.session_id.clone())
.collect::<Vec<_>>(),
vec!["a"],
"only the proj-a session matches the path prefix"
);
assert_eq!(
db.search_messages("keyword", &MessageFilters::default())
.unwrap()
.len(),
2
);
let meta = db
.session_metadata(&["a".to_string(), "b".to_string()])
.unwrap();
assert_eq!(meta["a"].cwd.as_deref(), Some("/Users/x/proj-a"));
assert_eq!(meta["a"].repo_root.as_deref(), Some("/Users/x/proj-a"));
assert_eq!(meta["a"].title.as_deref(), Some("Proj A"));
assert_eq!(meta["b"].title.as_deref(), Some("Proj B"));
}
#[test]
fn session_metadata_batches_use_one_bind_and_indexed_sql_above_parameter_limits() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let ids = (0..1_200)
.map(|index| format!("codex:metadata-{index}"))
.collect::<Vec<_>>();
{
let transaction = db.conn.unchecked_transaction().unwrap();
{
let mut insert = transaction
.prepare(
"insert into sessions (
id, provider, provider_session_id, title, preview_text, source_path,
parse_version, discovery_source, raw_metadata_json
) values (?1, 'codex', ?2, ?2, '', ?3, '1', 'test', ?4)",
)
.unwrap();
for (index, id) in ids.iter().enumerate() {
let provider_session_id = format!("metadata-{index}");
insert
.execute(params![
id,
provider_session_id,
format!("/transcripts/{provider_session_id}.jsonl"),
format!(r#"{{"index":{index}}}"#),
])
.unwrap();
}
}
transaction.commit().unwrap();
}
let metadata = db.session_metadata(&ids).unwrap();
let raw_metadata = db.session_raw_provider_metadata(&ids).unwrap();
assert_eq!(metadata.len(), ids.len());
assert_eq!(raw_metadata.len(), ids.len());
assert_eq!(
metadata[&ids[1_199]].title.as_deref(),
Some("metadata-1199")
);
let plan = {
let mut statement = db
.conn
.prepare(
"explain query plan
select s.id
from json_each(?) requested
join sessions s on s.id = requested.value",
)
.unwrap();
statement
.query_map([serde_json::to_string(&ids).unwrap()], |row| {
row.get::<_, String>(3)
})
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap()
.join(" | ")
};
assert!(
plan.contains("sqlite_autoindex_sessions_1"),
"session metadata join must probe indexed sessions.id: {plan}"
);
}
#[test]
fn search_messages_excludes_paths_and_sessions_before_limit() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let session = |id: &str, cwd: &str, repo: &str, source_path: &str| {
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, cwd, repo_root, \
preview_text, source_path, parse_version, discovery_source) \
values (?1,'claude',?1,?2,?3,'',?4,'1','test')",
params![id, cwd, repo, source_path],
)
.unwrap();
};
session("a", "/Users/x/proj-a", "/Users/x", "/logs/a.jsonl");
session("b", "/Users/x/proj-b", "/Users/x", "/logs/b.jsonl");
session("c", "/Users/x/proj-c", "/Users/x", "/tmp/noisy/c.jsonl");
for (id, seq) in [("a", 0), ("b", 0), ("c", 0)] {
db.conn
.execute(
"insert into messages (session_id, provider, seq, role, content) \
values (?1,'claude',?2,'user','shared needle')",
params![id, seq],
)
.unwrap();
}
for (mode, pattern) in MESSAGE_CONTENT_MODE_CASES {
let hits = mode
.search(
&db,
pattern,
MessageFilters {
path_prefix: Some("/Users/x".into()),
exclude_path_prefixes: vec!["/Users/x/proj-a".into(), "/tmp/noisy".into()],
limit: 1,
..Default::default()
},
)
.unwrap();
assert_eq!(
hits.iter()
.map(|hit| hit.session_id.as_str())
.collect::<Vec<_>>(),
vec!["b"],
"{mode:?}: exclusions apply before limit, including source_path exclusions"
);
}
for (mode, pattern) in MESSAGE_CONTENT_MODE_CASES {
let hits = mode
.search(
&db,
pattern,
MessageFilters {
exclude_session_ids: vec!["b".into()],
limit: 10,
..Default::default()
},
)
.unwrap();
assert_eq!(
hits.iter()
.map(|hit| hit.session_id.as_str())
.collect::<Vec<_>>(),
vec!["a", "c"],
"{mode:?}: session exclusions apply before content matching"
);
}
}
#[test]
fn search_messages_content_modes_share_structural_filters() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let session = |id: &str, provider: Provider, cwd: &str, repo: &str, source_path: &str| {
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, cwd, repo_root, \
preview_text, source_path, parse_version, discovery_source) \
values (?1,?2,?1,?3,?4,'',?5,'1','test')",
params![id, provider.as_str(), cwd, repo, source_path],
)
.unwrap();
};
session(
"target",
Provider::Claude,
"/Users/x/proj-a",
"/Users/x/proj-a",
"/logs/target.jsonl",
);
session(
"wrong-provider",
Provider::Codex,
"/Users/x/proj-a",
"/Users/x/proj-a",
"/logs/wrong-provider.jsonl",
);
session(
"wrong-path",
Provider::Claude,
"/Users/x/proj-b",
"/Users/x/proj-b",
"/logs/wrong-path.jsonl",
);
let msg =
|id: i64, session_id: &str, provider: Provider, seq: i64, role: Role, ts: &str| {
db.conn
.execute(
"insert into messages (id, session_id, provider, seq, role, ts, content) \
values (?1,?2,?3,?4,?5,?6,'shared needle phrase')",
params![id, session_id, provider.as_str(), seq, role.as_str(), ts],
)
.unwrap();
};
msg(
1,
"target",
Provider::Claude,
10,
Role::User,
"2026-01-02T00:00:00Z",
);
msg(
2,
"target",
Provider::Claude,
11,
Role::Assistant,
"2026-01-02T00:00:00Z",
);
msg(
3,
"wrong-provider",
Provider::Codex,
10,
Role::User,
"2026-01-02T00:00:00Z",
);
msg(
4,
"wrong-path",
Provider::Claude,
10,
Role::User,
"2026-01-02T00:00:00Z",
);
msg(
5,
"target",
Provider::Claude,
9,
Role::User,
"2025-12-31T00:00:00Z",
);
let filters = MessageFilters {
role: Some(Role::User),
providers: Some(vec![Provider::Claude]),
path_prefix: Some("/Users/x/proj-a".into()),
since: Some(utc("2026-01-01T00:00:00Z")),
until: Some(utc("2026-01-31T00:00:00Z")),
seq_from: Some(10),
seq_to: Some(10),
limit: 10,
..Default::default()
};
for (mode, pattern) in MESSAGE_CONTENT_MODE_CASES {
assert_eq!(
hit_keys(mode.search(&db, pattern, filters.clone()).unwrap()),
vec![("target".to_string(), 10)],
"{mode:?}: content mode must compose with role/provider/path/time/seq filters"
);
}
}
#[test]
fn read_session_messages_selects_by_order_within_range_and_offset() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, cwd, repo_root, \
preview_text, source_path, parse_version, discovery_source) \
values ('claude:s','claude','s','/p','/p','','/p','1','test')",
[],
)
.unwrap();
let msg = |seq: i64, role: &str, content: &str| {
db.conn
.execute(
"insert into messages (session_id, provider, seq, role, content) \
values ('claude:s','claude',?1,?2,?3)",
params![seq, role, content],
)
.unwrap();
};
msg(0, "user", "first user turn");
msg(1, "assistant", "assistant reply");
msg(2, "user", "second user turn");
msg(3, "slash", "/cmd make a plan");
let seqs = |hits: Vec<MessageHit>| hits.iter().map(|h| h.seq).collect::<Vec<i64>>();
let filters = |mutate: &dyn Fn(&mut MessageFilters)| {
let mut f = MessageFilters {
session_id: Some("claude:s".to_string()),
..Default::default()
};
mutate(&mut f);
f
};
assert_eq!(
seqs(
db.read_session_messages(&filters(&|f| f.limit = 2), MessageOrder::OldestFirst)
.unwrap()
),
vec![0, 1]
);
assert_eq!(
seqs(
db.read_session_messages(&filters(&|f| f.limit = 2), MessageOrder::NewestFirst)
.unwrap()
),
vec![2, 3]
);
assert_eq!(
seqs(
db.read_session_messages(&filters(&|f| f.limit = 0), MessageOrder::NewestFirst)
.unwrap()
),
vec![0, 1, 2, 3]
);
assert_eq!(
seqs(
db.read_session_messages(
&filters(&|f| {
f.role = Some(Role::User);
f.limit = 1;
}),
MessageOrder::NewestFirst
)
.unwrap()
),
vec![2]
);
assert_eq!(
seqs(
db.read_session_messages(
&filters(&|f| {
f.seq_from = Some(1);
f.seq_to = Some(2);
}),
MessageOrder::OldestFirst
)
.unwrap()
),
vec![1, 2]
);
assert_eq!(
seqs(
db.read_session_messages(
&filters(&|f| {
f.offset = 1;
f.limit = 2;
}),
MessageOrder::OldestFirst
)
.unwrap()
),
vec![1, 2]
);
}
#[test]
fn read_session_messages_requires_a_session_id() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
assert!(db
.read_session_messages(&MessageFilters::default(), MessageOrder::NewestFirst)
.is_err());
}
#[test]
fn read_session_messages_edge_cases_from_the_premortem_catalog() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, cwd, repo_root, \
preview_text, source_path, parse_version, discovery_source) \
values ('claude:s','claude','s','/p','/p','','/p','1','test')",
[],
)
.unwrap();
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, cwd, repo_root, \
preview_text, source_path, parse_version, discovery_source) \
values ('claude:one','claude','one','/p','/p','','/p','1','test')",
[],
)
.unwrap();
db.conn
.execute(
"insert into messages (session_id, provider, seq, role, content) \
values ('claude:one','claude',0,'user','only turn')",
[],
)
.unwrap();
for seq in 0..6 {
db.conn
.execute(
"insert into messages (session_id, provider, seq, role, content) \
values ('claude:s','claude',?1,'user',?2)",
params![seq, format!("turn {seq}")],
)
.unwrap();
}
let seqs = |hits: Vec<MessageHit>| hits.iter().map(|h| h.seq).collect::<Vec<i64>>();
let f = |mutate: &dyn Fn(&mut MessageFilters)| {
let mut m = MessageFilters {
session_id: Some("claude:s".to_string()),
..Default::default()
};
mutate(&mut m);
m
};
assert_eq!(
seqs(
db.read_session_messages(
&f(&|m| {
m.seq_from = Some(1);
m.seq_to = Some(3);
}),
MessageOrder::OldestFirst
)
.unwrap()
),
vec![1, 2, 3]
);
assert_eq!(
seqs(
db.read_session_messages(
&f(&|m| {
m.seq_from = Some(1);
m.seq_to = Some(4);
m.limit = 2;
}),
MessageOrder::NewestFirst
)
.unwrap()
),
vec![3, 4]
);
assert_eq!(
seqs(
db.read_session_messages(
&f(&|m| {
m.seq_from = Some(1);
m.seq_to = Some(4);
m.limit = 2;
}),
MessageOrder::OldestFirst
)
.unwrap()
),
vec![1, 2]
);
assert_eq!(
seqs(
db.read_session_messages(
&f(&|m| {
m.offset = 1;
m.limit = 2;
}),
MessageOrder::NewestFirst
)
.unwrap()
),
vec![3, 4]
);
assert_eq!(
seqs(
db.read_session_messages(&f(&|m| m.offset = 99), MessageOrder::OldestFirst)
.unwrap()
),
Vec::<i64>::new()
);
assert_eq!(
seqs(
db.read_session_messages(
&f(&|m| {
m.seq_from = Some(100);
m.seq_to = Some(200);
}),
MessageOrder::OldestFirst
)
.unwrap()
),
Vec::<i64>::new()
);
assert!(db
.read_session_messages(
&f(&|m| {
m.seq_from = Some(4);
m.seq_to = Some(1);
}),
MessageOrder::OldestFirst
)
.is_err());
assert_eq!(
seqs(
db.read_session_messages(
&f(&|m| m.role = Some(Role::Assistant)),
MessageOrder::NewestFirst
)
.unwrap()
),
Vec::<i64>::new()
);
let one = |order| {
db.read_session_messages(
&MessageFilters {
session_id: Some("claude:one".to_string()),
..Default::default()
},
order,
)
.unwrap()
};
assert_eq!(seqs(one(MessageOrder::OldestFirst)), vec![0]);
assert_eq!(seqs(one(MessageOrder::NewestFirst)), vec![0]);
}
#[test]
fn path_prefix_matches_directory_boundary_and_escapes_like_metacharacters() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let session = |id: &str, cwd: &str| {
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, cwd, repo_root, \
preview_text, source_path, parse_version, discovery_source) \
values (?1,'claude',?1,?2,?2,'','/p','1','test')",
params![id, cwd],
)
.unwrap();
};
session("root", "/tmp/proj");
session("child", "/tmp/proj/sub");
session("sibling", "/tmp/project2");
session("under", "/tmp/proj_under");
session("percent", "/tmp/proj%literal");
let msg = |id: i64, sid: &str| {
db.conn
.execute(
"insert into messages (id, session_id, provider, seq, role, content) \
values (?1,?2,'claude',0,'user','needle')",
params![id, sid],
)
.unwrap();
};
msg(1, "root");
msg(2, "child");
msg(3, "sibling");
msg(4, "under");
msg(5, "percent");
let ids = |prefix: &str| -> Vec<String> {
let mut ids: Vec<String> = db
.search_messages(
"needle",
&MessageFilters {
path_prefix: Some(prefix.into()),
..Default::default()
},
)
.unwrap()
.into_iter()
.map(|h| h.session_id)
.collect();
ids.sort();
ids
};
assert_eq!(ids("/tmp/proj"), vec!["child", "root"]);
assert_eq!(ids("/tmp/proj%literal"), vec!["percent"]);
session("windows-root", r"C:\Users\x\proj");
session("windows-child", r"C:\Users\x\proj\sub");
session("windows-sibling", r"C:\Users\x\project2");
msg(6, "windows-root");
msg(7, "windows-child");
msg(8, "windows-sibling");
assert_eq!(
ids(r"C:\Users\x\proj"),
vec!["windows-child", "windows-root"]
);
}
#[test]
fn list_recent_returns_newest_sessions_in_exact_directory_or_descendants() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions (
id, provider, provider_session_id, cwd, updated_at, preview_text,
source_path, parse_version, discovery_source
) values
('claude:root-old', 'claude', 'root-old', '/work/project',
'2026-01-01T00:00:00+00:00', '', '/root-old', 'v1', 'test'),
('claude:child-new', 'claude', 'child-new', '/work/project/sub/dir',
'2026-03-01T00:00:00+00:00', '', '/child-new', 'v1', 'test'),
('claude:sibling-newer', 'claude', 'sibling-newer', '/work/project-other',
'2026-04-01T00:00:00+00:00', '', '/sibling-newer', 'v1', 'test');",
)
.unwrap();
let query = |limit| SearchFilters {
path_prefix: Some("/work/project".to_string()),
limit,
..SearchFilters::default()
};
assert_eq!(
db.list_recent(&query(1))
.unwrap()
.into_iter()
.map(|session| session.id)
.collect::<Vec<_>>(),
["claude:child-new"],
"limit applies after exact-directory/descendant filtering and newest-first ordering"
);
assert_eq!(
db.list_recent(&query(0))
.unwrap()
.into_iter()
.map(|session| session.id)
.collect::<Vec<_>>(),
["claude:child-new", "claude:root-old"]
);
}
#[test]
fn restricted_fuzzy_search_scores_only_authorized_messages() {
use crate::config::{SearchScopeConfig, SearchScopeMode};
use crate::search_scope::{EffectiveAccessScope, TrustedAccessInputs};
let dir = tempfile::tempdir().unwrap();
let allowed = dir.path().join("allowed");
let hidden = dir.path().join("hidden");
fs::create_dir_all(&allowed).unwrap();
fs::create_dir_all(&hidden).unwrap();
let mut db = Db::open(&dir.path().join("index.db")).unwrap();
db.mark_schema_current().unwrap();
let transaction = db.conn.unchecked_transaction().unwrap();
{
let mut insert_session = transaction
.prepare(
"insert into sessions (
id, provider, provider_session_id, cwd, repo_root, preview_text,
source_path, parse_version, discovery_source
) values (?, 'claude', ?, ?, ?, '', '/fixture', '1', 'test')",
)
.unwrap();
let mut insert_message = transaction
.prepare(
"insert into messages (
id, session_id, provider, seq, role, kind, content
) values (?, ?, 'claude', 0, 'user', 'conversation', 'needle')",
)
.unwrap();
for index in 0..1_201_i64 {
let id = format!("hidden-{index:04}");
let workspace = hidden.to_string_lossy();
insert_session
.execute(params![id, id, workspace.as_ref(), workspace.as_ref()])
.unwrap();
insert_message.execute(params![index + 1, id]).unwrap();
}
let allowed_id = "allowed";
let workspace = allowed.to_string_lossy();
insert_session
.execute(params![
allowed_id,
allowed_id,
workspace.as_ref(),
workspace.as_ref()
])
.unwrap();
insert_message
.execute(params![1_202_i64, allowed_id])
.unwrap();
}
transaction.commit().unwrap();
db.set_access_scope(
EffectiveAccessScope::resolve(
&SearchScopeConfig {
mode: SearchScopeMode::AllowedRoots,
roots: vec![allowed.to_string_lossy().into_owned()],
include_invocation_directory: false,
},
TrustedAccessInputs::default(),
)
.unwrap(),
);
let hits = db
.search_messages(
"needle",
&MessageFilters {
match_mode: MessageSearchMode::Fuzzy,
limit: 1,
..Default::default()
},
)
.unwrap();
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].session_id, "allowed");
}
#[test]
fn exact_session_filter_does_not_merge_substring_matches() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
for id in ["abc", "xabcx"] {
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values (?1,'claude',?1,'','/p','1','test')",
params![id],
)
.unwrap();
}
db.conn
.execute(
"insert into messages (id, session_id, provider, seq, role, content) \
values (1,'abc','claude',0,'user','same')",
[],
)
.unwrap();
db.conn
.execute(
"insert into messages (id, session_id, provider, seq, role, content) \
values (2,'xabcx','claude',0,'user','same')",
[],
)
.unwrap();
let exact = db
.search_messages(
"",
&MessageFilters {
session_id: Some("abc".into()),
..Default::default()
},
)
.unwrap();
assert_eq!(exact.len(), 1);
assert_eq!(exact[0].session_id, "abc");
}
#[test]
fn open_with_busy_timeout_sets_sqlite_busy_timeout() {
let dir = tempfile::tempdir().unwrap();
let db =
Db::open_with_busy_timeout(&dir.path().join("index.db"), TEST_BUSY_TIMEOUT_MS).unwrap();
assert_eq!(db.busy_timeout_ms().unwrap(), TEST_BUSY_TIMEOUT_MS);
}
#[test]
fn scoped_busy_timeout_restores_previous_value() {
let dir = tempfile::tempdir().unwrap();
let db =
Db::open_with_busy_timeout(&dir.path().join("index.db"), TEST_BUSY_TIMEOUT_MS).unwrap();
let observed = db
.with_busy_timeout_ms(TEST_NO_WAIT_BUSY_TIMEOUT_MS, || db.busy_timeout_ms())
.unwrap();
assert_eq!(observed, TEST_NO_WAIT_BUSY_TIMEOUT_MS);
assert_eq!(db.busy_timeout_ms().unwrap(), TEST_BUSY_TIMEOUT_MS);
}
#[test]
fn sqlite_busy_error_detection_matches_locked_writer() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("index.db");
let writer = Db::open(&path).unwrap();
let contender = Db::open_with_busy_timeout(&path, TEST_NO_WAIT_BUSY_TIMEOUT_MS).unwrap();
writer.conn.execute_batch("begin immediate").unwrap();
let err = contender
.conn
.execute(
"insert into sessions (id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values ('busy','claude','busy','','/p','1','test')",
[],
)
.unwrap_err();
let err = anyhow::Error::from(err);
assert!(Db::is_sqlite_busy_error(&err));
writer.conn.execute_batch("rollback").unwrap();
}
#[test]
fn auto_reindex_completion_timestamp_controls_shared_freshness_window() {
const COMPLETED_MS: i64 = 20_000;
const INTERVAL_MS: u64 = 1_000;
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("index.db");
let first = Db::open(&path).unwrap();
let second = Db::open(&path).unwrap();
assert!(!second
.auto_reindex_is_fresh_at(COMPLETED_MS, INTERVAL_MS)
.unwrap());
let completed_at = Utc.timestamp_millis_opt(COMPLETED_MS).single().unwrap()
+ chrono::Duration::nanoseconds(456_789);
first.mark_auto_reindex_complete_at(completed_at).unwrap();
assert_eq!(
second.auto_reindex_completed_at().unwrap(),
Some(completed_at)
);
assert!(second
.auto_reindex_is_fresh_at(COMPLETED_MS + 999, INTERVAL_MS)
.unwrap());
assert!(!second
.auto_reindex_is_fresh_at(COMPLETED_MS + 1_000, INTERVAL_MS)
.unwrap());
}
#[test]
fn auto_reindex_freshness_uses_only_metadata_primary_keys() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.mark_auto_reindex_complete().unwrap();
let plan = db
.conn
.prepare(&format!("explain query plan {AUTO_REINDEX_FRESHNESS_SQL}"))
.unwrap()
.query_map(
params![
AUTO_REINDEX_COMPLETED_MS_KEY,
AUTO_REINDEX_PARSER_CONTRACT_KEYS[0],
AUTO_REINDEX_PARSER_CONTRACT_KEYS[1],
AUTO_REINDEX_PARSER_CONTRACT_KEYS[2],
AUTO_REINDEX_PARSER_CONTRACT_KEYS[3],
],
|row| row.get::<_, String>(3),
)
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap()
.join("\n");
assert!(!plan.contains("SCAN"), "{plan}");
assert_eq!(
plan.matches("sqlite_autoindex_index_metadata_1").count(),
5,
"{plan}"
);
}
#[test]
fn message_search_filters_by_session_local_seq_range() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
for id in ["claude:s1", "claude:s2"] {
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values (?1,'claude',?1,'','/p','1','test')",
params![id],
)
.unwrap();
for seq in 0..5 {
db.conn
.execute(
"insert into messages (session_id, provider, seq, role, content) \
values (?1,'claude',?2,'user',?3)",
params![id, seq, format!("needle {id} {seq}")],
)
.unwrap();
}
}
let bounded = db
.search_messages(
"needle",
&MessageFilters {
session_id: Some("claude:s1".into()),
seq_from: Some(1),
seq_to: Some(3),
..Default::default()
},
)
.unwrap();
assert_eq!(
bounded.iter().map(|h| h.seq).collect::<Vec<_>>(),
vec![1, 2, 3]
);
assert!(bounded.iter().all(|h| h.session_id == "claude:s1"));
let open_ended = db
.search_messages(
"",
&MessageFilters {
session_id: Some("claude:s2".into()),
seq_from: Some(3),
..Default::default()
},
)
.unwrap();
assert_eq!(
open_ended.iter().map(|h| h.seq).collect::<Vec<_>>(),
vec![3, 4]
);
}
#[test]
fn find_corrections_honors_path_prefix() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let session = |id: &str, cwd: &str| {
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, cwd, repo_root, \
preview_text, source_path, parse_version, discovery_source) \
values (?1,'claude',?1,?2,?2,'','/p','1','test')",
params![id, cwd],
)
.unwrap();
};
session("a", "/Users/x/proj-a");
session("b", "/Users/x/proj-b");
let user_msg = |id: i64, sid: &str| {
db.conn
.execute(
"insert into messages (
id, session_id, provider, seq, role, content,
authorship, record_relation
) values (
?1,?2,'claude',0,'user','that is wrong, please revert',
'human','original'
)",
params![id, sid],
)
.unwrap();
};
user_msg(1, "a");
user_msg(2, "b");
let policies = test_policies(&[("misc", &["wrong"])]);
let scoped = db
.find_corrections(
&policies,
&MessageFilters {
path_prefix: Some("/Users/x/proj-a".into()),
..Default::default()
},
)
.unwrap();
assert_eq!(
scoped
.iter()
.map(|c| c.session_id.clone())
.collect::<Vec<_>>(),
vec!["a"],
"path_prefix must scope corrections to the matching session"
);
assert_eq!(
db.find_corrections(&policies, &MessageFilters::default())
.unwrap()
.len(),
2
);
}
#[test]
fn analytics_queries_honor_exact_session_id() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let session = |id: &str| {
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values (?1,'claude',?1,'','/p','1','test')",
params![id],
)
.unwrap();
};
session("s1");
session("s10");
let message = |id: i64, sid: &str, seq: i64, role: &str, content: &str| {
db.conn
.execute(
"insert into messages (
id, session_id, provider, seq, role, content,
authorship, record_relation
) values (?1,?2,'claude',?3,?4,?5,'human','original')",
params![id, sid, seq, role, content],
)
.unwrap();
};
message(1, "s1", 0, "user", "that is wrong");
message(2, "s1", 1, "slash", "/cmd-a");
message(3, "s10", 0, "user", "that is wrong");
message(4, "s10", 1, "slash", "/cmd-b");
let filters = MessageFilters {
session_id: Some("s1".into()),
..Default::default()
};
let policies = test_policies(&[("misc", &["wrong"])]);
assert_eq!(
db.find_corrections(&policies, &filters)
.unwrap()
.iter()
.map(|row| row.session_id.as_str())
.collect::<Vec<_>>(),
vec!["s1"],
"exact session_id must not match s10"
);
assert_eq!(
db.message_role_counts(&filters).unwrap(),
vec![("slash".to_string(), 1), ("user".to_string(), 1)]
);
let planning = db.planning_usage(&filters, &[]).unwrap();
assert_eq!(planning.len(), 1);
assert_eq!(planning[0].command, "/cmd-a");
}
#[test]
fn search_messages_filters_by_tool_name_and_surfaces_it() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values ('s1','claude','s1','','/p','1','test')",
[],
)
.unwrap();
let insert = |id: i64, seq: i64, role: &str, tool: Option<&str>, content: &str| {
db.conn
.execute(
"insert into messages (id, session_id, provider, seq, role, tool_name, content) \
values (?1,'s1','claude',?2,?3,?4,?5)",
params![id, seq, role, tool, content],
)
.unwrap();
};
insert(1, 0, "user", None, "run the build");
insert(2, 1, "tool", Some("Bash"), "build ok");
insert(3, 2, "tool", Some("Edit"), "edited the file");
insert(4, 3, "tool", Some("ÄTool"), "unicode tool");
let tools = db
.search_messages(
"",
&MessageFilters {
role: Some(Role::Tool),
..Default::default()
},
)
.unwrap();
let bash = tools
.iter()
.find(|h| h.seq == 1)
.expect("Bash tool message");
assert_eq!(bash.tool_name.as_deref(), Some("Bash"));
let only = db
.search_messages(
"",
&MessageFilters {
tool: Some("bash".into()),
..Default::default()
},
)
.unwrap();
assert_eq!(only.len(), 1);
assert_eq!(only[0].seq, 1);
let unicode = db
.search_messages(
"ät",
&MessageFilters {
field: Some(SearchField::ToolName),
limit: 1,
..Default::default()
},
)
.unwrap();
assert_eq!(unicode.len(), 1);
assert_eq!(unicode[0].tool_name.as_deref(), Some("ÄTool"));
db.conn
.execute_batch(
"with recursive n(value) as (
values(1) union all select value + 1 from n where value < 100
)
insert into messages (
session_id, provider, seq, role, tool_name, content
)
select 's1', 'claude', 100 + value, 'tool',
printf('other_tool_%03d', value), 'unrelated'
from n;",
)
.unwrap();
let (fuzzy, explain) = db
.search_messages_with_explain(
"edt",
&MessageFilters {
field: Some(SearchField::ToolName),
match_mode: MessageSearchMode::Fuzzy,
limit: 5,
..Default::default()
},
true,
)
.unwrap();
assert_eq!(fuzzy.len(), 1);
assert_eq!(fuzzy[0].tool_name.as_deref(), Some("Edit"));
let explain = explain.unwrap();
assert_eq!(explain.prefilter, None);
assert_eq!(explain.candidates, Some(1));
assert_eq!(
explain.prefilter_skipped.as_deref(),
Some("complete filtered tool-name vocabulary scored before matching rows are loaded")
);
assert!(explain.candidates.unwrap() < explain.corpus);
let vocabulary_plan = db
.conn
.prepare(
"explain query plan select tool_name from messages
where tool_name is not null order by tool_name",
)
.unwrap()
.query_map([], |row| row.get::<_, String>(3))
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap()
.join("\n");
assert!(
vocabulary_plan.contains("idx_messages_tool_name"),
"{vocabulary_plan}"
);
let none = db
.search_messages(
"",
&MessageFilters {
tool: Some("zzz".into()),
..Default::default()
},
)
.unwrap();
assert!(
none.is_empty(),
"unknown tool matches nothing (incl. NULL-tool rows)"
);
}
#[test]
#[ignore = "deterministic release-mode tool-name vocabulary performance measurement"]
fn fuzzy_tool_name_vocabulary_benchmark() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions (
id, provider, provider_session_id, preview_text, source_path,
parse_version, discovery_source
) values ('s1','claude','s1','','/p','1','test')",
)
.unwrap();
let tx = db.conn.unchecked_transaction().unwrap();
{
let mut insert = tx
.prepare(
"insert into messages (
session_id, provider, seq, role, tool_name, content
) values ('s1', 'claude', ?1, 'tool', ?2, ?3)",
)
.unwrap();
for seq in 0..250_000_i64 {
let name = if seq % 25_000 == 0 {
"exec_command".to_string()
} else {
format!("unrelated_tool_{:03}", seq % 250)
};
insert
.execute(params![seq, name, "x".repeat(2_048)])
.unwrap();
}
}
tx.commit().unwrap();
let filters = MessageFilters {
field: Some(SearchField::ToolName),
match_mode: MessageSearchMode::Fuzzy,
limit: 20,
..Default::default()
};
let measure = |optimized: bool| {
let search = || {
if optimized {
db.search_messages("exec", &filters).unwrap()
} else {
db.search_tool_name_fuzzy_direct_for_test("exec", &filters)
.unwrap()
}
};
let expected = serde_json::to_vec(&search()).unwrap();
let digest = crate::hashing::sha256(&expected);
let mut elapsed = Vec::new();
for _ in 0..7 {
let started = Instant::now();
assert_eq!(serde_json::to_vec(&search()).unwrap(), expected);
elapsed.push(started.elapsed().as_micros());
}
elapsed.sort_unstable();
(
elapsed[elapsed.len() / 2],
elapsed[elapsed.len() - 1],
digest,
)
};
let direct = measure(false);
let optimized = measure(true);
assert_eq!(direct.2, optimized.2);
eprintln!(
"tool_name_vocabulary_benchmark rows=250000 vocabulary=251 direct_median_us={} direct_p95_us={} optimized_median_us={} optimized_p95_us={} digest=sha256:{}",
direct.0, direct.1, optimized.0, optimized.1, direct.2
);
}
#[test]
fn global_tool_name_vocabulary_matches_distinct_index_scan() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions (
id, provider, provider_session_id, preview_text, source_path,
parse_version, discovery_source
) values ('s1','claude','s1','','/p','1','test');
insert into messages (
session_id, provider, seq, role, tool_name, content
) values
('s1', 'claude', 0, 'tool', 'ÄTool', ''),
('s1', 'claude', 1, 'tool', 'exec_command', ''),
('s1', 'claude', 2, 'tool', 'exec_command', ''),
('s1', 'claude', 3, 'user', null, '');",
)
.unwrap();
let expected = db
.conn
.prepare(
"select distinct tool_name from messages
where tool_name is not null order by tool_name",
)
.unwrap()
.query_map([], |row| row.get::<_, String>(0))
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap();
assert_eq!(db.global_tool_name_vocabulary().unwrap(), expected);
}
#[test]
fn fuzzy_tool_name_vocabulary_path_matches_direct_row_scoring() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions (
id, provider, provider_session_id, preview_text, source_path,
parse_version, discovery_source
) values
('s1','claude','s1','','/p','1','test'),
('s2','codex','s2','','/q','1','test');
insert into messages (
session_id, provider, seq, role, tool_name, content
) values
('s1', 'claude', 0, 'tool', 'exec_command', 'large ignored payload a'),
('s1', 'claude', 1, 'tool', 'Edit', 'large ignored payload b'),
('s1', 'claude', 2, 'tool', 'mcp:memory:execute', 'large ignored payload c'),
('s1', 'claude', 3, 'tool', 'ExecCommand', 'equal-score spelling payload'),
('s2', 'codex', 0, 'tool', 'exec_command', 'large ignored payload d'),
('s2', 'codex', 1, 'tool', 'write_stdin', 'large ignored payload e');",
)
.unwrap();
let filters = MessageFilters {
field: Some(SearchField::ToolName),
match_mode: MessageSearchMode::Fuzzy,
providers: Some(vec![Provider::Claude]),
limit: 2,
offset: 1,
..Default::default()
};
let optimized = db.search_messages("exec", &filters).unwrap();
let direct = db
.search_tool_name_fuzzy_direct_for_test("exec", &filters)
.unwrap();
assert_eq!(
optimized
.iter()
.map(|hit| (&hit.session_id, hit.seq, hit.fuzzy_score))
.collect::<Vec<_>>(),
direct
.iter()
.map(|hit| (&hit.session_id, hit.seq, hit.fuzzy_score))
.collect::<Vec<_>>()
);
}
#[test]
fn fuzzy_tool_name_searches_the_complete_filtered_vocabulary() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions (
id, provider, provider_session_id, preview_text, source_path,
parse_version, discovery_source
) values ('s1','claude','s1','','/p','1','test')",
)
.unwrap();
let tx = db.conn.unchecked_transaction().unwrap();
{
let mut insert = tx
.prepare(
"insert into messages (
session_id, provider, seq, role, tool_name, content
) values ('s1', 'claude', ?1, 'tool', ?2, '')",
)
.unwrap();
for seq in 0..=10_000_usize {
insert
.execute(params![seq as i64, format!("tool_{seq:05}")])
.unwrap();
}
}
tx.commit().unwrap();
db.conn
.execute(
"insert into messages (
session_id, provider, seq, role, tool_name, content
) values ('s1', 'claude', ?1, 'tool', 'tol', '')",
params![10_001_i64],
)
.unwrap();
let (hits, explain) = db
.search_messages_with_explain(
"tol",
&MessageFilters {
field: Some(SearchField::ToolName),
match_mode: MessageSearchMode::Fuzzy,
limit: 1,
..Default::default()
},
true,
)
.unwrap();
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].tool_name.as_deref(), Some("tol"));
let explain = explain.unwrap();
assert_eq!(explain.corpus, 10_002);
assert_eq!(explain.candidates, Some(10_002));
}
#[test]
fn tool_name_literal_receipt_reports_the_named_tool_corpus_independent_of_the_page() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions (
id, provider, provider_session_id, preview_text, source_path,
parse_version, discovery_source
) values ('s1','claude','s1','','/p','1','test');
insert into messages (session_id, provider, seq, role, kind, tool_name, content)
values ('s1', 'claude', 0, 'tool', 'tool_call', 'exec_command', ''),
('s1', 'claude', 1, 'tool', 'tool_call', 'exec_command', ''),
('s1', 'claude', 2, 'tool', 'tool_call', 'apply_patch', ''),
('s1', 'claude', 3, 'user', 'conversation', null, 'no tool');",
)
.unwrap();
for limit in [1, 2, 10] {
let (hits, explain) = db
.search_messages_with_explain(
"exec",
&MessageFilters {
field: Some(SearchField::ToolName),
limit,
..Default::default()
},
true,
)
.unwrap();
assert_eq!(hits.len(), 2.min(limit));
let explain = explain.unwrap();
assert_eq!(
explain.corpus, 3,
"limit {limit}: the denominator is every named-tool row the filters admit"
);
assert_eq!(explain.candidates, None, "nothing was prefiltered");
assert!(explain.prefilter_skipped.is_some());
}
}
#[test]
fn fuzzy_batches_score_at_the_row_cap_or_the_byte_cap_whichever_comes_first() {
assert!(fuzzy_batch_is_full(FUZZY_SCORE_BATCH_SIZE, 0));
assert!(fuzzy_batch_is_full(3, FUZZY_BATCH_BYTES));
assert!(fuzzy_batch_is_full(3, FUZZY_BATCH_BYTES + 1));
assert!(!fuzzy_batch_is_full(
FUZZY_SCORE_BATCH_SIZE - 1,
FUZZY_BATCH_BYTES - 1
));
assert!(!fuzzy_batch_is_full(1, 1));
}
#[test]
fn fuzzy_tool_name_offset_beyond_the_corpus_returns_empty_without_reserving_the_offset() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions (
id, provider, provider_session_id, preview_text, source_path,
parse_version, discovery_source
) values ('s1','claude','s1','','/p','1','test');
insert into messages (session_id, provider, seq, role, tool_name, content)
values ('s1', 'claude', 0, 'tool', 'exec_command', '');",
)
.unwrap();
for offset in [100_000_000_000_000_000_usize, 1_000_000_000] {
let hits = db
.search_messages(
"exec",
&MessageFilters {
field: Some(SearchField::ToolName),
match_mode: MessageSearchMode::Fuzzy,
limit: 1,
offset,
..Default::default()
},
)
.unwrap();
assert!(hits.is_empty(), "offset {offset} lies past the only row");
}
}
#[test]
fn a_fuzzy_window_past_the_corpus_is_answered_without_scoring_any_row() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions (
id, provider, provider_session_id, preview_text, source_path,
parse_version, discovery_source
) values ('s1','claude','s1','','/p','1','test');",
)
.unwrap();
for seq in 0..4 {
db.conn
.execute(
"insert into messages (session_id, provider, seq, role, tool_name, content) \
values ('s1', 'claude', ?, 'tool', 'exec_command', 'exec command output')",
[seq],
)
.unwrap();
}
for field in [
None,
Some(SearchField::ToolName),
Some(SearchField::Content),
] {
let filters = |offset: usize| MessageFilters {
field,
match_mode: MessageSearchMode::Fuzzy,
limit: 2,
offset,
..Default::default()
};
let (hits, explain) = db
.search_messages_with_explain("exec", &filters(1_000_000_000), true)
.unwrap();
assert!(hits.is_empty(), "{field:?}");
let explain = explain.expect("an explained request returns a receipt");
assert_eq!(explain.corpus, 4, "{field:?}");
assert_eq!(
explain.candidates, None,
"{field:?}: no row scored, so no candidate count was measured"
);
let (hits, explain) = db
.search_messages_with_explain("exec", &filters(0), true)
.unwrap();
assert_eq!(hits.len(), 2, "{field:?}");
assert_eq!(
explain.expect("receipt").candidates,
Some(4),
"{field:?}: every row scored"
);
}
}
#[test]
fn the_fuzzy_window_boundary_keeps_the_last_row_and_drops_the_first_one_past_it() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions (
id, provider, provider_session_id, preview_text, source_path,
parse_version, discovery_source
) values ('s1','claude','s1','','/p','1','test');",
)
.unwrap();
let corpus = 4_usize;
for seq in 0..corpus as i64 {
db.conn
.execute(
"insert into messages (session_id, provider, seq, role, content) \
values ('s1', 'claude', ?, 'user', 'exec command output')",
[seq],
)
.unwrap();
}
let filters = |offset: usize| MessageFilters {
match_mode: MessageSearchMode::Fuzzy,
limit: FUZZY_SCORE_BATCH_SIZE + 100,
offset,
..Default::default()
};
let (hits, explain) = db
.search_messages_with_explain("exec", &filters(corpus - 1), true)
.unwrap();
assert_eq!(hits.len(), 1, "the last row is inside the corpus");
assert_eq!(explain.expect("receipt").candidates, Some(corpus as i64));
let (hits, explain) = db
.search_messages_with_explain("exec", &filters(corpus), true)
.unwrap();
assert!(hits.is_empty(), "one past the last row is an empty page");
let explain = explain.expect("receipt");
assert_eq!(explain.corpus, corpus as i64);
assert_eq!(
explain.candidates, None,
"an empty page from the count scored no row"
);
}
#[test]
fn a_literal_search_finds_greek_text_ending_in_sigma() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions (
id, provider, provider_session_id, preview_text, source_path,
parse_version, discovery_source
) values ('s1','claude','s1','','/p','1','test');
insert into messages (session_id, provider, seq, role, kind, content)
values ('s1','claude',0,'user','message','η ΟΔΟΣΣ που ψαχνω'),
('s1','claude',1,'user','message','η οδοσς που ψαχνω επισης');",
)
.unwrap();
for query in ["ΟΔΟΣΣ", "οδοσς", "οδοσσ"] {
let hits = db
.search_messages(
query,
&MessageFilters {
match_mode: MessageSearchMode::Literal,
limit: 10,
..Default::default()
},
)
.unwrap();
assert_eq!(hits.len(), 2, "query {query:?} finds both spellings");
}
}
#[test]
fn a_pre_v4_literal_search_finds_greek_text_ending_in_sigma() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
enable_v3_custom_trigram_compatibility(&db);
seed_messages(
&db,
&[
("user", "η ΟΔΟΣΣ που ψαχνω"),
("user", "η οδοσς που ψαχνω επισης"),
],
);
for query in ["ΟΔΟΣΣ", "οδοσς", "οδοσσ"] {
let (hits, explain) = db
.search_messages_with_explain(
query,
&MessageFilters {
match_mode: MessageSearchMode::Literal,
limit: 10,
..Default::default()
},
true,
)
.unwrap();
assert_eq!(hits.len(), 2, "query {query:?} finds both spellings");
let explain = explain.expect("receipt");
assert_eq!(explain.prefilter, None, "query {query:?} indexed nothing");
assert!(
explain
.prefilter_skipped
.as_deref()
.is_some_and(|reason| reason.contains("sigma")),
"query {query:?} reports why it scanned: {:?}",
explain.prefilter_skipped
);
}
}
#[test]
fn a_tool_argument_window_counts_only_tool_call_rows() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions (
id, provider, provider_session_id, preview_text, source_path,
parse_version, discovery_source
) values ('s1','claude','s1','','/p','1','test');
insert into messages (session_id, provider, seq, role, kind, tool_name, content)
values ('s1','claude',0,'tool','tool_call','exec_command','{\"args\":{\"cmd\":\"needle\"}}');
insert into messages (session_id, provider, seq, role, kind, content)
values ('s1','claude',1,'user','message','needle in ordinary prose'),
('s1','claude',2,'assistant','message','needle again'),
('s1','claude',3,'user','message','needle once more');",
)
.unwrap();
let filters = |offset: usize| MessageFilters {
field: Some(SearchField::ToolArgument),
argument_path: Some("/cmd".to_string()),
match_mode: MessageSearchMode::Fuzzy,
limit: 1,
offset,
..Default::default()
};
let (hits, explain) = db
.search_messages_with_explain("needle", &filters(0), true)
.unwrap();
assert_eq!(hits.len(), 1, "only the tool call has an argument");
assert_eq!(
explain.expect("receipt").corpus,
1,
"corpus is the rows the searched field admits"
);
let (hits, explain) = db
.search_messages_with_explain("needle", &filters(1_000_000_000), true)
.unwrap();
assert!(hits.is_empty());
let explain = explain.expect("an explained request returns a receipt");
assert_eq!(explain.corpus, 1, "ordinary rows are not tool arguments");
assert_eq!(
explain.candidates, None,
"no row scored, so no candidate count was measured"
);
}
#[test]
fn an_empty_corpus_reports_the_same_receipt_whatever_the_page_size() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let receipt = |limit: usize| {
let (hits, explain) = db
.search_messages_with_explain(
"anything",
&MessageFilters {
match_mode: MessageSearchMode::Fuzzy,
limit,
..Default::default()
},
true,
)
.unwrap();
assert!(hits.is_empty(), "limit {limit}");
explain.expect("an explained request returns a receipt")
};
let small = receipt(100);
let large = receipt(FUZZY_SCORE_BATCH_SIZE + 1);
assert_eq!(small.corpus, 0);
assert_eq!(large.corpus, 0);
assert_eq!(small.candidates, Some(0));
assert_eq!(
large.candidates, small.candidates,
"an empty corpus scored nothing either way; the count is 0, not absent"
);
assert_eq!(large.prefilter_skipped, small.prefilter_skipped);
}
#[test]
fn date_filter_until_covers_sub_second_tail_of_final_second() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values ('s1','claude','s1','','/p','1','test')",
[],
)
.unwrap();
db.conn
.execute(
"insert into messages (id, session_id, provider, seq, role, ts, content) \
values (1,'s1','claude',0,'user','2026-01-15T23:59:59.123456789+00:00','late')",
[],
)
.unwrap();
let until =
crate::dates::parse_bound("2026-01-15", crate::dates::Bound::End, Utc::now()).unwrap();
let hits = db
.search_messages(
"",
&MessageFilters {
until: Some(until),
..Default::default()
},
)
.unwrap();
assert_eq!(
hits.len(),
1,
"an inclusive --until must cover sub-second timestamps in its final second"
);
}
#[test]
fn date_filter_preserves_exact_fractional_upper_bound() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values ('s1','claude','s1','','/p','1','test')",
[],
)
.unwrap();
for (id, seq, ts) in [
(1, 0, "2026-07-07T02:53:22.357999999+00:00"),
(2, 1, "2026-07-07T02:53:22.358000000+00:00"),
(3, 2, "2026-07-07T02:53:22.358000001+00:00"),
] {
db.conn
.execute(
"insert into messages (id, session_id, provider, seq, role, ts, content) \
values (?1,'s1','claude',?2,'user',?3,'event')",
rusqlite::params![id, seq, ts],
)
.unwrap();
}
let until = crate::dates::parse_bound(
"2026-07-07T02:53:22.358Z",
crate::dates::Bound::End,
Utc::now(),
)
.unwrap();
let hits = db
.search_messages(
"",
&MessageFilters {
until: Some(until),
..Default::default()
},
)
.unwrap();
assert_eq!(
hits.iter().map(|hit| hit.seq).collect::<Vec<_>>(),
vec![0, 1],
"an exact fractional --until must exclude later events in the same second"
);
}
#[test]
fn date_filter_keeps_messages_with_unknown_timestamp() {
use chrono::TimeZone;
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values ('s1','claude','s1','','/p','1','test')",
[],
)
.unwrap();
db.conn
.execute(
"insert into messages (id, session_id, provider, seq, role, ts, content) \
values (1,'s1','claude',0,'user',NULL,'undated correction')",
[],
)
.unwrap();
let since = Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).single().unwrap();
let until = Utc
.with_ymd_and_hms(2026, 12, 31, 23, 59, 59)
.single()
.unwrap();
let hits = db
.search_messages(
"zebracode",
&MessageFilters {
since: Some(since),
until: Some(until),
..Default::default()
},
)
.unwrap();
assert!(
hits.is_empty(),
"a NULL-timestamp message must not match every date filter; index a fallback timestamp instead"
);
}
#[test]
fn session_search_zero_is_unlimited_and_large_limit_does_not_overflow() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
for id in ["s1", "s2"] {
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values (?1,'claude',?1,'alpha preview',?2,'1','test')",
params![id, format!("/{id}.jsonl")],
)
.unwrap();
}
let scoring = crate::config::ScoringConfig::default();
let mut filters = SearchFilters::default();
assert_eq!(
db.search("alpha", &filters, None, &scoring).unwrap().len(),
2
);
filters.limit = 1;
assert_eq!(
db.search("alpha", &filters, None, &scoring).unwrap().len(),
1
);
filters.limit = usize::MAX;
assert_eq!(
db.search("alpha", &filters, None, &scoring).unwrap().len(),
2
);
}
#[test]
fn session_search_requires_a_text_match_before_ranking_bonuses() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute(
"insert into sessions (
id, provider, provider_session_id, cwd, repo_root, preview_text,
source_path, updated_at, parse_version, discovery_source
) values ('s1','claude','s1','/repo','/repo','ordinary text','/p',?1,'1','test')",
params![Utc::now().to_rfc3339()],
)
.unwrap();
let hits = db
.search(
"⸘⸘⸘",
&SearchFilters::default(),
Some("/repo"),
&crate::config::ScoringConfig::default(),
)
.unwrap();
assert!(
hits.is_empty(),
"recency and repository bonuses must not admit an unrelated session"
);
}
#[test]
fn session_search_combines_query_term_coverage_across_one_session() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute(
"insert into sessions (
id, provider, provider_session_id, title, summary, preview_text,
source_path, parse_version, discovery_source
) values ('s1','claude','s1','alpha','beta','','/p','1','test')",
[],
)
.unwrap();
let scoring = crate::config::ScoringConfig {
title_score: 0,
summary_score: 0,
path_score: 0,
preview_score: 0,
other_score: 0,
token_bonus: 0,
all_tokens_bonus: 150,
recency_weight: 0,
current_repo_bonus: 0,
..Default::default()
};
let hits = db
.search("alpha beta", &SearchFilters::default(), None, &scoring)
.unwrap();
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].session.id, "s1");
assert_eq!(hits[0].score, 150);
}
#[test]
fn session_search_positive_limit_is_a_prefix_of_unlimited_ranking() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
for (id, title, preview) in [("s1", "", "alpha"), ("s2", "alpha", "")] {
db.conn
.execute(
"insert into sessions (
id, provider, provider_session_id, title, preview_text,
source_path, parse_version, discovery_source
) values (?1,'claude',?1,?2,?3,?4,'1','test')",
params![id, title, preview, format!("/{id}.jsonl")],
)
.unwrap();
}
let scoring = crate::config::ScoringConfig {
recency_weight: 0,
..Default::default()
};
let all_filters = SearchFilters {
limit: 0,
..Default::default()
};
let all = db.search("alpha", &all_filters, None, &scoring).unwrap();
let one_filter = SearchFilters {
limit: 1,
..all_filters
};
let one = db.search("alpha", &one_filter, None, &scoring).unwrap();
assert_eq!(one.len(), 1);
assert_eq!(one[0].session.id, all[0].session.id);
assert_eq!(one[0].session.id, "s2");
}
#[test]
fn session_search_uses_the_application_worker_runtime_without_changing_ranking() {
let root = tempfile::tempdir().unwrap();
let one = Db::open_with_threads(
&root.path().join("one.db"),
TEST_BUSY_TIMEOUT_MS,
NonZeroUsize::new(1).unwrap(),
)
.unwrap();
let four = Db::open_with_threads(
&root.path().join("four.db"),
TEST_BUSY_TIMEOUT_MS,
NonZeroUsize::new(4).unwrap(),
)
.unwrap();
for db in [&one, &four] {
for index in 0..1_100 {
let id = format!("s{index:04}");
db.conn
.execute(
"insert into sessions (
id, provider, provider_session_id, title, summary, cwd, repo_root,
preview_text, source_path, updated_at, parse_version, discovery_source
) values (?1, 'claude', ?1, ?2, ?3, ?4, ?4, ?5, ?6, ?7, '1', 'test')",
params![
id,
(index % 7 == 0).then_some("database migration"),
format!("summary {index}"),
format!("/repo/{index}"),
format!("preview database work item {index}"),
format!("/{index}.jsonl"),
format!("2026-01-{:02}T00:00:00Z", index % 28 + 1),
],
)
.unwrap();
db.conn
.execute(
"insert into transcripts(session_id, transcript_text) values (?1, ?2)",
params![
format!("s{index:04}"),
format!(
"transcript {index} discusses database schema migration planning"
)
],
)
.unwrap();
}
}
let filters = SearchFilters {
limit: 40,
..Default::default()
};
let scoring = crate::config::ScoringConfig {
recency_weight: 0,
..Default::default()
};
let identities = |hits: Vec<SearchHit>| {
hits.into_iter()
.map(|hit| {
(
hit.session.id,
hit.score,
hit.match_source,
hit.match_snippet,
)
})
.collect::<Vec<_>>()
};
assert_eq!(one.worker_pool_build_count(), 0);
assert_eq!(four.worker_pool_build_count(), 0);
let expected = identities(
one.search("databse migraton", &filters, Some("/repo/7"), &scoring)
.unwrap(),
);
let actual = identities(
four.search("databse migraton", &filters, Some("/repo/7"), &scoring)
.unwrap(),
);
assert_eq!(
actual, expected,
"worker count must not change session ranking"
);
assert_eq!(one.worker_pool_build_count(), 1);
assert_eq!(four.worker_pool_build_count(), 1);
}
#[test]
#[ignore = "deterministic release-mode session-search performance measurement"]
fn session_search_parallel_scoring_benchmark() {
let root = tempfile::tempdir().unwrap();
let path = root.path().join("index.db");
{
let db =
Db::open_with_threads(&path, TEST_BUSY_TIMEOUT_MS, NonZeroUsize::new(1).unwrap())
.unwrap();
let transcript_padding = "database schema migration planning ".repeat(250);
let tx = db.conn.unchecked_transaction().unwrap();
{
let mut session = tx
.prepare(
"insert into sessions (
id, provider, provider_session_id, title, preview_text, source_path,
parse_version, discovery_source
) values (?1, 'claude', ?1, ?2, ?3, ?4, '1', 'test')",
)
.unwrap();
let mut transcript = tx
.prepare("insert into transcripts(session_id, transcript_text) values (?1, ?2)")
.unwrap();
for index in 0..4_000 {
let id = format!("s{index:04}");
session
.execute(params![
id,
(index % 11 == 0).then_some("database migration"),
format!("preview {index}"),
format!("/{index}.jsonl"),
])
.unwrap();
transcript
.execute(params![
format!("s{index:04}"),
format!("{transcript_padding}{index}")
])
.unwrap();
}
}
tx.commit().unwrap();
}
let filters = SearchFilters {
limit: 50,
..Default::default()
};
let scoring = crate::config::ScoringConfig {
recency_weight: 0,
..Default::default()
};
let measure = |threads| {
let db = Db::open_with_threads(
&path,
TEST_BUSY_TIMEOUT_MS,
NonZeroUsize::new(threads).unwrap(),
)
.unwrap();
let search = || {
db.search("databse migraton", &filters, None, &scoring)
.unwrap()
};
let expected = serde_json::to_vec(&search()).unwrap();
let digest = crate::hashing::sha256(&expected);
let mut elapsed = Vec::new();
for _ in 0..7 {
let started = Instant::now();
assert_eq!(serde_json::to_vec(&search()).unwrap(), expected);
elapsed.push(started.elapsed().as_micros());
}
elapsed.sort_unstable();
(
elapsed[elapsed.len() / 2],
elapsed[elapsed.len() - 1],
digest,
)
};
let one = measure(1);
let four = measure(4);
assert_eq!(one.2, four.2);
eprintln!(
"session_parallel_benchmark sessions=4000 transcript_bytes_per_session={} one_worker_median_us={} one_worker_p95_us={} four_worker_median_us={} four_worker_p95_us={} digest=sha256:{}",
"database schema migration planning ".len() * 250,
one.0,
one.1,
four.0,
four.1,
one.2
);
}
#[test]
#[ignore = "deterministic release-mode temporal predicate measurement"]
fn session_temporal_overlap_predicate_benchmark() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let tx = db.conn.unchecked_transaction().unwrap();
{
let mut insert = tx
.prepare(
"insert into sessions (id, provider, provider_session_id, created_at, updated_at, preview_text, source_path, parse_version, discovery_source) values (?1, 'claude', ?1, ?2, ?3, '', ?4, '1', 'test')",
)
.unwrap();
for index in 0..100_000 {
let created = if index % 2 == 0 {
"2026-01-01T00:00:00+00:00"
} else {
"2025-12-01T00:00:00+00:00"
};
insert
.execute(params![
format!("s{index:06}"),
created,
"2026-01-15T00:00:00+00:00",
format!("/{index}.jsonl")
])
.unwrap();
}
}
tx.commit().unwrap();
let measure = |predicate: &str| {
let sql = format!("select count(*) from sessions s where {predicate}");
let expected: i64 = db.conn.query_row(&sql, [], |row| row.get(0)).unwrap();
let mut elapsed = Vec::new();
for _ in 0..9 {
let started = Instant::now();
assert_eq!(
db.conn
.query_row::<i64, _, _>(&sql, [], |row| row.get(0))
.unwrap(),
expected
);
elapsed.push(started.elapsed().as_micros());
}
elapsed.sort_unstable();
let plan = db
.conn
.prepare(&format!("explain query plan {sql}"))
.unwrap()
.query_map([], |row| row.get::<_, String>(3))
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap()
.join(" | ");
assert!(
!plan.contains("messages") && !plan.contains("transcripts"),
"{plan}"
);
(
elapsed[elapsed.len() / 2],
elapsed[elapsed.len() - 1],
expected,
plan,
)
};
let terminal = measure(
"coalesce(s.updated_at, s.created_at) >= '2025-12-15T00:00:00+00:00' and coalesce(s.updated_at, s.created_at) <= '2025-12-31T23:59:59+00:00'",
);
let overlap = measure(
"coalesce(s.updated_at, s.created_at) >= '2025-12-01T00:00:00+00:00' and coalesce(s.created_at, s.updated_at) <= '2025-12-31T23:59:59+00:00'",
);
eprintln!(
"session_temporal_predicate_benchmark sessions=100000 terminal_median_us={} terminal_p95_us={} terminal_matches={} overlap_median_us={} overlap_p95_us={} overlap_matches={} terminal_plan={:?} overlap_plan={:?}",
terminal.0, terminal.1, terminal.2, overlap.0, overlap.1, overlap.2, terminal.3, overlap.3
);
}
#[test]
fn message_update_keeps_fts_in_sync() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values ('s1','claude','s1','','/p','1','test')",
[],
)
.unwrap();
db.conn
.execute(
"insert into messages (id, session_id, provider, seq, role, content) \
values (1,'s1','claude',0,'user','alpha original')",
[],
)
.unwrap();
let count = |term: &str| -> i64 {
db.conn
.query_row(
"select count(*) from messages_fts where messages_fts match ?1",
params![term],
|r| r.get(0),
)
.unwrap()
};
assert_eq!(count("original"), 1, "inserted content is searchable");
db.conn
.execute("update messages set content='beta updated' where id=1", [])
.unwrap();
assert_eq!(count("updated"), 1, "new content searchable after update");
assert_eq!(count("original"), 0, "stale term dropped after update");
}
#[test]
fn open_retires_the_unread_session_fts_copy_once_and_keeps_the_empty_table_for_older_builds() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("index.db");
drop(Db::open(&path).unwrap());
let fill = || {
rusqlite::Connection::open(&path)
.unwrap()
.execute_batch(
"insert into sessions_fts(rowid, title, summary, preview_text, transcript_text)
values (1, 'older build wrote this', '', '', '');",
)
.unwrap();
};
let fts_rows = |db: &Db| -> i64 {
db.conn
.query_row("select count(*) from sessions_fts", [], |row| row.get(0))
.unwrap()
};
let db = Db::open(&path).unwrap();
assert_eq!(fts_rows(&db), 0);
assert!(
crate::indexer::current_schema_layout_problem(&db.conn)
.unwrap()
.is_none(),
"the empty table satisfies the layout check an older build performs"
);
seed_messages(&db, &[("user", "hello")]);
assert_eq!(fts_rows(&db), 0, "no write path fills it");
db.clear_all().unwrap();
drop(db);
fill();
let reopened = Db::open(&path).unwrap();
assert_eq!(
fts_rows(&reopened),
1,
"a refill by an older writer is left alone after the one-time retirement"
);
drop(reopened);
rusqlite::Connection::open(&path)
.unwrap()
.execute_batch("delete from index_metadata where key = 'sessions_fts_retired';")
.unwrap();
let retired = Db::open(&path).unwrap();
assert_eq!(
fts_rows(&retired),
0,
"the earlier build's copy is emptied once"
);
}
#[test]
fn resolve_session_errors_are_actionable() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let insert = |id: &str| {
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values (?1,'claude',?1,'','/p','1','test')",
params![id],
)
.unwrap();
};
insert("claude:abc123");
insert("claude:abc456");
let err = db.resolve_session("zzz").unwrap_err().to_string();
assert!(err.contains("no session matches"));
assert!(err.contains("aise list") || err.contains("aise search"));
let err = db.resolve_session_record("zzz").unwrap_err().to_string();
assert!(err.contains("no session matches"));
assert!(err.contains("aise list") || err.contains("aise search"));
let err = db.resolve_session("claude:abc").unwrap_err().to_string();
assert!(err.contains("ambiguous"));
assert!(
err.contains("claude:abc123") && err.contains("claude:abc456"),
"ambiguous error must list candidates: {err}"
);
let err = db
.resolve_session_record("claude:abc")
.unwrap_err()
.to_string();
assert!(err.contains("ambiguous"));
assert!(
err.contains("claude:abc123") && err.contains("claude:abc456"),
"ambiguous error must list candidates: {err}"
);
}
#[test]
fn exact_session_resolution_uses_indexes_and_bare_collisions_name_canonical_candidates() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
for provider in [Provider::Claude, Provider::Codex] {
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values (?1, ?2, 'shared-native-id', '', '/p', '1', 'test')",
params![
format!("{}:shared-native-id", provider.as_str()),
provider.as_str()
],
)
.unwrap();
}
for canonical in ["claude:shared-native-id", "codex:shared-native-id"] {
assert_eq!(db.resolve_session_record(canonical).unwrap().id, canonical);
}
let collision = db
.resolve_session_record("shared-native-id")
.unwrap_err()
.to_string();
assert!(collision.contains("ambiguous"), "{collision}");
assert!(collision.contains("claude:shared-native-id"), "{collision}");
assert!(collision.contains("codex:shared-native-id"), "{collision}");
let mut statement = db
.conn
.prepare(&format!(
"explain query plan {RESOLVE_SESSION_RECORD_EXACT_SQL}"
))
.unwrap();
let plan = statement
.query_map(["claude:shared-native-id"], |row| row.get::<_, String>(3))
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap()
.join("\n");
assert!(plan.contains("SEARCH"), "{plan}");
assert!(!plan.contains("SCAN s"), "{plan}");
}
#[test]
fn session_prefix_resolution_treats_sql_wildcards_as_literal_text() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
for id in ["claude:percent", "claude:under_score"] {
db.conn
.execute(
"insert into sessions (id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values (?1, 'claude', ?1, '', '/p', '1', 'test')",
params![id],
)
.unwrap();
}
for literal in ["%", "_"] {
let error = db.resolve_session_record(literal).unwrap_err().to_string();
assert!(error.contains("no session matches"), "{error}");
}
}
#[test]
fn open_adds_typed_message_columns_to_version_one_database() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("index.db");
{
let legacy = Connection::open(&path).unwrap();
legacy
.execute_batch(
"create table messages (
id integer primary key,
session_id text not null,
provider text not null,
seq integer not null,
role text not null,
ts text,
tool_name text,
is_compaction integer not null default 0,
content text not null
);
insert into messages
(session_id, provider, seq, role, content)
values ('claude:legacy', 'claude', 0, 'user', 'legacy text');
pragma user_version = 1;",
)
.unwrap();
}
let db = Db::open(&path).unwrap();
let columns = db
.conn
.prepare("pragma table_info(messages)")
.unwrap()
.query_map([], |row| row.get::<_, String>(1))
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap();
assert!(columns.iter().any(|column| column == "kind"));
assert!(columns.iter().any(|column| column == "tool_call_id"));
assert!(columns.iter().any(|column| column == "authorship"));
assert!(columns
.iter()
.any(|column| column == "correlation_authority"));
assert!(columns.iter().any(|column| column == "correlation_scope"));
assert!(columns.iter().any(|column| column == "correlation_id"));
assert!(columns.iter().any(|column| column == "record_relation"));
let legacy_authorship: String = db
.conn
.query_row("select authorship from messages limit 1", [], |row| {
row.get(0)
})
.unwrap();
assert_eq!(legacy_authorship, "unknown");
let parts_table_exists: bool = db
.conn
.query_row(
"select exists(
select 1 from sqlite_schema
where type = 'table' and name = 'message_content_parts'
)",
[],
|row| row.get(0),
)
.unwrap();
assert!(parts_table_exists);
assert!(db.needs_backfill().unwrap());
}
#[test]
fn message_provenance_round_trips_without_copying_part_text() {
use crate::models::{
ContentPartAuthorship, ContentPartOrigin, MessageAuthorship, MessageContentPart,
MessageCorrelationAuthority, MessageCorrelationIdentity, MessageProvenance,
MessageRecordRelation,
};
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let mut parsed = crate::util::minimal_record(
Provider::GeminiCli,
Path::new("/fixture/mixed.json"),
String::new(),
);
parsed.session.id = "gemini-cli:mixed".to_string();
parsed.session.provider_session_id = "mixed".to_string();
parsed.messages = vec![crate::models::Message {
seq: 0,
role: Role::User,
ts: None,
tool_name: None,
kind: crate::models::MessageKind::Conversation,
tool_call_id: None,
is_compaction: false,
content: "askAI".to_string(),
provenance: MessageProvenance {
authorship: MessageAuthorship::Mixed,
record_relation: MessageRecordRelation::Mirror,
correlation_identity: Some(MessageCorrelationIdentity {
authority: MessageCorrelationAuthority::Google,
scope: "conversation_event".to_string(),
id: "event-7".to_string(),
}),
content_parts: vec![
MessageContentPart {
ordinal: 0,
start_char: 0,
end_char: 3,
authorship: ContentPartAuthorship::Human,
origin: ContentPartOrigin::DirectInput,
},
MessageContentPart {
ordinal: 1,
start_char: 3,
end_char: 5,
authorship: ContentPartAuthorship::Generated,
origin: ContentPartOrigin::QuotedContent,
},
],
},
}];
db.upsert_session(&parsed, 0, 0).unwrap();
let stored: (String, String, String, String, String) = db
.conn
.query_row(
"select authorship, correlation_authority, correlation_scope,
correlation_id, record_relation
from messages where session_id = ?1 and seq = 0",
[&parsed.session.id],
|row| {
Ok((
row.get(0)?,
row.get(1)?,
row.get(2)?,
row.get(3)?,
row.get(4)?,
))
},
)
.unwrap();
assert_eq!(
stored,
(
"mixed".to_string(),
"google".to_string(),
"conversation_event".to_string(),
"event-7".to_string(),
"mirror".to_string(),
)
);
let parts = db
.conn
.prepare(
"select ordinal, start_char, end_char, authorship, origin
from message_content_parts order by ordinal",
)
.unwrap()
.query_map([], |row| {
Ok((
row.get::<_, i64>(0)?,
row.get::<_, i64>(1)?,
row.get::<_, i64>(2)?,
row.get::<_, String>(3)?,
row.get::<_, String>(4)?,
))
})
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap();
assert_eq!(
parts,
vec![
(0, 0, 3, "human".to_string(), "direct_input".to_string()),
(
1,
3,
5,
"generated".to_string(),
"quoted_content".to_string()
),
]
);
let part_columns = db
.conn
.prepare("pragma table_info(message_content_parts)")
.unwrap()
.query_map([], |row| row.get::<_, String>(1))
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap();
assert!(!part_columns.iter().any(|column| column == "content"));
}
#[test]
fn invalid_message_provenance_rolls_back_the_whole_session_upsert() {
use crate::models::{MessageAuthorship, MessageProvenance};
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let mut parsed = crate::util::minimal_record(
Provider::Claude,
Path::new("/fixture/invalid.jsonl"),
String::new(),
);
parsed.session.id = "claude:invalid-provenance".to_string();
parsed.session.provider_session_id = "invalid-provenance".to_string();
parsed.messages = vec![crate::models::Message {
seq: 0,
role: Role::User,
ts: None,
tool_name: None,
kind: crate::models::MessageKind::Conversation,
tool_call_id: None,
is_compaction: false,
content: "text".to_string(),
provenance: MessageProvenance {
authorship: MessageAuthorship::Mixed,
..Default::default()
},
}];
let error = db.upsert_session(&parsed, 0, 0).unwrap_err().to_string();
assert!(error.contains("message sequence 0"), "{error}");
assert!(error.contains("claude:invalid-provenance"), "{error}");
assert!(error.contains("requires content_parts"), "{error}");
assert_eq!(
db.conn
.query_row(
"select count(*) from sessions where id = ?1",
[&parsed.session.id],
|row| row.get::<_, i64>(0),
)
.unwrap(),
0,
"the transaction must not publish a session whose message provenance is invalid"
);
}
#[test]
fn replacing_a_mixed_message_cascades_its_part_ranges() {
use crate::models::{
ContentPartAuthorship, ContentPartOrigin, MessageAuthorship, MessageContentPart,
MessageProvenance,
};
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let mut parsed = parsed_with_messages("claude:parts-cascade", &["ab"]);
parsed.messages[0].provenance = MessageProvenance {
authorship: MessageAuthorship::Mixed,
content_parts: vec![
MessageContentPart {
ordinal: 0,
start_char: 0,
end_char: 1,
authorship: ContentPartAuthorship::Human,
origin: ContentPartOrigin::DirectInput,
},
MessageContentPart {
ordinal: 1,
start_char: 1,
end_char: 2,
authorship: ContentPartAuthorship::Generated,
origin: ContentPartOrigin::QuotedContent,
},
],
..Default::default()
};
db.upsert_session(&parsed, 0, 0).unwrap();
assert_eq!(
db.conn
.query_row("select count(*) from message_content_parts", [], |row| {
row.get::<_, i64>(0)
})
.unwrap(),
2
);
parsed.messages[0].provenance = MessageProvenance {
authorship: MessageAuthorship::Human,
..Default::default()
};
db.replace_session(&parsed, 1, 1).unwrap();
assert_eq!(
db.conn
.query_row("select count(*) from message_content_parts", [], |row| {
row.get::<_, i64>(0)
})
.unwrap(),
0
);
}
#[test]
fn provenance_queries_use_session_and_behavioral_analysis_indexes() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
for (sql, expected_index) in [
(
"select seq from messages
where session_id = 'claude:s' and authorship = 'human'
and record_relation = 'original'
order by seq",
"idx_messages_session_authorship_relation_seq",
),
(
"select session_id, seq from messages
where authorship = 'human' and record_relation = 'original'
order by ts desc, session_id, seq",
"idx_messages_authorship_relation_ts",
),
] {
let plan = db
.conn
.prepare(&format!("explain query plan {sql}"))
.unwrap()
.query_map([], |row| row.get::<_, String>(3))
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap()
.join("\n");
assert!(plan.contains(expected_index), "{plan}");
assert!(!plan.contains("SCAN messages"), "{plan}");
}
for sql in [
ANALYSIS_MESSAGE_COUNTS_SQL,
ANALYSIS_ATTRIBUTABLE_HUMAN_MESSAGES_SQL,
] {
let plan = db
.conn
.prepare(&format!("explain query plan {sql}"))
.unwrap()
.query_map(["claude:s"], |row| row.get::<_, String>(3))
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap()
.join("\n");
assert!(
plan.contains("idx_messages_session_authorship_relation_seq"),
"analysis must select candidate messages through the composite provenance index:\n{plan}"
);
assert!(
!plan.contains("SCAN m"),
"analysis must not full-scan messages for one session:\n{plan}"
);
}
}
#[test]
fn correction_candidate_and_part_queries_use_provenance_indexes() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let candidates = format!(
"explain query plan {CORRECTION_CANDIDATES_SQL}
order by m.ts desc, m.session_id asc, m.seq asc"
);
let candidate_plan = db
.conn
.prepare(&candidates)
.unwrap()
.query_map([], |row| row.get::<_, String>(3))
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap()
.join("\n");
assert!(
candidate_plan.contains("idx_messages_authorship_relation_ts"),
"{candidate_plan}"
);
assert!(!candidate_plan.contains("SCAN m"), "{candidate_plan}");
let parts = format!(
"explain query plan {CORRECTION_HUMAN_PARTS_SQL}
order by m.id, p.ordinal"
);
let parts_plan = db
.conn
.prepare(&parts)
.unwrap()
.query_map([], |row| row.get::<_, String>(3))
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap()
.join("\n");
assert!(
parts_plan.contains("idx_messages_authorship_relation_ts"),
"{parts_plan}"
);
assert!(
parts_plan.contains("SEARCH p USING PRIMARY KEY"),
"{parts_plan}"
);
assert!(!parts_plan.contains("SCAN m"), "{parts_plan}");
assert!(!parts_plan.contains("SCAN p"), "{parts_plan}");
}
#[test]
fn tool_argument_search_uses_explicit_json_pointer_and_excludes_results() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
seed_messages(&db, &[("tool", "call"), ("tool", "cargo test output")]);
db.conn
.execute_batch(
r#"update messages set
kind = 'tool_call', tool_name = 'exec_command',
content = '{"args":{"cmd":"cargo test","request":{"path":"src/lib.rs"}},"kind":"tool_call","tool_name":"exec_command"}'
where seq = 0;
update messages set kind = 'tool_result', tool_name = 'exec_command'
where seq = 1;"#,
)
.unwrap();
let hits = db
.search_messages(
"cargo test",
&MessageFilters {
field: Some(SearchField::ToolArgument),
argument_path: Some("/cmd".to_string()),
..Default::default()
},
)
.unwrap();
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].seq, 0);
let nested = db
.search_messages(
"src/lib.rs",
&MessageFilters {
field: Some(SearchField::ToolArgument),
argument_path: Some("/request/path".to_string()),
..Default::default()
},
)
.unwrap();
assert_eq!(nested.len(), 1);
db.conn
.execute(
"insert into messages (
session_id, provider, seq, role, kind, content
) values (
'claude:s1', 'claude', 2, 'user', 'conversation',
'{\"args\":{\"cmd\":\"cargo test\"}}'
)",
[],
)
.unwrap();
for (query, match_mode) in [
("", MessageSearchMode::Literal),
("cargo test", MessageSearchMode::Literal),
(r"cargo t.st", MessageSearchMode::Regex),
("crgo tst", MessageSearchMode::Fuzzy),
] {
let hits = db
.search_messages(
query,
&MessageFilters {
kinds: Some(vec![
crate::models::MessageKind::Conversation,
crate::models::MessageKind::ToolCall,
]),
field: Some(SearchField::ToolArgument),
argument_path: Some("/cmd".to_string()),
match_mode,
limit: 10,
..Default::default()
},
)
.unwrap();
assert_eq!(
hits.iter().map(|hit| hit.seq).collect::<Vec<_>>(),
vec![0],
"tool-argument {match_mode:?} search must intersect an explicit kind union with tool_call"
);
}
db.conn
.execute_batch(
r#"with recursive n(value) as (
values(1) union all select value + 1 from n where value < 100
)
insert into messages (
session_id, provider, seq, role, tool_name, kind, content
)
select 'claude:s1', 'claude', 10 + value, 'tool', 'exec_command',
'tool_call',
printf('{"args":{"cmd":"unrelated payload %d"}}', value)
from n;"#,
)
.unwrap();
for (query, match_mode) in [
("cargo test", MessageSearchMode::Literal),
(r"cargo t.st", MessageSearchMode::Regex),
] {
let filters = MessageFilters {
field: Some(SearchField::ToolArgument),
argument_path: Some("/cmd".to_string()),
match_mode,
..Default::default()
};
let (hits, explain) = db
.search_messages_with_explain(query, &filters, true)
.unwrap();
assert_eq!(hits.iter().map(|hit| hit.seq).collect::<Vec<_>>(), vec![0]);
let explain = explain.unwrap();
assert!(
explain.prefilter.is_some(),
"tool-argument {match_mode:?} search should prefilter raw tool-call JSON: {explain:?}"
);
assert!(
explain.corpus >= 101,
"corpus is the structural tool-call denominator: {explain:?}"
);
let candidates = explain.candidates.unwrap();
assert!(
candidates >= hits.len() as i64 && candidates <= explain.corpus,
"candidates are the prefilter superset of the hits: {explain:?}"
);
let summary = explain.summary(true);
assert!(
!summary.contains("low selectivity"),
"one hit in a hundred is not low selectivity: {summary}"
);
}
db.conn
.execute(
"update messages set content = ?1 where seq = 0",
[crate::util::tool_call_message_content(
"exec_command",
serde_json::json!({"cmd": "quoted \"target\" and slash \\ value"}),
)],
)
.unwrap();
for (query, expected, expect_prefilter) in [
("quoted \"target\"", vec![0], true),
(r#"quoted \"target\""#, Vec::new(), true),
("slash \\ value", vec![0], true),
("d\\ ", Vec::new(), false),
] {
let filters = MessageFilters {
field: Some(SearchField::ToolArgument),
argument_path: Some("/cmd".to_string()),
..Default::default()
};
let (hits, explain) = db
.search_messages_with_explain(query, &filters, true)
.unwrap();
assert_eq!(
hits.iter().map(|hit| hit.seq).collect::<Vec<_>>(),
expected,
"JSON escaping must not create a trigram false negative for {query:?}"
);
assert_eq!(
explain.unwrap().prefilter.is_some(),
expect_prefilter,
"only JSON-stable required trigrams may prefilter {query:?}"
);
}
db.conn
.execute(
"update messages set content = ?1 where seq = 0",
[crate::util::tool_call_message_content(
"exec_command",
serde_json::json!({"cmd": "cargo test"}),
)],
)
.unwrap();
db.conn
.execute(
"insert into messages (
session_id, provider, seq, role, tool_name, kind, content
) values ('claude:s1', 'claude', 500, 'tool', 'exec_command', 'tool_call', ?1)",
[crate::util::tool_call_message_content(
"exec_command",
serde_json::json!({"cmd": "incremental trigram target"}),
)],
)
.unwrap();
let incremental_filters = MessageFilters {
field: Some(SearchField::ToolArgument),
argument_path: Some("/cmd".to_string()),
..Default::default()
};
let find_incremental = |query: &str| {
db.search_messages(query, &incremental_filters)
.unwrap()
.into_iter()
.map(|hit| hit.seq)
.collect::<Vec<_>>()
};
assert_eq!(find_incremental("incremental trigram target"), vec![500]);
db.conn
.execute(
"update messages set content = ?1 where seq = 500",
[crate::util::tool_call_message_content(
"exec_command",
serde_json::json!({"cmd": "replacement trigram target"}),
)],
)
.unwrap();
assert!(find_incremental("incremental trigram target").is_empty());
assert_eq!(find_incremental("replacement trigram target"), vec![500]);
db.conn
.execute("delete from messages where seq = 500", [])
.unwrap();
assert!(find_incremental("replacement trigram target").is_empty());
let fuzzy_filters = MessageFilters {
field: Some(SearchField::ToolArgument),
argument_path: Some("/cmd".to_string()),
match_mode: MessageSearchMode::Fuzzy,
limit: 5,
..Default::default()
};
let (fuzzy, explain) = db
.search_messages_with_explain("crgo tst", &fuzzy_filters, true)
.unwrap();
assert_eq!(fuzzy.len(), 1);
assert_eq!(fuzzy[0].seq, 0);
let explain = explain.unwrap();
assert_eq!(
explain.prefilter_skipped.as_deref(),
Some("complete filtered corpus scored, retaining at most min(offset + limit, corpus) ranked rows")
);
assert!(
explain.candidates.unwrap() < explain.corpus,
"only matching tool-argument projections should count as fuzzy candidates: {explain:?}"
);
let plan = db
.conn
.prepare(
"explain query plan select session_id, seq from messages
where kind = 'tool_call' order by session_id, seq",
)
.unwrap()
.query_map([], |row| row.get::<_, String>(3))
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap()
.join("\n");
assert!(plan.contains("idx_messages_tool_calls"), "{plan}");
let error = db
.search_messages(
"cargo",
&MessageFilters {
field: Some(SearchField::ToolArgument),
argument_path: Some("cmd".to_string()),
..Default::default()
},
)
.unwrap_err()
.to_string();
assert!(error.contains("RFC 6901"));
}
#[test]
fn tool_argument_trigram_prefilter_preserves_direct_scan_order_and_pages() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
seed_messages(&db, &[("tool", "placeholder")]);
db.conn
.execute_batch(
r#"update messages set kind = 'tool_call', tool_name = 'exec_command',
content = '{"args":{"cmd":"cargo test --workspace"},"kind":"tool_call","tool_name":"exec_command"}'
where seq = 0;
with recursive n(value) as (
values(1) union all select value + 1 from n where value < 250
)
insert into messages (
session_id, provider, seq, role, tool_name, kind, content
)
select 'claude:s1', 'claude', value, 'tool', 'exec_command', 'tool_call',
case when value in (70, 140, 210)
then printf('{"args":{"cmd":"cargo test target %d"},"kind":"tool_call","tool_name":"exec_command"}', value)
else printf('{"args":{"cmd":"unrelated payload %d"},"kind":"tool_call","tool_name":"exec_command"}', value)
end
from n;"#,
)
.unwrap();
for (query, mode) in [
("cargo test", MessageSearchMode::Literal),
(r"cargo t.st", MessageSearchMode::Regex),
] {
for (limit, offset) in [(0, 0), (2, 0), (2, 1), (1, 3)] {
let filters = MessageFilters {
field: Some(SearchField::ToolArgument),
argument_path: Some("/cmd".to_string()),
match_mode: mode,
limit,
offset,
..Default::default()
};
let accelerated = db.search_messages(query, &filters).unwrap();
let predicate = match mode {
MessageSearchMode::Literal => {
"unicode_lower_contains(rust_json_pointer('/cmd', m.content), ?1)"
}
MessageSearchMode::Regex => {
"rust_regexp(?1, rust_json_pointer('/cmd', m.content))"
}
MessageSearchMode::Fuzzy => unreachable!(),
};
let mut sql = format!(
"select m.session_id, m.seq from messages m \
where m.kind = 'tool_call' and {predicate} \
order by m.session_id, m.seq"
);
if limit > 0 {
sql.push_str(" limit ?2 offset ?3");
} else if offset > 0 {
sql.push_str(" limit -1 offset ?3");
}
let query_arg = if mode == MessageSearchMode::Literal {
fold_caseless(query)
} else {
query.to_string()
};
let mut statement = db.conn.prepare(&sql).unwrap();
let direct = if limit > 0 {
statement
.query_map(params![query_arg, limit as i64, offset as i64], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
})
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap()
} else if offset > 0 {
statement
.query_map(params![query_arg, limit as i64, offset as i64], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
})
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap()
} else {
statement
.query_map([query_arg], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
})
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap()
};
assert_eq!(
hit_keys(accelerated),
direct,
"accelerated {mode:?} page limit={limit} offset={offset} must equal the independent direct projection scan"
);
}
}
}
#[test]
#[ignore = "deterministic release-mode performance measurement"]
fn tool_argument_trigram_prefilter_benchmark() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
seed_messages(&db, &[("tool", "placeholder")]);
let tx = db.conn.unchecked_transaction().unwrap();
{
let mut insert = tx
.prepare(
"insert into messages (session_id, provider, seq, role, tool_name, kind, content) \
values ('claude:s1', 'claude', ?1, 'tool', 'exec_command', 'tool_call', ?2)",
)
.unwrap();
for seq in 1..=25_000_i64 {
let cmd = if seq % 5_000 == 0 {
format!("selective trigram benchmark target {seq}")
} else {
format!("ordinary deterministic payload {seq} with JSON parsing padding")
};
insert
.execute(params![
seq,
crate::util::tool_call_message_content(
"exec_command",
serde_json::json!({"cmd": cmd})
)
])
.unwrap();
}
}
tx.commit().unwrap();
let filters = MessageFilters {
field: Some(SearchField::ToolArgument),
argument_path: Some("/cmd".to_string()),
..Default::default()
};
let query = "selective trigram benchmark target";
let accelerated = || db.search_messages(query, &filters).unwrap();
let direct = || {
let mut statement = db
.conn
.prepare(
"select m.session_id, m.seq from messages m \
where m.kind = 'tool_call' and \
unicode_lower_contains(rust_json_pointer('/cmd', m.content), ?1) \
order by m.session_id, m.seq",
)
.unwrap();
statement
.query_map([fold_caseless(query)], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
})
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap()
};
let accelerated_keys = hit_keys(accelerated());
let direct_keys = direct();
assert_eq!(accelerated_keys, direct_keys);
let digest = crate::hashing::sha256(&serde_json::to_vec(&direct_keys).unwrap());
let mut accelerated_us = Vec::new();
let mut direct_us = Vec::new();
for _ in 0..7 {
let started = Instant::now();
assert_eq!(hit_keys(accelerated()), direct_keys);
accelerated_us.push(started.elapsed().as_micros());
let started = Instant::now();
assert_eq!(direct(), direct_keys);
direct_us.push(started.elapsed().as_micros());
}
accelerated_us.sort_unstable();
direct_us.sort_unstable();
let (_, explain) = db
.search_messages_with_explain(query, &filters, true)
.unwrap();
let explain = explain.unwrap();
eprintln!(
"tool_argument_prefilter_benchmark rows={} candidates={} hits={} accelerated_median_us={} accelerated_p95_us={} direct_median_us={} direct_p95_us={} digest=sha256:{}",
explain.corpus,
explain.candidates.unwrap(),
direct_keys.len(),
accelerated_us[accelerated_us.len() / 2],
accelerated_us[accelerated_us.len() - 1],
direct_us[direct_us.len() / 2],
direct_us[direct_us.len() - 1],
digest
);
}
#[test]
fn fuzzy_tool_argument_search_uses_workers_without_changing_scores_or_order() {
let root = tempfile::tempdir().unwrap();
let seed = |db: &Db| {
seed_messages(db, &[("tool", "placeholder")]);
let tx = db.conn.unchecked_transaction().unwrap();
{
let mut insert = tx
.prepare(
"insert into messages (
session_id, provider, seq, role, tool_name, kind, content
) values ('claude:s1', 'claude', ?1, 'tool', 'exec_command',
'tool_call', ?2)",
)
.unwrap();
for seq in 1..=1_100_i64 {
let cmd = if seq % 137 == 0 {
format!("cargo test workspace target {seq}")
} else {
format!("ordinary tool argument payload {seq}")
};
insert
.execute(params![
seq,
crate::util::tool_call_message_content(
"exec_command",
serde_json::json!({"cmd": cmd})
)
])
.unwrap();
}
}
tx.commit().unwrap();
};
let one = Db::open_with_threads(
&root.path().join("one.db"),
TEST_BUSY_TIMEOUT_MS,
NonZeroUsize::new(1).unwrap(),
)
.unwrap();
let four = Db::open_with_threads(
&root.path().join("four.db"),
TEST_BUSY_TIMEOUT_MS,
NonZeroUsize::new(4).unwrap(),
)
.unwrap();
seed(&one);
seed(&four);
let filters = MessageFilters {
field: Some(SearchField::ToolArgument),
argument_path: Some("/cmd".to_string()),
match_mode: MessageSearchMode::Fuzzy,
limit: 20,
..Default::default()
};
let identity = |hits: Vec<MessageHit>| {
hits.into_iter()
.map(|hit| (hit.session_id, hit.seq, hit.fuzzy_score))
.collect::<Vec<_>>()
};
assert_eq!(one.worker_pool_build_count(), 0);
assert_eq!(four.worker_pool_build_count(), 0);
let expected = identity(one.search_messages("crgo tst wrkspce", &filters).unwrap());
let actual = identity(four.search_messages("crgo tst wrkspce", &filters).unwrap());
assert_eq!(actual, expected);
assert_eq!(one.worker_pool_build_count(), 1);
assert_eq!(four.worker_pool_build_count(), 1);
}
#[test]
#[ignore = "deterministic release-mode derived fuzzy performance measurement"]
fn fuzzy_tool_argument_parallel_scoring_benchmark() {
let root = tempfile::tempdir().unwrap();
let path = root.path().join("index.db");
{
let db =
Db::open_with_threads(&path, TEST_BUSY_TIMEOUT_MS, NonZeroUsize::new(1).unwrap())
.unwrap();
seed_messages(&db, &[("tool", "placeholder")]);
let tx = db.conn.unchecked_transaction().unwrap();
{
let mut insert = tx
.prepare(
"insert into messages (
session_id, provider, seq, role, tool_name, kind, content
) values ('claude:s1', 'claude', ?1, 'tool', 'exec_command',
'tool_call', ?2)",
)
.unwrap();
for seq in 1..=30_000_i64 {
let cmd = if seq % 3_000 == 0 {
format!("cargo test workspace target {seq}")
} else {
format!("ordinary deterministic tool argument payload {seq}")
};
insert
.execute(params![
seq,
crate::util::tool_call_message_content(
"exec_command",
serde_json::json!({"cmd": cmd})
)
])
.unwrap();
}
}
tx.commit().unwrap();
}
let filters = MessageFilters {
field: Some(SearchField::ToolArgument),
argument_path: Some("/cmd".to_string()),
match_mode: MessageSearchMode::Fuzzy,
limit: 50,
..Default::default()
};
let measure = |threads| {
let db = Db::open_with_threads(
&path,
TEST_BUSY_TIMEOUT_MS,
NonZeroUsize::new(threads).unwrap(),
)
.unwrap();
let search = || db.search_messages("crgo tst wrkspce", &filters).unwrap();
let expected = serde_json::to_vec(&search()).unwrap();
let digest = crate::hashing::sha256(&expected);
let mut elapsed = Vec::new();
for _ in 0..7 {
let started = Instant::now();
assert_eq!(serde_json::to_vec(&search()).unwrap(), expected);
elapsed.push(started.elapsed().as_micros());
}
elapsed.sort_unstable();
(
elapsed[elapsed.len() / 2],
elapsed[elapsed.len() - 1],
digest,
)
};
let one = measure(1);
let four = measure(4);
assert_eq!(one.2, four.2);
eprintln!(
"tool_argument_fuzzy_parallel_benchmark rows=30000 one_worker_median_us={} one_worker_p95_us={} four_worker_median_us={} four_worker_p95_us={} digest=sha256:{}",
one.0, one.1, four.0, four.1, one.2
);
}
#[test]
fn message_search_three_modes_by_three_fields_share_one_result_contract() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
seed_messages(
&db,
&[
("tool", "placeholder"),
("tool", "distractor"),
("tool", "second target"),
],
);
db.conn
.execute_batch(
r#"update messages set
kind = 'tool_call', tool_name = 'exec_command',
content = '{"args":{"cmd":"cargo test --workspace"},"kind":"tool_call","tool_name":"exec_command"}'
where seq = 0;
update messages set
kind = 'tool_call', tool_name = 'read_file',
content = '{"args":{"cmd":"open notes.md"},"kind":"tool_call","tool_name":"read_file"}'
where seq = 1;
update messages set
kind = 'tool_call', tool_name = 'exec_command',
content = '{"args":{"cmd":"cargo test --workspace"},"kind":"tool_call","tool_name":"exec_command"}'
where seq = 2;"#,
)
.unwrap();
let cases = [
(
SearchField::Content,
MessageSearchMode::Literal,
"cargo test",
),
(
SearchField::Content,
MessageSearchMode::Regex,
r"cargo\s+test",
),
(SearchField::Content, MessageSearchMode::Fuzzy, "crgo tst"),
(SearchField::ToolName, MessageSearchMode::Literal, "exec"),
(SearchField::ToolName, MessageSearchMode::Regex, r"^exec_"),
(SearchField::ToolName, MessageSearchMode::Fuzzy, "excmd"),
(
SearchField::ToolArgument,
MessageSearchMode::Literal,
"cargo test",
),
(
SearchField::ToolArgument,
MessageSearchMode::Regex,
r"cargo\s+test",
),
(
SearchField::ToolArgument,
MessageSearchMode::Fuzzy,
"crgo tst",
),
];
for (field, match_mode, query) in cases {
let (hits, explain) = db
.search_messages_with_explain(
query,
&MessageFilters {
kinds: Some(vec![crate::models::MessageKind::ToolCall]),
field: Some(field),
argument_path: (field == SearchField::ToolArgument)
.then(|| "/cmd".to_string()),
match_mode,
limit: 10,
..Default::default()
},
true,
)
.unwrap_or_else(|error| panic!("{field:?}/{match_mode:?}: {error:#}"));
if match_mode == MessageSearchMode::Fuzzy {
assert_eq!(
hits[0].fuzzy_score, hits[1].fuzzy_score,
"identical projected values must exercise the stable identity tie-break"
);
}
assert_eq!(
hit_keys(hits),
vec![("claude:s1".into(), 0), ("claude:s1".into(), 2)],
"{field:?}/{match_mode:?}"
);
assert!(explain.is_some(), "{field:?}/{match_mode:?}");
let page = db
.search_messages(
query,
&MessageFilters {
kinds: Some(vec![crate::models::MessageKind::ToolCall]),
field: Some(field),
argument_path: (field == SearchField::ToolArgument)
.then(|| "/cmd".to_string()),
match_mode,
limit: 1,
offset: 1,
..Default::default()
},
)
.unwrap_or_else(|error| panic!("paged {field:?}/{match_mode:?}: {error:#}"));
assert_eq!(
hit_keys(page),
vec![("claude:s1".into(), 2)],
"paged {field:?}/{match_mode:?}"
);
}
}
#[test]
fn non_fuzzy_message_batches_match_materialized_search_across_fields_and_offsets() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
seed_messages(
&db,
&[
("user", "alpha needle"),
("tool", "placeholder"),
("user", "omega needle"),
],
);
db.conn
.execute(
"update messages
set kind = 'tool_call',
tool_name = 'exec_command',
content = '{\"args\":{\"cmd\":\"cargo test needle\"}}'
where seq = 1",
[],
)
.unwrap();
for (field, match_mode, query, argument_path) in [
(
SearchField::Content,
MessageSearchMode::Literal,
"needle",
None,
),
(
SearchField::Content,
MessageSearchMode::Regex,
"n[e]+dle",
None,
),
(SearchField::Content, MessageSearchMode::Literal, "", None),
(
SearchField::ToolName,
MessageSearchMode::Literal,
"exec",
None,
),
(
SearchField::ToolArgument,
MessageSearchMode::Literal,
"cargo test",
Some("/cmd".to_string()),
),
] {
let filters = MessageFilters {
field: Some(field),
argument_path,
match_mode,
limit: 0,
offset: 1,
..Default::default()
};
let materialized = db
.search_messages_ordered(query, &filters, MessageOrder::OldestFirst)
.unwrap();
for batch_size in [1, 2, 17] {
let mut batched = Vec::new();
let outcome = db
.visit_messages_in_batches(
query,
&filters,
true,
MessageOrder::OldestFirst,
NonZeroUsize::new(batch_size).unwrap(),
|batch| {
assert!(batch.len() <= batch_size);
batched.extend(batch);
Ok(MessageBatchControl::Continue)
},
)
.unwrap_or_else(|error| {
panic!("{field:?}/{match_mode:?}/batch={batch_size}: {error:#}")
});
assert!(outcome.exhausted);
assert_eq!(outcome.rows_visited, materialized.len());
assert!(outcome.explain.is_some());
assert_eq!(
hit_keys(batched),
hit_keys(materialized.clone()),
"{field:?}/{match_mode:?}/batch={batch_size}"
);
}
}
}
#[test]
fn newest_edge_batches_stream_the_post_offset_selection_in_chronological_order() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
seed_messages(
&db,
&[
("user", "needle zero"),
("user", "needle one"),
("user", "needle two"),
("user", "needle three"),
("user", "needle four"),
],
);
let filters = MessageFilters {
session_id: Some("claude:s1".into()),
limit: 0,
offset: 2,
..Default::default()
};
let (mut materialized, materialized_explain) = db
.search_messages_with_explain_order("needle", &filters, true, MessageOrder::NewestFirst)
.unwrap();
materialized.reverse();
for batch_size in [1, 2, 17] {
let mut batched = Vec::new();
let outcome = db
.visit_messages_in_batches(
"needle",
&filters,
true,
MessageOrder::NewestFirst,
NonZeroUsize::new(batch_size).unwrap(),
|batch| {
batched.extend(batch);
Ok(MessageBatchControl::Continue)
},
)
.unwrap();
assert!(outcome.exhausted);
assert_eq!(hit_keys(batched), hit_keys(materialized.clone()));
assert_eq!(
outcome.explain.as_ref().map(|explain| explain.corpus),
materialized_explain.as_ref().map(|explain| explain.corpus)
);
assert_eq!(
outcome
.explain
.as_ref()
.and_then(|explain| explain.candidates),
materialized_explain
.as_ref()
.and_then(|explain| explain.candidates)
);
}
}
#[test]
fn non_fuzzy_message_batches_hold_one_snapshot_and_stop_without_draining() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("index.db");
let db = Db::open(&path).unwrap();
let writer = Db::open(&path).unwrap();
seed_messages(
&db,
&[
("user", "needle zero"),
("user", "needle one"),
("user", "needle two"),
],
);
let filters = MessageFilters {
limit: 0,
..Default::default()
};
let mut seen = Vec::new();
let snapshot = db
.visit_messages_in_batches(
"needle",
&filters,
false,
MessageOrder::OldestFirst,
NonZeroUsize::new(1).unwrap(),
|batch| {
seen.extend(batch.into_iter().map(|hit| hit.seq));
if seen.len() == 1 {
writer
.conn
.execute(
"insert into messages (
session_id, provider, seq, role, content
) values ('claude:s1', 'claude', 3, 'user', 'needle late')",
[],
)
.unwrap();
}
Ok(MessageBatchControl::Continue)
},
)
.unwrap();
assert!(snapshot.exhausted);
assert_eq!(seen, [0, 1, 2]);
assert_eq!(db.search_messages("needle", &filters).unwrap().len(), 4);
let mut delivered = 0;
let stopped = db
.visit_messages_in_batches(
"needle",
&filters,
false,
MessageOrder::OldestFirst,
NonZeroUsize::new(2).unwrap(),
|batch| {
delivered += batch.len();
Ok(MessageBatchControl::Stop)
},
)
.unwrap();
assert_eq!(delivered, 2);
assert_eq!(stopped.rows_visited, 2);
assert!(!stopped.exhausted);
assert!(stopped.explain.is_none());
let checkpoint_busy: i64 = writer
.conn
.query_row("pragma wal_checkpoint(passive)", [], |row| row.get(0))
.expect("early stop drops the read transaction without draining unread rows");
assert_eq!(checkpoint_busy, 0);
}
#[test]
fn non_fuzzy_message_batches_finish_legacy_index_maintenance_before_the_snapshot() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
enable_v3_custom_trigram_compatibility(&db);
seed_messages(&db, &[("user", "socket failed with ECONNRESET today")]);
let filters = MessageFilters {
match_mode: MessageSearchMode::Regex,
..Default::default()
};
let mut seen = Vec::new();
let outcome = db
.visit_messages_in_batches(
"ECONNRESET",
&filters,
false,
MessageOrder::OldestFirst,
NonZeroUsize::new(1).unwrap(),
|batch| {
seen.extend(batch);
Ok(MessageBatchControl::Continue)
},
)
.unwrap();
assert!(outcome.exhausted);
assert_eq!(seen.len(), 1);
assert!(seen[0].content.contains("ECONNRESET"));
assert!(
crate::trigram_index::base_max_id(&db.conn).unwrap() > 0,
"legacy trigram maintenance must finish before the read transaction starts"
);
let error = db
.visit_messages_in_batches(
"ECONNRESET",
&filters,
true,
MessageOrder::OldestFirst,
NonZeroUsize::new(1).unwrap(),
|_| Ok(MessageBatchControl::Continue),
)
.unwrap_err()
.to_string();
assert!(
error.contains("planner counts and results share one snapshot"),
"{error}"
);
}
#[test]
fn message_search_validation_is_identical_for_all_three_fields() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
for field in [
SearchField::Content,
SearchField::ToolName,
SearchField::ToolArgument,
] {
let filters = |match_mode, limit, offset| MessageFilters {
field: Some(field),
argument_path: (field == SearchField::ToolArgument).then(|| "/cmd".into()),
match_mode,
limit,
offset,
..Default::default()
};
let malformed = db
.search_messages("[", &filters(MessageSearchMode::Regex, 10, 0))
.unwrap_err()
.to_string();
assert!(
malformed.contains("invalid regex"),
"{field:?}: {malformed}"
);
let short = db
.search_messages("ab", &filters(MessageSearchMode::Fuzzy, 10, 0))
.unwrap_err()
.to_string();
assert!(
short.contains("at least 3 characters"),
"{field:?}: {short}"
);
let unlimited = db
.search_messages("abc", &filters(MessageSearchMode::Fuzzy, 0, 0))
.unwrap_err()
.to_string();
assert!(
unlimited.contains("finite non-zero limit"),
"{field:?}: {unlimited}"
);
let large_offset =
db.search_messages("abc", &filters(MessageSearchMode::Fuzzy, 2, 9_999));
assert!(
large_offset.is_ok(),
"{field:?}: finite fuzzy offsets have no arbitrary result window: {large_offset:?}"
);
}
}
#[test]
fn tool_argument_search_handles_json_shapes_unicode_punctuation_and_large_values() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
seed_messages(
&db,
&[
("tool", "malformed"),
("tool", "missing args"),
("tool", "scalar args"),
("tool", "null"),
("tool", "array"),
("tool", "nested"),
("tool", "large"),
],
);
let large = format!("{} cargo test --workspace", "x".repeat(128 * 1024));
let rows = [
"{malformed".to_string(),
r#"{"kind":"tool_call"}"#.to_string(),
r#"{"args":"scalar","kind":"tool_call"}"#.to_string(),
r#"{"args":{"cmd":null},"kind":"tool_call"}"#.to_string(),
r#"{"args":{"cmd":["café","C++","--path"]},"kind":"tool_call"}"#.to_string(),
r#"{"args":{"request":{"path":"src/quoted file.rs"}},"kind":"tool_call"}"#.to_string(),
serde_json::json!({"args": {"cmd": large}, "kind": "tool_call"}).to_string(),
];
for (seq, content) in rows.iter().enumerate() {
db.conn
.execute(
"update messages set kind='tool_call', tool_name='ÄTool::C++', content=?1
where seq=?2",
params![content, seq as i64],
)
.unwrap();
}
let argument = |path: &str, mode| MessageFilters {
field: Some(SearchField::ToolArgument),
argument_path: Some(path.into()),
match_mode: mode,
limit: 10,
..Default::default()
};
assert_eq!(
hit_keys(
db.search_messages("café", &argument("/cmd", MessageSearchMode::Literal))
.unwrap()
),
vec![("claude:s1".into(), 4)]
);
assert_eq!(
hit_keys(
db.search_messages(
r#"C\+\+.*--path"#,
&argument("/cmd", MessageSearchMode::Regex),
)
.unwrap()
),
vec![("claude:s1".into(), 4)]
);
assert_eq!(
hit_keys(
db.search_messages(
"quoted file.rs",
&argument("/request/path", MessageSearchMode::Literal),
)
.unwrap()
),
vec![("claude:s1".into(), 5)]
);
assert_eq!(
hit_keys(
db.search_messages("null", &argument("/cmd", MessageSearchMode::Literal))
.unwrap()
),
vec![("claude:s1".into(), 3)]
);
assert_eq!(
hit_keys(
db.search_messages(
"cargo test --workspace",
&argument("/cmd", MessageSearchMode::Literal),
)
.unwrap()
),
vec![("claude:s1".into(), 6)]
);
assert!(
db.search_messages("scalar", &argument("/cmd", MessageSearchMode::Literal))
.unwrap()
.is_empty(),
"a pointer below scalar args and malformed/missing envelopes must project NULL"
);
for mode in [MessageSearchMode::Literal, MessageSearchMode::Regex] {
let query = if mode == MessageSearchMode::Literal {
"ätool::c++"
} else {
r#"(?i)^ätool::c\+\+$"#
};
assert_eq!(
db.search_messages(
query,
&MessageFilters {
field: Some(SearchField::ToolName),
match_mode: mode,
limit: 10,
..Default::default()
},
)
.unwrap()
.len(),
7,
"{mode:?} must preserve Unicode and punctuation in tool names"
);
}
}
#[test]
fn fuzzy_content_reports_complete_filtered_corpus_scoring() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute(
"insert into sessions (
id, provider, provider_session_id, preview_text, source_path,
parse_version, discovery_source
) values ('s1','claude','s1','','/p','1','test')",
[],
)
.unwrap();
let tx = db.conn.unchecked_transaction().unwrap();
{
let mut insert = tx
.prepare(
"insert into messages(session_id, provider, seq, role, content)
values ('s1','claude',?1,'user',?2)",
)
.unwrap();
for seq in 0..1_205_i64 {
insert
.execute(params![seq, format!("abc candidate {seq:04}")])
.unwrap();
}
}
tx.commit().unwrap();
let (hits, explain) = db
.search_messages_with_explain(
"abc",
&MessageFilters {
match_mode: MessageSearchMode::Fuzzy,
limit: 1,
..Default::default()
},
true,
)
.unwrap();
assert_eq!(hits.len(), 1);
let explain = explain.unwrap();
assert_eq!(
explain.prefilter_skipped.as_deref(),
Some("complete filtered corpus scored, retaining at most min(offset + limit, corpus) ranked rows")
);
assert_eq!(explain.candidates, Some(1_205));
assert_eq!(explain.corpus, 1_205);
}
#[test]
fn v4_prefilter_work_stays_selective_across_one_two_four_x_corpora() {
for corpus in [200_i64, 400, 800] {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute(
"insert into sessions (
id, provider, provider_session_id, preview_text, source_path,
parse_version, discovery_source
) values ('s1','claude','s1','','/p','1','test')",
[],
)
.unwrap();
let tx = db.conn.unchecked_transaction().unwrap();
{
let mut insert = tx
.prepare(
"insert into messages(session_id, provider, seq, role, content)
values ('s1','claude',?1,'user',?2)",
)
.unwrap();
insert
.execute(params![0_i64, "rare fuzzy anchor C++ --path"])
.unwrap();
for seq in 1..corpus {
insert
.execute(params![seq, format!("unrelated filler row {seq:04}")])
.unwrap();
}
}
tx.commit().unwrap();
for (mode, query) in [
(MessageSearchMode::Literal, "fuzzy anchor C++"),
(MessageSearchMode::Regex, r"fuzzy\s+anchor\s+C\+\+"),
(MessageSearchMode::Fuzzy, "fzzy anchr C++"),
] {
let (hits, explain) = db
.search_messages_with_explain(
query,
&MessageFilters {
match_mode: mode,
limit: 10,
..Default::default()
},
true,
)
.unwrap();
assert_eq!(hit_keys(hits), vec![("s1".into(), 0)], "{corpus}/{mode:?}");
let explain = explain.unwrap();
assert_eq!(explain.corpus, corpus, "{corpus}/{mode:?}");
assert!(
explain.candidates.is_some_and(|count| count < corpus / 4),
"candidate work must stay selective as the unrelated corpus scales: \
corpus={corpus} mode={mode:?} explain={explain:?}"
);
}
}
}
#[test]
fn message_kind_call_id_and_offset_share_one_query_contract() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
seed_messages(&db, &[("tool", "first call"), ("tool", "first result")]);
db.conn
.execute_batch(
"update messages set kind = 'tool_call', tool_call_id = 'call-1' where seq = 0;
update messages set kind = 'tool_result', tool_call_id = 'call-1' where seq = 1;",
)
.unwrap();
let calls = db
.search_messages(
"",
&MessageFilters {
kinds: Some(vec![crate::models::MessageKind::ToolCall]),
..Default::default()
},
)
.unwrap();
assert_eq!(calls.len(), 1);
assert_eq!(calls[0].kind, crate::models::MessageKind::ToolCall);
assert_eq!(calls[0].tool_call_id.as_deref(), Some("call-1"));
let page = db
.search_messages(
"",
&MessageFilters {
limit: 1,
offset: 1,
..Default::default()
},
)
.unwrap();
assert_eq!(page.len(), 1);
assert_eq!(page[0].seq, 1);
assert_eq!(page[0].kind, crate::models::MessageKind::ToolResult);
}
#[test]
fn message_context_rejects_negative_window_bounds() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let error = db
.message_context("missing", 0, -1, 0)
.unwrap_err()
.to_string();
assert!(error.contains("before must be non-negative"), "{error}");
let error = db
.message_context("missing", 0, 0, -1)
.unwrap_err()
.to_string();
assert!(error.contains("after must be non-negative"), "{error}");
}
#[test]
fn query_timeout_interrupts_expensive_work_and_resets_the_handler() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let error = with_sqlite_query_timeout(
&db.conn,
std::num::NonZeroU64::new(1),
"read-only SQL query",
"increase timeout_ms or pass 0 for no timeout",
|| {
db.conn
.query_row(
"with recursive n(x) as (values(0) union all select x + 1 from n where x < 100000000) select sum(x) from n",
[],
|row| row.get::<_, i64>(0),
)
.map_err(Into::into)
},
)
.unwrap_err()
.to_string();
assert!(error.contains("timed out after 1 ms"), "{error}");
assert_eq!(
db.conn
.query_row("select 1", [], |row| row.get::<_, i64>(0))
.unwrap(),
1,
"the RAII guard must clear SQLite's progress handler after interruption"
);
}
#[test]
fn persistent_interrupt_cancels_a_statement_started_after_the_flag_is_set() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let cancellation = Arc::new(AtomicBool::new(false));
db.interrupt_while(Arc::clone(&cancellation))
.expect("arming cancellation is a precondition of this test");
cancellation.store(true, Ordering::Release);
let error = db
.conn
.query_row(
"with recursive n(value) as (
values(1)
union all
select value + 1 from n where value < 1000000
)
select sum(value) from n",
[],
|row| row.get::<_, i64>(0),
)
.unwrap_err();
assert!(matches!(
error,
rusqlite::Error::SqliteFailure(inner, _)
if inner.code == rusqlite::ErrorCode::OperationInterrupted
));
}
#[test]
fn query_cancellation_registered_after_cancel_still_interrupts_the_next_statement() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let cancellation = QueryCancellation::new();
cancellation.cancel();
db.install_query_cancellation(&cancellation)
.expect("arming cancellation is a precondition of this test");
let error = db
.conn
.query_row(
"with recursive n(value) as (
values(1)
union all
select value + 1 from n where value < 1000000
)
select sum(value) from n",
[],
|row| row.get::<_, i64>(0),
)
.unwrap_err();
assert!(matches!(
error,
rusqlite::Error::SqliteFailure(inner, _)
if inner.code == rusqlite::ErrorCode::OperationInterrupted
));
}
#[test]
fn message_batch_allocation_failure_names_the_unit_and_recovery() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
seed_messages(&db, &[("user", "needle")]);
let error = db
.visit_messages_in_batches(
"needle",
&MessageFilters {
limit: 0,
..Default::default()
},
false,
MessageOrder::OldestFirst,
NonZeroUsize::new(usize::MAX).unwrap(),
|_| Ok(MessageBatchControl::Continue),
)
.unwrap_err()
.to_string();
assert!(error.contains("batch_rows"));
assert!(error.contains(&usize::MAX.to_string()));
assert!(error.contains("pass a smaller positive row count"));
}
#[test]
fn query_timeout_never_panics_when_deadline_exceeds_platform_instant_range() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let result = with_sqlite_query_timeout(
&db.conn,
std::num::NonZeroU64::new(u64::MAX),
"read-only SQL query",
"increase timeout_ms or pass 0 for no timeout",
|| {
db.conn
.query_row("select 1", [], |row| row.get::<_, i64>(0))
.map_err(Into::into)
},
);
match result {
Ok(value) => assert_eq!(value, 1),
Err(error) => assert!(
error.to_string().contains("timeout_ms is too large"),
"{error}"
),
}
assert_eq!(
db.conn
.query_row("select 2", [], |row| row.get::<_, i64>(0))
.unwrap(),
2,
"an unrepresentable timeout must not leave a progress handler installed"
);
}
#[test]
fn message_context_windows_batch_preserves_anchor_order_and_empty_windows() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
seed_messages(
&db,
&[
("user", "zero"),
("assistant", "one"),
("user", "two"),
("assistant", "three"),
],
);
let anchors = vec![
("claude:s1".to_string(), 3),
("missing".to_string(), 10),
("claude:s1".to_string(), 0),
];
let windows = db.message_context_windows(&anchors, 1, 1).unwrap();
assert_eq!(windows.len(), 3);
assert_eq!(
windows[0].iter().map(|hit| hit.seq).collect::<Vec<_>>(),
vec![2, 3]
);
assert!(windows[1].is_empty());
assert_eq!(
windows[2].iter().map(|hit| hit.seq).collect::<Vec<_>>(),
vec![0, 1]
);
}
#[test]
fn session_time_profile_is_bounded_and_uses_typed_message_metadata() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
seed_messages(
&db,
&[
("user", "start"),
("tool", "call"),
("tool", "result"),
("assistant", "end"),
],
);
db.conn
.execute_batch(
"update messages set ts = '2026-07-10T10:00:00Z' where seq = 0;
update messages set ts = '2026-07-10T10:00:05Z', kind = 'tool_call' where seq = 1;
update messages set ts = '2026-07-10T10:00:20Z', kind = 'tool_result' where seq = 2;",
)
.unwrap();
let profile = db.session_time_profile("claude:s1").unwrap();
assert_eq!(profile.messages, 4);
assert_eq!(profile.timestamped_messages, 3);
assert_eq!(profile.undated_messages, 1);
assert_eq!(profile.observed_span_seconds, Some(20));
assert_eq!(profile.max_message_gap_seconds, Some(15));
assert_eq!((profile.tool_calls, profile.tool_results), (1, 1));
}
#[test]
fn session_time_profile_uses_intermediate_events_across_months() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
seed_messages(
&db,
&[
("user", "january"),
("assistant", "february"),
("user", "april"),
("assistant", "undated tail"),
],
);
db.conn
.execute_batch(
"update sessions
set created_at = '2026-04-30T00:00:00Z',
updated_at = '2026-04-30T00:00:01Z',
last_message_at = '2026-04-30T00:00:01Z'
where id = 'claude:s1';
update messages set ts = '2026-01-01T00:00:00.100Z' where seq = 0;
update messages set ts = '2026-02-01T00:00:00.200Z' where seq = 1;
update messages set ts = '2026-04-30T00:00:00.300Z' where seq = 2;
update messages set ts = null where seq = 3;",
)
.unwrap();
let profile = db.session_time_profile("claude:s1").unwrap();
assert_eq!(profile.timestamped_messages, 3);
assert_eq!(profile.undated_messages, 1);
assert_eq!(
profile.first_timestamp.unwrap().to_rfc3339(),
"2026-01-01T00:00:00.100+00:00"
);
assert_eq!(
profile.last_timestamp.unwrap().to_rfc3339(),
"2026-04-30T00:00:00.300+00:00"
);
assert_eq!(profile.observed_span_seconds, Some(10_281_600));
assert_eq!(profile.max_message_gap_seconds, Some(7_603_200));
}
#[test]
fn parser_health_uses_shared_provider_versions_and_consistent_totals() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.mark_schema_current().unwrap();
db.conn
.execute_batch(
"insert into sessions(id, provider, provider_session_id, preview_text, source_path, parse_version, parse_warning, discovery_source) values
('claude:current','claude','current','','/a','claude-v4',null,'jsonl'),
('claude:stale','claude','stale','','/b','claude-v1','old parser','jsonl');",
)
.unwrap();
db.conn
.execute(
"insert into sessions(id, provider, provider_session_id, preview_text, source_path, parse_version, discovery_source)
values ('codex:current','codex','current','','/c',?1,'jsonl')",
[crate::util::provider_parse_version(Provider::Codex)],
)
.unwrap();
let health = db.parser_health().unwrap();
assert!(health.schema_current);
assert_eq!(health.indexed_sessions, 3);
assert_eq!(health.current_sessions, 2);
assert_eq!(health.stale_sessions, 1);
assert_eq!(health.parse_warnings, 1);
let claude = health
.providers
.iter()
.find(|item| item.provider == Provider::Claude)
.unwrap();
assert_eq!(claude.expected_parse_version, "claude-v4");
assert_eq!((claude.current_sessions, claude.stale_sessions), (1, 1));
assert_eq!(
db.stale_session_sources().unwrap(),
vec![(Provider::Claude, "/b".to_string())]
);
}
#[test]
fn complete_parse_replaces_superseded_identity_for_the_same_source() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let source = std::path::Path::new("/fixture/changing-identity.jsonl");
let mut old = crate::util::minimal_record(Provider::ClaudeDesktop, source, String::new());
old.session.id = "claude-desktop:old".into();
old.session.provider_session_id = "old".into();
old.session.parse_version = "claude-desktop-local-agent-v1".into();
old.messages = vec![crate::models::Message {
seq: 0,
role: Role::User,
kind: crate::models::MessageKind::Conversation,
ts: None,
tool_name: None,
tool_call_id: None,
is_compaction: false,
content: "obsolete".into(),
provenance: crate::models::MessageProvenance::default(),
}];
db.upsert_session(&old, 1, 1).unwrap();
let mut current =
crate::util::minimal_record(Provider::ClaudeDesktop, source, String::new());
current.session.id = "claude-desktop:current".into();
current.session.provider_session_id = "current".into();
current.messages = vec![crate::models::Message {
seq: 0,
role: Role::User,
kind: crate::models::MessageKind::Conversation,
ts: None,
tool_name: None,
tool_call_id: None,
is_compaction: false,
content: "replacement".into(),
provenance: crate::models::MessageProvenance::default(),
}];
db.upsert_session(¤t, 2, 2).unwrap();
assert!(db.resolve_session_record("claude-desktop:old").is_err());
assert_eq!(
db.resolve_session_record("claude-desktop:current")
.unwrap()
.source_path,
source.to_string_lossy()
);
assert!(db
.search_messages("obsolete", &MessageFilters::default())
.unwrap()
.is_empty());
assert!(db
.is_file_current(
Provider::ClaudeDesktop,
&source.to_string_lossy(),
2,
2,
crate::util::provider_parse_version(Provider::ClaudeDesktop),
)
.unwrap());
}
#[test]
fn source_checkpoint_version_backfill_requires_a_matching_source_path() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let source = dir.path().join("session.jsonl");
let parsed = crate::util::minimal_record(Provider::Claude, &source, String::new());
db.upsert_session(&parsed, 10, 20).unwrap();
db.conn
.execute("update files_seen set parse_version = ''", [])
.unwrap();
db.backfill_source_parse_versions().unwrap();
assert!(db
.is_file_current(
Provider::Claude,
&source.to_string_lossy(),
10,
20,
crate::util::provider_parse_version(Provider::Claude),
)
.unwrap());
db.conn
.execute_batch(
"update sessions set source_path = '/different/source.jsonl';
update files_seen set parse_version = '';",
)
.unwrap();
db.backfill_source_parse_versions().unwrap();
assert!(!db
.is_file_current(
Provider::Claude,
&source.to_string_lossy(),
10,
20,
crate::util::provider_parse_version(Provider::Claude),
)
.unwrap());
}
#[test]
fn complete_parse_reconciles_verified_source_aliases() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let alias = std::path::Path::new("/fixture/alias/session.jsonl");
let canonical = std::path::Path::new("/fixture/canonical/session.jsonl");
let mut old = crate::util::minimal_record(Provider::Claude, alias, String::new());
old.session.id = "claude:old-alias".into();
old.session.provider_session_id = "old-alias".into();
db.upsert_session(&old, 1, 1).unwrap();
let mut current = crate::util::minimal_record(Provider::Claude, canonical, String::new());
current.session.id = "claude:current".into();
current.session.provider_session_id = "current".into();
db.upsert_session_reconciling_sources(
¤t,
2,
2,
&[alias.to_string_lossy().into_owned()],
true,
)
.unwrap();
assert!(db.resolve_session_record("claude:old-alias").is_err());
assert_eq!(db.indexed_source_identities().unwrap().len(), 1);
assert_eq!(
db.indexed_source_identities().unwrap()[0].1,
canonical.to_string_lossy()
);
}
#[test]
fn schema_backfill_flag_round_trips() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn.pragma_update(None, "user_version", 2).unwrap();
assert!(db.needs_backfill().unwrap());
db.mark_schema_current().unwrap();
assert!(
db.needs_backfill().unwrap(),
"parser rows are backfilled, but the pre-v4 FTS layout is not yet current"
);
assert_eq!(db.schema_version().unwrap(), PRE_FTS_PARSER_SCHEMA_VERSION);
db.migrate_message_search_schema_exclusive().unwrap();
assert!(!db.needs_backfill().unwrap());
assert_eq!(db.schema_version().unwrap(), SCHEMA_VERSION);
}
#[test]
fn mark_schema_current_preserves_the_v4_fts_gate_before_promoting_schema_5() {
let dir = tempfile::tempdir().unwrap();
for start in [0_i64, 1, 2, PRE_FTS_PARSER_SCHEMA_VERSION] {
let path = dir.path().join(format!("v{start}.db"));
let db = Db::open(&path).unwrap();
db.conn.pragma_update(None, "user_version", start).unwrap();
db.mark_schema_current().unwrap();
assert!(
db.schema_version().unwrap() < SCHEMA_VERSION,
"start={start}: mark_schema_current must not promote a pre-v{SCHEMA_VERSION} index to current",
);
}
let v4 = Db::open(&dir.path().join("v4.db")).unwrap();
v4.conn.pragma_update(None, "user_version", 4).unwrap();
v4.mark_schema_current().unwrap();
assert_eq!(v4.schema_version().unwrap(), SCHEMA_VERSION);
}
#[test]
fn clear_all_is_atomic_and_rolls_back_on_failure() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions (id, provider, provider_session_id, preview_text, source_path,
parse_version, discovery_source)
values ('s', 'claude', 's', '', '/s.jsonl', 'test', 'fixture');
insert into messages (session_id, provider, seq, role, kind, tool_name, content)
values ('s', 'claude', 0, 'user', 'message', null, 'keep me');",
)
.unwrap();
db.conn.execute_batch("drop table files_seen;").unwrap();
assert!(
db.clear_all().is_err(),
"clear_all should surface the failing delete"
);
let (sessions, messages): (i64, i64) = db
.conn
.query_row(
"select (select count(*) from sessions), (select count(*) from messages)",
[],
|row| Ok((row.get(0)?, row.get(1)?)),
)
.unwrap();
assert_eq!(
(sessions, messages),
(1, 1),
"clear_all must roll back all deletes when one fails"
);
}
#[test]
fn direct_db_open_self_enforces_v4_layout_after_triggers_dropped() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("index.db");
{
let db = Db::open(&path).unwrap();
db.conn
.execute_batch(
"insert into sessions (id, provider, provider_session_id, preview_text,
source_path, parse_version, discovery_source)
values ('s', 'claude', 's', '', '/s.jsonl', 'test', 'fixture');
insert into messages (session_id, provider, seq, role, kind, tool_name, content)
values ('s', 'claude', 0, 'user', 'message', null, 'existing-row');
drop trigger messages_ai;
drop trigger messages_ad;
drop trigger messages_au;",
)
.unwrap();
}
let db = Db::open(&path).unwrap();
assert!(
crate::indexer::current_schema_layout_problem(&db.conn)
.unwrap()
.is_none(),
"Db::open must self-enforce a consistent v4 layout"
);
let terms_before: i64 = db
.conn
.query_row("select count(*) from messages_trigram_terms", [], |row| {
row.get(0)
})
.unwrap();
db.conn
.execute_batch(
"insert into messages (session_id, provider, seq, role, kind, tool_name, content)
values ('s', 'claude', 1, 'user', 'message', null, 'freshtrigramrow');",
)
.unwrap();
let terms_after: i64 = db
.conn
.query_row("select count(*) from messages_trigram_terms", [], |row| {
row.get(0)
})
.unwrap();
assert!(
terms_after > terms_before,
"new insert must be maintained in messages_trigram (terms {terms_before} -> {terms_after})"
);
}
#[test]
fn open_drops_legacy_fts5_trigram_and_self_heals() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("index.db");
{
let db = Db::open(&path).unwrap();
db.conn
.execute_batch(
"drop trigger messages_ai;
drop trigger messages_ad;
drop trigger messages_au;
drop table messages_trigram_terms;
drop table messages_trigram_vocab;
drop table messages_trigram;
create virtual table messages_trigram using fts5(content, \
content='messages', content_rowid='id', tokenize='trigram', detail='none');",
)
.unwrap();
crate::fts::install_released_message_word_index(&db.conn).unwrap();
db.conn.pragma_update(None, "user_version", 3).unwrap();
}
let db = Db::open(&path).unwrap();
let legacy: i64 = db
.conn
.query_row(
"select count(*) from sqlite_master where name='messages_trigram'",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(legacy, 0, "legacy FTS5 messages_trigram dropped on open");
let custom: i64 = db
.conn
.query_row(
"select count(*) from sqlite_master where name='trigram_postings'",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(custom, 1, "custom trigram_postings present after open");
seed_messages(&db, &[("user", "an econnreset row")]);
let hits = db
.search_messages(
"econnreset",
&MessageFilters {
match_mode: MessageSearchMode::Regex,
..Default::default()
},
)
.unwrap();
assert_eq!(hits.len(), 1, "regex search works after the self-heal");
}
#[test]
fn messages_indexes_drop_redundant_singles_and_keep_composites() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("index.db");
let count_index = |db: &Db, name: &str| -> i64 {
db.conn
.query_row(
"select count(*) from sqlite_master where type='index' and name=?1",
[name],
|row| row.get(0),
)
.unwrap()
};
{
let db = Db::open(&path).unwrap();
db.conn
.execute_batch(
"create index if not exists idx_messages_session on messages(session_id);
create index if not exists idx_messages_role on messages(role);",
)
.unwrap();
assert_eq!(count_index(&db, "idx_messages_session"), 1, "precondition");
assert_eq!(count_index(&db, "idx_messages_role"), 1, "precondition");
}
let db = Db::open(&path).unwrap();
assert_eq!(
count_index(&db, "idx_messages_session"),
0,
"redundant (session_id) dropped"
);
assert_eq!(
count_index(&db, "idx_messages_role"),
0,
"redundant (role) dropped"
);
for idx in [
"idx_messages_session_seq",
"idx_messages_role_ts",
"idx_messages_ts",
] {
assert_eq!(count_index(&db, idx), 1, "{idx} must exist");
}
}
#[test]
fn hot_message_queries_use_indexes_not_full_scans() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions(id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values('claude:s1','claude','s1','','/x','claude-v1','jsonl');",
)
.unwrap();
{
let tx = db.conn.unchecked_transaction().unwrap();
{
let mut stmt = tx
.prepare(
"insert into messages(session_id, provider, seq, role, ts, content) \
values('claude:s1','claude',?1,?2,?3,?4)",
)
.unwrap();
for i in 0..2000i64 {
let role = if i % 7 == 0 { "user" } else { "assistant" };
let ts = format!("2026-06-{:02}T00:00:00+00:00", (i % 28) + 1);
stmt.execute(params![i, role, ts, format!("message number {i} alpha")])
.unwrap();
}
}
tx.commit().unwrap();
}
db.conn.execute_batch("analyze").unwrap();
let plan = |sql: &str| -> String {
let mut stmt = db
.conn
.prepare(&format!("explain query plan {sql}"))
.unwrap();
let rows = stmt.query_map([], |r| r.get::<_, String>(3)).unwrap();
rows.filter_map(Result::ok).collect::<Vec<_>>().join(" | ")
};
let p = plan(
"select m.id from messages_fts f join messages m on m.id = f.rowid \
where messages_fts match 'alpha'",
);
assert!(
p.contains("VIRTUAL TABLE INDEX"),
"content search must be driven by the messages_fts index: {p}"
);
assert!(
p.contains("USING INTEGER PRIMARY KEY") && !p.contains("SCAN m "),
"messages rows must be reached by rowid from the FTS matches, not scanned: {p}"
);
let p = plan("select content from messages where role = 'user' order by ts desc");
assert!(
p.contains("idx_messages_role_ts"),
"role/ts query must use the composite: {p}"
);
let p = plan(
"select content from messages where session_id = 'claude:s1' \
and seq between 10 and 20 order by seq",
);
assert!(
p.contains("idx_messages_session_seq"),
"session/seq query must use the composite: {p}"
);
let p = plan(
"select content from messages where provider = 'claude' \
order by session_id, seq limit 2",
);
assert!(
p.contains("idx_messages_provider_session_seq") && !p.contains("USE TEMP B-TREE"),
"provider-scoped pages must use the ordered provider composite without a sort: {p}"
);
}
#[test]
fn messages_fts_is_rebuilt_when_empty_but_messages_exist() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("index.db");
{
let db = Db::open(&path).unwrap();
db.conn
.execute_batch(
"insert into sessions(id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values('claude:s1','claude','s1','','/x','claude-v1','jsonl'); \
insert into messages(session_id, provider, seq, role, content) \
values('claude:s1','claude',0,'user','findthisneedle');",
)
.unwrap();
let hit: i64 = db
.conn
.query_row(
"select count(*) from messages_fts where messages_fts match 'findthisneedle'",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(hit, 1, "precondition: triggers index the inserted message");
db.conn
.execute_batch(
"drop trigger messages_ai; drop trigger messages_ad; drop trigger messages_au; \
drop table messages_fts;",
)
.unwrap();
}
let db = Db::open(&path).unwrap();
let hit: i64 = db
.conn
.query_row(
"select count(*) from messages_fts where messages_fts match 'findthisneedle'",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(
hit, 1,
"messages_fts rebuilt on open when empty but messages exist"
);
}
#[test]
fn messages_fts_count_reports_index_documents_not_content_rows() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions(id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values('claude:s1','claude','s1','','/x','claude-v1','jsonl'); \
insert into messages(session_id, provider, seq, role, content) \
values('claude:s1','claude',0,'user','indexedtoken');",
)
.unwrap();
assert_eq!(db.message_count().unwrap(), 1);
assert_eq!(db.messages_fts_count().unwrap(), 1);
db.conn
.execute("delete from messages_fts_docsize", [])
.unwrap();
let external_content_rows: i64 = db
.conn
.query_row("select count(*) from messages_fts", [], |r| r.get(0))
.unwrap();
assert_eq!(external_content_rows, 1);
assert_eq!(
db.messages_fts_count().unwrap(),
0,
"helper must report indexed docs, not external content rows"
);
}
#[test]
fn substring_search_matches_inside_tokens_via_custom_index() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
seed_messages(
&db,
&[
("user", "the socket failed with ECONNRESET) today"),
("user", "you forgot the tests again"),
("assistant", "an unrelated message"),
],
);
let find = |needle: &str| -> usize {
db.search_messages(
needle,
&MessageFilters {
match_mode: MessageSearchMode::Regex,
..Default::default()
},
)
.unwrap()
.len()
};
assert_eq!(find("ECONNRESET"), 1, "substring inside a token");
assert_eq!(find("you forgot"), 1, "multi-word phrase substring");
assert_eq!(find("nonexistent_zz"), 0, "no false positives");
}
#[test]
fn trigram_base_rebuild_restores_searchability_including_short_docs() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
enable_v3_custom_trigram_compatibility(&db);
seed_messages(
&db,
&[
("user", "alpha contains zebracode here"),
("user", "bravo contains zebracode too"),
("user", "charlie has zebracode as well"),
("user", "ok"), ],
);
let hits = db
.search_messages(
"zebracode",
&MessageFilters {
match_mode: MessageSearchMode::Regex,
..Default::default()
},
)
.unwrap();
assert_eq!(
hits.len(),
3,
"every zebracode row searchable after base build"
);
assert_eq!(
crate::trigram_index::base_max_id(&db.conn).unwrap(),
4,
"base covers all messages including the zero-trigram short doc"
);
}
#[test]
fn messages_fts_updates_are_transactional_with_messages() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions(id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values('claude:s1','claude','s1','','/x','claude-v1','jsonl');",
)
.unwrap();
let before = db.messages_fts_count().unwrap();
{
let tx = db.conn.unchecked_transaction().unwrap();
tx.execute(
"insert into messages(session_id, provider, seq, role, content) \
values('claude:s1','claude',0,'user','rollbackme token here')",
[],
)
.unwrap();
}
assert_eq!(
db.messages_fts_count().unwrap(),
before,
"rolled-back insert leaves no messages_fts entry"
);
let hit: i64 = db
.conn
.query_row(
"select count(*) from messages_fts where messages_fts match 'rollbackme'",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(
hit, 0,
"rolled-back content is not searchable via messages_fts"
);
}
#[test]
fn storage_allocation_reports_freed_pages_without_scanning_content() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"create table allocation_probe (payload blob not null);
insert into allocation_probe(payload) values(zeroblob(2097152));
delete from allocation_probe;",
)
.unwrap();
let allocation = db.storage_allocation().unwrap();
assert!(allocation.total_bytes >= allocation.reclaimable_bytes);
assert!(
allocation.reclaimable_bytes > 0,
"deleted overflow pages must be reported as reclaimable"
);
}
#[test]
fn replacement_freelist_compaction_preserves_rows_fts_and_search_results() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let large_suffix = " allocation-padding".repeat(32_768);
for index in 0..6 {
let id = format!("claude:allocation-{index}");
let content = format!("original-token-{index}{large_suffix}");
let mut parsed = parsed_with_messages(&id, &[&content]);
parsed.session.provider_session_id = format!("allocation-{index}");
parsed.session.source_path = format!("/allocation/{index}.jsonl");
db.replace_session(&parsed, index, index).unwrap();
}
assert_eq!(db.message_count().unwrap(), 6);
assert_eq!(db.messages_fts_count().unwrap(), 6);
for index in 0..3 {
let id = format!("claude:allocation-{index}");
let content = format!("replacement-token-{index}");
let mut parsed = parsed_with_messages(&id, &[&content]);
parsed.session.provider_session_id = format!("allocation-{index}");
parsed.session.source_path = format!("/allocation/{index}.jsonl");
db.replace_session(&parsed, 10 + index, 10 + index).unwrap();
}
db.optimize_fts().unwrap();
let before = db.storage_allocation().unwrap();
assert!(before.reclaimable_bytes > 0);
assert_eq!(db.message_count().unwrap(), 6);
assert_eq!(db.messages_fts_count().unwrap(), 6);
for index in 0..3 {
assert_eq!(
db.search_messages(
&format!("replacement-token-{index}"),
&MessageFilters::default(),
)
.unwrap()
.len(),
1
);
}
db.vacuum().unwrap();
db.checkpoint_truncate().unwrap();
let after = db.storage_allocation().unwrap();
assert_eq!(after.reclaimable_bytes, 0);
assert!(after.total_bytes < before.total_bytes);
assert_eq!(db.message_count().unwrap(), 6);
assert_eq!(db.messages_fts_count().unwrap(), 6);
for index in 0..3 {
assert_eq!(
db.search_messages(
&format!("replacement-token-{index}"),
&MessageFilters::default(),
)
.unwrap()
.len(),
1
);
}
}
#[test]
fn session_replacement_rolls_back_every_index_when_sqlite_is_full() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let mut original = parsed_with_messages("claude:s1", &["original durable token"]);
original.session.title = Some("original title".into());
db.replace_session(&original, 1, 1).unwrap();
db.checkpoint_truncate().unwrap();
let original_max_pages: i64 = db
.conn
.query_row("pragma max_page_count", [], |row| row.get(0))
.unwrap();
let current_pages: i64 = db
.conn
.query_row("pragma page_count", [], |row| row.get(0))
.unwrap();
let free_pages: i64 = db
.conn
.query_row("pragma freelist_count", [], |row| row.get(0))
.unwrap();
let page_size: i64 = db
.conn
.query_row("pragma page_size", [], |row| row.get(0))
.unwrap();
db.conn
.pragma_update(None, "max_page_count", current_pages)
.unwrap();
let mut replacement = original.clone();
replacement.session.title = Some("replacement title".into());
const REQUIRED_GROWTH_PAGES: i64 = 2;
const REPLACEMENT_TOKEN: &str = "replacement-full-token ";
let required_bytes = (free_pages + REQUIRED_GROWTH_PAGES) * page_size;
let repeats = required_bytes as usize / REPLACEMENT_TOKEN.len() + 1;
replacement.transcript_text = REPLACEMENT_TOKEN.repeat(repeats);
replacement.messages[0].content = replacement.transcript_text.clone();
let error = db.replace_session(&replacement, 2, 2).unwrap_err();
db.conn
.pragma_update(None, "max_page_count", original_max_pages)
.unwrap();
assert!(error.chain().any(|cause| {
cause
.downcast_ref::<rusqlite::Error>()
.and_then(rusqlite::Error::sqlite_error_code)
== Some(ErrorCode::DiskFull)
}));
let persisted = db.resolve_session("claude:s1").unwrap();
assert_eq!(persisted.session.title.as_deref(), Some("original title"));
assert_eq!(persisted.transcript_text, "original durable token");
assert_eq!(
db.search_messages("original durable token", &MessageFilters::default())
.unwrap()
.len(),
1
);
assert!(db
.search_messages("replacement-full-token", &MessageFilters::default())
.unwrap()
.is_empty());
}
#[test]
fn generated_trigram_query_is_a_superset_in_the_custom_index() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions(id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values('claude:s1','claude','s1','','/x','claude-v1','jsonl');",
)
.unwrap();
let rows = [
"you forgot the tests",
"well You Forgot it",
"no, that's wrong",
"we also need more coverage",
"socket hang up ECONNRESET here",
"please stop doing that",
"totally unrelated message",
"scatter the cats", ];
{
let tx = db.conn.unchecked_transaction().unwrap();
{
let mut stmt = tx
.prepare(
"insert into messages(session_id, provider, seq, role, content) \
values('claude:s1','claude',?1,'user',?2)",
)
.unwrap();
for (i, row) in rows.iter().enumerate() {
stmt.execute(params![i as i64, row]).unwrap();
}
}
tx.commit().unwrap();
}
crate::trigram_index::build(&db.conn, &db.runtime).unwrap();
let patterns = [
r"\byou forgot\b",
r"\bno,?\s+that'?s\b",
r"\balso need\b",
"ECONNRESET",
r"\bstop doing\b",
r"\bcat\b", ];
for pat in patterns {
let regex = regex::Regex::new(&format!("(?i){pat}")).unwrap();
let expected: Vec<i64> = {
let mut stmt = db.conn.prepare("select id, content from messages").unwrap();
let iter = stmt
.query_map([], |row| {
Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?))
})
.unwrap();
iter.filter_map(Result::ok)
.filter(|(_, c)| regex.is_match(c))
.map(|(id, _)| id)
.collect()
};
if let Some(groups) = crate::trigram::trigram_prefilter_groups(pat) {
let candidates = crate::trigram_index::candidates(&db.conn, &groups).unwrap();
for id in &expected {
assert!(
candidates.contains(id),
"SUPERSET VIOLATION: {pat:?} -> {groups:?} missed id {id}",
);
}
}
}
}
#[test]
fn detail_mode_comparison_like_on_external_content() {
let dir = tempfile::tempdir().unwrap();
let size_of = |variant: &str, detail_clause: &str| -> (i64, rusqlite::Connection) {
let conn =
rusqlite::Connection::open(dir.path().join(format!("{variant}.db"))).unwrap();
conn.execute_batch(
"create table messages(id integer primary key, content text not null);",
)
.unwrap();
{
let tx = conn.unchecked_transaction().unwrap();
{
let mut stmt = tx
.prepare("insert into messages(content) values(?1)")
.unwrap();
stmt.execute(["the socket failed with ECONNRESET) today"])
.unwrap();
stmt.execute(["you forgot the tests again"]).unwrap();
stmt.execute(["an unrelated assistant message"]).unwrap();
for i in 0..3000 {
stmt.execute([format!(
"filler row {i} lorem ipsum dolor sit amet consectetur adipiscing"
)])
.unwrap();
}
}
tx.commit().unwrap();
}
conn.execute_batch(&format!(
"create virtual table tri using fts5(content, content='messages', \
content_rowid='id', tokenize='trigram'{detail_clause}); \
insert into tri(tri) values('rebuild');",
))
.unwrap();
let pages: i64 = conn
.query_row("pragma page_count", [], |r| r.get(0))
.unwrap();
let page_size: i64 = conn
.query_row("pragma page_size", [], |r| r.get(0))
.unwrap();
(pages * page_size, conn)
};
let (full_bytes, full) = size_of("full", "");
let (none_bytes, none) = size_of("none", ", detail='none'");
let full_match = |q: &str| -> i64 {
full.query_row("select count(*) from tri where tri match ?1", [q], |r| {
r.get(0)
})
.unwrap()
};
assert_eq!(
full_match("\"econnreset\""),
1,
"detail=full MATCH substring-in-token"
);
assert_eq!(
full_match("\"you forgot\""),
1,
"detail=full MATCH multi-word phrase"
);
let none_like = |needle: &str| -> i64 {
none.query_row(
"select count(*) from tri where content like ?1 escape '\\'",
[format!("%{needle}%")],
|r| r.get(0),
)
.unwrap()
};
assert_eq!(
none_like("econnreset"),
1,
"detail=none LIKE substring-in-token MUST work on external content",
);
assert_eq!(
none_like("you forgot"),
1,
"detail=none LIKE multi-word phrase MUST work on external content",
);
assert_eq!(
none_like("zzqqxx_absent"),
0,
"detail=none LIKE rejects absent substring"
);
let none_match = |needle: &str| -> i64 {
let query = crate::trigram::trigram_prefilter(needle)
.unwrap_or_else(|| panic!("needle {needle:?} must be prefilterable"));
none.query_row(
"select count(*) from tri where tri match ?1",
[query],
|r| r.get(0),
)
.unwrap()
};
assert!(
none_match("econnreset") >= 1,
"detail=none AND-of-trigrams MATCH finds substring-in-token on external content",
);
assert!(
none_match("you forgot") >= 1,
"detail=none AND-of-trigrams MATCH finds multi-word substring on external content",
);
assert_eq!(
none_match("zzqqwwxx"),
0,
"detail=none AND-of-trigrams MATCH rejects an absent needle",
);
let plan: String = {
let mut stmt = none
.prepare(
"explain query plan select rowid from tri \
where content like '%econnreset%' escape '\\'",
)
.unwrap();
let rows = stmt
.query_map([], |r| r.get::<_, String>(3))
.unwrap()
.filter_map(Result::ok)
.collect::<Vec<_>>()
.join(" | ");
rows
};
eprintln!(
"[#230] index size: detail=full={}KB detail=none={}KB (full is {:.1}x none)",
full_bytes / 1024,
none_bytes / 1024,
full_bytes as f64 / none_bytes.max(1) as f64,
);
eprintln!("[#230] detail=none LIKE query plan: {plan}");
assert!(
!plan.to_lowercase().contains("scan messages"),
"detail=none LIKE should not linear-scan the messages table; plan was: {plan}",
);
}
#[test]
fn regex_search_composes_with_role_and_session_scope() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions(id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) values \
('claude:a','claude','a','','/x','v1','jsonl'), \
('claude:b','claude','b','','/x','v1','jsonl');",
)
.unwrap();
{
let tx = db.conn.unchecked_transaction().unwrap();
{
let mut stmt = tx
.prepare(
"insert into messages(session_id, provider, seq, role, content) \
values(?1,'claude',?2,?3,?4)",
)
.unwrap();
stmt.execute(params!["claude:a", 0, "user", "needle_xyz in a user"])
.unwrap();
stmt.execute(params![
"claude:a",
1,
"assistant",
"needle_xyz in a assistant"
])
.unwrap();
stmt.execute(params!["claude:b", 0, "user", "needle_xyz in b user"])
.unwrap();
}
tx.commit().unwrap();
}
let count = |role: Option<Role>, session_id: Option<&str>| -> usize {
db.search_messages(
"needle_xyz",
&MessageFilters {
match_mode: MessageSearchMode::Regex,
role,
session_id: session_id.map(str::to_string),
..Default::default()
},
)
.unwrap()
.len()
};
assert_eq!(count(None, None), 3, "unscoped: all three rows");
assert_eq!(
count(Some(Role::User), None),
2,
"role scope narrows to user rows"
);
assert_eq!(
count(None, Some("claude:a")),
2,
"session scope narrows to session a"
);
assert_eq!(
count(Some(Role::User), Some("claude:a")),
1,
"role+session scope composes",
);
}
#[test]
fn trigram_search_correct_across_all_providers() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
for p in [
"claude",
"claude-desktop",
"codex",
"cursor",
"antigravity",
"pi",
] {
db.conn
.execute(
"insert into sessions(id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values(?1, ?2, 's', '', '/x', 'v1', 'jsonl')",
params![format!("{p}:s"), p],
)
.unwrap();
}
let rows: &[(&str, &str, &str)] = &[
(
"claude",
"tool",
r#"{"type":"tool_result","content":"net error ECONNRESET) deploy"}"#,
),
(
"claude-desktop",
"assistant",
"Desktop local agent saw ECONNRESET too",
),
(
"codex",
"assistant",
"```rust\nconnect()?; // ECONNRESET retry\n```",
),
("cursor", "user", "hey you forgot the ECONNRESET retry path"),
(
"antigravity",
"assistant",
"MODEL: ECONNRESET observed — naïve café résumé",
),
("pi", "user", "ECONNRESET again"),
];
{
let tx = db.conn.unchecked_transaction().unwrap();
{
let mut stmt = tx
.prepare(
"insert into messages(session_id, provider, seq, role, content) \
values(?1,?2,?3,?4,?5)",
)
.unwrap();
for (i, (p, role, content)) in rows.iter().enumerate() {
stmt.execute(params![format!("{p}:s"), p, i as i64, role, content])
.unwrap();
}
}
tx.commit().unwrap();
}
let providers_for = |query: &str, filters: MessageFilters| -> Vec<String> {
let mut got: Vec<String> = db
.search_messages(query, &filters)
.unwrap()
.into_iter()
.map(|h| h.provider.as_str().to_string())
.collect();
got.sort();
got
};
let all = providers_for(
"ECONNRESET",
MessageFilters {
match_mode: MessageSearchMode::Regex,
..Default::default()
},
);
assert_eq!(
all.len(),
6,
"every provider's ECONNRESET found regardless of content shape"
);
let claude = providers_for(
"ECONNRESET",
MessageFilters {
match_mode: MessageSearchMode::Regex,
providers: Some(vec![Provider::Claude]),
..Default::default()
},
);
assert_eq!(
claude,
vec!["claude"],
"provider scope restricts to the claude message"
);
let claude_desktop = providers_for(
"ECONNRESET",
MessageFilters {
match_mode: MessageSearchMode::Regex,
providers: Some(vec![Provider::ClaudeDesktop]),
..Default::default()
},
);
assert_eq!(
claude_desktop,
vec!["claude-desktop"],
"provider scope restricts to the claude-desktop message"
);
let forgot = providers_for(
r"\byou forgot\b",
MessageFilters {
match_mode: MessageSearchMode::Regex,
..Default::default()
},
);
assert_eq!(
forgot,
vec!["cursor"],
"you-forgot regex selects exactly cursor"
);
}
#[cfg(test)]
fn seed_messages(db: &Db, rows: &[(&str, &str)]) {
db.conn
.execute_batch(
"insert into sessions(id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values('claude:s1','claude','s1','','/x','v1','jsonl');",
)
.unwrap();
let tx = db.conn.unchecked_transaction().unwrap();
{
let mut stmt = tx
.prepare(
"insert into messages(session_id, provider, seq, role, content) \
values('claude:s1','claude',?1,?2,?3)",
)
.unwrap();
for (i, (role, content)) in rows.iter().enumerate() {
stmt.execute(params![i as i64, role, content]).unwrap();
}
}
tx.commit().unwrap();
}
#[test]
fn purge_injected_messages_removes_only_leading_marker_user_rows() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
seed_messages(
&db,
&[
(
"user",
"<local-command-stdout>Set model to Opus</local-command-stdout>",
), ("user", "<local-command-stderr>boom</local-command-stderr>"), (
"user",
"<environment_context>\n<current_date>2026</current_date>",
), (
"user",
"<turn_aborted>\nThe user interrupted the previous turn on purpose.",
), ("user", "fix the failing login test before release"), ("user", "what does <local-command-stdout> mean in the logs"), (
"user",
"what does <turn_aborted> mean in a Codex transcript",
), (
"assistant",
"<local-command-stdout>tool output</local-command-stdout>",
), ],
);
let before = db.message_count().unwrap();
let purged = db.purge_injected_messages().unwrap();
assert_eq!(purged, 4, "the four leading-marker USER rows are deleted");
assert_eq!(db.message_count().unwrap(), before - 4);
assert_eq!(
db.messages_fts_count().unwrap(),
db.message_count().unwrap()
);
let users: Vec<String> = db
.search_messages(
"",
&MessageFilters {
role: Some(Role::User),
..Default::default()
},
)
.unwrap()
.into_iter()
.map(|h| h.content)
.collect();
assert!(
users
.iter()
.any(|c| c.contains("fix the failing login test")),
"real prompt kept"
);
assert!(
users
.iter()
.any(|c| c.starts_with("what does <local-command-stdout>")),
"a non-leading mention is kept"
);
assert_eq!(
users.len(),
3,
"exactly the three legitimate user messages remain"
);
}
#[test]
fn regex_search_lazily_builds_custom_trigram_base_and_is_correct() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
enable_v3_custom_trigram_compatibility(&db);
seed_messages(&db, &[("user", "socket failed with ECONNRESET) today")]);
assert_eq!(
crate::trigram_index::base_max_id(&db.conn).unwrap(),
0,
"precondition: custom trigram base not built before first regex use"
);
let filters = MessageFilters {
match_mode: MessageSearchMode::Regex,
..Default::default()
};
let hits = db.search_messages("ECONNRESET", &filters).unwrap();
assert_eq!(
hits.len(),
1,
"regex search returns the match despite an unbuilt index"
);
assert!(
crate::trigram_index::base_max_id(&db.conn).unwrap() > 0,
"regex search lazily built the custom trigram base index"
);
}
#[test]
fn lazy_trigram_build_busy_writer_falls_back_to_delta_scan() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("index.db");
let writer = Db::open(&path).unwrap();
let reader = Db::open_with_busy_timeout(&path, TEST_NO_WAIT_BUSY_TIMEOUT_MS).unwrap();
seed_messages(
&writer,
&[
(
"user",
"the deploy hit ECONNRESET while the trigram base was empty",
),
("assistant", "ack"),
],
);
assert_eq!(
crate::trigram_index::base_max_id(&reader.conn).unwrap(),
0,
"precondition: custom trigram base starts empty"
);
writer.conn.execute_batch("begin immediate").unwrap();
let filters = MessageFilters {
match_mode: MessageSearchMode::Regex,
..Default::default()
};
let hits = reader.search_messages("ECONNRESET", &filters).unwrap();
writer.conn.execute_batch("rollback").unwrap();
assert_eq!(hits.len(), 1, "busy lazy build must not drop regex hits");
assert_eq!(
crate::trigram_index::base_max_id(&reader.conn).unwrap(),
0,
"busy fallback serves the existing base and leaves rebuild for a later query"
);
}
#[test]
fn progress_reporter_fires_on_lazy_build_only_and_is_silent_when_unset() {
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::Arc;
let dir = tempfile::tempdir().unwrap();
let silent = Db::open(&dir.path().join("a.db")).unwrap();
enable_v3_custom_trigram_compatibility(&silent);
seed_messages(&silent, &[("user", "econnreset here")]);
assert_eq!(silent.ensure_trigram_base().unwrap(), 1);
let dir2 = tempfile::tempdir().unwrap();
let mut db = Db::open(&dir2.path().join("b.db")).unwrap();
enable_v3_custom_trigram_compatibility(&db);
let calls = Arc::new(AtomicU32::new(0));
let counter = Arc::clone(&calls);
db.set_progress_reporter(move |_msg| {
counter.fetch_add(1, Ordering::Relaxed);
});
seed_messages(&db, &[("user", "econnreset here")]);
assert_eq!(db.ensure_trigram_base().unwrap(), 1);
assert_eq!(
calls.load(Ordering::Relaxed),
1,
"reporter fires once for the one-time build"
);
db.ensure_trigram_base().unwrap();
assert_eq!(
calls.load(Ordering::Relaxed),
1,
"no report when the base is already current"
);
}
#[test]
fn search_messages_regex_prunes_lookaround_and_falls_back() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
seed_messages(
&db,
&[
("user", "socket failed with ECONNRESET) today"),
("user", "you forgot the tests"),
("assistant", "scatter the cats"), ("user", "a cat sat here"), ("assistant", "totally unrelated text 1234"),
],
);
let run = |pattern: &str| -> Vec<String> {
let filters = MessageFilters {
match_mode: MessageSearchMode::Regex,
..Default::default()
};
let mut got: Vec<String> = db
.search_messages(pattern, &filters)
.unwrap()
.into_iter()
.map(|h| h.content)
.collect();
got.sort();
got
};
assert_eq!(
run(r"\bcat\b"),
vec!["a cat sat here".to_string()],
"look-around pruned"
);
assert_eq!(
run("ECONNRESET"),
vec!["socket failed with ECONNRESET) today".to_string()],
"substring inside a token",
);
assert_eq!(
run(r"\byou forgot\b"),
vec!["you forgot the tests".to_string()],
"phrase"
);
assert_eq!(
run(r"\d{4}"),
vec!["totally unrelated text 1234".to_string()],
"non-prefilterable pattern falls back to scan, still correct",
);
assert!(run(r"[0-9]{9}").is_empty(), "non-prefilterable, no match");
}
#[test]
fn search_messages_provider_filter_scopes() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
for p in ["claude", "codex"] {
db.conn
.execute(
"insert into sessions(id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values(?1, ?2, 's', '', '/x', 'v1', 'jsonl')",
params![format!("{p}:s"), p],
)
.unwrap();
db.conn
.execute(
"insert into messages(session_id, provider, seq, role, content) \
values(?1, ?2, 0, 'user', 'shared ECONNRESET token')",
params![format!("{p}:s"), p],
)
.unwrap();
}
let scoped = |provider: Option<Provider>| -> usize {
let filters = MessageFilters {
match_mode: MessageSearchMode::Regex,
providers: provider.map(|provider| vec![provider]),
..Default::default()
};
db.search_messages("ECONNRESET", &filters).unwrap().len()
};
assert_eq!(scoped(None), 2, "unscoped: both providers");
assert_eq!(scoped(Some(Provider::Claude)), 1, "scoped to claude");
assert_eq!(scoped(Some(Provider::Codex)), 1, "scoped to codex");
}
#[test]
fn find_corrections_requires_human_authorship_after_session_kind_selection() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions(id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values('claude:human','claude','human','','/x','v1','jsonl'); \
insert into sessions(id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source, parent_session_id) \
values('claude:spawned','claude','spawned','','/x','v1','jsonl','claude:human');",
)
.unwrap();
db.conn
.execute_batch(
"insert into messages(
session_id, provider, seq, role, content, authorship, record_relation
) values
('claude:human','claude',0,'user','don''t forget to check the tests',
'human','original'),
('claude:spawned','claude',0,'user','don''t forget to check the tests',
'agent','original'),
('claude:spawned','claude',1,'user','you forgot the human follow-up',
'human','original');",
)
.unwrap();
let policies = default_policies();
let matches_of = |filters: &MessageFilters| -> Vec<(String, i64)> {
let mut matches: Vec<(String, i64)> = db
.find_corrections(&policies, filters)
.unwrap()
.into_iter()
.map(|hit| (hit.session_id, hit.message_seq))
.collect();
matches.sort();
matches
};
assert_eq!(
matches_of(&MessageFilters::default()),
[("claude:human".to_string(), 0)],
"by default a correction is what a person said, so the delegation prompt is excluded"
);
assert_eq!(
matches_of(&MessageFilters {
session_kinds: Some(vec![crate::models::SessionKind::Subagent]),
..Default::default()
}),
[("claude:spawned".to_string(), 1)],
"session scope reopens genuine human content without reclassifying an agent prompt"
);
assert_eq!(
matches_of(&MessageFilters {
session_kinds: Some(vec![
crate::models::SessionKind::User,
crate::models::SessionKind::Subagent,
]),
..Default::default()
}),
[
("claude:human".to_string(), 0),
("claude:spawned".to_string(), 1),
],
"the union contains attributable human content from both session classes"
);
assert!(
matches_of(&MessageFilters {
session_kinds: Some(Vec::new()),
..Default::default()
})
.is_empty(),
"an empty class set matches nothing rather than silently matching all"
);
}
#[test]
fn find_corrections_honors_offset_before_limit_in_newest_first_order() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions(id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values('claude:s1','claude','s1','','/x','v1','jsonl');",
)
.unwrap();
for seq in 0..5i64 {
db.conn
.execute(
"insert into messages(
session_id, provider, seq, role, ts, content,
authorship, record_relation
) values(
'claude:s1','claude',?1,'user',?2,?3,'human','original'
)",
params![
seq,
format!("2026-01-0{}T00:00:00Z", seq + 1),
format!("you forgot item {seq}")
],
)
.unwrap();
}
let policies = default_policies();
let contents = |offset: usize, limit: usize| -> Vec<String> {
db.find_corrections(
&policies,
&MessageFilters {
offset,
limit,
..Default::default()
},
)
.unwrap()
.into_iter()
.map(|hit| hit.content)
.collect()
};
assert_eq!(contents(0, 2), ["you forgot item 4", "you forgot item 3"]);
assert_eq!(
contents(2, 2),
["you forgot item 2", "you forgot item 1"],
"offset skips the newest N, then limit takes: one page, not a different sort"
);
assert_eq!(
contents(3, 0),
["you forgot item 1", "you forgot item 0"],
"offset applies even when limit is 0 (unbounded)"
);
assert!(
contents(99, 0).is_empty(),
"an offset past the end is empty, not a panic or a wrap-around"
);
}
#[test]
fn find_corrections_pages_timestamp_ties_and_nulls_in_one_stable_total_order() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
for session_id in ["claude:a", "claude:b"] {
db.conn
.execute(
"insert into sessions(id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values(?1,'claude',?1,'','/x','v1','jsonl')",
[session_id],
)
.unwrap();
}
for (session_id, seq, timestamp, content) in [
(
"claude:b",
0_i64,
Some("2026-01-01T00:00:00Z"),
"you forgot b0",
),
("claude:a", 1, Some("2026-01-01T00:00:00Z"), "you forgot a1"),
("claude:a", 0, Some("2026-01-01T00:00:00Z"), "you forgot a0"),
("claude:b", 1, None, "you forgot b1-null"),
("claude:a", 2, None, "you forgot a2-null"),
] {
db.conn
.execute(
"insert into messages(
session_id, provider, seq, role, ts, content,
authorship, record_relation
) values(?1,'claude',?2,'user',?3,?4,'human','original')",
params![session_id, seq, timestamp, content],
)
.unwrap();
}
let policies = default_policies();
let page = |offset: usize, limit: usize| {
db.find_corrections(
&policies,
&MessageFilters {
offset,
limit,
..Default::default()
},
)
.unwrap()
.into_iter()
.map(|hit| hit.content)
.collect::<Vec<_>>()
};
let first = page(0, 2);
let second = page(2, 2);
let third = page(4, 2);
let combined = first
.iter()
.chain(&second)
.chain(&third)
.cloned()
.collect::<Vec<_>>();
assert_eq!(
combined,
[
"you forgot a0",
"you forgot a1",
"you forgot b0",
"you forgot a2-null",
"you forgot b1-null",
],
"equal and missing timestamps use session id then sequence as stable tie-breakers"
);
assert_eq!(combined, page(0, 0), "pages must partition the full result");
}
#[test]
fn find_corrections_scans_user_rows_and_classifies() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions(id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values('claude:s1','claude','s1','','/x','v1','jsonl');",
)
.unwrap();
let rows = [
("user", "you forgot the unit tests again"), ("user", "we also need integration coverage"), ("user", "looks great, ship it"), ("assistant", "you forgot nothing, here is the fix"), ("user", "the deploy hit econnreset once more"), ];
{
let tx = db.conn.unchecked_transaction().unwrap();
{
let mut stmt = tx
.prepare(
"insert into messages(
session_id, provider, seq, role, content, authorship, record_relation
) values('claude:s1','claude',?1,?2,?3,'human','original')",
)
.unwrap();
for (i, (role, content)) in rows.iter().enumerate() {
stmt.execute(params![i as i64, role, content]).unwrap();
}
}
tx.commit().unwrap();
}
let policies = test_policies(&[
("skip_step", &[r"\byou forgot\b"]),
("incomplete", &[r"\balso need\b"]),
]);
let scan = db
.find_corrections(&policies, &MessageFilters::default())
.unwrap();
assert_eq!(
scan.len(),
2,
"exactly the two user corrections, assistant ignored"
);
assert!(scan.iter().any(|c| c.category == "skip_step"));
assert!(scan.iter().any(|c| c.category == "incomplete"));
assert!(
!scan
.iter()
.any(|c| c.content.contains("you forgot nothing")),
"assistant-role match excluded by the role='user' filter",
);
}
#[test]
fn find_corrections_classifies_only_original_human_content_with_absolute_coordinates() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions (
id, provider, provider_session_id, preview_text,
source_path, parse_version, discovery_source
) values (
'claude:provenance', 'claude', 'provenance', '',
'/provenance', 'claude-v4', 'test'
);
insert into messages (
id, session_id, provider, seq, role, kind, content,
authorship, record_relation
) values
(201, 'claude:provenance', 'claude', 0, 'user', 'conversation',
'human original you forgot', 'human', 'original'),
(202, 'claude:provenance', 'claude', 1, 'user', 'conversation',
'agent original you forgot', 'agent', 'original'),
(203, 'claude:provenance', 'claude', 2, 'user', 'conversation',
'human mirror you forgot', 'human', 'mirror');",
)
.unwrap();
let generated_prefix = "agent you forgot | ";
let human_request = "human α you forgot the tests";
let mixed_content = format!("{generated_prefix}{human_request}");
db.conn
.execute(
"insert into messages (
id, session_id, provider, seq, role, kind, content,
authorship, record_relation
) values (
204, 'claude:provenance', 'claude', 3, 'user', 'conversation', ?1,
'mixed', 'original'
)",
[&mixed_content],
)
.unwrap();
let generated_end = generated_prefix.chars().count() as i64;
let mixed_end = mixed_content.chars().count() as i64;
db.conn
.execute(
"insert into message_content_parts (
message_id, ordinal, start_char, end_char, authorship, origin
) values
(204, 0, 0, ?1, 'generated', 'generated_summary'),
(204, 1, ?1, ?2, 'human', 'direct_input')",
params![generated_end, mixed_end],
)
.unwrap();
let left = "left";
let generated_bridge = " generated bridge ";
let right = "right";
let split_content = format!("{left}{generated_bridge}{right}");
db.conn
.execute(
"insert into messages (
id, session_id, provider, seq, role, kind, content,
authorship, record_relation
) values (
205, 'claude:provenance', 'claude', 4, 'user', 'conversation', ?1,
'mixed', 'original'
)",
[&split_content],
)
.unwrap();
let left_end = left.chars().count() as i64;
let bridge_end = (left.chars().count() + generated_bridge.chars().count()) as i64;
let split_end = split_content.chars().count() as i64;
db.conn
.execute(
"insert into message_content_parts (
message_id, ordinal, start_char, end_char, authorship, origin
) values
(205, 0, 0, ?1, 'human', 'direct_input'),
(205, 1, ?1, ?2, 'generated', 'generated_summary'),
(205, 2, ?2, ?3, 'human', 'direct_input')",
params![left_end, bridge_end, split_end],
)
.unwrap();
let policies = test_policies(&[
("skip_step", &[r"\byou forgot\b"]),
("cross_part", &[r"\bleft\s*right\b"]),
]);
let hits = db
.find_corrections(&policies, &MessageFilters::default())
.unwrap();
assert_eq!(
hits.iter().map(|hit| hit.message_seq).collect::<Vec<_>>(),
[0, 3],
"agent-authored originals, mirrors, and artificial cross-part matches are excluded"
);
let mixed_hit = &hits[1];
assert_eq!(mixed_hit.content, mixed_content);
assert_eq!(mixed_hit.matched_text, "you forgot");
let expected_start = generated_prefix.chars().count() + "human α ".chars().count();
assert_eq!(mixed_hit.match_start_char, expected_start);
assert_eq!(
mixed_hit.match_end_char_exclusive,
expected_start + "you forgot".chars().count(),
"mixed-message coordinates remain absolute in the original full content"
);
}
#[test]
fn find_corrections_reads_candidates_and_parts_from_one_snapshot() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("index.db");
let db = Db::open(&path).unwrap();
let writer = Db::open(&path).unwrap();
db.conn
.execute_batch(
"insert into sessions (
id, provider, provider_session_id, preview_text,
source_path, parse_version, discovery_source
) values (
'claude:snapshot', 'claude', 'snapshot', '',
'/snapshot', 'claude-v4', 'test'
);
insert into messages (
id, session_id, provider, seq, role, kind, content,
authorship, record_relation
) values (
301, 'claude:snapshot', 'claude', 0, 'user', 'conversation',
'human you forgot', 'mixed', 'original'
);
insert into message_content_parts (
message_id, ordinal, start_char, end_char, authorship, origin
) values (
301, 0, 0, 16, 'human', 'direct_input'
);",
)
.unwrap();
let policies = test_policies(&[("skip_step", &[r"\byou forgot\b"])]);
let hits = db
.find_corrections_with_after_candidates(&policies, &MessageFilters::default(), || {
writer
.conn
.execute_batch(
"begin immediate;
update messages
set content = 'generated you forgot | human okay'
where id = 301;
delete from message_content_parts where message_id = 301;
insert into message_content_parts (
message_id, ordinal, start_char, end_char, authorship, origin
) values
(301, 0, 0, 23, 'generated', 'generated_summary'),
(301, 1, 23, 33, 'human', 'direct_input');
commit;",
)
.unwrap();
})
.unwrap();
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].content, "human you forgot");
assert_eq!(hits[0].matched_text, "you forgot");
assert_eq!(
writer
.conn
.query_row("select content from messages where id = 301", [], |row| {
row.get::<_, String>(0)
})
.unwrap(),
"generated you forgot | human okay",
"the writer committed while the reader retained the older snapshot"
);
}
#[test]
fn find_corrections_honors_provider_scope() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
for provider in [Provider::Claude, Provider::Codex] {
let session_id = format!("{}:s1", provider.as_str());
db.conn
.execute(
"insert into sessions(id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values(?1, ?2, 's1', '', '/x', 'v1', 'jsonl')",
params![session_id, provider.as_str()],
)
.unwrap();
db.conn
.execute(
"insert into messages(
session_id, provider, seq, role, content, authorship, record_relation
) values(
?1, ?2, 0, 'user', 'you forgot the provider filter', 'human', 'original'
)",
params![session_id, provider.as_str()],
)
.unwrap();
}
let policies = test_policies(&[("skip_step", &[r"\byou forgot\b"])]);
let hits = db
.find_corrections(
&policies,
&MessageFilters {
providers: Some(vec![Provider::Claude]),
..Default::default()
},
)
.unwrap();
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].provider, Provider::Claude);
assert_eq!(hits[0].session_id, "claude:s1");
}
#[test]
fn find_corrections_parallel_matches_sequential() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.execute_batch(
"insert into sessions(id, provider, provider_session_id, preview_text, \
source_path, parse_version, discovery_source) \
values('claude:s1','claude','s1','','/x','v1','jsonl');",
)
.unwrap();
let base = chrono::DateTime::parse_from_rfc3339("2026-01-01T00:00:00Z").unwrap();
let n = 600i64;
{
let tx = db.conn.unchecked_transaction().unwrap();
{
let mut stmt = tx
.prepare(
"insert into messages(
session_id, provider, seq, role, ts, content,
authorship, record_relation
) values(
'claude:s1','claude',?1,'user',?2,?3,'human','original'
)",
)
.unwrap();
for i in 0..n {
let ts = (base + chrono::Duration::seconds(i)).to_rfc3339();
let content = if i % 5 == 0 {
format!("row-{i} you forgot the tests")
} else {
format!("row-{i} all good here")
};
stmt.execute(params![i, ts, content]).unwrap();
}
}
tx.commit().unwrap();
}
let policies = test_policies(&[("skip_step", &[r"\byou forgot\b"])]);
let expected: Vec<i64> = (0..n).rev().filter(|i| i % 5 == 0).collect();
let all = db
.find_corrections(&policies, &MessageFilters::default())
.unwrap();
assert_eq!(all.len(), expected.len(), "match count");
for (hit, want_i) in all.iter().zip(&expected) {
assert_eq!(hit.category, "skip_step");
assert_eq!(hit.matched_text, "you forgot");
assert!(
hit.content.starts_with(&format!("row-{want_i} ")),
"order mismatch: got {:?}, expected row-{want_i}",
hit.content
);
}
let limited = db
.find_corrections(
&policies,
&MessageFilters {
limit: 10,
..Default::default()
},
)
.unwrap();
assert_eq!(limited.len(), 10);
for (hit, want_i) in limited.iter().zip(expected.iter().take(10)) {
assert!(hit.content.starts_with(&format!("row-{want_i} ")));
}
}
#[test]
fn regex_search_corpus_gate_is_result_equivalent() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
seed_messages(
&db,
&[
("user", "the deploy hit ECONNRESET in prod"),
("assistant", "ECONNRESET means the socket closed"),
("tool", "stderr: ECONNRESET at line 42"),
("user", "an unrelated note about apples"),
],
);
let search = |role: Option<Role>| -> Vec<(Role, String)> {
let filters = MessageFilters {
match_mode: MessageSearchMode::Regex,
role,
..Default::default()
};
db.search_messages("ECONNRESET", &filters)
.unwrap()
.into_iter()
.map(|hit| (hit.role, hit.content))
.collect()
};
let all = search(None);
assert_eq!(
all.len(),
3,
"ECONNRESET in the user + assistant + tool rows"
);
let user_only = search(Some(Role::User));
let expected_user: Vec<(Role, String)> = all
.iter()
.filter(|(role, _)| *role == Role::User)
.cloned()
.collect();
assert_eq!(
user_only, expected_user,
"gate's scan path agrees with the prefilter path restricted to role=user"
);
assert_eq!(user_only.len(), 1, "exactly the one user ECONNRESET row");
}
#[test]
fn explain_message_search_counts_candidates_within_corpus() {
let dir = tempfile::tempdir().unwrap();
let mut db = Db::open(&dir.path().join("index.db")).unwrap();
db.prefilter_min_corpus = 1;
db.upsert_session(
&parsed_with_messages(
"claude:s1",
&[
"zebracode appears here once",
"common text alpha",
"common text bravo",
"common text charlie",
],
),
1,
100,
)
.unwrap();
let selective = MessageFilters {
role: Some(Role::User),
match_mode: MessageSearchMode::Regex,
..Default::default()
};
let ex = db
.explain_message_search("(?i)zebra.ode", &selective)
.unwrap();
assert_eq!(
ex.corpus, 4,
"all four user messages form the selectivity denominator"
);
assert!(
ex.prefilter.is_some(),
"a >=3-char literal yields a trigram prefilter"
);
let candidates = ex
.candidates
.expect("the regex path reports a candidate count");
assert!(
candidates <= ex.corpus,
"candidates are always a subset of the corpus"
);
assert_eq!(
candidates, 1,
"only the zebracode row survives the trigram prefilter"
);
let no_anchor = MessageFilters {
role: Some(Role::User),
match_mode: MessageSearchMode::Regex,
..Default::default()
};
let ex2 = db.explain_message_search("a.b", &no_anchor).unwrap();
assert!(ex2.prefilter.is_none(), "no >=3-char anchor → no prefilter");
assert!(
ex2.candidates.is_none(),
"no prefilter → no candidate count"
);
assert_eq!(ex2.corpus, 4);
}
#[test]
fn schema_v4_search_explain_uses_sqlite_prefilter_without_legacy_threshold() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.upsert_session(
&parsed_with_messages("claude:s1", &["zebracode appears here once"]),
1,
100,
)
.unwrap();
let filters = MessageFilters {
role: Some(Role::User),
match_mode: MessageSearchMode::Regex,
..Default::default()
};
let (hits, explain) = db
.search_messages_with_explain("zebracode", &filters, true)
.unwrap();
let explain = explain.expect("explain requested");
assert_eq!(hits.len(), 1);
assert!(explain.prefilter.is_some(), "anchor is available");
assert_eq!(explain.candidates, Some(1));
assert!(explain.prefilter_skipped.is_none());
}
#[test]
fn internal_message_order_is_typed_scoped_and_reversible() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.upsert_session(
&parsed_with_messages("claude:ordered", &["zero", "one", "two", "three", "four"]),
1,
1,
)
.unwrap();
let newest = db
.search_messages_ordered(
"",
&MessageFilters {
session_id: Some("claude:ordered".to_string()),
limit: 2,
..Default::default()
},
MessageOrder::NewestFirst,
)
.unwrap();
assert_eq!(
newest.iter().map(|hit| hit.seq).collect::<Vec<_>>(),
vec![4, 3]
);
let oldest = db
.search_messages(
"",
&MessageFilters {
session_id: Some("claude:ordered".to_string()),
limit: 2,
..Default::default()
},
)
.unwrap();
assert_eq!(
oldest.iter().map(|hit| hit.seq).collect::<Vec<_>>(),
vec![0, 1]
);
let error = db
.search_messages_ordered("", &MessageFilters::default(), MessageOrder::NewestFirst)
.unwrap_err();
assert!(error
.to_string()
.contains("newest-first message order requires session_id"));
}
#[cfg(test)]
fn parsed_with_messages(id: &str, contents: &[&str]) -> crate::models::ParsedSession {
use crate::models::{Message, ParsedSession, SessionRecord};
let messages = contents
.iter()
.enumerate()
.map(|(i, c)| Message {
seq: i as i64,
role: Role::User,
ts: None,
tool_name: None,
kind: crate::models::MessageKind::Conversation,
tool_call_id: None,
is_compaction: false,
content: c.to_string(),
provenance: crate::models::MessageProvenance::default(),
})
.collect();
ParsedSession {
session: SessionRecord {
id: id.to_string(),
provider: Provider::Claude,
provider_session_id: "s".into(),
title: None,
summary: None,
cwd: None,
repo_root: None,
created_at: None,
updated_at: None,
last_message_at: None,
preview_text: String::new(),
source_path: "/x".into(),
message_count: Some(contents.len() as i64),
parse_version: "v1".into(),
raw_metadata_json: None,
parse_warning: None,
discovery_source: "jsonl".into(),
parent_session_id: None,
agent_label: None,
},
transcript_text: contents.join("\n\n"),
messages,
file_edits: Vec::new(),
}
}
#[test]
fn upsert_appends_only_new_messages_when_session_grows() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let contents = |db: &Db| -> Vec<String> {
let mut s = db
.conn
.prepare("select content from messages where session_id='claude:s1' order by seq")
.unwrap();
s.query_map([], |r| r.get(0))
.unwrap()
.filter_map(Result::ok)
.collect()
};
let tagged = |db: &Db| -> i64 {
db.conn
.query_row(
"select count(*) from messages where session_id='claude:s1' \
and tool_name='SENTINEL'",
[],
|r| r.get(0),
)
.unwrap()
};
db.upsert_session(
&parsed_with_messages("claude:s1", &["alpha", "bravo", "charlie"]),
1,
100,
)
.unwrap();
db.conn
.execute(
"update messages set tool_name='SENTINEL' where session_id='claude:s1'",
[],
)
.unwrap();
db.upsert_session(
&parsed_with_messages("claude:s1", &["alpha", "bravo", "charlie", "delta", "echo"]),
2,
200,
)
.unwrap();
assert_eq!(
contents(&db),
["alpha", "bravo", "charlie", "delta", "echo"]
);
assert_eq!(
tagged(&db),
3,
"prefix rows RETAINED the sentinel (appended, not re-indexed)"
);
let new_found = db
.search_messages(
"delta",
&MessageFilters {
match_mode: MessageSearchMode::Regex,
..Default::default()
},
)
.unwrap();
assert_eq!(new_found.len(), 1, "the appended message is searchable");
db.upsert_session(
&parsed_with_messages("claude:s1", &["alpha", "bravo"]),
3,
60,
)
.unwrap();
assert_eq!(
contents(&db),
["alpha", "bravo"],
"shrink re-replaces fully"
);
assert_eq!(tagged(&db), 0, "shrink did a full replace");
db.upsert_session(
&parsed_with_messages("claude:s1", &["alpha", "boundary"]),
4,
70,
)
.unwrap();
db.conn
.execute(
"update messages set tool_name='SENTINEL' where session_id='claude:s1'",
[],
)
.unwrap();
db.upsert_session(
&parsed_with_messages("claude:s1", &["alpha", "CHANGED", "extra"]),
5,
90,
)
.unwrap();
assert_eq!(
contents(&db),
["alpha", "CHANGED", "extra"],
"boundary content changed → full replace keeps content correct",
);
assert_eq!(tagged(&db), 0, "boundary mismatch forced a full replace");
}
#[test]
fn replace_session_rewrites_matching_prefix_metadata_on_full_reindex() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.replace_session(
&parsed_with_messages("claude:s1", &["alpha", "bravo"]),
1,
100,
)
.unwrap();
db.conn
.execute(
"update messages set role='tool', tool_name='STALE' where session_id='claude:s1' and seq=0",
[],
)
.unwrap();
db.replace_session(
&parsed_with_messages("claude:s1", &["alpha", "bravo", "charlie"]),
2,
150,
)
.unwrap();
let row: (String, Option<String>) = db
.conn
.query_row(
"select role, tool_name from messages where session_id='claude:s1' and seq=0",
[],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.unwrap();
assert_eq!(
row,
("user".to_string(), None),
"full replace must repair stale prefix metadata even when content is unchanged"
);
}
#[test]
fn tail_flow_appends_without_reparsing_prefix() {
use crate::providers::claude::ClaudeAdapter;
use std::io::Write;
let dir = tempfile::tempdir().unwrap();
let file = dir.path().join("probe.jsonl");
let line = |ts: &str, role: &str, text: &str| {
format!(
"{{\"type\":\"{role}\",\"sessionId\":\"probe\",\"timestamp\":\"{ts}\",\
\"message\":{{\"role\":\"{role}\",\"content\":[{{\"type\":\"text\",\"text\":\"{text}\"}}]}}}}\n"
)
};
let initial = format!(
"{}{}",
line("2026-06-01T10:00:00Z", "user", "first prompt"),
line("2026-06-01T10:00:05Z", "assistant", "first reply"),
);
std::fs::write(&file, &initial).unwrap();
let claude = ClaudeAdapter::new(vec![dir.path().to_path_buf()]);
let source = crate::models::SourceFile {
provider: Provider::Claude,
path: file.clone(),
mtime_ns: 1,
size_bytes: std::fs::metadata(&file).unwrap().len() as i64,
};
let source_path = crate::util::normalize_path(&file);
let db = Db::open(&dir.path().join("index.db")).unwrap();
let mut parsed = claude.parse(&source);
crate::util::backfill_session_dates(&mut parsed.session, source.mtime_ns);
db.upsert_session(&parsed, source.mtime_ns, source.size_bytes)
.unwrap();
db.set_file_checkpoint(
Provider::Claude,
&source_path,
crate::tail::complete_prefix_offset(&file).unwrap(),
&crate::tail::prefix_fingerprint(&file).unwrap(),
)
.unwrap();
assert_eq!(db.message_count().unwrap(), 2);
db.conn
.execute(
"update messages set content='CORRUPTED_PROBE' \
where session_id='claude:probe' and seq=0",
[],
)
.unwrap();
std::fs::OpenOptions::new()
.append(true)
.open(&file)
.unwrap()
.write_all(line("2026-06-01T10:01:00Z", "user", "second prompt").as_bytes())
.unwrap();
let new_size = std::fs::metadata(&file).unwrap().len() as i64;
let (offset, stored_fp) = db
.file_checkpoint(
Provider::Claude,
&source_path,
crate::util::provider_parse_version(Provider::Claude),
)
.unwrap()
.unwrap();
assert!(
crate::tail::fingerprint_matches(&file, &stored_fp).unwrap(),
"an append must keep the head fingerprint matching"
);
let tail = crate::tail::tail_parse(&file, offset, |cursor, path| {
claude.parse_reader(cursor, path)
})
.unwrap()
.expect("a new complete line was appended");
db.append_tail(&tail, 2, new_size).unwrap();
assert_eq!(
db.message_count().unwrap(),
3,
"the appended turn is indexed"
);
let seq2: String = db
.conn
.query_row(
"select content from messages where session_id='claude:probe' and seq=2",
[],
|row| row.get(0),
)
.unwrap();
assert_eq!(
seq2, "second prompt",
"new message appended at the next seq"
);
let seq0: String = db
.conn
.query_row(
"select content from messages where session_id='claude:probe' and seq=0",
[],
|row| row.get(0),
)
.unwrap();
assert_eq!(
seq0, "CORRUPTED_PROBE",
"tail append must NOT reparse/replace the prefix rows"
);
assert_eq!(
db.messages_fts_count().unwrap(),
db.message_count().unwrap(),
"FTS in sync"
);
}
#[test]
fn vocabulary_reports_term_frequencies() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
seed_messages(
&db,
&[
("user", "alpha alpha bravo"), ("user", "alpha charlie"), ],
);
let vocab = db.vocabulary(false, 0, None).unwrap();
let alpha = vocab
.iter()
.find(|(t, _, _)| t == "alpha")
.expect("alpha present");
assert_eq!(alpha.1, 2, "alpha appears in 2 documents");
assert_eq!(alpha.2, 3, "alpha occurs 3 times total");
assert_eq!(vocab[0].0, "alpha", "most frequent term first");
let schema_before = schema_fingerprint(&db.conn);
let row_vocab_exists: bool = db
.conn
.query_row(
"select exists(
select 1 from sqlite_schema
where type = 'table' and name = 'messages_trigram_terms'
)",
[],
|row| row.get(0),
)
.unwrap();
assert!(
row_vocab_exists,
"v4 must provide the row-level trigram vocabulary used for bounded output"
);
let tri = db.vocabulary(true, 0, None).unwrap();
assert!(
tri.iter().any(|(t, _, _)| t == "alp"),
"trigram vocab has 3-gram terms"
);
assert_eq!(
schema_fingerprint(&db.conn),
schema_before,
"reading v4 trigram vocabulary must not mutate any schema object"
);
}
#[test]
fn a_vocabulary_prefix_ending_in_sigma_finds_the_term_the_index_folded_it_to() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
seed_messages(&db, &[("user", "ΟΔΟΣ ΟΔΟΣΣΑ")]);
let stored: Vec<String> = db
.vocabulary(false, 0, Some("οδοσ"))
.unwrap()
.into_iter()
.map(|(term, _, _)| term)
.collect();
assert_eq!(
stored,
vec!["οδοσ".to_string(), "οδοσσα".to_string()],
"the folded spelling is what the index stores"
);
for prefix in ["ΟΔΟΣ", "οδος", "Οδος"] {
let terms: Vec<String> = db
.vocabulary(false, 0, Some(prefix))
.unwrap()
.into_iter()
.map(|(term, _, _)| term)
.collect();
assert_eq!(
terms,
vec!["οδοσ".to_string(), "οδοσσα".to_string()],
"prefix {prefix:?} reaches the terms the index folded it to"
);
}
}
#[test]
fn vocabulary_prefix_selects_only_terms_starting_with_it() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
seed_messages(
&db,
&[
("user", "alpha alpha bravo alphabet"),
("user", "alpha charlie alphabet"),
],
);
let terms = |prefix: Option<&str>, limit: usize| -> Vec<String> {
db.vocabulary(false, limit, prefix)
.unwrap()
.into_iter()
.map(|(term, _, _)| term)
.collect()
};
let mut matched = terms(Some("alph"), 0);
matched.sort();
assert_eq!(
matched,
vec!["alpha".to_string(), "alphabet".to_string()],
"a prefix keeps every term extending it and drops the rest"
);
assert_eq!(
terms(Some("alphabet"), 0),
vec!["alphabet".to_string()],
"a prefix equal to a whole term still matches that term"
);
assert!(
terms(Some("zulu"), 0).is_empty(),
"an unindexed prefix reports nothing rather than falling back to every term"
);
let mut every = terms(None, 0);
let mut empty = terms(Some(""), 0);
every.sort();
empty.sort();
assert_eq!(every, empty, "an empty prefix selects every term");
assert_eq!(
terms(Some("alph"), 1),
vec!["alpha".to_string()],
"limit applies after the prefix, most frequent first"
);
let alpha = db.vocabulary(false, 0, Some("alpha")).unwrap();
let alpha = alpha
.iter()
.find(|(term, _, _)| term == "alpha")
.expect("alpha present");
assert_eq!((alpha.1, alpha.2), (2, 3), "2 documents, 3 occurrences");
assert!(
db.vocabulary(true, 0, Some("alp"))
.unwrap()
.iter()
.all(|(term, _, _)| term.starts_with("alp")),
"the prefix reaches the trigram source too, not only word tokens"
);
}
#[test]
fn schema_v4_rejects_custom_trigram_maintenance_without_mutation() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
let schema_before = schema_fingerprint(&db.conn);
let error = db.ensure_trigram_base().unwrap_err().to_string();
assert!(error.contains("unavailable on schema v4"), "{error}");
assert_eq!(schema_fingerprint(&db.conn), schema_before);
}
#[test]
fn ensure_trigram_base_self_heals_corrupt_metadata_instead_of_failing_every_search() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
enable_v3_custom_trigram_compatibility(&db);
seed_messages(&db, &[("user", "socket failed with ECONNRESET today")]);
db.conn
.execute_batch(
"drop table trigram_meta; create table trigram_meta (key text primary key);",
)
.unwrap();
let base_max = db
.ensure_trigram_base()
.expect("corruption must self-heal, not propagate an error");
assert_eq!(
base_max, 1,
"self-heal rebuilds the base from the message table"
);
assert_eq!(
crate::trigram_index::base_max_id(&db.conn).unwrap(),
1,
"trigram_meta is valid and queryable again after self-heal"
);
let groups = crate::trigram::trigram_prefilter_groups("ECONNRESET").unwrap();
let cands = crate::trigram_index::candidates(&db.conn, &groups).unwrap();
assert!(
cands.contains(&1),
"the rebuilt index actually finds the seeded message, not just an empty shell"
);
}
#[test]
fn ensure_trigram_base_does_not_repair_metadata_when_maintenance_is_disabled() {
let dir = tempfile::tempdir().unwrap();
let mut db = Db::open(&dir.path().join("index.db")).unwrap();
enable_v3_custom_trigram_compatibility(&db);
seed_messages(&db, &[("user", "socket failed with ECONNRESET today")]);
db.conn
.execute_batch(
"drop table trigram_meta; create table trigram_meta (key text primary key);",
)
.unwrap();
db.set_implicit_index_maintenance(false);
let error = db.ensure_trigram_base().unwrap_err().to_string();
assert!(
error.contains("automatic maintenance is disabled"),
"{error}"
);
assert!(
!crate::trigram_index::schema_is_compatible(&db.conn).unwrap(),
"a maintenance-disabled read must not replace the malformed table"
);
}
#[test]
fn read_only_open_does_not_repair_incompatible_trigram_tables() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("index.db");
let db = Db::open(&path).unwrap();
enable_v3_custom_trigram_compatibility(&db);
seed_messages(&db, &[("user", "socket failed with ECONNRESET today")]);
db.conn
.execute_batch(
"drop table trigram_meta; create table trigram_meta (key text primary key);",
)
.unwrap();
let schema_before = schema_fingerprint(&db.conn);
drop(db);
let read_only = Db::open_existing_read_only_with_threads(
&path,
TEST_BUSY_TIMEOUT_MS,
NonZeroUsize::MIN,
)
.unwrap();
let error = read_only.ensure_trigram_base().unwrap_err().to_string();
assert!(
error.contains("automatic maintenance is disabled"),
"{error}"
);
assert_eq!(
schema_fingerprint(&read_only.conn),
schema_before,
"SQLite read-only mode must leave the incompatible derived schema unchanged"
);
}
#[test]
fn concurrent_read_only_connections_share_one_fixed_worker_runtime() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("index.db");
drop(Db::open(&path).unwrap());
let runtime = Arc::new(ExecutionRuntime::new(
NonZeroUsize::new(2).expect("nonzero worker count"),
));
let first = Db::open_existing_read_only_with_runtime(
&path,
TEST_BUSY_TIMEOUT_MS,
Arc::clone(&runtime),
)
.unwrap();
let second = Db::open_existing_read_only_with_runtime(
&path,
TEST_BUSY_TIMEOUT_MS,
Arc::clone(&runtime),
)
.unwrap();
assert!(Arc::ptr_eq(&first.runtime, &runtime));
assert!(Arc::ptr_eq(&second.runtime, &runtime));
assert_eq!(first.worker_threads(), 2);
assert_eq!(second.worker_threads(), 2);
}
#[test]
fn ensure_trigram_base_does_not_create_missing_tables_when_maintenance_is_disabled() {
let dir = tempfile::tempdir().unwrap();
let mut db = Db::open(&dir.path().join("index.db")).unwrap();
enable_v3_custom_trigram_compatibility(&db);
db.conn
.execute_batch("drop table trigram_postings; drop table trigram_meta;")
.unwrap();
db.set_implicit_index_maintenance(false);
let error = db.ensure_trigram_base().unwrap_err().to_string();
assert!(
error.contains("automatic maintenance is disabled"),
"{error}"
);
let derived_tables: i64 = db
.conn
.query_row(
"select count(*) from sqlite_master where name in ('trigram_postings', 'trigram_meta')",
[],
|row| row.get(0),
)
.unwrap();
assert_eq!(
derived_tables, 0,
"a read-only probe must not create schema"
);
}
#[test]
fn ensure_trigram_base_rolls_back_schema_repair_when_rebuild_fails() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
enable_v3_custom_trigram_compatibility(&db);
db.conn
.execute_batch(
"drop table trigram_meta;
create table trigram_meta (key text primary key);
drop table messages;",
)
.unwrap();
let error = db.ensure_trigram_base().unwrap_err().to_string();
assert!(error.contains("no such table: messages"), "{error}");
assert!(
!crate::trigram_index::schema_is_compatible(&db.conn).unwrap(),
"the failed transaction must restore the original incompatible schema"
);
}
#[test]
fn ensure_trigram_base_busy_repair_leaves_incompatible_tables_untouched() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("index.db");
let writer = Db::open(&path).unwrap();
enable_v3_custom_trigram_compatibility(&writer);
seed_messages(&writer, &[("user", "socket failed with ECONNRESET today")]);
writer
.conn
.execute_batch(
"drop table trigram_meta; create table trigram_meta (key text primary key);",
)
.unwrap();
let schema_before = schema_fingerprint(&writer.conn);
let contender = Db::open_with_busy_timeout(&path, TEST_NO_WAIT_BUSY_TIMEOUT_MS).unwrap();
writer.conn.execute_batch("begin immediate").unwrap();
let error = contender
.ensure_trigram_base()
.expect_err("a competing writer must prevent derived-table repair");
writer.conn.execute_batch("rollback").unwrap();
assert!(Db::is_sqlite_busy_error(&error), "{error:#}");
assert!(
!crate::trigram_index::schema_is_compatible(&contender.conn).unwrap(),
"a failed repair attempt must not replace either derived table"
);
assert_eq!(
schema_fingerprint(&contender.conn),
schema_before,
"the failed repair must preserve the entire preexisting schema"
);
}
#[test]
fn regex_prefilter_reaches_rows_by_primary_key_not_full_scan() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
enable_v3_custom_trigram_compatibility(&db);
seed_messages(&db, &[("user", "socket failed with ECONNRESET) today")]);
let base_max = db.ensure_trigram_base().unwrap();
let groups = crate::trigram::trigram_prefilter_groups("ECONNRESET").unwrap();
let cands = crate::trigram_index::candidates(&db.conn, &groups).unwrap();
db.stage_candidates(base_max, &cands).unwrap();
let plan = {
let mut stmt = db
.conn
.prepare(
"explain query plan select m.id from messages m \
where m.id in (select id from _trigram_cand) order by m.session_id, m.seq",
)
.unwrap();
stmt.query_map([], |r| r.get::<_, String>(3))
.unwrap()
.filter_map(Result::ok)
.collect::<Vec<_>>()
.join(" | ")
};
assert!(
plan.contains("PRIMARY KEY") || plan.contains("USING INTEGER PRIMARY KEY"),
"messages must be reached by primary key, not scanned: {plan}",
);
assert!(
!plan.contains("SCAN m "),
"no full scan of the messages table: {plan}",
);
}
#[test]
fn checkpoint_truncate_is_safe_and_preserves_data() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
seed_messages(&db, &[("user", "the deploy hit ECONNRESET) again")]);
db.checkpoint_truncate().unwrap();
db.checkpoint_truncate().unwrap();
let hits = db
.search_messages(
"ECONNRESET",
&MessageFilters {
match_mode: MessageSearchMode::Regex,
..Default::default()
},
)
.unwrap();
assert_eq!(hits.len(), 1, "data intact and searchable after checkpoint");
}
#[test]
fn schema_state_classification_is_total_and_preserves_readable_older_policy() {
assert_eq!(
SchemaState::from_version(SCHEMA_VERSION),
SchemaState::Current
);
assert_eq!(
SchemaState::from_version(SCHEMA_VERSION - 1),
SchemaState::Older {
current: SCHEMA_VERSION - 1,
required: SCHEMA_VERSION,
}
);
assert_eq!(
SchemaState::from_version(SCHEMA_VERSION + 1),
SchemaState::Newer {
current: SCHEMA_VERSION + 1,
supported: SCHEMA_VERSION,
}
);
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
db.conn
.pragma_update(None, "user_version", MIN_READABLE_SCHEMA_VERSION)
.unwrap();
assert!(db.needs_backfill().unwrap());
assert!(db.schema_is_readable().unwrap());
}
#[test]
fn corpus_threshold_probe_stops_at_the_decision_boundary() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
seed_messages(
&db,
&[
("user", "one"),
("user", "two"),
("user", "three"),
("user", "four"),
("assistant", "five"),
],
);
let filters = MessageFilters {
role: Some(Role::User),
..Default::default()
};
assert!(db.filtered_corpus_reaches(&filters, 3).unwrap());
assert!(db.filtered_corpus_reaches(&filters, 4).unwrap());
assert!(!db.filtered_corpus_reaches(&filters, 5).unwrap());
assert_eq!(db.filtered_corpus_count(&filters).unwrap(), 4);
}
fn seed_every_message_class(db: &Db) {
seed_messages(db, &[("user", "one"), ("assistant", "two")]);
db.conn
.execute_batch(
"update messages set kind = 'conversation', ts = '2026-08-01T00:00:00Z';
insert into messages(session_id, provider, seq, role, kind, ts, tool_name, content) values
('claude:s1', 'claude', 10, 'user', 'harness_notice', '2026-08-02T00:00:00Z', null, 'notice'),
('claude:s1', 'claude', 11, 'user', 'compaction', '2026-08-02T00:00:00Z', null, 'summary'),
('claude:s1', 'claude', 12, 'assistant', 'message', '2026-08-03T00:00:00Z', null, 'legacy kind'),
('claude:s1', 'claude', 13, 'assistant', 'tool_call', '2026-08-03T00:00:00Z', 'Bash', 'ls'),
('claude:s1', 'claude', 14, 'tool', 'tool_result', '2026-08-03T00:00:00Z', 'Bash', 'ok');",
)
.unwrap();
}
fn corpus_count_filter_cases() -> Vec<(&'static str, MessageFilters)> {
vec![
("default", MessageFilters::default()),
(
"role",
MessageFilters {
role: Some(Role::User),
..Default::default()
},
),
(
"provider",
MessageFilters {
providers: Some(vec![Provider::Claude]),
..Default::default()
},
),
(
"since",
MessageFilters {
since: Some(utc("2026-08-02T12:00:00Z")),
..Default::default()
},
),
(
"session",
MessageFilters {
session_id: Some("claude:s1".to_string()),
..Default::default()
},
),
(
"workspace_path",
MessageFilters {
workspace_path_prefix: Some("/x".to_string()),
..Default::default()
},
),
(
"tool",
MessageFilters {
tool: Some("bash".to_string()),
..Default::default()
},
),
(
"no_compaction",
MessageFilters {
no_compaction: true,
..Default::default()
},
),
(
"explicit_excludable_kind",
MessageFilters {
kinds: Some(vec![crate::models::MessageKind::HarnessNotice]),
..Default::default()
},
),
]
}
fn explicit_row_scanning_kind_filters() -> MessageFilters {
MessageFilters {
kinds: Some(vec![
crate::models::MessageKind::Conversation,
crate::models::MessageKind::ToolCall,
]),
..Default::default()
}
}
fn naive_corpus_count(db: &Db, filters: &MessageFilters) -> i64 {
let mut sql = String::from("select count(*) from messages m where 1 = 1");
let mut args = Vec::new();
append_message_filters(&mut sql, &mut args, filters, &db.access_scope);
db.conn
.query_row(&sql, rusqlite::params_from_iter(args.iter()), |row| {
row.get(0)
})
.unwrap()
}
fn query_plan(db: &Db, sql: &str, args: &[rusqlite::types::Value]) -> String {
let mut stmt = db
.conn
.prepare(&format!("explain query plan {sql}"))
.unwrap();
stmt.query_map(rusqlite::params_from_iter(args.iter()), |row| {
row.get::<_, String>(3)
})
.unwrap()
.filter_map(Result::ok)
.collect::<Vec<_>>()
.join(" | ")
}
#[test]
fn corpus_count_equals_the_naive_count_for_every_class_predicate() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
seed_every_message_class(&db);
for (name, filters) in corpus_count_filter_cases() {
assert_eq!(
db.filtered_corpus_count(&filters).unwrap(),
naive_corpus_count(&db, &filters),
"{name}: the decomposed count must equal the single-statement count"
);
}
let explicit = explicit_row_scanning_kind_filters();
assert_eq!(
db.filtered_corpus_count(&explicit).unwrap(),
naive_corpus_count(&db, &explicit),
"explicit non-excludable classes"
);
assert_eq!(
db.filtered_corpus_count(&MessageFilters::default())
.unwrap(),
6
);
}
#[test]
fn corpus_count_statements_are_served_from_covering_indexes() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open(&dir.path().join("index.db")).unwrap();
seed_every_message_class(&db);
let mut cases = corpus_count_filter_cases()
.into_iter()
.map(|(name, filters)| (name, "1 = 1", filters))
.collect::<Vec<_>>();
cases.push((
"tool_argument",
"m.kind = 'tool_call'",
MessageFilters::default(),
));
for (name, base, filters) in cases {
let statements = db.corpus_count_statements(base, &filters);
assert!(!statements.is_empty(), "{name}: no count statements");
for (sql, args, _sign) in &statements {
let plan = query_plan(&db, sql, args);
for step in plan.split(" | ") {
if step.contains("SCAN m") || step.contains("SEARCH m") {
assert!(
step.contains("USING COVERING INDEX"),
"{name}: corpus count must not read message rows\nsql: {sql}\nplan: {plan}"
);
}
}
assert!(
plan.contains("COVERING INDEX"),
"{name}: corpus count must be index-only\nsql: {sql}\nplan: {plan}"
);
}
}
}
#[test]
fn message_kind_index_is_created_free_on_an_empty_database_and_deferred_on_a_populated_one() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("index.db");
let db = Db::open(&path).unwrap();
assert!(
db.message_kind_index_exists().unwrap(),
"an empty database gets the class index at open, where building it is free"
);
seed_every_message_class(&db);
db.conn
.execute_batch(&format!("drop index {MESSAGE_KIND_INDEX}"))
.unwrap();
drop(db);
let reopened = Db::open(&path).unwrap();
assert!(
!reopened.message_kind_index_exists().unwrap(),
"open must defer the class index build on a populated database"
);
assert!(
reopened.ensure_message_kind_index().unwrap(),
"the maintenance path reports that it built the index"
);
assert!(reopened.message_kind_index_exists().unwrap());
assert!(
!reopened.ensure_message_kind_index().unwrap(),
"a second call is a no-op"
);
for (name, filters) in corpus_count_filter_cases() {
assert_eq!(
reopened.filtered_corpus_count(&filters).unwrap(),
naive_corpus_count(&reopened, &filters),
"{name}"
);
}
}
}