1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
/// Enrich functions with contract metadata from SQLite (optional columns).
///
/// Handles old schema gracefully — if contract_level/contract_equation
/// columns don't exist, silently skips enrichment.
fn enrich_contract_metadata(conn: &rusqlite::Connection, functions: &mut [FunctionEntry]) {
let query = "SELECT id, contract_level, contract_equation FROM functions WHERE contract_level IS NOT NULL ORDER BY id";
let Ok(mut stmt) = conn.prepare(query) else {
return; // Column doesn't exist in old schema — skip
};
let Ok(rows) = stmt.query_map([], |row| {
Ok((
row.get::<_, i64>(0)? as usize,
row.get::<_, Option<String>>(1)?,
row.get::<_, Option<String>>(2)?,
))
}) else {
return;
};
for row in rows.flatten() {
let (id, level, equation) = row;
if id >= 1 && id <= functions.len() {
functions[id - 1].quality.contract_level = level;
functions[id - 1].quality.contract_equation = equation;
}
}
}
impl AgentContextIndex {
/// Save index to directory
#[provable_contracts_macros::contract("pmat-core.yaml", equation = "path_exists")]
pub fn save(&self, index_path: &Path) -> Result<(), String> {
fs::create_dir_all(index_path)
.map_err(|e| format!("Failed to create index directory: {e}"))?;
// Save manifest
let manifest_json = serde_json::to_string_pretty(&self.manifest)
.map_err(|e| format!("Failed to serialize manifest: {e}"))?;
fs::write(index_path.join("manifest.json"), manifest_json)
.map_err(|e| format!("Failed to write manifest: {e}"))?;
// Phase 3 (#159): Write only SQLite + FTS5 index (no more LZ4 blob)
// context.db lives alongside context.idx directory
let db_path = index_path.with_extension("db");
super::sqlite_backend::save_to_sqlite(
&db_path,
&self.functions,
&self.calls,
&self.graph_metrics,
&self.manifest,
&self.coverage_off_files,
)?;
Ok(())
}
/// Load index from directory.
///
/// Prefers SQLite `context.db` when available (v2.0), falls back to
/// LZ4+bincode blob `context.idx/functions.lz4` (v1.x).
#[provable_contracts_macros::contract("pmat-core.yaml", equation = "path_exists")]
pub fn load(index_path: &Path) -> Result<Self, String> {
// R30: ONE scale check, ahead of both backends, with ONE remediation.
//
// A pre-v3.30.0 index has every required table and every required
// column, so the structural check below passes it happily — while its
// `tdg_score` column holds 0-10 lower-is-better debt numbers that
// today's readers report as 0-100 quality scores, turning a stored 0.12
// (the BEST legacy score) into an F. Structure valid does not mean
// contents readable.
//
// The remediation is to DISCARD both artifacts, not merely to report.
// This used to be split: the SQLite branch deleted its own stale `.db`
// and let the blob path produce the error, so `pmat query` self-healed
// while MCP's `IndexManager` — which propagates the error — answered
// `-32603 … rebuild required` on every default call, forever. With the
// whole stale index removed, the next `load()` sees no index at all and
// every caller (CLI, MCP, comply) takes its ordinary build path.
if let Err(reason) = super::scale_guard::verify_index_scale(index_path) {
eprintln!(
" Index at {} is stale: {reason} (scores are now 0-100, higher is better).",
index_path.display()
);
super::scale_guard::discard_stale_index(index_path);
return Err(reason);
}
// Try SQLite path first (v2.0)
let db_candidate = index_path.with_extension("db");
if db_candidate.exists() {
// Validate schema before attempting full load — stale DBs from older
// versions may lack required tables, producing confusing warnings.
let conn = super::sqlite_backend::open_db(&db_candidate).ok();
let schema_ok = conn
.as_ref()
.is_some_and(super::sqlite_backend::has_valid_schema);
drop(conn);
if schema_ok {
match Self::load_from_sqlite(&db_candidate) {
Ok(index) => return Ok(index),
Err(e) => {
eprintln!(" Warning: SQLite load failed, falling back to blob: {e}");
}
}
} else {
// Delete broken DB so next save() regenerates it
let _ = std::fs::remove_file(&db_candidate);
}
}
Self::load_from_blob(index_path)
}
/// Load index from SQLite database (v2.0 fast path).
///
/// Reads functions (without source) and graph metrics from `context.db`.
/// Skips corpus (FTS5 handles search), call graph (queried on-demand),
/// and source code (loaded on-demand for display or regex/literal search).
fn load_from_sqlite(db_path: &Path) -> Result<Self, String> {
use super::sqlite_backend::{
load_functions_lightweight, load_graph_metrics, load_metadata, open_db,
};
let conn = open_db(db_path)?;
let manifest = load_metadata(&conn)?;
let mut functions = load_functions_lightweight(&conn)?;
// Enrich with contract metadata (optional columns, may not exist in old schemas)
enrich_contract_metadata(&conn, &mut functions);
let graph_metrics = load_graph_metrics(&conn)?;
// Call graph loaded on-demand via get_calls()/get_called_by() SQLite fallback
let calls = HashMap::new();
let called_by = HashMap::new();
// Build name_index + file_index only (no corpus — FTS5 handles search)
let indices = build_indices_without_corpus(&functions);
let name_frequency = compute_name_frequency(&indices.name_index, functions.len());
let project_root = PathBuf::from(&manifest.project_root);
// Load cached coverage_off_files from SQLite metadata
let coverage_off_files = load_coverage_off_files(&conn);
Ok(Self {
functions,
name_index: indices.name_index,
file_index: indices.file_index,
corpus: Vec::new(),
corpus_lower: Vec::new(),
name_frequency,
calls,
called_by,
graph_metrics,
project_root,
manifest,
db_path: Some(db_path.to_path_buf()),
coverage_off_files,
})
}
/// Load index from LZ4+bincode blob (v1.x legacy path).
fn load_from_blob(index_path: &Path) -> Result<Self, String> {
// Load manifest
let manifest_str = fs::read_to_string(index_path.join("manifest.json"))
.map_err(|e| format!("Failed to read manifest: {e}"))?;
let manifest: IndexManifest = serde_json::from_str(&manifest_str)
.map_err(|e| format!("Failed to parse manifest: {e}"))?;
// R30: the manifest's scale marker is checked by `load()` via
// `scale_guard::verify_index_scale`, which also discards the stale
// index. A second comparison here would be a second decision point —
// exactly the duplication that let `pmat sql` drift.
// Load and decompress blob
let compressed = fs::read(index_path.join("functions.lz4"))
.map_err(|e| format!("Failed to read functions: {e}"))?;
let decompressed = lz4_flex::decompress_size_prepended(&compressed)
.map_err(|e| format!("Failed to decompress functions: {e}"))?;
// Deserialize payload (v1.3.0+ has cached indices)
let payload: IndexPayload = rmp_serde::from_slice(&decompressed)
.map_err(|e| format!("Failed to parse payload: {e}"))?;
let functions = payload.functions;
let corpus = payload.corpus;
let calls = payload.calls;
let called_by = payload.called_by;
// Check if we have cached indices (v1.3.0+) - avoids expensive PageRank recomputation
let has_cached_indices = !payload.name_index.is_empty();
let (name_index, file_index, graph_metrics, corpus_lower, name_frequency) =
if has_cached_indices {
let corpus_lower = if payload.corpus_lower.is_empty() {
corpus.iter().map(|d| d.to_lowercase()).collect()
} else {
payload.corpus_lower
};
(
payload.name_index,
payload.file_index,
payload.graph_metrics,
corpus_lower,
payload.name_frequency,
)
} else {
// Slow path: rebuild indices for legacy formats (v1.0-v1.2)
let is_legacy =
manifest.version.starts_with("1.0") || manifest.version.starts_with("1.1");
let indices = build_indices(&functions);
let (calls_rebuilt, called_by_rebuilt) = if is_legacy && calls.is_empty() {
build_call_graph(&functions, &indices.name_index)
} else {
(calls.clone(), called_by.clone())
};
let graph_metrics =
compute_graph_metrics(functions.len(), &calls_rebuilt, &called_by_rebuilt);
let name_frequency = compute_name_frequency(&indices.name_index, functions.len());
let corpus_lower: Vec<String> = corpus.iter().map(|d| d.to_lowercase()).collect();
(
indices.name_index,
indices.file_index,
graph_metrics,
corpus_lower,
name_frequency,
)
};
let project_root = PathBuf::from(&manifest.project_root);
// Detect SQLite FTS5 database alongside blob
let db_candidate = index_path.with_extension("db");
let db_path = if db_candidate.exists() {
Some(db_candidate)
} else {
None
};
Ok(Self {
functions,
name_index,
file_index,
corpus,
corpus_lower,
name_frequency,
calls,
called_by,
graph_metrics,
project_root,
manifest,
db_path,
coverage_off_files: HashSet::new(), // Legacy blob path — no cached data
})
}
}