rto_graph/query.rs
1//! The agent- and human-facing query surface over the graph.
2//!
3//! Everything here is a read-only view built from the store's typed queries,
4//! serialised under a **stable, versioned** JSON schema ([`SCHEMA`]) so agents
5//! can depend on the shape. The primitives are [`explain`] (a node and its
6//! provenance-labelled neighbourhood), [`list_kind`] (all nodes of a kind),
7//! [`path`] (a shortest path between two nodes), [`debt`] (the intent-debt marker
8//! inventory), [`debt_density`] (that inventory per file, normalised by file
9//! length), [`coupling`] (directed fan-in/fan-out over `Calls` edges),
10//! [`config_secrets`] (secret-named config keys and their redaction state), and
11//! [`search`] (relevance-ranked node search). All return
12//! mixed-provenance results — the "one query surface" from ADR-0001 — with every
13//! edge carrying its `provenance`.
14
15use std::collections::{BTreeMap, BTreeSet, VecDeque};
16
17use serde::Serialize;
18
19use crate::store::{Store, StoreError};
20use crate::{Edge, EdgeKind, NodeKind, Provenance};
21
22/// The versioned schema tag emitted on every query result. Bump the version on
23/// any breaking change to the shape.
24pub const SCHEMA: &str = "roteiro.query/v1";
25
26/// Cut an already-ordered, already-materialised list down to the window a caller
27/// asked for: skip `offset` items from the front, then keep at most `limit`.
28///
29/// **This is the one place that decides what `limit` and `offset` mean** for the
30/// graph's list lenses — [`debt_density`], [`config_secrets`] and [`coupling`]
31/// here, and the `/nodes` and `/hotspots` endpoints in the `roteiro` binary. It
32/// exists because the parameter previously had two implementations: three lenses
33/// truncated here and treated `0` as "no limit", while two HTTP handlers used
34/// [`Iterator::take`] and so returned nothing for `0` — the same parameter name
35/// with opposite meanings, and nothing that could make the disagreement visible
36/// (issue #375). A sixth list lens should call this rather than write a third.
37/// One did write a third — see *Episodic recall* below — and the warning is left
38/// standing because the next one will too.
39///
40/// The contract:
41///
42/// - **`limit == 0` means unlimited** — every item that survives `offset` is
43/// kept. This is the reading the CLI already documents (`roteiro
44/// config-secrets --help`: *"0 shows every secret-named key"*), so no
45/// published promise is withdrawn by making it universal; and it is the safer
46/// of the two, because a caller who passes an unset variable then gets more
47/// data than they meant to ask for rather than an empty page that reads like a
48/// truthful "nothing found".
49/// - **`offset` applies first, and `limit` to what remains.** So `offset = 20,
50/// limit = 0` is "skip the first 20, then every remaining item", not "skip 20,
51/// then nothing". An `offset` past the end yields an empty window rather than
52/// panicking — a page beyond the last one is empty, not an error.
53/// - A caller's reported `total` must be taken **before** this runs: every
54/// surface reports the pre-windowing population, so a cut page still says what
55/// it was cut from.
56///
57/// Ordering is the caller's job; this only removes, and only from the ends, so a
58/// deterministically-ordered input yields a deterministic window.
59///
60/// # The search channels call this too, and the unit there is *per channel*
61///
62/// [`search`] and the generated/memory channels behind [`search_channels`] used
63/// to keep a `limit == 0 => no hits` guard of their own — a third reading of one
64/// parameter name, in the same crate as the two #375 reconciled (issue #393).
65/// They now window here like every other lens, so `limit` has one definition and
66/// not a second implementation of it, which is exactly how the first two drifted.
67///
68/// What differs is the **unit**, not the rule: a search `limit` bounds *each
69/// channel* independently, so `0` is "every match, in every channel that was
70/// asked for", not "every match overall". That is a bounded request rather than
71/// "dump the graph": a channel's ranking only *orders* a set the query has
72/// already filtered — every token must appear in a hit — and a query with no
73/// tokens returns nothing at any limit, `0` included. Measured on this
74/// repository at 6,685 nodes: an unbounded one-token search returned ~2.7k hits
75/// in 0.24 s and a two-token one returned 3, against a full-population scan that
76/// every limit pays anyway, so unlimited costs no more than the default does.
77///
78/// The **MCP tools are the one surface that cannot ask for it**, deliberately:
79/// they clamp `limit` into `1..=25` and advertise `"minimum": 1`, because their
80/// results are spent against a model's context window and `0` would be the one
81/// value that escaped the ceiling those clamps exist to impose. That is a
82/// surface declining to offer a value, not a second meaning for it — a model
83/// that sends `0` anyway gets the smallest page, never the silent empty answer
84/// this issue is about. The reasoning is restated where each clamp lives, in
85/// `rto_render::mcp::GraphServer::search` and the served-chat `search` arm in
86/// the `roteiro` binary; if this rule changes, those two must change with it.
87///
88/// # Episodic recall — the third implementation this doc predicted (issue #447)
89///
90/// [`crate::Store::recall_memory`] ranked, then called
91/// [`Vec::truncate`] directly, so `limit = 0` emptied the result: `recall
92/// --limit 0` returned nothing on a store where five other surfaces returned
93/// everything, and the JSON said `"live": 8000` beside `"results": []` — not a
94/// lie, and no help at all. It calls this now. Two details are worth keeping:
95///
96/// - **`None` and `Some(0)` had to collapse onto one meaning, not two.**
97/// `RecallOptions::limit` is an `Option`, so "unlimited" was already sayable
98/// twice; the fix maps `None` to `0` rather than adding a branch, because two
99/// spellings of one request are how the first divergence started.
100/// - **`memory list` cuts in SQL and so cannot call this.** `LIMIT 0` in SQL
101/// means the *opposite* of what this function means, so `memory::records`
102/// omits the clause entirely for `0`. That is the contract translated, and it
103/// is the only place in the crate where the rule is re-expressed rather than
104/// called — worth knowing if it ever has to change.
105pub fn window<T>(items: &mut Vec<T>, offset: usize, limit: usize) {
106 // `min(len)` rather than a bounds check: `drain(..offset)` panics past the
107 // end of the vector, and "page 900 of 3" is an empty page, not a 500.
108 items.drain(..offset.min(items.len()));
109 if limit > 0 {
110 items.truncate(limit);
111 }
112}
113
114/// A compact node summary (used in listings and as the subject of an
115/// [`Explanation`]).
116#[derive(Debug, Clone, PartialEq, Serialize)]
117pub struct NodeSummary {
118 /// Natural key.
119 pub key: String,
120 /// Kind token (e.g. `fn`, `adr`).
121 pub kind: String,
122 /// Human-facing name.
123 pub name: String,
124 /// Repository-relative path, if any.
125 pub path: Option<String>,
126 /// Language token, if any.
127 pub lang: Option<String>,
128}
129
130impl NodeSummary {
131 fn from_node(node: &crate::Node) -> Self {
132 Self {
133 key: node.key.clone(),
134 kind: node.kind.as_str().to_owned(),
135 name: node.name.clone(),
136 path: node.path.clone(),
137 lang: node.lang.clone(),
138 }
139 }
140}
141
142/// One end of an edge as seen from a subject node: the relationship, how it was
143/// produced, and the node on the other end.
144#[derive(Debug, Clone, PartialEq, Serialize)]
145pub struct EdgeRef {
146 /// Edge kind token (e.g. `calls`, `references`).
147 pub kind: String,
148 /// How the edge was produced (`derived` | `authored` | `inferred`).
149 pub provenance: &'static str,
150 /// Confidence score, present only for inferred edges.
151 pub confidence: Option<f64>,
152 /// The natural key of the node at the other end.
153 pub node: String,
154}
155
156/// A node together with its provenance-labelled neighbourhood.
157#[derive(Debug, Clone, PartialEq, Serialize)]
158pub struct Explanation {
159 /// Stable schema tag ([`SCHEMA`]).
160 pub schema: &'static str,
161 /// The subject node.
162 pub node: NodeSummary,
163 /// Structured metadata attached to the node.
164 pub meta: serde_json::Value,
165 /// Edges where the subject is the source.
166 pub outgoing: Vec<EdgeRef>,
167 /// Edges where the subject is the destination.
168 pub incoming: Vec<EdgeRef>,
169}
170
171/// A listing of all nodes of one kind.
172#[derive(Debug, Clone, PartialEq, Serialize)]
173pub struct Listing {
174 /// Stable schema tag ([`SCHEMA`]).
175 pub schema: &'static str,
176 /// The kind that was listed.
177 pub kind: String,
178 /// Matching nodes, ordered by key.
179 pub nodes: Vec<NodeSummary>,
180}
181
182/// One intent-debt finding in a [`DebtReport`].
183#[derive(Debug, Clone, PartialEq, Serialize)]
184pub struct DebtItem {
185 /// Natural key of the marker node (`marker:<path>#<line>`).
186 pub key: String,
187 /// Category token (`todo` | `fixme` | `hack` | `stub` | `deferred`).
188 pub category: String,
189 /// The marker text (the trimmed source line).
190 pub text: String,
191 /// Repository-relative path of the source file, if any.
192 pub path: Option<String>,
193 /// 1-based line number, if recorded.
194 pub line: Option<u32>,
195}
196
197/// The intent-debt inventory: every `marker` node, grouped and listed. A
198/// deterministic, provenance-`derived` view of what is incomplete or postponed.
199#[derive(Debug, Clone, PartialEq, Serialize)]
200pub struct DebtReport {
201 /// Stable schema tag ([`SCHEMA`]).
202 pub schema: &'static str,
203 /// Total markers in the report (after any category filter).
204 pub total: usize,
205 /// Count per category, ordered by category token.
206 pub by_category: BTreeMap<String, usize>,
207 /// The markers, ordered by `(path, line, key)`.
208 pub items: Vec<DebtItem>,
209}
210
211/// Inventory intent-debt markers in the graph, optionally restricted to the
212/// given `categories` (empty means all) and excluding markers whose file path
213/// matches any `ignore` glob (config `[debt] ignore` — empty means keep all).
214/// Ordered by `(path, line)` so output is stable and reads top-to-bottom per
215/// file; `total` and `by_category` reflect the retained markers only.
216///
217/// # Errors
218/// Returns [`StoreError`] on query failure.
219pub fn debt(
220 store: &Store,
221 categories: &[String],
222 ignore: &[String],
223) -> Result<DebtReport, StoreError> {
224 let filter: std::collections::BTreeSet<&str> = categories.iter().map(String::as_str).collect();
225 let mut items = Vec::new();
226 let mut by_category: BTreeMap<String, usize> = BTreeMap::new();
227 for node in store.nodes_by_kind(&NodeKind::Marker)? {
228 let category = node
229 .meta
230 .get("category")
231 .and_then(serde_json::Value::as_str)
232 .unwrap_or("other")
233 .to_owned();
234 if !filter.is_empty() && !filter.contains(category.as_str()) {
235 continue;
236 }
237 // Drop markers under an ignored path (e.g. `vendor/**`) before counting.
238 if let Some(path) = node.path.as_deref()
239 && ignore.iter().any(|glob| glob_match(glob, path))
240 {
241 continue;
242 }
243 let text = node
244 .meta
245 .get("text")
246 .and_then(serde_json::Value::as_str)
247 .unwrap_or(node.name.as_str())
248 .to_owned();
249 let line = node
250 .meta
251 .get("line")
252 .and_then(serde_json::Value::as_u64)
253 .and_then(|l| u32::try_from(l).ok());
254 *by_category.entry(category.clone()).or_default() += 1;
255 items.push(DebtItem {
256 key: node.key.clone(),
257 category,
258 text,
259 path: node.path.clone(),
260 line,
261 });
262 }
263 items.sort_by(|a, b| (&a.path, a.line, &a.key).cmp(&(&b.path, b.line, &b.key)));
264 Ok(DebtReport {
265 schema: SCHEMA,
266 total: items.len(),
267 by_category,
268 items,
269 })
270}
271
272/// How a [`DebtDensityReport`]'s files are ranked.
273#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
274pub enum DensityOrder {
275 /// By `per_kloc` — markers relative to file length. The lens's own question.
276 #[default]
277 Density,
278 /// By `markers` — the raw count, which [`debt`] already reports per marker.
279 /// Offered so the two rankings can be compared on one report rather than the
280 /// reader being asked to trust that they differ.
281 Markers,
282 /// By `lines` — longest file first. Not a debt ranking; the control that
283 /// shows *which* files the denominator is large for.
284 Lines,
285}
286
287impl DensityOrder {
288 /// The stable token for this order, as accepted by [`from_token`](Self::from_token).
289 #[must_use]
290 pub fn as_str(self) -> &'static str {
291 match self {
292 Self::Density => "density",
293 Self::Markers => "markers",
294 Self::Lines => "lines",
295 }
296 }
297
298 /// Parse an order token. `None` for anything else — callers surface an error
299 /// rather than silently ranking by something the caller did not ask for.
300 #[must_use]
301 pub fn from_token(s: &str) -> Option<Self> {
302 match s {
303 "density" => Some(Self::Density),
304 "markers" => Some(Self::Markers),
305 "lines" => Some(Self::Lines),
306 _ => None,
307 }
308 }
309
310 /// The tokens [`from_token`](Self::from_token) accepts, for error messages
311 /// and argument schemas — so the accepted set is stated in exactly one place.
312 #[must_use]
313 pub fn tokens() -> [&'static str; 3] {
314 [
315 Self::Density.as_str(),
316 Self::Markers.as_str(),
317 Self::Lines.as_str(),
318 ]
319 }
320}
321
322/// The default `min_lines` floor for [`debt_density`]: files shorter than this
323/// are counted but not ranked (see [`DebtDensityReport::min_lines`] for why the
324/// floor exists at all).
325///
326/// 50 because that is where one marker stops dominating: a single marker in a
327/// 50-line file scores 20 per kloc, which is already near the top of this
328/// repository's real ranking, so any shorter file with a marker is guaranteed a
329/// high placement by its length alone. It is a default, not a rule — `0` ranks
330/// every file.
331pub const DEFAULT_MIN_LINES: u32 = 50;
332
333/// One file's intent-debt density in a [`DebtDensityReport`].
334#[derive(Debug, Clone, PartialEq, Serialize)]
335pub struct DensityItem {
336 /// Repository-relative path of the file.
337 pub path: String,
338 /// Retained markers in this file (after category and `ignore` filtering).
339 pub markers: u32,
340 /// The file's length in lines — the denominator. See [`debt_density`] for
341 /// exactly what this counts and what it does not.
342 pub lines: u32,
343 /// Markers per 1,000 lines, rounded to two decimals. Per *kilo*-line rather
344 /// than per line because per-line densities are all leading zeroes: this
345 /// repository's worst file is 0.06 markers per line, and `60.0` per kloc is
346 /// a number a reader can hold.
347 pub per_kloc: f64,
348 /// Count per category within this file, ordered by category token — so a
349 /// dense file can be read as "twelve `todo`" or "twelve `stub`", which are
350 /// not the same finding.
351 pub by_category: BTreeMap<String, usize>,
352}
353
354/// Intent-debt **density**: markers per file normalised by file length, ranked.
355/// The counterpart to [`debt`], which reports markers and therefore ranks large
356/// files first by construction.
357#[derive(Debug, Clone, PartialEq, Serialize)]
358pub struct DebtDensityReport {
359 /// Stable schema tag ([`SCHEMA`]).
360 pub schema: &'static str,
361 /// The ranking that produced `items` ([`DensityOrder::as_str`]).
362 pub order: &'static str,
363 /// The requested cap on `items`; `0` means unlimited.
364 pub limit: usize,
365 /// The `min_lines` floor applied. Files shorter than this are excluded from
366 /// the ranking and counted in `short_files`; `0` disables the floor.
367 ///
368 /// The floor exists because density is unstable in the denominator's tail: a
369 /// 3-line stub file with one marker scores 333 per kloc, which is true and
370 /// tells the reader nothing. Excluding those files is a ranking decision,
371 /// not a suppression — they stay in `files_with_markers` and `total_markers`.
372 pub min_lines: u32,
373 /// Distinct files carrying at least one retained marker, before the
374 /// `min_lines` floor. The population `items` is drawn from.
375 pub files_with_markers: usize,
376 /// Files that passed the `min_lines` floor and were therefore ranked.
377 /// `ranked_files > items.len()` means `limit` truncated the list.
378 pub ranked_files: usize,
379 /// Files excluded from the ranking by the `min_lines` floor — reported, not
380 /// silently dropped, so a short-file-heavy repository cannot read as a clean one.
381 pub short_files: usize,
382 /// Files whose marker count is known but whose length is **not**: no `file`
383 /// node, or one carrying no `meta.lines`. Excluded from the ranking, because
384 /// a density with no denominator is not a number — and reported, because
385 /// silently omitting them would understate the inventory.
386 pub unknown_length_files: usize,
387 /// Retained markers across every file in `files_with_markers`, including
388 /// those the floor excluded. Matches [`DebtReport::total`] for the same
389 /// filters, minus any marker with no `path`.
390 pub total_markers: usize,
391 /// Summed `lines` of the ranked files. The denominator behind
392 /// `overall_per_kloc`.
393 pub total_lines: u64,
394 /// Markers per 1,000 lines across the **ranked** files taken together — the
395 /// baseline a single file's `per_kloc` should be read against. `0.0` when
396 /// nothing was ranked.
397 pub overall_per_kloc: f64,
398 /// The ranked files: by `order` descending, ties broken by `path` ascending.
399 pub items: Vec<DensityItem>,
400}
401
402/// Rank files by intent-debt **density** — retained markers per 1,000 lines —
403/// most-dense first by `order`, capped at `limit` (`0` = unlimited). `categories`
404/// and `ignore` filter markers exactly as [`debt`] does, so the two lenses always
405/// agree about which markers exist.
406///
407/// # Why this is not [`debt`] with a division
408///
409/// A raw marker count ranks by file size: the biggest file wins because it has
410/// the most lines to put a marker on. Density asks the different question — *how
411/// concentrated is the debt* — and a 40-marker file of 4,000 lines and a
412/// 40-marker file of 200 lines separate by a factor of twenty under it while
413/// being indistinguishable under [`debt`].
414///
415/// # The denominator, and why it is this one
416///
417/// **`lines` is the `file` node's `meta.lines`**, recorded at extraction time as
418/// the count of `\n` bytes in the blob. It is read straight from the graph, so
419/// this lens adds **no extraction metadata and needs no `EXTRACT_VERSION` bump**.
420///
421/// It is deliberately *not* derived from [`crate::Span`]: a node's span is a pair
422/// of **byte offsets**, not line numbers, and there is no line index in the store
423/// to convert one to the other. Anything span-derived would be a byte density,
424/// which is not the quantity anyone means by "debt density".
425///
426/// Three alternatives were rejected, each for the same reason:
427///
428/// - **Source lines of code** (blank and comment lines removed) is the denominator
429/// a reader probably imagines. It does not exist in the graph and cannot be
430/// computed from it: producing it means counting lines per language at
431/// extraction, which is net-new derived metadata and would move this lens into
432/// the batch that pays for an `EXTRACT_VERSION` bump.
433/// - **Per symbol** rather than per file would be the finer-grained view — markers
434/// already attach to their innermost enclosing symbol. But a symbol's length in
435/// *lines* is exactly what `Span`'s byte offsets cannot give.
436/// - **The highest marker line in the file** is available (`meta.line`), and is a
437/// lower bound on the file's length rather than the length: a file whose only
438/// marker is on line 3 would score 333 per kloc however long it is.
439///
440/// So `lines` is what the graph honestly has. What it counts, stated plainly
441/// because the name invites over-reading:
442///
443/// - **Every line, including blanks, comments, imports and licence headers.** It
444/// is *file length*, not "lines of code". Density figures here are therefore
445/// systematically lower than an SLOC-based tool's, and by a different factor
446/// per language and per file.
447/// - **Newline bytes.** A file not ending in a newline is counted one line short,
448/// and a file of a single unterminated line counts as `0` lines and is reported
449/// under `unknown_length_files` rather than divided by zero.
450/// - **The whole blob, vendored code included.** A minified bundle is one enormous
451/// line and will look flawless. Use `ignore` (the shared `[debt] ignore` globs)
452/// rather than a second exclusion vocabulary.
453///
454/// # Confidence, and why there is no CI gate
455///
456/// Density inherits every false positive of the marker scan beneath it — the
457/// prose rules (`for now`, `deferred`, `tbd`) fire on ordinary writing, so a
458/// design document rich in the word "deferred" ranks as dense debt. It then adds
459/// one of its own: the denominator is file length, so a language or a file with
460/// low information per line (verbose config, generated code, wide indentation) is
461/// systematically flattered, and a dense language is systematically penalised.
462/// Neither is a defect being reported; both move the number. A gate would fail
463/// builds on prose and on formatting. So this lens **offers no CI gate**, and its
464/// suppression story is the one that already exists: `[debt] ignore` globs and
465/// the `roteiro:ignore` / `roteiro:ignore-file` source directives, both applied
466/// before anything is counted here.
467///
468/// Ordering is total and deterministic: by the chosen metric descending, then by
469/// `path` ascending, so identical input yields byte-identical output.
470///
471/// # Errors
472/// Returns [`StoreError`] on query failure.
473pub fn debt_density(
474 store: &Store,
475 categories: &[String],
476 ignore: &[String],
477 order: DensityOrder,
478 limit: usize,
479 min_lines: u32,
480) -> Result<DebtDensityReport, StoreError> {
481 // Reuse `debt` rather than re-walking the markers: the two lenses must never
482 // disagree about which markers exist, and the only way to guarantee that is
483 // for one to be built from the other's output.
484 let inventory = debt(store, categories, ignore)?;
485 let mut per_file: BTreeMap<String, BTreeMap<String, usize>> = BTreeMap::new();
486 let mut total_markers = 0usize;
487 for item in &inventory.items {
488 // A marker with no `path` cannot be attributed to a file, so it cannot
489 // have a density. Extraction always records one; this is defence in
490 // depth, and such a marker is left out of `total_markers` too so the
491 // report's own arithmetic stays consistent.
492 let Some(path) = item.path.as_deref() else {
493 continue;
494 };
495 *per_file
496 .entry(path.to_owned())
497 .or_default()
498 .entry(item.category.clone())
499 .or_default() += 1;
500 total_markers += 1;
501 }
502 let files_with_markers = per_file.len();
503
504 // Only files that actually carry a marker are read back, so the denominator
505 // costs one node lookup per such file rather than a whole-graph file scan —
506 // which matters because `file` nodes carry captured `meta.content`.
507 let mut ranked: Vec<(String, u32, u32, BTreeMap<String, usize>)> = Vec::new();
508 let mut short_files = 0usize;
509 let mut unknown_length_files = 0usize;
510 for (path, by_category) in per_file {
511 let markers = u32::try_from(by_category.values().sum::<usize>()).unwrap_or(u32::MAX);
512 let Some(lines) = file_lines(store, &path)? else {
513 unknown_length_files += 1;
514 continue;
515 };
516 if lines < min_lines {
517 short_files += 1;
518 continue;
519 }
520 ranked.push((path, markers, lines, by_category));
521 }
522 let ranked_files = ranked.len();
523 let total_lines: u64 = ranked
524 .iter()
525 .map(|(_, _, lines, _)| u64::from(*lines))
526 .sum();
527 let ranked_markers: u64 = ranked.iter().map(|(_, m, _, _)| u64::from(*m)).sum();
528
529 ranked.sort_by(|a, b| {
530 // Each order yields the metric as an exact `numerator / denominator`, so
531 // the comparison can cross-multiply. Ranking on the rounded `per_kloc`
532 // instead would make genuinely different densities tie and then break on
533 // `path`, silently reordering the ranking the caller asked for; ranking
534 // on `f64` at full precision would make the order depend on the last bit
535 // of a division. `u128` so the cross-product cannot overflow whatever the
536 // file lengths are, rather than relying on repositories staying small.
537 let metric =
538 |&(_, markers, lines, _): &(String, u32, u32, BTreeMap<String, usize>)| match order {
539 DensityOrder::Density => (u128::from(markers) * 1000, u128::from(lines)),
540 DensityOrder::Markers => (u128::from(markers), 1),
541 DensityOrder::Lines => (u128::from(lines), 1),
542 };
543 let (an, ad) = metric(a);
544 let (bn, bd) = metric(b);
545 (bn * ad).cmp(&(an * bd)).then_with(|| a.0.cmp(&b.0))
546 });
547 window(&mut ranked, 0, limit);
548
549 let items = ranked
550 .into_iter()
551 .map(|(path, markers, lines, by_category)| DensityItem {
552 path,
553 markers,
554 lines,
555 per_kloc: per_kloc(u64::from(markers), u64::from(lines)),
556 by_category,
557 })
558 .collect();
559
560 Ok(DebtDensityReport {
561 schema: SCHEMA,
562 order: order.as_str(),
563 limit,
564 min_lines,
565 files_with_markers,
566 ranked_files,
567 short_files,
568 unknown_length_files,
569 total_markers,
570 total_lines,
571 overall_per_kloc: per_kloc(ranked_markers, total_lines),
572 items,
573 })
574}
575
576/// A file's length in lines from its `file` node's `meta.lines`, or `None` when
577/// the node is absent, carries no `lines`, or reports **zero** lines.
578///
579/// Zero is folded into `None` deliberately: it is not a length that can be
580/// divided by, and the two cases a reader would want distinguished — a genuinely
581/// empty file and a single line with no terminating newline — are
582/// indistinguishable in a newline count. Reporting both as "length unknown" is
583/// the honest reading; reporting either as a density is not.
584fn file_lines(store: &Store, path: &str) -> Result<Option<u32>, StoreError> {
585 let Some(node) = store.get_node(&format!("file:{path}"))? else {
586 return Ok(None);
587 };
588 Ok(node
589 .meta
590 .get("lines")
591 .and_then(serde_json::Value::as_u64)
592 .and_then(|n| u32::try_from(n).ok())
593 .filter(|&n| n > 0))
594}
595
596/// Markers per 1,000 lines, rounded to two decimals; `0.0` for a zero
597/// denominator, which callers have already excluded from any ranking.
598fn per_kloc(markers: u64, lines: u64) -> f64 {
599 if lines == 0 {
600 return 0.0;
601 }
602 // `u64 as f64` is lossy above 2^53; a line count or marker count that large
603 // is not reachable from a repository on disk, and the precision lost would be
604 // below the two decimals this rounds to anyway.
605 #[expect(clippy::cast_precision_loss, reason = "counts are far below 2^53")]
606 let ratio = (markers as f64) * 1000.0 / (lines as f64);
607 round2(ratio)
608}
609
610/// The redaction state of one config key in a [`ConfigSecretReport`]. Three
611/// states, because collapsing them would misreport two of them: "declared in
612/// code" is not a redaction, and "value present" is not a safe one.
613#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
614#[serde(rename_all = "snake_case")]
615pub enum RedactionState {
616 /// The value was read from a source file and **replaced** with the redaction
617 /// placeholder before anything was persisted. The expected state for a
618 /// secret-named key extracted from a config file.
619 Redacted,
620 /// The key carries **no value at all** — a struct-derived key
621 /// (`meta.source = "struct"`), a config *field* declared in Rust with no
622 /// literal in the code to redact.
623 ///
624 /// Not a redaction and not a leak: it records that a setting by this name
625 /// exists, which is the inventory's job, and nothing about any value.
626 Declared,
627 /// The key carries a value that is **not** the redaction placeholder.
628 ///
629 /// Extraction redacts every secret-named key, so this state is unreachable
630 /// from extraction alone. It is reachable through
631 /// [`Store::apply_import_layer`](crate::Store::apply_import_layer), which
632 /// upserts whatever nodes an imported factset carries — so an import produced
633 /// by another tool, or by an older Roteiro, can put an unredacted value in the
634 /// store. That is worth reporting loudly, and it is a finding about **this
635 /// store**, not about the source repository.
636 Present,
637}
638
639impl RedactionState {
640 /// The stable token for this state, as serialised.
641 #[must_use]
642 pub fn as_str(self) -> &'static str {
643 match self {
644 Self::Redacted => "redacted",
645 Self::Declared => "declared",
646 Self::Present => "present",
647 }
648 }
649}
650
651/// One secret-named config key in a [`ConfigSecretReport`].
652#[derive(Debug, Clone, PartialEq, Serialize)]
653pub struct ConfigSecretItem {
654 /// Natural key of the config-key node (`cfgkey:<path>#<dotted>`).
655 pub key: String,
656 /// Repository-relative path of the config file or Rust source it came from.
657 pub path: Option<String>,
658 /// The dotted key name (e.g. `serve.api_token`). **Names only** — no value is
659 /// carried here, by construction as much as by choice: the value in the store
660 /// is the redaction placeholder.
661 pub name: String,
662 /// Whether the value was redacted, absent, or present.
663 pub state: RedactionState,
664 /// `meta.source`, when the node records one — `struct` for a key synthesised
665 /// from a `@rto:config` Rust struct. Absent for a file-derived key.
666 pub source: Option<String>,
667}
668
669/// An inventory of **secret-named** config keys and their redaction state.
670///
671/// See [`config_secrets`] for what this is and — more importantly — what it is
672/// not.
673#[derive(Debug, Clone, PartialEq, Serialize)]
674pub struct ConfigSecretReport {
675 /// Stable schema tag ([`SCHEMA`]).
676 pub schema: &'static str,
677 /// The requested cap on `items`; `0` means unlimited.
678 pub limit: usize,
679 /// Every `config_key` node in the graph, secret-named or not — the population
680 /// the inventory was drawn from.
681 pub config_keys: usize,
682 /// Config keys whose **name** matched the secret-name heuristic. `items` is
683 /// the first `limit` of these, so `secret_named > items.len()` means truncation.
684 pub secret_named: usize,
685 /// Of `secret_named`: how many carry the redaction placeholder.
686 pub redacted: usize,
687 /// Of `secret_named`: how many carry no value at all (struct-derived).
688 pub declared: usize,
689 /// Of `secret_named`: how many carry a value that is **not** the placeholder.
690 ///
691 /// **Expected to be zero.** A non-zero count is a finding about this store —
692 /// see [`RedactionState::Present`] for the one path that reaches it.
693 pub unredacted: usize,
694 /// Config keys that are redacted but whose name is **not** secret-looking: a
695 /// Kubernetes `Secret`'s `data`, redacted because of where it lives rather
696 /// than what it is called.
697 ///
698 /// Reported so the redaction counts reconcile against the graph: without it a
699 /// reader comparing `redacted` to the number of `<redacted>` values in the
700 /// store would find an unexplained surplus.
701 pub redacted_not_secret_named: usize,
702 /// Distinct files carrying at least one secret-named key.
703 pub files: usize,
704 /// The secret-named keys, ordered by `(path, name, key)`.
705 pub items: Vec<ConfigSecretItem>,
706}
707
708/// Inventory the **secret-named** config keys in the graph — where they are, what
709/// they are called, and whether their values were redacted before persistence —
710/// capped at `limit` (`0` = unlimited).
711///
712/// # What this reports
713///
714/// Config extraction (ADR-0009) flattens TOML/JSON/YAML/`.env` into `config_key`
715/// nodes, and **redacts the value of any secret-named key before it reaches the
716/// store** (see [`crate::config_keys::REDACTED`] and the redaction sites it
717/// names). This lens reads that back: *secret-named config keys are present, here
718/// are their paths and names, and here is their redaction state*. It is an
719/// **inventory with an invariant check**, and it is useful for exactly two
720/// questions: which of my config surfaces deal in credentials, and did anything
721/// unredacted get into this graph.
722///
723/// # What this CANNOT do — read this before extending it
724///
725/// **It is not a secret scanner and this architecture cannot make it one.** The
726/// lens is named for the inventory it can be, not the scanner the shortlist's
727/// original title promised.
728///
729/// - **It cannot detect a hardcoded credential in source code.** It reads
730/// `config_key` nodes, which come only from config *files* and from
731/// `@rto:config` struct declarations. An AWS key pasted into a `.rs` string
732/// literal produces no `config_key` node and is invisible here. Nothing about
733/// the node kinds this reads can change that.
734/// - **It cannot judge validity.** It never sees a value: by the time anything is
735/// in the store, a secret-named value has already been replaced. There is no
736/// entropy test, no format check, no liveness probe, and there cannot be one
737/// without persisting the very thing extraction exists to redact.
738/// - **It cannot tell a real secret from a placeholder.** `API_TOKEN=changeme` in
739/// a committed `.env.example` and a genuine token in an uncommitted `.env` are
740/// the same row here: same key name, same redacted value, same state.
741/// - **It cannot say a repository has no secrets.** An empty report means "no
742/// secret-*named* config key", which is a statement about naming. A credential
743/// under an innocuous key (`endpoint`, `dsn`, `url`) is not secret-named, is not
744/// redacted, and does not appear.
745///
746/// If you find yourself wanting to widen this toward detecting real credentials,
747/// **that instinct is what the rename exists to prevent**: the widening cannot be
748/// built on these inputs, and a tool that half-does it while being named for the
749/// whole job is worse than one that does the inventory honestly. Every surface
750/// carries this limitation in its own words, so a model calling the tool passes it
751/// on rather than reporting a security guarantee that was never offered.
752///
753/// # The heuristic, stated
754///
755/// "Secret-named" is [`crate::config_keys::is_secret_key`]: the key's
756/// ASCII-alphanumerics, lowercased, containing any of `secret`, `password`,
757/// `passwd`, `passphrase`, `token`, `apikey`, `credential`, `privatekey`,
758/// `accesskey`, `pwd`. So it matches `API_TOKEN`, `db.passwordFile` and
759/// `serve.apiKey`, and misses `dsn`, `connection_string` and `auth` — and it
760/// false-positives on `token_bucket_size` and `csrf_token_header`, which are
761/// settings, not secrets. Both directions of error are inherent to matching on
762/// names; neither is reported as a finding.
763///
764/// # Ordering
765///
766/// By `(path, name, key)` ascending — an inventory, like [`debt`], not a ranking.
767/// There is deliberately no ordering knob: nothing here is a magnitude worth
768/// sorting by, and offering one would suggest some keys are more secret than
769/// others. Identical input yields byte-identical output.
770///
771/// # Errors
772/// Returns [`StoreError`] on query failure.
773pub fn config_secrets(store: &Store, limit: usize) -> Result<ConfigSecretReport, StoreError> {
774 let nodes = store.nodes_by_kind(&NodeKind::Other(crate::config_keys::KIND.to_owned()))?;
775 let config_keys = nodes.len();
776 let mut items = Vec::new();
777 let mut redacted = 0usize;
778 let mut declared = 0usize;
779 let mut unredacted = 0usize;
780 let mut redacted_not_secret_named = 0usize;
781 let mut files: BTreeSet<String> = BTreeSet::new();
782 for node in nodes {
783 // Prefer `meta.key` — the dotted key as the source spelled it — and fall
784 // back to the node's name, which extraction sets to the same string.
785 let name = node
786 .meta
787 .get("key")
788 .and_then(serde_json::Value::as_str)
789 .unwrap_or(node.name.as_str())
790 .to_owned();
791 let value = node.meta.get("value").and_then(serde_json::Value::as_str);
792 if !crate::config_keys::is_secret_key(&name) {
793 // A redacted value under a non-secret name is a k8s `Secret`'s data:
794 // counted so the report's redaction figures reconcile with the graph,
795 // but not listed — this lens's subject is secret-*named* keys.
796 if value == Some(crate::config_keys::REDACTED) {
797 redacted_not_secret_named += 1;
798 }
799 continue;
800 }
801 let state = match value {
802 Some(v) if v == crate::config_keys::REDACTED => {
803 redacted += 1;
804 RedactionState::Redacted
805 }
806 // A struct-derived key omits `meta.value` entirely: there is no
807 // literal in the code to redact, so "absent" is the honest state
808 // rather than folding it in with a successful redaction.
809 None => {
810 declared += 1;
811 RedactionState::Declared
812 }
813 Some(_) => {
814 unredacted += 1;
815 RedactionState::Present
816 }
817 };
818 if let Some(path) = node.path.as_deref() {
819 files.insert(path.to_owned());
820 }
821 items.push(ConfigSecretItem {
822 key: node.key,
823 path: node.path,
824 name,
825 state,
826 source: node
827 .meta
828 .get("source")
829 .and_then(serde_json::Value::as_str)
830 .map(str::to_owned),
831 });
832 }
833 let secret_named = items.len();
834 items.sort_by(|a, b| (&a.path, &a.name, &a.key).cmp(&(&b.path, &b.name, &b.key)));
835 window(&mut items, 0, limit);
836
837 Ok(ConfigSecretReport {
838 schema: SCHEMA,
839 limit,
840 config_keys,
841 secret_named,
842 redacted,
843 declared,
844 unredacted,
845 redacted_not_secret_named,
846 files: files.len(),
847 items,
848 })
849}
850
851/// Match a slash-separated `path` against a glob `pattern`, anchored end-to-end.
852/// `?` matches one non-`/` character, `*` matches any run within a single path
853/// segment, and `**` matches zero or more whole segments. Used for config
854/// `[debt] ignore` patterns (e.g. `vendor/**`, `**/generated/*`).
855#[must_use]
856fn glob_match(pattern: &str, path: &str) -> bool {
857 let pat: Vec<&str> = pattern.split('/').collect();
858 let seg: Vec<&str> = path.split('/').collect();
859 match_segments(&pat, &seg)
860}
861
862/// Anchored match of glob segments `pat` against path segments `seg`, with `**`
863/// consuming zero or more segments.
864fn match_segments(pat: &[&str], seg: &[&str]) -> bool {
865 match pat.first() {
866 None => seg.is_empty(),
867 Some(&"**") => (0..=seg.len()).any(|i| match_segments(&pat[1..], &seg[i..])),
868 Some(token) => {
869 !seg.is_empty() && match_token(token, seg[0]) && match_segments(&pat[1..], &seg[1..])
870 }
871 }
872}
873
874/// Match a single path segment `s` against a `pattern` token containing `*`
875/// (any run, no `/`) and `?` (one char, no `/`).
876fn match_token(pattern: &str, s: &str) -> bool {
877 let pat: Vec<char> = pattern.chars().collect();
878 let chars: Vec<char> = s.chars().collect();
879 match_token_chars(&pat, &chars)
880}
881
882/// Recursive char-slice matcher backing [`match_token`].
883fn match_token_chars(pat: &[char], chars: &[char]) -> bool {
884 match pat.first() {
885 None => chars.is_empty(),
886 Some('*') => (0..=chars.len()).any(|i| match_token_chars(&pat[1..], &chars[i..])),
887 Some('?') => !chars.is_empty() && match_token_chars(&pat[1..], &chars[1..]),
888 Some(&ch) => {
889 !chars.is_empty() && chars[0] == ch && match_token_chars(&pat[1..], &chars[1..])
890 }
891 }
892}
893
894/// How a [`CouplingReport`]'s items are ranked. The three orders answer three
895/// different questions, which a single undirected degree cannot tell apart.
896#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
897pub enum CouplingOrder {
898 /// By `fan_in + fan_out` — overall call coupling.
899 #[default]
900 Total,
901 /// By `fan_in` — the most depended-on symbols ("what calls this?").
902 FanIn,
903 /// By `fan_out` — the symbols that reach furthest ("what does this call?").
904 FanOut,
905}
906
907impl CouplingOrder {
908 /// The stable token for this order, as accepted by [`from_token`](Self::from_token).
909 #[must_use]
910 pub fn as_str(self) -> &'static str {
911 match self {
912 Self::Total => "total",
913 Self::FanIn => "fan_in",
914 Self::FanOut => "fan_out",
915 }
916 }
917
918 /// Parse an order token. `None` for anything else — callers surface an error
919 /// rather than silently ranking by something the caller did not ask for.
920 #[must_use]
921 pub fn from_token(s: &str) -> Option<Self> {
922 match s {
923 "total" => Some(Self::Total),
924 "fan_in" => Some(Self::FanIn),
925 "fan_out" => Some(Self::FanOut),
926 _ => None,
927 }
928 }
929
930 /// The tokens [`from_token`](Self::from_token) accepts, for error messages
931 /// and argument schemas — so the accepted set is stated in exactly one place.
932 #[must_use]
933 pub fn tokens() -> [&'static str; 3] {
934 [
935 Self::Total.as_str(),
936 Self::FanIn.as_str(),
937 Self::FanOut.as_str(),
938 ]
939 }
940}
941
942/// One node's **directed** call coupling in a [`CouplingReport`].
943#[derive(Debug, Clone, PartialEq, Serialize)]
944pub struct CouplingItem {
945 /// Natural key of the node.
946 pub key: String,
947 /// Kind token (e.g. `fn`).
948 pub kind: String,
949 /// Human-facing name.
950 pub name: String,
951 /// Repository-relative path, if any.
952 pub path: Option<String>,
953 /// How many **distinct** other nodes call this one.
954 pub fan_in: u32,
955 /// How many **distinct** other nodes this one calls.
956 pub fan_out: u32,
957 /// `fan_in + fan_out` — the directed equivalent of the undirected degree.
958 pub total: u32,
959 /// Martin's instability, `fan_out / (fan_in + fan_out)`, rounded to two
960 /// decimals. `0.0` = purely depended-on (stable); `1.0` = purely depending
961 /// (unstable). The denominator is never zero: an item exists only when it
962 /// has at least one non-self call edge.
963 pub instability: f64,
964}
965
966/// Directed call coupling: per-node fan-in and fan-out over `Calls` edges,
967/// ranked. The counterpart to an undirected degree ranking, which cannot tell
968/// "everything calls this" from "this calls everything".
969#[derive(Debug, Clone, PartialEq, Serialize)]
970pub struct CouplingReport {
971 /// Stable schema tag ([`SCHEMA`]).
972 pub schema: &'static str,
973 /// The edge kind measured. Always `calls` — the only edge kind whose
974 /// direction carries a caller/callee meaning.
975 pub edge_kind: &'static str,
976 /// The ranking that produced `items` ([`CouplingOrder::as_str`]).
977 pub order: &'static str,
978 /// The requested cap on `items`; `0` means unlimited.
979 pub limit: usize,
980 /// Total `Calls` edges scanned, including duplicates and self-calls.
981 pub call_edges: usize,
982 /// Self-referential `Calls` edges (recursion), counted in `call_edges` but
983 /// excluded from every fan — see [`coupling`].
984 pub self_calls: usize,
985 /// `Calls` edges whose endpoints are in two different languages: name
986 /// collisions from simple-name call resolution, not calls. Counted in
987 /// `call_edges` but excluded from every fan — see [`coupling`].
988 pub cross_language_calls: usize,
989 /// Distinct nodes with at least one non-self `Calls` edge. `items` is the
990 /// top `limit` of these, so `coupled_nodes > items.len()` means truncation.
991 pub coupled_nodes: usize,
992 /// The ranked nodes: by `order` descending, ties broken by `key` ascending.
993 pub items: Vec<CouplingItem>,
994}
995
996/// Rank nodes by **directed** call coupling — fan-in (distinct callers) and
997/// fan-out (distinct callees) over `Calls` edges — most-coupled first by
998/// `order`, capped at `limit` (`0` = unlimited).
999///
1000/// Three deliberate counting rules, all of which change the numbers:
1001///
1002/// - **Distinct counterparts, not edges.** Edges are a set per `(src, dst, kind,
1003/// provenance)`, which still admits *parallel* `Calls` edges between one pair
1004/// at different provenances — a `derived` extraction and an `inferred`
1005/// suggestion of the same call. Counting distinct counterpart keys makes
1006/// `fan_in` mean "how many things depend on this", which is the coupling
1007/// question, rather than "how many layers asserted the dependency".
1008/// - **Self-calls are excluded from both fans.** Recursion is a real edge but
1009/// couples a node to nothing outside itself, and counting it would inflate
1010/// `fan_in` *and* `fan_out` for the same node. It is reported separately as
1011/// `self_calls` rather than silently dropped.
1012/// - **Cross-language call edges are excluded.** Roteiro extracts no FFI, so a
1013/// `Calls` edge between two languages is never a call — see
1014/// [`same_language`]. Reported as `cross_language_calls`.
1015///
1016/// Ordering is total and deterministic: by the chosen metric descending, then by
1017/// `key` ascending, so identical input yields byte-identical output.
1018///
1019/// # Precision
1020///
1021/// `fan_in` is exactly as precise as the `Calls` edges beneath it, and those are
1022/// resolved by **simple name**: a callee that is unique by bare name anywhere in
1023/// the repository binds to that definition, wherever it lives. So a single
1024/// same-language helper with a very common name absorbs every call to that name,
1025/// and its `fan_in` reads high for a reason that has nothing to do with design.
1026/// Excluding cross-language edges removes the worst of this, but not all of it.
1027/// Treat a large `fan_in` on a short, generically-named function as a question,
1028/// not a finding — which is also why this lens offers no CI gate.
1029///
1030/// # Errors
1031/// Returns [`StoreError`] on query failure.
1032pub fn coupling(
1033 store: &Store,
1034 order: CouplingOrder,
1035 limit: usize,
1036) -> Result<CouplingReport, StoreError> {
1037 // `inbound`: dst key -> distinct src keys. `outbound`: src key -> distinct dst
1038 // keys. Named for the direction rather than caller/callee, which read alike.
1039 let mut inbound: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
1040 let mut outbound: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
1041 let mut call_edges = 0usize;
1042 let mut self_calls = 0usize;
1043 let mut cross_language_calls = 0usize;
1044 for edge in store.all_edges()? {
1045 if edge.kind != EdgeKind::Calls {
1046 continue;
1047 }
1048 call_edges += 1;
1049 if edge.src == edge.dst {
1050 self_calls += 1;
1051 continue;
1052 }
1053 if !same_language(&edge.src, &edge.dst) {
1054 cross_language_calls += 1;
1055 continue;
1056 }
1057 inbound
1058 .entry(edge.dst.clone())
1059 .or_default()
1060 .insert(edge.src.clone());
1061 outbound.entry(edge.src).or_default().insert(edge.dst);
1062 }
1063
1064 // Rank on the counts alone, so only the nodes that survive the cap are read
1065 // back from the store — a whole-graph node scan is not needed to answer a
1066 // top-N question.
1067 let keys: BTreeSet<&String> = inbound.keys().chain(outbound.keys()).collect();
1068 let coupled_nodes = keys.len();
1069 let mut ranked: Vec<(u32, u32, &String)> = keys
1070 .into_iter()
1071 .map(|key| {
1072 let fan_in = count_of(&inbound, key);
1073 let fan_out = count_of(&outbound, key);
1074 (fan_in, fan_out, key)
1075 })
1076 .collect();
1077 ranked.sort_by(|a, b| {
1078 let metric = |&(fan_in, fan_out, _): &(u32, u32, &String)| match order {
1079 CouplingOrder::Total => fan_in + fan_out,
1080 CouplingOrder::FanIn => fan_in,
1081 CouplingOrder::FanOut => fan_out,
1082 };
1083 metric(b).cmp(&metric(a)).then_with(|| a.2.cmp(b.2))
1084 });
1085 window(&mut ranked, 0, limit);
1086
1087 let mut items = Vec::with_capacity(ranked.len());
1088 for (fan_in, fan_out, key) in ranked {
1089 // `edges.src`/`edges.dst` are foreign keys into `nodes`, so a node behind
1090 // a call edge always exists; the guard is defence in depth, not a case.
1091 let Some(node) = store.get_node(key)? else {
1092 continue;
1093 };
1094 let total = fan_in + fan_out;
1095 items.push(CouplingItem {
1096 key: node.key,
1097 kind: node.kind.as_str().to_owned(),
1098 name: node.name,
1099 path: node.path,
1100 fan_in,
1101 fan_out,
1102 total,
1103 instability: round2(f64::from(fan_out) / f64::from(total)),
1104 });
1105 }
1106
1107 Ok(CouplingReport {
1108 schema: SCHEMA,
1109 edge_kind: EdgeKind::Calls.as_str(),
1110 order: order.as_str(),
1111 limit,
1112 call_edges,
1113 self_calls,
1114 cross_language_calls,
1115 coupled_nodes,
1116 items,
1117 })
1118}
1119
1120/// The language token of a symbol key (`sym:<lang>:<path>#<name>` → `<lang>`),
1121/// or `None` for any other key shape.
1122fn sym_lang(key: &str) -> Option<&str> {
1123 let rest = key.strip_prefix("sym:")?;
1124 let (lang, _) = rest.split_once(':')?;
1125 (!lang.is_empty()).then_some(lang)
1126}
1127
1128/// Whether a call edge's two endpoints are in the same language — `true` unless
1129/// both keys carry a language token and the tokens differ.
1130///
1131/// Cross-file call resolution binds a callee by **simple name** across every
1132/// `Fn` node in the repository, language included. Roteiro extracts no FFI, so
1133/// nothing in the graph can legitimately record a JavaScript function calling a
1134/// Rust one; such an edge is a name collision — a lone Rust `join` helper
1135/// absorbing every JavaScript `.join(…)` in the tree. Excluding them keeps a
1136/// language's coupling figures about that language.
1137///
1138/// Unknown-shaped keys (anything that is not `sym:<lang>:…`) are **kept**: this
1139/// filter removes edges it can prove span two languages, and never guesses.
1140fn same_language(src: &str, dst: &str) -> bool {
1141 match (sym_lang(src), sym_lang(dst)) {
1142 (Some(a), Some(b)) => a == b,
1143 _ => true,
1144 }
1145}
1146
1147/// The size of `key`'s counterpart set, as a `u32` (a node cannot have more
1148/// distinct counterparts than there are nodes, so the cast cannot realistically
1149/// saturate; saturating beats wrapping if it ever did).
1150fn count_of(map: &BTreeMap<String, BTreeSet<String>>, key: &str) -> u32 {
1151 map.get(key)
1152 .map_or(0, |set| u32::try_from(set.len()).unwrap_or(u32::MAX))
1153}
1154
1155/// Round to two decimals, so the serialised ratio is short and stable rather
1156/// than carrying the full binary expansion of a division.
1157fn round2(v: f64) -> f64 {
1158 (v * 100.0).round() / 100.0
1159}
1160
1161/// One step along a [`Path`]: the edge traversed and the node it leads to.
1162#[derive(Debug, Clone, PartialEq, Serialize)]
1163pub struct PathHop {
1164 /// Edge kind token (e.g. `calls`, `contains`).
1165 pub kind: String,
1166 /// How the edge was produced.
1167 pub provenance: &'static str,
1168 /// Confidence score, present only for inferred edges.
1169 pub confidence: Option<f64>,
1170 /// The direction the edge was traversed relative to the previous node
1171 /// (`outgoing` = along the edge, `incoming` = against it).
1172 pub direction: &'static str,
1173 /// The natural key of the node this hop arrives at.
1174 pub node: String,
1175}
1176
1177/// A shortest path between two nodes. Edges are followed in either direction
1178/// (the graph is treated as undirected for reachability), and each hop records
1179/// the actual direction and provenance of the edge used.
1180#[derive(Debug, Clone, PartialEq, Serialize)]
1181pub struct Path {
1182 /// Stable schema tag ([`SCHEMA`]).
1183 pub schema: &'static str,
1184 /// Natural key of the start node.
1185 pub from: String,
1186 /// Natural key of the goal node.
1187 pub to: String,
1188 /// Whether a path (including the trivial empty one) was found.
1189 pub found: bool,
1190 /// Number of hops (edges) in the path; `0` when `from == to`.
1191 pub length: usize,
1192 /// The hops from `from` to `to`, in order.
1193 pub hops: Vec<PathHop>,
1194}
1195
1196fn out_ref(edge: &Edge) -> EdgeRef {
1197 EdgeRef {
1198 kind: edge.kind.as_str().to_owned(),
1199 provenance: edge.provenance.as_str(),
1200 confidence: edge.confidence,
1201 node: edge.dst.clone(),
1202 }
1203}
1204
1205fn in_ref(edge: &Edge) -> EdgeRef {
1206 EdgeRef {
1207 kind: edge.kind.as_str().to_owned(),
1208 provenance: edge.provenance.as_str(),
1209 confidence: edge.confidence,
1210 node: edge.src.clone(),
1211 }
1212}
1213
1214fn sort_refs(refs: &mut [EdgeRef]) {
1215 // Include provenance so edges differing only in provenance have a total,
1216 // stable order; with the edge-uniqueness constraint this key is unique.
1217 refs.sort_by(|a, b| (&a.kind, &a.node, a.provenance).cmp(&(&b.kind, &b.node, b.provenance)));
1218}
1219
1220/// Explain a node: its record plus every incoming and outgoing edge, each
1221/// labelled with provenance. Returns `None` if no node has that key.
1222///
1223/// # Errors
1224/// Returns [`StoreError`] on query failure.
1225pub fn explain(store: &Store, key: &str) -> Result<Option<Explanation>, StoreError> {
1226 let Some(node) = store.get_node(key)? else {
1227 return Ok(None);
1228 };
1229 let mut outgoing: Vec<EdgeRef> = store.edges_from(key)?.iter().map(out_ref).collect();
1230 let mut incoming: Vec<EdgeRef> = store.edges_to(key)?.iter().map(in_ref).collect();
1231 sort_refs(&mut outgoing);
1232 sort_refs(&mut incoming);
1233 Ok(Some(Explanation {
1234 schema: SCHEMA,
1235 node: NodeSummary::from_node(&node),
1236 meta: node.meta,
1237 outgoing,
1238 incoming,
1239 }))
1240}
1241
1242/// List every node of the given `kind`, ordered by key.
1243///
1244/// # Errors
1245/// Returns [`StoreError`] on query failure.
1246pub fn list_kind(store: &Store, kind: &NodeKind) -> Result<Listing, StoreError> {
1247 let nodes = store
1248 .nodes_by_kind(kind)?
1249 .iter()
1250 .map(NodeSummary::from_node)
1251 .collect();
1252 Ok(Listing {
1253 schema: SCHEMA,
1254 kind: kind.as_str().to_owned(),
1255 nodes,
1256 })
1257}
1258
1259/// A relevance-ranked search hit: a node summary plus its score.
1260#[derive(Debug, Clone, PartialEq, Serialize)]
1261pub struct SearchHit {
1262 /// Relevance score (higher is better); see [`search`] for how it is derived.
1263 pub score: u32,
1264 /// The matching node.
1265 #[serde(flatten)]
1266 pub node: NodeSummary,
1267 /// A short, whitespace-collapsed excerpt of the node's captured
1268 /// `meta.content` (see [`content_snippet`]), so a model that never calls
1269 /// [`explain`] still has real grounding text. `None` for pure symbol/config
1270 /// nodes with no content — the summary (name/kind/path) is the grounding then.
1271 #[serde(skip_serializing_if = "Option::is_none")]
1272 pub snippet: Option<String>,
1273}
1274
1275/// Max **chars** of a search-hit content snippet, counting the trailing ellipsis
1276/// when truncated (so the total length never exceeds this). Bounded so many hits
1277/// cannot bloat the tool response or blow the served model's context window.
1278const SNIPPET_MAX: usize = 300;
1279
1280/// Build a bounded, whitespace-collapsed snippet from a node's captured
1281/// `meta.content`, or `None` when the node has no textual content (pure symbol/
1282/// config nodes). Runs of whitespace collapse to single spaces, and the result is
1283/// at most [`SNIPPET_MAX`] chars *including* a trailing `…` when the content was
1284/// truncated, so a search hit carries grounding text even when the model never
1285/// calls [`explain`].
1286///
1287/// Processes `content` **lazily**: it collapses whitespace on the fly and stops
1288/// after ~`SNIPPET_MAX` chars, so a large content-bearing node never materialises
1289/// more than the bound regardless of how big its content is.
1290fn content_snippet(meta: &serde_json::Value) -> Option<String> {
1291 let content = meta.get("content").and_then(|v| v.as_str())?;
1292
1293 // Collect at most SNIPPET_MAX + 1 collapsed chars: the one extra char only
1294 // tells us whether the content overflowed the bound (→ needs an ellipsis);
1295 // we never buffer more than that, however large `content` is.
1296 let mut collapsed: Vec<char> = Vec::with_capacity(SNIPPET_MAX + 1);
1297 let mut pending_space = false;
1298 for ch in content.chars() {
1299 if ch.is_whitespace() {
1300 // A run of whitespace becomes a single separator, but only once a
1301 // real char has been emitted (this also drops any leading whitespace).
1302 pending_space = !collapsed.is_empty();
1303 continue;
1304 }
1305 if pending_space {
1306 collapsed.push(' ');
1307 pending_space = false;
1308 if collapsed.len() > SNIPPET_MAX {
1309 break;
1310 }
1311 }
1312 collapsed.push(ch);
1313 if collapsed.len() > SNIPPET_MAX {
1314 break;
1315 }
1316 }
1317
1318 if collapsed.is_empty() {
1319 return None;
1320 }
1321 // Overflowed the bound: truncate to SNIPPET_MAX - 1 chars and append the
1322 // ellipsis, so the total length (ellipsis included) is exactly SNIPPET_MAX.
1323 if collapsed.len() > SNIPPET_MAX {
1324 let snippet: String = collapsed[..SNIPPET_MAX - 1].iter().collect();
1325 Some(format!("{snippet}…"))
1326 } else {
1327 Some(collapsed.into_iter().collect())
1328 }
1329}
1330
1331/// Deterministically search nodes for `query`, ranked by relevance, returning at
1332/// most `limit` hits — or **every match when `limit == 0`**, which is [`window`]'s
1333/// rule and the one every list lens follows (issue #393). An empty result
1334/// therefore always means "nothing matched", never "you asked for nothing".
1335///
1336/// Case-insensitive; every whitespace/`::`-separated token must appear
1337/// somewhere in the node's **name, key, path, or captured `meta.content`**
1338/// (so a question's words find the *description*, e.g. a README/ADR, not only a
1339/// same-named symbol). Scoring favours an exact name match, then a name/content
1340/// substring, then per-token hits; it then **boosts curated intent** (`authored`
1341/// ADRs/blueprints) and READMEs/overviews and **penalises test scaffolding**, so
1342/// "what/why" questions land on the real answer rather than a same-named test
1343/// helper. Ties break by key so results are stable.
1344///
1345/// # Errors
1346/// Returns [`StoreError`] on query failure.
1347pub fn search(store: &Store, query: &str, limit: usize) -> Result<Vec<SearchHit>, StoreError> {
1348 let q = query.trim().to_lowercase();
1349 // Tokens are separated by whitespace or the `::` path separator; a lone `:`
1350 // (as in a `sym:rust:…` key) does not split a token.
1351 let tokens: Vec<&str> = q.split("::").flat_map(str::split_whitespace).collect();
1352 if tokens.is_empty() {
1353 return Ok(Vec::new());
1354 }
1355
1356 let mut hits: Vec<SearchHit> = Vec::new();
1357 for node in store.all_nodes()? {
1358 let name = node.name.to_lowercase();
1359 let key = node.key.to_lowercase();
1360 let path = node.path.as_deref().unwrap_or("").to_lowercase();
1361 // The captured knowledge base (doc comments, prose, ADR/README/blueprint
1362 // text) is searchable too, so a question's words find the *description*,
1363 // not just a same-named symbol. Only lowercase when a node actually has
1364 // content — most nodes (code symbols) don't, so skip the allocation.
1365 let content = node
1366 .meta
1367 .get("content")
1368 .and_then(|v| v.as_str())
1369 .map(str::to_lowercase);
1370 let content = content.as_deref().unwrap_or("");
1371 // Require every token to appear somewhere (including content), so a
1372 // multi-word query narrows.
1373 if !tokens
1374 .iter()
1375 .all(|t| name.contains(t) || key.contains(t) || path.contains(t) || content.contains(t))
1376 {
1377 continue;
1378 }
1379 let mut relevance: i32 = 0;
1380 if name == q {
1381 relevance += 100;
1382 } else if name.contains(&q) {
1383 relevance += 60;
1384 } else if content.contains(&q) {
1385 relevance += 25;
1386 }
1387 for t in &tokens {
1388 if name.contains(t) {
1389 relevance += 12;
1390 } else if key.contains(t) {
1391 relevance += 6;
1392 } else if content.contains(t) {
1393 relevance += 8;
1394 } else if path.contains(t) {
1395 relevance += 3;
1396 }
1397 }
1398 // Curated intent (ADRs/blueprints — `authored`) is the best answer to a
1399 // "what/why" question; a README/overview is the natural landing page; and
1400 // test scaffolding should not outrank the real thing when it shares a name.
1401 //
1402 // A **peer's** curated prose earns half of it. What the boost measures is
1403 // "somebody wrote this deliberately", and an imported ADR passes that
1404 // test as squarely as a local one — so scoring it like a code symbol
1405 // would throw away the tier the import went to the trouble of carrying.
1406 // But a question asked in this repository is a question about this
1407 // repository, so our own decisions must stay ahead of another repo's when
1408 // both match: the boost is halved rather than shared or withheld.
1409 if node.provenance.tier() == Provenance::Authored {
1410 relevance += if node.provenance.is_external() {
1411 20
1412 } else {
1413 40
1414 };
1415 }
1416 if is_overview_path(&path) {
1417 relevance += 30;
1418 }
1419 if is_test_path(&path) {
1420 relevance -= 60;
1421 }
1422 hits.push(SearchHit {
1423 score: u32::try_from(relevance.max(0)).unwrap_or(0),
1424 snippet: content_snippet(&node.meta),
1425 node: NodeSummary::from_node(&node),
1426 });
1427 }
1428 // Highest score first; ties by key for a stable, deterministic order.
1429 hits.sort_by(|a, b| {
1430 b.score
1431 .cmp(&a.score)
1432 .then_with(|| a.node.key.cmp(&b.node.key))
1433 });
1434 // `window`, not `truncate`: `0` is unlimited here as it is everywhere else.
1435 // The scan above is full-population at every limit, so an unbounded search
1436 // costs the same as a bounded one — only the printing differs.
1437 window(&mut hits, 0, limit);
1438 Ok(hits)
1439}
1440
1441/// A hit in the **generated** channel: text a model produced about a media blob,
1442/// never a graph fact.
1443///
1444/// It is deliberately *not* a [`SearchHit`]. A generated hit has no node, no
1445/// provenance and no key, and giving it a [`NodeSummary`] would be the first step
1446/// towards it being treated like one — the exact mistake ADR-0015 exists to
1447/// correct. Everything a consumer needs to label it is on the struct, including
1448/// the literal `generated: true`, so a caller that reads nothing else still
1449/// cannot mistake it for extracted text.
1450#[derive(Debug, Clone, PartialEq, Serialize)]
1451pub struct GeneratedHit {
1452 /// Relevance within the generated channel. Not comparable with a
1453 /// [`SearchHit::score`]: the two are ranked by different scorers, in
1454 /// different channels, on purpose.
1455 pub score: u32,
1456 /// Always `true`. A marker a consumer cannot miss or forget to check.
1457 pub generated: bool,
1458 /// The producer identity that wrote the text — which model, at which
1459 /// quantisation, under which prompt (see [`crate::Producer::id`]).
1460 pub producer: String,
1461 /// The model's registry name, repeated for legibility.
1462 pub model: String,
1463 /// The modality (`audio` | `vision`).
1464 pub kind: &'static str,
1465 /// Git blob id of the source media.
1466 pub blob: String,
1467 /// Repository path the blob was seen at.
1468 pub path: String,
1469 /// A bounded, whitespace-collapsed excerpt of the generated text, on the same
1470 /// terms as [`SearchHit::snippet`].
1471 pub snippet: Option<String>,
1472}
1473
1474/// A hit in the **memory** channel: something a session learned, never a graph
1475/// fact and never a re-derivable one.
1476///
1477/// Deliberately *not* a [`SearchHit`], for the reason [`GeneratedHit`] is not: a
1478/// memory record has no node, no provenance and no key, and giving it a
1479/// [`NodeSummary`] would be the first step towards its being treated like one.
1480/// Unlike either of the other channels, it also carries **what the tree thinks of
1481/// it** — [`MemoryHit::applies`] and [`MemoryHit::anchor_state`] — because a
1482/// lesson about code that has since moved is worth reading and worth labelling,
1483/// and returning it unlabelled would be the worse of the two mistakes.
1484#[derive(Debug, Clone, PartialEq, Serialize)]
1485pub struct MemoryHit {
1486 /// Relevance within the memory channel. Not comparable with a
1487 /// [`SearchHit::score`] or a [`GeneratedHit::score`]: three channels, three
1488 /// scorers, on purpose.
1489 pub score: u32,
1490 /// Always `true`. A marker a consumer cannot miss or forget to check.
1491 pub memory: bool,
1492 /// The record's id — its generation, and what `roteiro memory forget` takes.
1493 pub id: i64,
1494 /// What kind of knowledge it is (`lesson` | `attempt` | …).
1495 pub kind: &'static str,
1496 /// The namespace it was recorded in. **Not a branch label.**
1497 pub scope: String,
1498 /// The node key it is anchored to, if any.
1499 #[serde(skip_serializing_if = "Option::is_none")]
1500 pub anchor: Option<String>,
1501 /// What that anchor is worth against the current tree (`valid` | `drifted` |
1502 /// `vanished` | `unverifiable` | `unanchored`).
1503 pub anchor_state: &'static str,
1504 /// **Whether this record applies to the tree being searched.** A `false` here
1505 /// is a label, never a reason to have withheld the hit.
1506 pub applies: bool,
1507 /// The evidence multiplier the record's own ranking gave it
1508 /// (`base_confidence × anchor_penalty`), reported so the channel's score can
1509 /// be taken apart.
1510 pub evidence: f64,
1511 /// A bounded, whitespace-collapsed excerpt of the body, on the same terms as
1512 /// [`SearchHit::snippet`].
1513 pub snippet: Option<String>,
1514}
1515
1516/// The three channels a search returns.
1517///
1518/// They are separate fields rather than one merged list because merging is
1519/// precisely what must not happen: generated text and remembered prose may both
1520/// be *retrievable*, but neither may ever be *indistinguishable* from a derived or
1521/// authored fact, and a single ranked list would make the distinction a matter of
1522/// reading each element carefully.
1523#[derive(Debug, Clone, PartialEq, Serialize)]
1524pub struct SearchResults {
1525 /// Stable schema tag ([`SCHEMA`]).
1526 pub schema: &'static str,
1527 /// The graph channel: ranked nodes, exactly what [`search`] returns.
1528 pub hits: Vec<SearchHit>,
1529 /// The generated channel. **Empty unless
1530 /// [`SearchOptions::include_generated`] was set** — off by default, so a
1531 /// silent clip's confabulated prose cannot reach a default search.
1532 pub generated: Vec<GeneratedHit>,
1533 /// The memory channel. **Empty unless [`SearchOptions::include_memory`] was
1534 /// set** — off by default, so unreviewed accumulated prose cannot reach a
1535 /// default search either.
1536 pub memory: Vec<MemoryHit>,
1537}
1538
1539/// How to search.
1540#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1541pub struct SearchOptions {
1542 /// Maximum hits **per channel**, where `0` is unlimited ([`window`]'s rule,
1543 /// applied per channel). Each channel is ranked and windowed independently,
1544 /// so opting in to another one never displaces a graph hit, and never
1545 /// silently returns fewer of them — and `0` is "all of each channel asked
1546 /// for", not "all of them merged and then cut".
1547 pub limit: usize,
1548 /// Fold in the generated channel. Off by default (see
1549 /// [`SearchOptions::default`]).
1550 pub include_generated: bool,
1551 /// Fold in the memory channel. Off by default, for the same reason: what an
1552 /// agent remembers is unreviewed, unredacted and accumulated, so it is
1553 /// something a caller asks for rather than something that arrives.
1554 pub include_memory: bool,
1555}
1556
1557impl Default for SearchOptions {
1558 /// Ten hits, graph channel only. The default is the safe answer: everything
1559 /// that is not an extracted or authored fact is opt-in, always.
1560 fn default() -> Self {
1561 Self {
1562 limit: 10,
1563 include_generated: false,
1564 include_memory: false,
1565 }
1566 }
1567}
1568
1569/// Search every channel: the graph, and — each only when asked for —
1570/// model-generated media content and episodic agent memory.
1571///
1572/// The graph channel is exactly [`search`]. The other two are ranked by scorers of
1573/// their own ([`generated_score`], [`memory_score`]) which have **no provenance
1574/// term at all**, so neither can acquire the `authored` boost that curated intent
1575/// gets. Neither could do so even by accident: neither record is a node, so
1576/// neither ever reaches the code that applies that boost.
1577///
1578/// The memory channel is scored with **no decay** regardless of what a caller
1579/// might prefer elsewhere, so a search is reproducible for a fixed store and a
1580/// fixed tree.
1581///
1582/// # Errors
1583/// Returns [`StoreError`] on query failure.
1584pub fn search_channels(
1585 store: &Store,
1586 query: &str,
1587 opts: SearchOptions,
1588) -> Result<SearchResults, StoreError> {
1589 let hits = search(store, query, opts.limit)?;
1590 let generated = if opts.include_generated {
1591 search_generated(store, query, opts.limit)?
1592 } else {
1593 Vec::new()
1594 };
1595 let memory = if opts.include_memory {
1596 search_memory(store, query, opts.limit)?
1597 } else {
1598 Vec::new()
1599 };
1600 Ok(SearchResults {
1601 schema: SCHEMA,
1602 hits,
1603 generated,
1604 memory,
1605 })
1606}
1607
1608/// Rank the memory channel alone.
1609///
1610/// Built on [`Store::recall_memory`] rather than on a query of its own, so the
1611/// channel inherits every promise recall makes without restating any of them: a
1612/// superseded record is already gone, an unanchored one is already labelled, and
1613/// nothing here writes anything. Decay is fixed at [`crate::Decay::None`] so a
1614/// search over an unchanged store and tree is reproducible.
1615///
1616/// Ties break by newest generation, so the order is total. `limit` follows
1617/// [`window`]: `0` is every matching record, not none of them.
1618fn search_memory(store: &Store, query: &str, limit: usize) -> Result<Vec<MemoryHit>, StoreError> {
1619 let q = query.trim().to_lowercase();
1620 let tokens: Vec<&str> = q.split("::").flat_map(str::split_whitespace).collect();
1621 if tokens.is_empty() {
1622 return Ok(Vec::new());
1623 }
1624 // Recall does the filtering, the anchor resolution and the evidence
1625 // weighting; this function only adds the lexical relevance a search wants.
1626 let recalled = store.recall_memory(&crate::RecallOptions {
1627 query: Some(query),
1628 decay: crate::Decay::None,
1629 ..crate::RecallOptions::default()
1630 })?;
1631
1632 let mut hits: Vec<MemoryHit> = recalled
1633 .results
1634 .into_iter()
1635 .map(|r| {
1636 let body = r.record.body.to_lowercase();
1637 let anchor = r
1638 .record
1639 .anchor
1640 .as_ref()
1641 .map(|a| a.key.to_lowercase())
1642 .unwrap_or_default();
1643 MemoryHit {
1644 score: memory_score(&q, &tokens, &body, &anchor, r.score),
1645 memory: true,
1646 id: r.record.id,
1647 kind: r.record.kind.as_str(),
1648 scope: r.record.scope.clone(),
1649 anchor: r.record.anchor.as_ref().map(|a| a.key.clone()),
1650 anchor_state: r.record.anchor_state.as_str(),
1651 applies: r.record.applies,
1652 evidence: r.score,
1653 snippet: content_snippet(&serde_json::json!({ "content": r.record.body })),
1654 }
1655 })
1656 .collect();
1657 hits.sort_by(|a, b| b.score.cmp(&a.score).then_with(|| b.id.cmp(&a.id)));
1658 window(&mut hits, 0, limit);
1659 Ok(hits)
1660}
1661
1662/// Relevance of one memory record: **lexical match, weighted by the record's own
1663/// evidence**, and nothing else.
1664///
1665/// The `evidence` factor is `base_confidence × anchor_penalty` from
1666/// [`crate::Store::recall_memory`] — so a lesson whose anchor still resolves in
1667/// this tree outranks an equally-worded one whose code has moved on, which is the
1668/// whole depreciation model showing up in search.
1669///
1670/// # The weight is in `[0, 1]`, and zero is reachable — deliberately
1671///
1672/// An earlier version of this comment said `(0, 1]`. That was wrong, and the
1673/// half-open interval hid a decision rather than describing one. The two factors
1674/// are not alike and the difference is the point:
1675///
1676/// - **[`crate::anchor_penalty`] can never be zero.** Its floor is `0.25`
1677/// ([`crate::AnchorState::Drifted`]), and
1678/// `memory::tests::anchor_penalty_demotes_without_ever_silencing` pins that
1679/// every state is `> 0`. So **drift can never drive evidence to zero** — which
1680/// is ADR-0013's "demote, never delete" rule holding *structurally*, not by
1681/// convention. Roteiro's own inference about a record is never allowed to
1682/// reduce it to nothing.
1683/// - **`base_confidence` can be exactly `0.0`**, because the writer can say so.
1684/// `roteiro memory add --confidence 0` is an operator stating "I am recording
1685/// this and I give it no credence." Flooring that would silently overrule an
1686/// explicit statement — and the value is a probability, where `0.0` is
1687/// legitimate rather than a boundary error.
1688///
1689/// So the asymmetry is exactly the right way round: **what Roteiro infers never
1690/// silences a record; what the operator explicitly states is honoured.**
1691///
1692/// # Zero relevance is not zero visibility
1693///
1694/// A zero score does **not** remove a hit. Nothing in this module or in
1695/// [`crate::Store::recall_memory`] filters on the score — it orders, and the
1696/// record comes back, is printed, and is labelled exactly as any other.
1697/// `a_zero_confidence_memory_is_ranked_last_and_still_returned` enforces that in
1698/// both surfaces, so the claim is a tested property rather than something this
1699/// comment asserts and nothing checks. (A limit can still truncate a
1700/// bottom-ranked hit — that is what a limit means, and it applies to every hit
1701/// regardless of score.)
1702///
1703/// The omissions are the point, and each is deliberate:
1704///
1705/// - **no `authored` boost** — this is the whole reason the channel exists. That
1706/// +40 is for intent a human deliberately wrote into a reviewed file;
1707/// accumulated, unreviewed, unredacted prose riding it would be trust-model
1708/// contamination by construction.
1709/// - **no overview boost** — a README's landing-page privilege is about authored
1710/// documentation.
1711/// - **no name or key term** — a memory record has neither.
1712///
1713/// Because this scorer shares no branch with the node scorer, "memory never
1714/// acquires the authored boost" is a structural fact rather than a condition to be
1715/// maintained.
1716fn memory_score(q: &str, tokens: &[&str], body: &str, anchor: &str, evidence: f64) -> u32 {
1717 let mut relevance: i32 = 0;
1718 if body.contains(q) {
1719 relevance += 25;
1720 }
1721 for t in tokens {
1722 if body.contains(t) {
1723 relevance += 8;
1724 } else if anchor.contains(t) {
1725 relevance += 3;
1726 }
1727 }
1728 // `[0.0, 1.0]`, closed at both ends: zero is reachable, and only ever because
1729 // a writer stated it. See the header — `anchor_penalty` cannot contribute a
1730 // zero, so drift can never land here.
1731 let weighted = f64::from(relevance.max(0)) * evidence.clamp(0.0, 1.0);
1732 #[expect(
1733 clippy::cast_possible_truncation,
1734 clippy::cast_sign_loss,
1735 reason = "the product of a small non-negative relevance and a weight in [0, 1]"
1736 )]
1737 let score = weighted.round() as u32;
1738 score
1739}
1740
1741/// Rank the generated channel alone. Ties break by `(producer, blob)` so results
1742/// are stable. `limit` follows [`window`]: `0` is every matching record, not none
1743/// of them.
1744fn search_generated(
1745 store: &Store,
1746 query: &str,
1747 limit: usize,
1748) -> Result<Vec<GeneratedHit>, StoreError> {
1749 let q = query.trim().to_lowercase();
1750 let tokens: Vec<&str> = q.split("::").flat_map(str::split_whitespace).collect();
1751 if tokens.is_empty() {
1752 return Ok(Vec::new());
1753 }
1754 let mut hits: Vec<GeneratedHit> = Vec::new();
1755 for record in store.media_records(&crate::MediaFilter::default())? {
1756 // A record the pre-generation gate refused holds a measurement, not text.
1757 // It is deliberately unsearchable: it has nothing to match on, and the
1758 // path *would* match — which would put a silent clip back into search
1759 // results as a hit with an empty snippet, which is the shape of the very
1760 // bug ADR-0015 exists to correct.
1761 let Some(generated_text) = record.outcome.text() else {
1762 continue;
1763 };
1764 let text = generated_text.to_lowercase();
1765 let path = record.path.to_lowercase();
1766 if !tokens.iter().all(|t| text.contains(t) || path.contains(t)) {
1767 continue;
1768 }
1769 hits.push(GeneratedHit {
1770 score: generated_score(&q, &tokens, &text, &path),
1771 generated: true,
1772 producer: record.producer_id.to_string(),
1773 model: record.producer.model.clone(),
1774 kind: record.producer.kind.as_str(),
1775 blob: record.blob_id.clone(),
1776 path: record.path.clone(),
1777 snippet: content_snippet(&serde_json::json!({ "content": generated_text })),
1778 });
1779 }
1780 hits.sort_by(|a, b| {
1781 b.score
1782 .cmp(&a.score)
1783 .then_with(|| (&a.producer, &a.blob).cmp(&(&b.producer, &b.blob)))
1784 });
1785 window(&mut hits, 0, limit);
1786 Ok(hits)
1787}
1788
1789/// Relevance of one generated record: whole-query and per-token matches over its
1790/// text and path, and **nothing else**.
1791///
1792/// The omissions are the point, and each is deliberate:
1793///
1794/// - **no `authored` boost** — generated text is not curated intent, and the
1795/// graph's +40 for an ADR must never land on a transcript;
1796/// - **no overview boost** — a README's landing-page privilege is about authored
1797/// documentation;
1798/// - **no name or key term** — a generated record has neither.
1799///
1800/// Because this scorer shares no branch with the node scorer, "generated content
1801/// never acquires the authored boost" is a structural fact rather than a
1802/// condition to be maintained.
1803fn generated_score(q: &str, tokens: &[&str], text: &str, path: &str) -> u32 {
1804 let mut relevance: i32 = 0;
1805 if text.contains(q) {
1806 relevance += 25;
1807 }
1808 for t in tokens {
1809 if text.contains(t) {
1810 relevance += 8;
1811 } else if path.contains(t) {
1812 relevance += 3;
1813 }
1814 }
1815 u32::try_from(relevance.max(0)).unwrap_or(0)
1816}
1817
1818/// Whether `path` (already lowercased) is a README/overview doc — the natural
1819/// landing for "what is this project" questions, so it is ranked up. Matches a
1820/// `readme*` or `overview*` basename (blueprints, the other overview docs, are
1821/// already boosted via their `authored` provenance).
1822fn is_overview_path(path: &str) -> bool {
1823 path.rsplit('/')
1824 .next()
1825 .is_some_and(|base| base.starts_with("readme") || base.starts_with("overview"))
1826}
1827
1828/// Whether `path` (already lowercased) is test scaffolding, which should not
1829/// outrank real content that happens to share a name.
1830fn is_test_path(path: &str) -> bool {
1831 path.contains("/tests/") || path.contains("/test/")
1832}
1833
1834/// A candidate step out of a node during traversal: the edge used and the node
1835/// on the other end. Ordered so BFS expansion is deterministic.
1836struct Step {
1837 node: String,
1838 hop: PathHop,
1839}
1840
1841/// All one-hop steps out of `key`, following edges in either direction, sorted
1842/// for deterministic traversal.
1843fn steps_from(store: &Store, key: &str) -> Result<Vec<Step>, StoreError> {
1844 let mut steps = Vec::new();
1845 for edge in store.edges_from(key)? {
1846 steps.push(Step {
1847 node: edge.dst.clone(),
1848 hop: hop(&edge, "outgoing", edge.dst.clone()),
1849 });
1850 }
1851 for edge in store.edges_to(key)? {
1852 steps.push(Step {
1853 node: edge.src.clone(),
1854 hop: hop(&edge, "incoming", edge.src.clone()),
1855 });
1856 }
1857 steps.sort_by(|a, b| {
1858 (&a.node, &a.hop.kind, a.hop.provenance, a.hop.direction).cmp(&(
1859 &b.node,
1860 &b.hop.kind,
1861 b.hop.provenance,
1862 b.hop.direction,
1863 ))
1864 });
1865 Ok(steps)
1866}
1867
1868fn hop(edge: &Edge, direction: &'static str, node: String) -> PathHop {
1869 PathHop {
1870 kind: edge.kind.as_str().to_owned(),
1871 provenance: edge.provenance.as_str(),
1872 confidence: edge.confidence,
1873 direction,
1874 node,
1875 }
1876}
1877
1878/// Find a shortest path from `from` to `to`, following edges in either
1879/// direction. Returns a [`Path`] with `found = false` (and no hops) if either
1880/// endpoint is absent or `to` is unreachable; `from == to` yields the trivial
1881/// zero-length path.
1882///
1883/// The search is breadth-first with deterministic neighbour ordering, so the
1884/// returned path is stable for a given graph.
1885///
1886/// # Errors
1887/// Returns [`StoreError`] on query failure.
1888pub fn path(store: &Store, from: &str, to: &str) -> Result<Path, StoreError> {
1889 let not_found = |found: bool, hops: Vec<PathHop>| Path {
1890 schema: SCHEMA,
1891 from: from.to_owned(),
1892 to: to.to_owned(),
1893 found,
1894 length: hops.len(),
1895 hops,
1896 };
1897
1898 // Both endpoints must exist in the graph.
1899 if store.get_node(from)?.is_none() || store.get_node(to)?.is_none() {
1900 return Ok(not_found(false, Vec::new()));
1901 }
1902 if from == to {
1903 return Ok(not_found(true, Vec::new()));
1904 }
1905
1906 // BFS, recording for each visited node the (predecessor, hop) that reached
1907 // it so the path can be reconstructed.
1908 let mut came_from: BTreeMap<String, (String, PathHop)> = BTreeMap::new();
1909 let mut queue: VecDeque<String> = VecDeque::new();
1910 queue.push_back(from.to_owned());
1911 came_from.insert(from.to_owned(), (String::new(), placeholder_hop()));
1912
1913 while let Some(current) = queue.pop_front() {
1914 if current == to {
1915 break;
1916 }
1917 for step in steps_from(store, ¤t)? {
1918 if came_from.contains_key(&step.node) {
1919 continue;
1920 }
1921 came_from.insert(step.node.clone(), (current.clone(), step.hop));
1922 queue.push_back(step.node);
1923 }
1924 }
1925
1926 // Walk predecessors back from `to` to `from`, then reverse. Every node in
1927 // `came_from` other than `from` has a real predecessor, so this terminates
1928 // at `from`. If the chain is ever broken (an invariant violation), treat it
1929 // as no path rather than silently returning a partial one.
1930 let mut hops = Vec::new();
1931 let mut cursor = to.to_owned();
1932 while cursor != from {
1933 let Some((prev, hop)) = came_from.get(&cursor) else {
1934 return Ok(not_found(false, Vec::new()));
1935 };
1936 hops.push(hop.clone());
1937 cursor = prev.clone();
1938 }
1939 hops.reverse();
1940 Ok(not_found(true, hops))
1941}
1942
1943/// A sentinel hop for the BFS start node (never emitted in a result).
1944fn placeholder_hop() -> PathHop {
1945 PathHop {
1946 kind: String::new(),
1947 provenance: "derived",
1948 confidence: None,
1949 direction: "outgoing",
1950 node: String::new(),
1951 }
1952}
1953
1954#[cfg(test)]
1955mod tests {
1956 use super::{
1957 ConfigSecretReport, CouplingItem, CouplingOrder, CouplingReport, DebtDensityReport,
1958 DensityItem, DensityOrder, RedactionState, SCHEMA, SNIPPET_MAX, SearchOptions,
1959 SearchResults, config_secrets, coupling, debt_density, explain, glob_match, list_kind,
1960 memory_score, path, search, search_channels, window,
1961 };
1962 use crate::{AnchorState, Edge, EdgeKind, FactSet, Node, NodeKind, Store};
1963
1964 fn seeded() -> Store {
1965 let mut store = Store::open_in_memory().expect("store");
1966 let facts = FactSet::new()
1967 .with_node(Node::new("sym:rust:a.rs#main", NodeKind::Fn, "main"))
1968 .with_node(Node::new("sym:rust:a.rs#helper", NodeKind::Fn, "helper"))
1969 .with_node(Node::new("adr:0001", NodeKind::Adr, "Build Roteiro"))
1970 .with_edge(Edge::derived(
1971 "sym:rust:a.rs#main",
1972 "sym:rust:a.rs#helper",
1973 EdgeKind::Calls,
1974 ))
1975 .with_edge(Edge::authored(
1976 "adr:0001",
1977 "sym:rust:a.rs#main",
1978 EdgeKind::References,
1979 ));
1980 store.apply_factset(&facts).expect("apply");
1981 store
1982 }
1983
1984 /// [`window`] is the single definition of `limit`/`offset` for every list
1985 /// lens, so its contract is pinned here rather than only through the lenses
1986 /// that call it.
1987 #[test]
1988 fn window_reads_zero_as_unlimited_and_offsets_before_limiting() {
1989 let ten = || (0..10).collect::<Vec<u8>>();
1990
1991 // `0` is unlimited, not empty — the whole point of #375.
1992 let mut all = ten();
1993 window(&mut all, 0, 0);
1994 assert_eq!(all, ten(), "limit 0 keeps everything");
1995
1996 // A non-zero limit cuts from the end, keeping the caller's order.
1997 let mut top = ten();
1998 window(&mut top, 0, 3);
1999 assert_eq!(top, vec![0, 1, 2]);
2000
2001 // A limit at or beyond the population is a no-op, so the boundary
2002 // between "bounded" and "unbounded" has no step in it.
2003 let mut exact = ten();
2004 window(&mut exact, 0, 10);
2005 assert_eq!(exact, ten());
2006 let mut over = ten();
2007 window(&mut over, 0, 99);
2008 assert_eq!(over, ten());
2009
2010 // `offset` applies first and `limit` to what remains.
2011 let mut paged = ten();
2012 window(&mut paged, 4, 3);
2013 assert_eq!(paged, vec![4, 5, 6]);
2014
2015 // The decision this fix had to make: offset with an unlimited limit is
2016 // "skip N, then every remaining item" — not "skip N, then nothing".
2017 let mut rest = ten();
2018 window(&mut rest, 7, 0);
2019 assert_eq!(rest, vec![7, 8, 9], "offset then unlimited");
2020
2021 // An offset at or past the end is an empty page, not a panic and not a
2022 // wrapped-around one.
2023 let mut at_end = ten();
2024 window(&mut at_end, 10, 0);
2025 assert!(at_end.is_empty());
2026 let mut past_end = ten();
2027 window(&mut past_end, 500, 0);
2028 assert!(past_end.is_empty());
2029 let mut past_end_limited = ten();
2030 window(&mut past_end_limited, 500, 5);
2031 assert!(past_end_limited.is_empty());
2032
2033 // An empty input stays empty under every combination.
2034 let mut empty: Vec<u8> = Vec::new();
2035 window(&mut empty, 0, 0);
2036 window(&mut empty, 3, 0);
2037 window(&mut empty, 0, 3);
2038 assert!(empty.is_empty());
2039 }
2040
2041 #[test]
2042 fn search_ranks_by_relevance_and_is_bounded() {
2043 let store = seeded();
2044 // An exact name match outranks a substring match.
2045 let hits = search(&store, "helper", 10).expect("search");
2046 assert_eq!(hits[0].node.key, "sym:rust:a.rs#helper");
2047 assert!(hits[0].score >= 100, "exact name match scores high");
2048
2049 // Every token must appear: "main roteiro" matches nothing (no node has both).
2050 assert!(
2051 search(&store, "main roteiro", 10)
2052 .expect("search")
2053 .is_empty()
2054 );
2055
2056 // A lone `:` does not split a token: `sym:rust` is one token matching the
2057 // code-symbol keys but not `adr:0001`.
2058 let by_prefix = search(&store, "sym:rust", 10).expect("search");
2059 assert!(!by_prefix.is_empty());
2060 assert!(
2061 by_prefix
2062 .iter()
2063 .all(|h| h.node.key.starts_with("sym:rust:"))
2064 );
2065
2066 // A blank query yields nothing; the limit is respected.
2067 assert!(search(&store, " ", 10).expect("search").is_empty());
2068 assert!(search(&store, "a.rs", 1).expect("search").len() <= 1);
2069 }
2070
2071 /// The population every issue-#393 test below works over: 12 in each of the
2072 /// three channels, each matching a term only its own channel carries.
2073 ///
2074 /// 12 is deliberately above the default of 10, so a `limit` of `0` that had
2075 /// quietly fallen back to the default could not pass for "unlimited".
2076 fn three_channels(population: usize) -> Store {
2077 use crate::{
2078 GeneratedContent, MediaKind, MediaOutcome, MediaWrite, MemoryKind, MemoryWrite,
2079 Producer,
2080 };
2081
2082 let mut store = Store::open_in_memory().expect("store");
2083 let mut facts = FactSet::new();
2084 for i in 0..population {
2085 facts = facts.with_node(Node::new(
2086 format!("sym:rust:a.rs#quokka{i}"),
2087 NodeKind::Fn,
2088 format!("quokka{i}"),
2089 ));
2090 }
2091 store.apply_factset(&facts).expect("apply");
2092
2093 let producer = Producer {
2094 kind: MediaKind::Audio,
2095 model: "voxtral-mini-3b".to_owned(),
2096 model_digest: "4705be8e".to_owned(),
2097 quantisation: "Q4_K_M".to_owned(),
2098 mmproj_digest: "4f24c4ef".to_owned(),
2099 prompt: "Transcribe this audio recording.".to_owned(),
2100 temperature: 0.0,
2101 max_tokens: 512,
2102 };
2103 for i in 0..population {
2104 store
2105 .record_memory(&MemoryWrite {
2106 scope: crate::DEFAULT_MEMORY_SCOPE,
2107 kind: MemoryKind::Lesson,
2108 anchor: None,
2109 body: &format!("wombat lesson number {i}"),
2110 confidence: None,
2111 supersedes: None,
2112 })
2113 .expect("memory write");
2114 assert!(
2115 store
2116 .record_media_content(&MediaWrite {
2117 blob_id: &format!("blob-{i}"),
2118 path: &format!("assets/clip{i}.wav"),
2119 producer: &producer,
2120 tool_version: "0.0.0",
2121 outcome: &MediaOutcome::Generated(GeneratedContent {
2122 text: format!("narwhal transcript number {i}"),
2123 confidence: None,
2124 }),
2125 replace: false,
2126 })
2127 .expect("media write"),
2128 "each clip is a fresh record",
2129 );
2130 }
2131 store
2132 }
2133
2134 /// Every channel asked for, at `limit`.
2135 fn all_channels(store: &Store, query: &str, limit: usize) -> SearchResults {
2136 search_channels(
2137 store,
2138 query,
2139 SearchOptions {
2140 limit,
2141 include_generated: true,
2142 include_memory: true,
2143 },
2144 )
2145 .expect("search")
2146 }
2147
2148 /// Issue #393: `limit == 0` reads as **unlimited on the graph channel**, and
2149 /// it is [`window`] that says so rather than a rule of `search`'s own — the
2150 /// third reading of one parameter name is gone, not relocated.
2151 #[test]
2152 fn search_reads_zero_as_unlimited_and_only_removes_the_cut() {
2153 const POPULATION: usize = 12;
2154 let store = three_channels(POPULATION);
2155
2156 let bounded = search(&store, "quokka", 10).expect("search");
2157 assert_eq!(bounded.len(), 10, "a positive limit still cuts");
2158
2159 let unlimited = search(&store, "quokka", 0).expect("search");
2160 assert_eq!(unlimited.len(), POPULATION, "0 is every match");
2161
2162 // An unlimited search is the same ranking uncut, not a different one:
2163 // the bounded page is the prefix of the unbounded one.
2164 assert_eq!(
2165 unlimited[..10]
2166 .iter()
2167 .map(|h| h.node.key.as_str())
2168 .collect::<Vec<_>>(),
2169 bounded
2170 .iter()
2171 .map(|h| h.node.key.as_str())
2172 .collect::<Vec<_>>(),
2173 "unlimited only removes the cut",
2174 );
2175 }
2176
2177 /// The unit is **per channel**: `0` is "all of each channel that was asked
2178 /// for", not "all of them merged and then cut". Each channel here matches a
2179 /// term the other two do not, so the three populations stay separable.
2180 #[test]
2181 fn each_search_channel_reads_zero_as_unlimited_over_its_own_population() {
2182 const POPULATION: usize = 12;
2183 let store = three_channels(POPULATION);
2184
2185 // Each channel matches a term the other two do not, so a bounded and an
2186 // unbounded read of one says nothing about the others.
2187 assert_eq!(all_channels(&store, "quokka", 10).hits.len(), 10);
2188 assert_eq!(
2189 all_channels(&store, "quokka", 0).hits.len(),
2190 POPULATION,
2191 "graph channel: 0 is unlimited",
2192 );
2193 assert_eq!(all_channels(&store, "wombat", 10).memory.len(), 10);
2194 assert_eq!(
2195 all_channels(&store, "wombat", 0).memory.len(),
2196 POPULATION,
2197 "memory channel: 0 is unlimited",
2198 );
2199 assert_eq!(all_channels(&store, "narwhal", 10).generated.len(), 10);
2200 assert_eq!(
2201 all_channels(&store, "narwhal", 0).generated.len(),
2202 POPULATION,
2203 "generated channel: 0 is unlimited",
2204 );
2205
2206 // And the unit really is per channel: an unbounded search of one term
2207 // leaves the channels it does not match empty rather than filling them.
2208 let graph_only = all_channels(&store, "quokka", 0);
2209 assert!(
2210 graph_only.memory.is_empty() && graph_only.generated.is_empty(),
2211 "unlimited is per channel, not a merged population",
2212 );
2213 }
2214
2215 /// What keeps "unlimited" from meaning "the whole store": a query with no
2216 /// tokens matches nothing, at `0` exactly as at any other limit. `--limit 0`
2217 /// is bounded by what was asked for, not by the population.
2218 #[test]
2219 fn a_tokenless_query_is_nothing_in_every_channel_at_every_limit() {
2220 let store = three_channels(12);
2221 for blank in ["", " ", "\t\n"] {
2222 for limit in [0, 10] {
2223 let nothing = all_channels(&store, blank, limit);
2224 assert!(
2225 nothing.hits.is_empty()
2226 && nothing.generated.is_empty()
2227 && nothing.memory.is_empty(),
2228 "a query with no tokens is nothing, not everything ({blank:?}, limit {limit})",
2229 );
2230 }
2231 }
2232 }
2233
2234 #[test]
2235 fn search_prefers_curated_content_over_same_named_test_symbols() {
2236 use crate::Provenance;
2237 let mut store = Store::open_in_memory().expect("store");
2238 // A same-named test helper (exact name, but test scaffolding)…
2239 let mut test_fn = Node::new(
2240 "sym:rust:crates/x/tests/cli.rs#roteiro",
2241 NodeKind::Fn,
2242 "roteiro",
2243 );
2244 test_fn.path = Some("crates/x/tests/cli.rs".into());
2245 // …the authored ADR that actually answers "what is roteiro"…
2246 let mut adr = Node::new("adr:0001", NodeKind::Adr, "Build Roteiro")
2247 .with_provenance(Provenance::Authored);
2248 adr.path = Some("docs/adr/0001.md".into());
2249 adr.meta = serde_json::json!({ "content": "Roteiro is a provenance-tagged codebase knowledge graph." });
2250 // …and a README whose *content* (not its name) describes the project.
2251 let mut readme = Node::new("file:README.md", NodeKind::File, "README.md");
2252 readme.path = Some("README.md".into());
2253 readme.meta =
2254 serde_json::json!({ "content": "Roteiro turns a repo into one knowledge graph." });
2255 store
2256 .apply_factset(
2257 &FactSet::new()
2258 .with_node(test_fn)
2259 .with_node(adr)
2260 .with_node(readme),
2261 )
2262 .expect("apply");
2263
2264 let hits = search(&store, "roteiro", 10).expect("search");
2265 let keys: Vec<&str> = hits.iter().map(|h| h.node.key.as_str()).collect();
2266 let idx = |k: &str| keys.iter().position(|x| *x == k).expect("present");
2267 // The authored ADR and the README (found *by content*) both outrank the
2268 // same-named test helper.
2269 assert!(
2270 idx("adr:0001") < idx("sym:rust:crates/x/tests/cli.rs#roteiro"),
2271 "authored ADR outranks the test symbol: {keys:?}"
2272 );
2273 assert!(
2274 idx("file:README.md") < idx("sym:rust:crates/x/tests/cli.rs#roteiro"),
2275 "README (matched via content) outranks the test symbol: {keys:?}"
2276 );
2277
2278 // A content-only term finds the node even though no name/key/path has it.
2279 let by_content = search(&store, "provenance-tagged", 10).expect("search");
2280 assert_eq!(
2281 by_content.first().map(|h| h.node.key.as_str()),
2282 Some("adr:0001"),
2283 "content search matches the ADR by its captured text"
2284 );
2285 }
2286
2287 #[test]
2288 fn search_hit_carries_a_bounded_content_snippet() {
2289 use crate::Provenance;
2290 let mut store = Store::open_in_memory().expect("store");
2291 // A content-bearing node whose content is longer than the cap and has
2292 // messy whitespace to collapse.
2293 let long = "word ".repeat(200);
2294 let mut adr =
2295 Node::new("adr:0001", NodeKind::Adr, "Overview").with_provenance(Provenance::Authored);
2296 adr.meta = serde_json::json!({ "content": format!("Roteiro is\n\na graph. {long}") });
2297 // A pure symbol node with no captured content.
2298 let sym = Node::new("sym:rust:a.rs#main", NodeKind::Fn, "main");
2299 store
2300 .apply_factset(&FactSet::new().with_node(adr).with_node(sym))
2301 .expect("apply");
2302
2303 let hits = search(&store, "roteiro", 10).expect("search");
2304 let adr_hit = hits
2305 .iter()
2306 .find(|h| h.node.key == "adr:0001")
2307 .expect("adr hit");
2308 let snippet = adr_hit
2309 .snippet
2310 .as_deref()
2311 .expect("a content-bearing node yields a snippet");
2312 // Whitespace is collapsed to single spaces (no runs, no newlines)…
2313 assert!(snippet.starts_with("Roteiro is a graph."), "got: {snippet}");
2314 assert!(!snippet.contains(" "));
2315 assert!(!snippet.contains('\n'));
2316 // …and the snippet is bounded to SNIPPET_MAX chars *including* the ellipsis.
2317 assert!(
2318 snippet.chars().count() <= SNIPPET_MAX,
2319 "snippet is bounded: {} chars",
2320 snippet.chars().count()
2321 );
2322 assert!(
2323 snippet.ends_with('…'),
2324 "over-long content is truncated with an ellipsis"
2325 );
2326
2327 // A node without content falls back cleanly: no snippet, so the summary
2328 // (name/kind/path) is the grounding.
2329 let hits = search(&store, "main", 10).expect("search");
2330 let sym_hit = hits
2331 .iter()
2332 .find(|h| h.node.key == "sym:rust:a.rs#main")
2333 .expect("sym hit");
2334 assert!(
2335 sym_hit.snippet.is_none(),
2336 "a node with no content has no snippet"
2337 );
2338 }
2339
2340 #[test]
2341 fn explain_reports_labelled_neighbourhood() {
2342 let store = seeded();
2343 let ex = explain(&store, "sym:rust:a.rs#main")
2344 .expect("query")
2345 .expect("present");
2346 assert_eq!(ex.schema, SCHEMA);
2347 assert_eq!(ex.node.kind, "fn");
2348
2349 // Outgoing: derived call to helper.
2350 assert_eq!(ex.outgoing.len(), 1);
2351 assert_eq!(ex.outgoing[0].kind, "calls");
2352 assert_eq!(ex.outgoing[0].provenance, "derived");
2353 assert_eq!(ex.outgoing[0].node, "sym:rust:a.rs#helper");
2354
2355 // Incoming: authored reference from the ADR.
2356 assert_eq!(ex.incoming.len(), 1);
2357 assert_eq!(ex.incoming[0].provenance, "authored");
2358 assert_eq!(ex.incoming[0].node, "adr:0001");
2359 }
2360
2361 #[test]
2362 fn explain_missing_node_is_none() {
2363 let store = seeded();
2364 assert!(explain(&store, "sym:rust:a.rs#ghost").expect("q").is_none());
2365 }
2366
2367 #[test]
2368 fn edges_differing_only_in_provenance_are_ordered() {
2369 // Two edges A->B with the same kind but different provenance must sort
2370 // into a stable, deterministic order (authored before derived).
2371 let mut store = Store::open_in_memory().expect("store");
2372 let facts = FactSet::new()
2373 .with_node(Node::new("a", NodeKind::Fn, "a"))
2374 .with_node(Node::new("b", NodeKind::Fn, "b"))
2375 .with_edge(Edge::derived("a", "b", EdgeKind::References))
2376 .with_edge(Edge::authored("a", "b", EdgeKind::References));
2377 store.apply_factset(&facts).expect("apply");
2378
2379 let ex = explain(&store, "a").expect("q").expect("present");
2380 let provs: Vec<_> = ex.outgoing.iter().map(|e| e.provenance).collect();
2381 assert_eq!(provs, ["authored", "derived"]);
2382 }
2383
2384 #[test]
2385 fn list_kind_is_ordered() {
2386 let store = seeded();
2387 let listing = list_kind(&store, &NodeKind::Fn).expect("list");
2388 let keys: Vec<_> = listing.nodes.iter().map(|n| n.key.as_str()).collect();
2389 assert_eq!(keys, ["sym:rust:a.rs#helper", "sym:rust:a.rs#main"]);
2390 }
2391
2392 #[test]
2393 fn json_schema_is_stable() {
2394 let store = seeded();
2395 let ex = explain(&store, "adr:0001").expect("q").expect("present");
2396 let json = serde_json::to_value(&ex).expect("json");
2397 assert_eq!(json["schema"], SCHEMA);
2398 assert_eq!(json["node"]["key"], "adr:0001");
2399 assert_eq!(json["node"]["kind"], "adr");
2400 // Outgoing authored reference is present with its provenance label.
2401 assert_eq!(json["outgoing"][0]["kind"], "references");
2402 assert_eq!(json["outgoing"][0]["provenance"], "authored");
2403 assert_eq!(json["outgoing"][0]["node"], "sym:rust:a.rs#main");
2404 assert!(json["outgoing"][0]["confidence"].is_null());
2405 }
2406
2407 #[test]
2408 fn path_crosses_provenance_and_direction() {
2409 // adr:0001 --authored/references--> main --derived/calls--> helper.
2410 // A path from the ADR to helper must traverse both, each hop labelled.
2411 let store = seeded();
2412 let p = path(&store, "adr:0001", "sym:rust:a.rs#helper").expect("path");
2413 assert!(p.found);
2414 assert_eq!(p.length, 2);
2415 assert_eq!(p.schema, SCHEMA);
2416
2417 assert_eq!(p.hops[0].kind, "references");
2418 assert_eq!(p.hops[0].provenance, "authored");
2419 assert_eq!(p.hops[0].direction, "outgoing");
2420 assert_eq!(p.hops[0].node, "sym:rust:a.rs#main");
2421
2422 assert_eq!(p.hops[1].kind, "calls");
2423 assert_eq!(p.hops[1].provenance, "derived");
2424 assert_eq!(p.hops[1].node, "sym:rust:a.rs#helper");
2425 }
2426
2427 #[test]
2428 fn path_follows_edges_against_direction() {
2429 // From helper back to the ADR: both edges are traversed against their
2430 // stored direction, so each hop is `incoming`.
2431 let store = seeded();
2432 let p = path(&store, "sym:rust:a.rs#helper", "adr:0001").expect("path");
2433 assert!(p.found);
2434 assert_eq!(p.length, 2);
2435 assert!(p.hops.iter().all(|h| h.direction == "incoming"));
2436 assert_eq!(p.hops.last().unwrap().node, "adr:0001");
2437 }
2438
2439 #[test]
2440 fn path_same_node_is_trivial() {
2441 let store = seeded();
2442 let p = path(&store, "adr:0001", "adr:0001").expect("path");
2443 assert!(p.found);
2444 assert_eq!(p.length, 0);
2445 assert!(p.hops.is_empty());
2446 }
2447
2448 #[test]
2449 fn path_missing_endpoint_or_unreachable_is_not_found() {
2450 let mut store = Store::open_in_memory().expect("store");
2451 // Two disconnected components: a-b and an isolated island.
2452 let facts = FactSet::new()
2453 .with_node(Node::new("a", NodeKind::Fn, "a"))
2454 .with_node(Node::new("b", NodeKind::Fn, "b"))
2455 .with_node(Node::new("island", NodeKind::Fn, "island"))
2456 .with_edge(Edge::derived("a", "b", EdgeKind::Calls));
2457 store.apply_factset(&facts).expect("apply");
2458
2459 // Absent endpoint.
2460 let missing = path(&store, "a", "ghost").expect("path");
2461 assert!(!missing.found);
2462 assert!(missing.hops.is_empty());
2463
2464 // Present but unreachable.
2465 let unreachable = path(&store, "a", "island").expect("path");
2466 assert!(!unreachable.found);
2467 assert!(unreachable.hops.is_empty());
2468 }
2469
2470 #[test]
2471 fn path_is_shortest() {
2472 // a-b-c-d chain plus a direct a-d edge: the path must take the shortcut.
2473 let mut store = Store::open_in_memory().expect("store");
2474 let facts = FactSet::new()
2475 .with_node(Node::new("a", NodeKind::Fn, "a"))
2476 .with_node(Node::new("b", NodeKind::Fn, "b"))
2477 .with_node(Node::new("c", NodeKind::Fn, "c"))
2478 .with_node(Node::new("d", NodeKind::Fn, "d"))
2479 .with_edge(Edge::derived("a", "b", EdgeKind::Calls))
2480 .with_edge(Edge::derived("b", "c", EdgeKind::Calls))
2481 .with_edge(Edge::derived("c", "d", EdgeKind::Calls))
2482 .with_edge(Edge::derived("a", "d", EdgeKind::Calls));
2483 store.apply_factset(&facts).expect("apply");
2484
2485 let p = path(&store, "a", "d").expect("path");
2486 assert!(p.found);
2487 assert_eq!(p.length, 1, "the direct a->d edge is the shortest path");
2488 assert_eq!(p.hops[0].node, "d");
2489 }
2490
2491 #[test]
2492 fn glob_matches_segments_and_wildcards() {
2493 // `**` spans segments (including zero) and anchors both ends.
2494 assert!(glob_match("vendor/**", "vendor/lib/a.rs"));
2495 assert!(glob_match("vendor/**", "vendor")); // zero trailing segments
2496 assert!(glob_match("**/generated/*", "src/gen/generated/x.rs"));
2497 assert!(glob_match("**/*.rs", "a/b/c.rs"));
2498 // `*` and `?` stay within one segment.
2499 assert!(glob_match("src/*.rs", "src/main.rs"));
2500 assert!(!glob_match("src/*.rs", "src/sub/main.rs"));
2501 assert!(glob_match("a?c.rs", "abc.rs"));
2502 assert!(!glob_match("a?c.rs", "ac.rs"));
2503 // Anchored: a bare name does not match a nested path.
2504 assert!(!glob_match("generated", "src/generated"));
2505 assert!(!glob_match("vendor/**", "third_party/vendor/a.rs"));
2506 }
2507
2508 /// **The evidence weight is closed at both ends**, and the two ends mean
2509 /// different things.
2510 ///
2511 /// The boundary the doc comment used to get wrong: it claimed `(0, 1]`, which
2512 /// would have made a zero unreachable. It is reachable, from a writer stating
2513 /// `--confidence 0` and from nowhere else — the lowest weight Roteiro can
2514 /// *infer* is `anchor_penalty(Drifted)`, and that still leaves a score
2515 /// standing, which is checked here against the real constant rather than a
2516 /// number copied from it.
2517 #[test]
2518 fn the_evidence_weight_is_closed_at_both_ends() {
2519 let score = |evidence| memory_score("batch", &["batch"], "a batch cursor", "", evidence);
2520 let full = score(1.0);
2521 assert!(full > 0, "a fully-evidenced hit scores");
2522 assert_eq!(score(0.0), 0, "and a zero weight takes it to zero");
2523 assert!(
2524 score(0.5) < full && score(0.5) > 0,
2525 "in between, in between"
2526 );
2527
2528 // The worst Roteiro can infer about a record still leaves it scoring —
2529 // "demote, never delete", holding as arithmetic.
2530 let worst_inferable = [
2531 AnchorState::Valid,
2532 AnchorState::Unanchored,
2533 AnchorState::Unverifiable,
2534 AnchorState::Vanished,
2535 AnchorState::Drifted,
2536 ]
2537 .into_iter()
2538 .map(crate::anchor_penalty)
2539 .fold(f64::INFINITY, f64::min);
2540 assert!(
2541 score(worst_inferable) > 0,
2542 "the most demoted anchor state ({worst_inferable}) must not silence a hit",
2543 );
2544
2545 // Out-of-range input is clamped rather than trusted, so a corrupt stored
2546 // confidence cannot manufacture a score above the honest ceiling.
2547 assert_eq!(score(2.0), full, "clamped at the top");
2548 assert_eq!(score(-1.0), 0, "and at the bottom");
2549 }
2550
2551 // -- coupling (Q3) -----------------------------------------------------
2552
2553 /// A graph whose two most-coupled nodes have the **same undirected degree**
2554 /// but opposite direction: `hub` is called by two callers and calls nothing;
2555 /// `spread` calls two callees and is called by nothing. An undirected degree
2556 /// ranking cannot tell them apart, which is the whole point of this lens.
2557 fn coupled() -> Store {
2558 let mut store = Store::open_in_memory().expect("store");
2559 let mut facts = FactSet::new();
2560 for name in ["hub", "spread", "a", "b", "x", "y"] {
2561 facts = facts.with_node(Node::new(
2562 format!("sym:rust:a.rs#{name}"),
2563 NodeKind::Fn,
2564 name,
2565 ));
2566 }
2567 for (src, dst) in [("a", "hub"), ("b", "hub"), ("spread", "x"), ("spread", "y")] {
2568 facts = facts.with_edge(Edge::derived(
2569 format!("sym:rust:a.rs#{src}"),
2570 format!("sym:rust:a.rs#{dst}"),
2571 EdgeKind::Calls,
2572 ));
2573 }
2574 store.apply_factset(&facts).expect("apply");
2575 store
2576 }
2577
2578 /// Find an item by symbol name, so assertions read by name not by index.
2579 fn item<'a>(report: &'a CouplingReport, name: &str) -> &'a CouplingItem {
2580 report
2581 .items
2582 .iter()
2583 .find(|i| i.name == name)
2584 .unwrap_or_else(|| panic!("`{name}` missing from {:?}", report.items))
2585 }
2586
2587 #[test]
2588 fn coupling_keeps_the_direction_an_undirected_degree_discards() {
2589 let report = coupling(&coupled(), CouplingOrder::Total, 0).expect("coupling");
2590 let hub = item(&report, "hub");
2591 let spread = item(&report, "spread");
2592
2593 // Identical undirected degree — what a both-ends-incremented ranking sees.
2594 assert_eq!(hub.total, spread.total, "same total coupling");
2595
2596 // …and opposite direction, which is what this lens exists to report.
2597 assert_eq!((hub.fan_in, hub.fan_out), (2, 0), "hub is depended upon");
2598 assert_eq!(
2599 (spread.fan_in, spread.fan_out),
2600 (0, 2),
2601 "spread depends on others"
2602 );
2603 assert!(
2604 (hub.instability - 0.0).abs() < f64::EPSILON,
2605 "a purely called node is maximally stable: {}",
2606 hub.instability
2607 );
2608 assert!(
2609 (spread.instability - 1.0).abs() < f64::EPSILON,
2610 "a purely calling node is maximally unstable: {}",
2611 spread.instability
2612 );
2613
2614 assert_eq!(report.edge_kind, "calls");
2615 assert_eq!(report.coupled_nodes, 6);
2616 assert_eq!(report.call_edges, 4);
2617 assert_eq!(report.self_calls, 0);
2618 assert_eq!(report.cross_language_calls, 0);
2619 }
2620
2621 #[test]
2622 fn coupling_excludes_cross_language_name_collisions() {
2623 // Cross-file call resolution binds a callee by simple name across every
2624 // `Fn` node regardless of language, and Roteiro extracts no FFI — so a
2625 // JavaScript function "calling" a Rust one is a name collision. On this
2626 // repository that single rule is the difference between a Rust helper
2627 // reading as the most depended-on symbol in the tree and not appearing
2628 // at all.
2629 let mut store = coupled();
2630 let mut facts = FactSet::new().with_node(Node::new(
2631 "sym:javascript:app.js#render",
2632 NodeKind::Fn,
2633 "render",
2634 ));
2635 facts = facts.with_edge(Edge::derived(
2636 "sym:javascript:app.js#render",
2637 "sym:rust:a.rs#hub",
2638 EdgeKind::Calls,
2639 ));
2640 store.apply_factset(&facts).expect("apply");
2641
2642 let report = coupling(&store, CouplingOrder::Total, 0).expect("coupling");
2643 assert_eq!(
2644 item(&report, "hub").fan_in,
2645 2,
2646 "a JavaScript caller is not a dependant of a Rust function"
2647 );
2648 assert_eq!(
2649 report.cross_language_calls, 1,
2650 "the excluded edge is reported, not silently dropped"
2651 );
2652 assert_eq!(report.call_edges, 5, "and still counted as scanned");
2653 }
2654
2655 #[test]
2656 fn same_language_never_guesses_about_unknown_key_shapes() {
2657 assert!(super::same_language("sym:rust:a.rs#f", "sym:rust:b.rs#g"));
2658 assert!(!super::same_language(
2659 "sym:javascript:a.js#f",
2660 "sym:rust:b.rs#g"
2661 ));
2662 // A key that is not `sym:<lang>:…` carries no language to compare, so the
2663 // edge is kept: this filter drops only what it can prove spans languages.
2664 assert!(super::same_language("file:a.md", "sym:rust:b.rs#g"));
2665 assert!(super::same_language("sym:", "sym:rust:b.rs#g"));
2666 assert_eq!(super::sym_lang("sym:rust:a.rs#f"), Some("rust"));
2667 assert_eq!(
2668 super::sym_lang("sym::a.rs#f"),
2669 None,
2670 "empty lang is no lang"
2671 );
2672 assert_eq!(super::sym_lang("marker:a.rs#7"), None);
2673 }
2674
2675 #[test]
2676 fn coupling_counts_distinct_callers_not_parallel_edges() {
2677 // Migration 3 makes edges a set per `(src, dst, kind, provenance)` — so
2678 // the way one caller contributes two `Calls` rows is by **provenance**:
2679 // an extractor's `derived` call and an inference layer's `inferred` one.
2680 // Two rows, one dependant.
2681 let mut store = coupled();
2682 let inferred = Edge::inferred("sym:rust:a.rs#a", "sym:rust:a.rs#hub", EdgeKind::Calls, 0.9);
2683 store
2684 .apply_factset(&FactSet::new().with_edge(inferred))
2685 .expect("apply");
2686
2687 let report = coupling(&store, CouplingOrder::Total, 0).expect("coupling");
2688 assert_eq!(
2689 item(&report, "hub").fan_in,
2690 2,
2691 "the same caller at two provenances is one dependant, not two"
2692 );
2693 // The raw edge is still counted, so the parallel edge stays visible
2694 // rather than being silently normalised away.
2695 assert_eq!(
2696 report.call_edges, 5,
2697 "the extra edge is reported as scanned"
2698 );
2699 }
2700
2701 #[test]
2702 fn coupling_excludes_self_calls_from_both_fans() {
2703 // Recursion is a real edge that couples a node to nothing outside itself;
2704 // counting it would inflate `fan_in` AND `fan_out` for the same node.
2705 let mut store = coupled();
2706 let recursive = Edge::derived("sym:rust:a.rs#hub", "sym:rust:a.rs#hub", EdgeKind::Calls);
2707 store
2708 .apply_factset(&FactSet::new().with_edge(recursive))
2709 .expect("apply");
2710
2711 let report = coupling(&store, CouplingOrder::Total, 0).expect("coupling");
2712 let hub = item(&report, "hub");
2713 assert_eq!(
2714 (hub.fan_in, hub.fan_out),
2715 (2, 0),
2716 "recursion changes neither fan"
2717 );
2718 assert_eq!(report.self_calls, 1, "but it is reported, not dropped");
2719 }
2720
2721 #[test]
2722 fn coupling_order_picks_the_question_being_asked() {
2723 let store = coupled();
2724 let top = |order| {
2725 coupling(&store, order, 1).expect("coupling").items[0]
2726 .name
2727 .clone()
2728 };
2729 assert_eq!(top(CouplingOrder::FanIn), "hub", "most depended-on");
2730 assert_eq!(top(CouplingOrder::FanOut), "spread", "reaches furthest");
2731
2732 // `total` cannot separate the two, so the tie must break on `key` —
2733 // a stable order rather than whatever the map iteration yields.
2734 let by_total = coupling(&store, CouplingOrder::Total, 2).expect("coupling");
2735 assert_eq!(
2736 by_total.items.iter().map(|i| &i.name).collect::<Vec<_>>(),
2737 ["hub", "spread"],
2738 "ties break by key ascending"
2739 );
2740 }
2741
2742 #[test]
2743 fn coupling_reports_truncation_and_is_deterministic() {
2744 let store = coupled();
2745 let capped = coupling(&store, CouplingOrder::Total, 2).expect("coupling");
2746 assert_eq!(capped.items.len(), 2);
2747 assert_eq!(
2748 capped.coupled_nodes, 6,
2749 "the population is reported, so a capped list cannot read as the whole graph"
2750 );
2751 assert_eq!(capped.limit, 2);
2752
2753 // Identical input → byte-identical output, including the ratio's rendering.
2754 let a = serde_json::to_string(&capped).expect("json");
2755 let b =
2756 serde_json::to_string(&coupling(&store, CouplingOrder::Total, 2).expect("coupling"))
2757 .expect("json");
2758 assert_eq!(a, b, "deterministic serialisation");
2759 }
2760
2761 #[test]
2762 fn coupling_ignores_edge_kinds_whose_direction_is_not_a_call() {
2763 // `references` is directed too, but an ADR referencing a symbol is not a
2764 // caller. Only `Calls` may move these numbers.
2765 let mut store = coupled();
2766 let mut facts = FactSet::new().with_node(Node::new("adr:0001", NodeKind::Adr, "A"));
2767 facts = facts.with_edge(Edge::authored(
2768 "adr:0001",
2769 "sym:rust:a.rs#hub",
2770 EdgeKind::References,
2771 ));
2772 store.apply_factset(&facts).expect("apply");
2773
2774 let report = coupling(&store, CouplingOrder::Total, 0).expect("coupling");
2775 assert_eq!(item(&report, "hub").fan_in, 2, "a reference is not a call");
2776 assert!(
2777 !report.items.iter().any(|i| i.key == "adr:0001"),
2778 "a node with no call edges is not in the population: {:?}",
2779 report.items
2780 );
2781 assert_eq!(report.call_edges, 4);
2782 }
2783
2784 #[test]
2785 fn coupling_order_tokens_round_trip() {
2786 for token in CouplingOrder::tokens() {
2787 let order = CouplingOrder::from_token(token)
2788 .unwrap_or_else(|| panic!("`{token}` is advertised but not accepted"));
2789 assert_eq!(order.as_str(), token);
2790 }
2791 assert!(
2792 CouplingOrder::from_token("degree").is_none(),
2793 "an unknown order is rejected, not silently defaulted"
2794 );
2795 }
2796
2797 // -- debt density (Q1) -------------------------------------------------
2798
2799 /// A `file` node carrying the `meta.lines` this lens divides by — the shape
2800 /// `extract::file_node` emits for every blob.
2801 fn file_of(path: &str, lines: u64) -> Node {
2802 let mut node = Node::new(format!("file:{path}"), NodeKind::File, path);
2803 node.path = Some(path.to_owned());
2804 node.meta = serde_json::json!({ "bytes": lines * 30, "lines": lines });
2805 node
2806 }
2807
2808 /// A marker node as `markers::augment` emits it.
2809 fn marker_of(path: &str, line: u32, category: &str) -> Node {
2810 let mut node = Node::new(
2811 format!("marker:{path}#{line}"),
2812 NodeKind::Marker,
2813 format!("TODO {line}"), // roteiro:ignore
2814 );
2815 node.path = Some(path.to_owned());
2816 node.meta = serde_json::json!({
2817 "category": category,
2818 "text": format!("TODO {line}"), // roteiro:ignore
2819 "line": line,
2820 });
2821 node
2822 }
2823
2824 /// Two files with the **same marker count** and very different lengths —
2825 /// indistinguishable under `debt`, twenty-fold apart under density. Plus a
2826 /// third, short file whose single marker would top the ranking on arithmetic
2827 /// alone.
2828 fn marked() -> Store {
2829 let mut store = Store::open_in_memory().expect("store");
2830 let mut facts = FactSet::new()
2831 .with_node(file_of("big.rs", 4000))
2832 .with_node(file_of("small.rs", 200))
2833 .with_node(file_of("tiny.rs", 10));
2834 for line in 1..=40 {
2835 facts = facts.with_node(marker_of("big.rs", line, "todo")); // roteiro:ignore
2836 facts = facts.with_node(marker_of("small.rs", line, "todo")); // roteiro:ignore
2837 }
2838 facts = facts.with_node(marker_of("tiny.rs", 3, "stub"));
2839 store.apply_factset(&facts).expect("apply");
2840 store
2841 }
2842
2843 /// Every default: no category filter, no ignore globs, unlimited, floored at
2844 /// [`super::DEFAULT_MIN_LINES`].
2845 fn density(store: &Store, order: DensityOrder) -> DebtDensityReport {
2846 debt_density(store, &[], &[], order, 0, super::DEFAULT_MIN_LINES).expect("density")
2847 }
2848
2849 /// Find an item by path, so assertions read by file not by index.
2850 fn at<'a>(report: &'a DebtDensityReport, path: &str) -> &'a DensityItem {
2851 report
2852 .items
2853 .iter()
2854 .find(|i| i.path == path)
2855 .unwrap_or_else(|| panic!("`{path}` missing from {:?}", report.items))
2856 }
2857
2858 #[test]
2859 fn density_separates_files_a_raw_marker_count_cannot() {
2860 let report = density(&marked(), DensityOrder::Density);
2861 let big = at(&report, "big.rs");
2862 let small = at(&report, "small.rs");
2863
2864 // Identical under `debt` — the same forty markers each.
2865 assert_eq!(big.markers, small.markers, "same raw count");
2866
2867 // …and twenty-fold apart under density, which is the whole lens.
2868 assert!(
2869 (big.per_kloc - 10.0).abs() < f64::EPSILON,
2870 "40 markers in 4000 lines is 10 per kloc, was {}",
2871 big.per_kloc
2872 );
2873 assert!(
2874 (small.per_kloc - 200.0).abs() < f64::EPSILON,
2875 "40 markers in 200 lines is 200 per kloc, was {}",
2876 small.per_kloc
2877 );
2878 assert_eq!(
2879 report.items.first().map(|i| i.path.as_str()),
2880 Some("small.rs"),
2881 "the dense file ranks first: {:?}",
2882 report.items
2883 );
2884
2885 // The per-file category split, so "forty todo" and "forty stub" stay
2886 // distinguishable in a report that otherwise shows one number per file.
2887 assert_eq!(small.by_category.get("todo"), Some(&40)); // roteiro:ignore
2888 assert_eq!(report.schema, SCHEMA);
2889 }
2890
2891 #[test]
2892 fn markers_order_ranks_the_way_debt_already_does() {
2893 // The control: on `markers` the two forty-marker files tie and break on
2894 // path, so density is demonstrably the thing that separated them — not
2895 // some other difference in the fixture.
2896 let report = density(&marked(), DensityOrder::Markers);
2897 assert_eq!(
2898 report.items.iter().map(|i| &i.path).collect::<Vec<_>>(),
2899 ["big.rs", "small.rs"],
2900 "equal counts tie and break on path ascending"
2901 );
2902 }
2903
2904 #[test]
2905 fn the_short_file_floor_excludes_without_hiding() {
2906 // `tiny.rs` is 1 marker in 10 lines = 100 per kloc, which would place it
2907 // second on arithmetic alone. The floor keeps it out of the *ranking*
2908 // while leaving it in the population and the totals.
2909 let report = density(&marked(), DensityOrder::Density);
2910 assert!(
2911 !report.items.iter().any(|i| i.path == "tiny.rs"),
2912 "a 10-line file is not ranked: {:?}",
2913 report.items
2914 );
2915 assert_eq!(report.short_files, 1, "and its exclusion is reported");
2916 assert_eq!(
2917 report.files_with_markers, 3,
2918 "the population still counts it"
2919 );
2920 assert_eq!(report.ranked_files, 2);
2921 assert_eq!(
2922 report.total_markers, 81,
2923 "and so do the totals: 40 + 40 + 1"
2924 );
2925
2926 // `min_lines = 0` disables the floor rather than merely lowering it.
2927 let unfloored =
2928 debt_density(&marked(), &[], &[], DensityOrder::Density, 0, 0).expect("density");
2929 assert_eq!(unfloored.short_files, 0);
2930 assert_eq!(unfloored.ranked_files, 3);
2931 let tiny = at(&unfloored, "tiny.rs").per_kloc;
2932 assert!(
2933 (tiny - 100.0).abs() < f64::EPSILON,
2934 "the arithmetic the floor exists to keep out of the ranking, was {tiny}"
2935 );
2936 }
2937
2938 #[test]
2939 fn a_file_with_no_recorded_length_is_reported_not_divided_by() {
2940 // Three ways a denominator goes missing, all of which must land in
2941 // `unknown_length_files` rather than in the ranking with a fabricated
2942 // density: no `file` node at all, a `file` node with no `meta.lines`, and
2943 // a `lines` of zero (an empty file, or one unterminated line — a newline
2944 // count cannot tell those apart, so neither does this).
2945 let mut store = Store::open_in_memory().expect("store");
2946 let mut no_lines = Node::new("file:b.rs", NodeKind::File, "b.rs");
2947 no_lines.path = Some("b.rs".into());
2948 no_lines.meta = serde_json::json!({ "bytes": 90 });
2949 let facts = FactSet::new()
2950 .with_node(marker_of("orphan.rs", 1, "todo")) // roteiro:ignore
2951 .with_node(no_lines)
2952 .with_node(marker_of("b.rs", 1, "todo")) // roteiro:ignore
2953 .with_node(file_of("empty.rs", 0))
2954 .with_node(marker_of("empty.rs", 1, "todo")); // roteiro:ignore
2955 store.apply_factset(&facts).expect("apply");
2956
2957 let report = density(&store, DensityOrder::Density);
2958 assert!(report.items.is_empty(), "nothing rankable: {report:?}");
2959 assert_eq!(report.unknown_length_files, 3);
2960 assert_eq!(
2961 report.total_markers, 3,
2962 "the markers are still inventoried, so the file cannot vanish silently"
2963 );
2964 assert!(
2965 (report.overall_per_kloc - 0.0).abs() < f64::EPSILON,
2966 "and no density is invented from a zero denominator"
2967 );
2968 }
2969
2970 #[test]
2971 fn density_shares_debt_s_filters_rather_than_adding_a_second_vocabulary() {
2972 let store = marked();
2973 // The `[debt] ignore` globs `debt` already honours.
2974 let ignored = debt_density(
2975 &store,
2976 &[],
2977 &["small.rs".into()],
2978 DensityOrder::Density,
2979 0,
2980 super::DEFAULT_MIN_LINES,
2981 )
2982 .expect("density");
2983 assert!(
2984 !ignored.items.iter().any(|i| i.path == "small.rs"),
2985 "an ignored path leaves the report entirely: {:?}",
2986 ignored.items
2987 );
2988 assert_eq!(
2989 ignored.files_with_markers, 2,
2990 "not merely unranked — it is not in the population either"
2991 );
2992
2993 // And the same category filter, so a `--kind stub` density is the density
2994 // of stubs and not of everything.
2995 let stubs = debt_density(&store, &["stub".into()], &[], DensityOrder::Density, 0, 0)
2996 .expect("density");
2997 assert_eq!(stubs.total_markers, 1);
2998 assert_eq!(
2999 stubs.items.iter().map(|i| &i.path).collect::<Vec<_>>(),
3000 ["tiny.rs"]
3001 );
3002 }
3003
3004 #[test]
3005 fn density_ranks_on_the_exact_ratio_not_the_rounded_one() {
3006 // Two files whose densities differ in the fourth decimal: 1/3000 is
3007 // 0.3333 per kloc and 1/3001 is 0.3332. Both round to 0.33, so a ranking
3008 // built on `per_kloc` would tie them and break on path — putting the
3009 // *less* dense file first, since `a.rs` sorts before `b.rs`.
3010 let mut store = Store::open_in_memory().expect("store");
3011 let facts = FactSet::new()
3012 .with_node(file_of("a.rs", 3001))
3013 .with_node(marker_of("a.rs", 1, "todo")) // roteiro:ignore
3014 .with_node(file_of("b.rs", 3000))
3015 .with_node(marker_of("b.rs", 1, "todo")); // roteiro:ignore
3016 store.apply_factset(&facts).expect("apply");
3017
3018 let report = density(&store, DensityOrder::Density);
3019 assert_eq!(
3020 report.items.iter().map(|i| &i.path).collect::<Vec<_>>(),
3021 ["b.rs", "a.rs"],
3022 "the shorter file is denser, however the figures round"
3023 );
3024 assert_eq!(
3025 (report.items[0].per_kloc, report.items[1].per_kloc),
3026 (0.33, 0.33),
3027 "and the rendered figures really are equal, so the order came from elsewhere"
3028 );
3029 }
3030
3031 #[test]
3032 fn density_reports_truncation_and_is_deterministic() {
3033 let store = marked();
3034 let capped = debt_density(&store, &[], &[], DensityOrder::Density, 1, 0).expect("density");
3035 assert_eq!(capped.items.len(), 1);
3036 assert_eq!(capped.limit, 1);
3037 assert_eq!(
3038 capped.ranked_files, 3,
3039 "the population is reported, so a capped list cannot read as the whole repository"
3040 );
3041 // `overall_per_kloc` is the baseline across every ranked file, not across
3042 // the ones that survived the cap — otherwise the top file's own density
3043 // would be its own baseline.
3044 assert_eq!(capped.total_lines, 4210);
3045 assert!(
3046 (capped.overall_per_kloc - 19.24).abs() < f64::EPSILON,
3047 "81 markers over 4210 lines, was {}",
3048 capped.overall_per_kloc
3049 );
3050
3051 let a = serde_json::to_string(&capped).expect("json");
3052 let b = serde_json::to_string(
3053 &debt_density(&store, &[], &[], DensityOrder::Density, 1, 0).expect("density"),
3054 )
3055 .expect("json");
3056 assert_eq!(a, b, "deterministic serialisation");
3057 }
3058
3059 #[test]
3060 fn density_order_tokens_round_trip() {
3061 for token in DensityOrder::tokens() {
3062 let order = DensityOrder::from_token(token)
3063 .unwrap_or_else(|| panic!("`{token}` is advertised but not accepted"));
3064 assert_eq!(order.as_str(), token);
3065 }
3066 assert!(
3067 DensityOrder::from_token("count").is_none(),
3068 "an unknown order is rejected, not silently defaulted"
3069 );
3070 }
3071
3072 // -- config-secret inventory (S1) --------------------------------------
3073
3074 /// A `config_key` node as `extract::config_facts` emits it: `meta.value`
3075 /// present (already redacted, if the key name called for it).
3076 fn cfgkey(path: &str, dotted: &str, value: &str) -> Node {
3077 let mut node = Node::new(
3078 format!("cfgkey:{path}#{dotted}"),
3079 NodeKind::Other("config_key".to_owned()),
3080 dotted,
3081 );
3082 node.path = Some(path.to_owned());
3083 node.meta = serde_json::json!({ "key": dotted, "value": value });
3084 node
3085 }
3086
3087 /// A **struct-derived** `config_key` node as `synthesize_config_keys` emits
3088 /// it: `meta.value` OMITTED, because a Rust field declares no literal value.
3089 fn struct_cfgkey(path: &str, dotted: &str) -> Node {
3090 let mut node = Node::new(
3091 format!("cfgkey:{path}#{dotted}"),
3092 NodeKind::Other("config_key".to_owned()),
3093 dotted,
3094 );
3095 node.path = Some(path.to_owned());
3096 node.meta = serde_json::json!({
3097 "key": dotted,
3098 "source": "struct",
3099 "struct": "AppConfig",
3100 });
3101 node
3102 }
3103
3104 /// One of each state extraction can produce, plus a non-secret key and a
3105 /// k8s-`Secret`-style redaction under an innocuous name.
3106 fn configured() -> Store {
3107 let mut store = Store::open_in_memory().expect("store");
3108 let facts = FactSet::new()
3109 // Secret-named, redacted by extraction — the expected state.
3110 .with_node(cfgkey(".env", "API_TOKEN", "<redacted>"))
3111 .with_node(cfgkey("config.toml", "db.password", "<redacted>"))
3112 // Secret-named, struct-derived — no value to redact.
3113 .with_node(struct_cfgkey("src/config.rs", "serve.api_key"))
3114 // Not secret-named — not this lens's subject at all.
3115 .with_node(cfgkey("config.toml", "serve.addr", "127.0.0.1:8017"))
3116 // A k8s `Secret`'s data: redacted for where it lives, not what it is
3117 // called, so it is counted but not listed.
3118 .with_node(cfgkey("k8s/secret.yaml", "database-url", "<redacted>"));
3119 store.apply_factset(&facts).expect("apply");
3120 store
3121 }
3122
3123 /// Find an item by dotted name.
3124 fn secret<'a>(report: &'a ConfigSecretReport, name: &str) -> &'a super::ConfigSecretItem {
3125 report
3126 .items
3127 .iter()
3128 .find(|i| i.name == name)
3129 .unwrap_or_else(|| panic!("`{name}` missing from {:?}", report.items))
3130 }
3131
3132 #[test]
3133 fn the_inventory_reports_presence_and_redaction_not_values() {
3134 let report = config_secrets(&configured(), 0).expect("config_secrets");
3135
3136 assert_eq!(report.config_keys, 5, "the population it drew from");
3137 assert_eq!(report.secret_named, 3, "{:?}", report.items);
3138 assert_eq!(report.files, 3);
3139 assert_eq!(report.schema, SCHEMA);
3140
3141 // Paths, key names and state, which is what the lens is for.
3142 assert_eq!(secret(&report, "API_TOKEN").path.as_deref(), Some(".env"));
3143 assert_eq!(
3144 secret(&report, "db.password").key,
3145 "cfgkey:config.toml#db.password"
3146 );
3147 // The state comes from comparing the stored value against the redactor's
3148 // own constant, so asserting it is what keeps reader and writer from
3149 // drifting apart on a spelling.
3150 assert_eq!(
3151 secret(&report, "API_TOKEN").state,
3152 RedactionState::Redacted,
3153 "the placeholder extraction wrote is recognised as a redaction"
3154 );
3155 assert_eq!(report.redacted, 2, "{report:?}");
3156
3157 // No value is carried on any item — there is no field for one. The
3158 // serialised shape is the contract, so assert against that, not the type.
3159 let json = serde_json::to_value(&report).expect("json");
3160 let text = serde_json::to_string(&report).expect("json");
3161 assert!(
3162 json["items"][0].get("value").is_none(),
3163 "an item carries no value field: {text}"
3164 );
3165 assert!(
3166 !text.contains("<redacted>"),
3167 "not even the placeholder is echoed back: {text}"
3168 );
3169
3170 // Ordering is `(path, name, key)` — an inventory, not a ranking.
3171 assert_eq!(
3172 report.items.iter().map(|i| &i.name).collect::<Vec<_>>(),
3173 ["API_TOKEN", "db.password", "serve.api_key"]
3174 );
3175 }
3176
3177 #[test]
3178 fn a_struct_declared_key_is_neither_redacted_nor_a_leak() {
3179 // A `@rto:config` struct field has no literal value in code, so extraction
3180 // omits `meta.value` entirely. Folding that in with a successful redaction
3181 // would claim a redaction that never happened; calling it unredacted would
3182 // report a leak that does not exist.
3183 let report = config_secrets(&configured(), 0).expect("config_secrets");
3184 let declared = secret(&report, "serve.api_key");
3185 assert_eq!(declared.state, RedactionState::Declared);
3186 assert_eq!(declared.source.as_deref(), Some("struct"));
3187
3188 assert_eq!(report.redacted, 2, "the two file-derived keys");
3189 assert_eq!(report.declared, 1);
3190 assert_eq!(
3191 report.unredacted, 0,
3192 "the invariant extraction maintains: {report:?}"
3193 );
3194 }
3195
3196 #[test]
3197 fn an_unredacted_secret_named_value_is_reported_as_a_finding() {
3198 // Extraction redacts every secret-named key, so this state is unreachable
3199 // from extraction — but `apply_import_layer` upserts whatever nodes an
3200 // imported factset carries, so another tool's import can put an unredacted
3201 // value in the store. That is the one path worth reporting, and it is a
3202 // finding about THIS STORE, not about the source repository.
3203 let mut store = configured();
3204 store
3205 .apply_import_layer(
3206 "other-tool",
3207 &FactSet::new().with_node(cfgkey("imported.env", "AWS_SECRET", "AKIAnot-redacted")),
3208 )
3209 .expect("import");
3210
3211 let report = config_secrets(&store, 0).expect("config_secrets");
3212 assert_eq!(report.unredacted, 1, "{report:?}");
3213 assert_eq!(secret(&report, "AWS_SECRET").state, RedactionState::Present);
3214 // And still no value in the report: the lens says *that* something is
3215 // unredacted, and never repeats it.
3216 let text = serde_json::to_string(&report).expect("json");
3217 assert!(
3218 !text.contains("AKIA"),
3219 "the value is not echoed back: {text}"
3220 );
3221 }
3222
3223 #[test]
3224 fn a_redaction_under_an_innocuous_name_is_counted_but_not_listed() {
3225 // A k8s `Secret`'s `data` is redacted because of where it lives, whatever
3226 // the key is called. It is not secret-*named*, so it is not this lens's
3227 // subject — but it is counted, so a reader comparing `redacted` against the
3228 // number of `<redacted>` values in the graph does not find a surplus they
3229 // cannot explain.
3230 let report = config_secrets(&configured(), 0).expect("config_secrets");
3231 assert_eq!(report.redacted_not_secret_named, 1);
3232 assert!(
3233 !report.items.iter().any(|i| i.name == "database-url"),
3234 "not listed: {:?}",
3235 report.items
3236 );
3237 assert_eq!(
3238 report.redacted + report.redacted_not_secret_named,
3239 3,
3240 "and the two figures together account for every redacted value"
3241 );
3242 }
3243
3244 #[test]
3245 fn the_inventory_cannot_see_a_credential_that_is_not_a_config_key() {
3246 // The load-bearing limitation, asserted rather than only documented: a
3247 // credential in a Rust string literal produces no `config_key` node, so it
3248 // is invisible here. No extension of this lens can change that — which is
3249 // why it is named for the inventory it is, not the scanner it is not.
3250 let mut store = configured();
3251 let mut hardcoded = Node::new("sym:rust:src/main.rs#connect", NodeKind::Fn, "connect");
3252 hardcoded.path = Some("src/main.rs".into());
3253 // Split at the prefix for the same reason as `FAKE_TOKEN` in
3254 // `roteiro/tests/config_secrets_cli.rs`: assembled, this is AWS's own
3255 // documentation placeholder, but it matches the canonical access-key-id
3256 // rule exactly and a regex-rule scanner cannot know the difference. The
3257 // assembled value is unchanged; no assertion here matches on its text.
3258 hardcoded.meta = serde_json::json!({
3259 "content": concat!("let token = \"AKIA", "IOSFODNN7EXAMPLE\";"),
3260 });
3261 store
3262 .apply_factset(&FactSet::new().with_node(hardcoded))
3263 .expect("apply");
3264
3265 let report = config_secrets(&store, 0).expect("config_secrets");
3266 assert_eq!(
3267 report.secret_named, 3,
3268 "a hardcoded credential does not appear: {:?}",
3269 report.items
3270 );
3271 assert_eq!(report.config_keys, 5, "and is not a config key at all");
3272 }
3273
3274 #[test]
3275 fn the_inventory_reports_truncation_and_is_deterministic() {
3276 let store = configured();
3277 let capped = config_secrets(&store, 1).expect("config_secrets");
3278 assert_eq!(capped.items.len(), 1);
3279 assert_eq!(capped.limit, 1);
3280 assert_eq!(
3281 capped.secret_named, 3,
3282 "the population is reported, so a capped list cannot read as a clean repository"
3283 );
3284 // The state counts are over the whole population too, not the shown rows —
3285 // otherwise a cap could hide an `unredacted` finding.
3286 assert_eq!((capped.redacted, capped.declared), (2, 1));
3287
3288 let a = serde_json::to_string(&capped).expect("json");
3289 let b = serde_json::to_string(&config_secrets(&store, 1).expect("config_secrets"))
3290 .expect("json");
3291 assert_eq!(a, b, "deterministic serialisation");
3292 }
3293
3294 #[test]
3295 fn an_empty_report_means_no_secret_named_key_not_no_secret() {
3296 // The distinction the lens must never blur: a credential under an
3297 // innocuous key name (`dsn`) is not secret-named, is not redacted, and does
3298 // not appear. So "nothing found" is a statement about naming.
3299 let mut store = Store::open_in_memory().expect("store");
3300 store
3301 .apply_factset(&FactSet::new().with_node(cfgkey(
3302 ".env",
3303 "DSN",
3304 "postgres://u:pw@host/db",
3305 )))
3306 .expect("apply");
3307
3308 let report = config_secrets(&store, 0).expect("config_secrets");
3309 assert_eq!(report.secret_named, 0, "nothing is secret-*named*");
3310 assert_eq!(report.redacted_not_secret_named, 0);
3311 assert_eq!(
3312 report.config_keys, 1,
3313 "while the graph does hold a config key with a credential in it"
3314 );
3315 }
3316
3317 #[test]
3318 fn redaction_state_tokens_match_their_serialisation() {
3319 // The token and the wire form are the same string, so a caller matching on
3320 // the JSON and a caller matching on `as_str` cannot disagree.
3321 for state in [
3322 RedactionState::Redacted,
3323 RedactionState::Declared,
3324 RedactionState::Present,
3325 ] {
3326 let json = serde_json::to_string(&state).expect("json");
3327 assert_eq!(json, format!("\"{}\"", state.as_str()));
3328 }
3329 }
3330}