rto_graph/query.rs
1//! The agent- and human-facing query surface over the graph.
2//!
3//! Everything here is a read-only view built from the store's typed queries,
4//! serialised under a **stable, versioned** JSON schema ([`SCHEMA`]) so agents
5//! can depend on the shape. The primitives are [`explain`] (a node and its
6//! provenance-labelled neighbourhood), [`list_kind`] (all nodes of a kind),
7//! [`path`] (a shortest path between two nodes), [`debt`] (the intent-debt marker
8//! inventory), [`debt_density`] (that inventory per file, normalised by file
9//! length), [`coupling`] (directed fan-in/fan-out over `Calls` edges),
10//! [`config_secrets`] (secret-named config keys and their redaction state), and
11//! [`search`] (relevance-ranked node search). All return
12//! mixed-provenance results — the "one query surface" from ADR-0001 — with every
13//! edge carrying its `provenance`.
14
15use crate::paths::glob_match;
16use std::collections::{BTreeMap, BTreeSet, VecDeque};
17
18use serde::Serialize;
19
20use crate::store::{Store, StoreError};
21use crate::{Edge, EdgeKind, NodeKind, Provenance};
22
23/// The versioned schema tag emitted on every query result. Bump the version on
24/// any breaking change to the shape.
25pub const SCHEMA: &str = "roteiro.query/v1";
26
27/// Cut an already-ordered, already-materialised list down to the window a caller
28/// asked for: skip `offset` items from the front, then keep at most `limit`.
29///
30/// **This is the one place that decides what `limit` and `offset` mean** for the
31/// graph's list lenses — [`debt_density`], [`config_secrets`] and [`coupling`]
32/// here, and the `/nodes` and `/hotspots` endpoints in the `roteiro` binary. It
33/// exists because the parameter previously had two implementations: three lenses
34/// truncated here and treated `0` as "no limit", while two HTTP handlers used
35/// [`Iterator::take`] and so returned nothing for `0` — the same parameter name
36/// with opposite meanings, and nothing that could make the disagreement visible
37/// (issue #375). A sixth list lens should call this rather than write a third.
38/// One did write a third — see *Episodic recall* below — and the warning is left
39/// standing because the next one will too.
40///
41/// The contract:
42///
43/// - **`limit == 0` means unlimited** — every item that survives `offset` is
44/// kept. This is the reading the CLI already documents (`roteiro
45/// config-secrets --help`: *"0 shows every secret-named key"*), so no
46/// published promise is withdrawn by making it universal; and it is the safer
47/// of the two, because a caller who passes an unset variable then gets more
48/// data than they meant to ask for rather than an empty page that reads like a
49/// truthful "nothing found".
50/// - **`offset` applies first, and `limit` to what remains.** So `offset = 20,
51/// limit = 0` is "skip the first 20, then every remaining item", not "skip 20,
52/// then nothing". An `offset` past the end yields an empty window rather than
53/// panicking — a page beyond the last one is empty, not an error.
54/// - A caller's reported `total` must be taken **before** this runs: every
55/// surface reports the pre-windowing population, so a cut page still says what
56/// it was cut from.
57///
58/// Ordering is the caller's job; this only removes, and only from the ends, so a
59/// deterministically-ordered input yields a deterministic window.
60///
61/// # The search channels call this too, and the unit there is *per channel*
62///
63/// [`search`] and the generated/memory channels behind [`search_channels`] used
64/// to keep a `limit == 0 => no hits` guard of their own — a third reading of one
65/// parameter name, in the same crate as the two #375 reconciled (issue #393).
66/// They now window here like every other lens, so `limit` has one definition and
67/// not a second implementation of it, which is exactly how the first two drifted.
68///
69/// What differs is the **unit**, not the rule: a search `limit` bounds *each
70/// channel* independently, so `0` is "every match, in every channel that was
71/// asked for", not "every match overall". That is a bounded request rather than
72/// "dump the graph": a channel's ranking only *orders* a set the query has
73/// already filtered — every token must appear in a hit — and a query with no
74/// tokens returns nothing at any limit, `0` included. Measured on this
75/// repository at 6,685 nodes: an unbounded one-token search returned ~2.7k hits
76/// in 0.24 s and a two-token one returned 3, against a full-population scan that
77/// every limit pays anyway, so unlimited costs no more than the default does.
78///
79/// The **MCP tools are the one surface that cannot ask for it**, deliberately:
80/// they clamp `limit` into `1..=25` and advertise `"minimum": 1`, because their
81/// results are spent against a model's context window and `0` would be the one
82/// value that escaped the ceiling those clamps exist to impose. That is a
83/// surface declining to offer a value, not a second meaning for it — a model
84/// that sends `0` anyway gets the smallest page, never the silent empty answer
85/// this issue is about. The reasoning is restated where each clamp lives, in
86/// `rto_render::mcp::GraphServer::search` and the served-chat `search` arm in
87/// the `roteiro` binary; if this rule changes, those two must change with it.
88///
89/// # Episodic recall — the third implementation this doc predicted (issue #447)
90///
91/// [`crate::Store::recall_memory`] ranked, then called
92/// [`Vec::truncate`] directly, so `limit = 0` emptied the result: `recall
93/// --limit 0` returned nothing on a store where five other surfaces returned
94/// everything, and the JSON said `"live": 8000` beside `"results": []` — not a
95/// lie, and no help at all. It calls this now. Two details are worth keeping:
96///
97/// - **`None` and `Some(0)` had to collapse onto one meaning, not two.**
98/// `RecallOptions::limit` is an `Option`, so "unlimited" was already sayable
99/// twice; the fix maps `None` to `0` rather than adding a branch, because two
100/// spellings of one request are how the first divergence started.
101/// - **`memory list` cuts in SQL and so cannot call this.** `LIMIT 0` in SQL
102/// means the *opposite* of what this function means, so `memory::records`
103/// omits the clause entirely for `0`. That is the contract translated, and it
104/// is the only place in the crate where the rule is re-expressed rather than
105/// called — worth knowing if it ever has to change.
106pub fn window<T>(items: &mut Vec<T>, offset: usize, limit: usize) {
107 // `min(len)` rather than a bounds check: `drain(..offset)` panics past the
108 // end of the vector, and "page 900 of 3" is an empty page, not a 500.
109 items.drain(..offset.min(items.len()));
110 if limit > 0 {
111 items.truncate(limit);
112 }
113}
114
115/// A compact node summary (used in listings and as the subject of an
116/// [`Explanation`]).
117#[derive(Debug, Clone, PartialEq, Serialize)]
118pub struct NodeSummary {
119 /// Natural key.
120 pub key: String,
121 /// Kind token (e.g. `fn`, `adr`).
122 pub kind: String,
123 /// Human-facing name.
124 pub name: String,
125 /// Repository-relative path, if any.
126 pub path: Option<String>,
127 /// Language token, if any.
128 pub lang: Option<String>,
129}
130
131impl NodeSummary {
132 fn from_node(node: &crate::Node) -> Self {
133 Self {
134 key: node.key.clone(),
135 kind: node.kind.as_str().to_owned(),
136 name: node.name.clone(),
137 path: node.path.clone(),
138 lang: node.lang.clone(),
139 }
140 }
141}
142
143/// One end of an edge as seen from a subject node: the relationship, how it was
144/// produced, and the node on the other end.
145#[derive(Debug, Clone, PartialEq, Serialize)]
146pub struct EdgeRef {
147 /// Edge kind token (e.g. `calls`, `references`).
148 pub kind: String,
149 /// How the edge was produced (`derived` | `authored` | `inferred`).
150 pub provenance: &'static str,
151 /// Confidence score, present only for inferred edges.
152 pub confidence: Option<f64>,
153 /// The natural key of the node at the other end.
154 pub node: String,
155}
156
157/// A node together with its provenance-labelled neighbourhood.
158#[derive(Debug, Clone, PartialEq, Serialize)]
159pub struct Explanation {
160 /// Stable schema tag ([`SCHEMA`]).
161 pub schema: &'static str,
162 /// The subject node.
163 pub node: NodeSummary,
164 /// Structured metadata attached to the node.
165 pub meta: serde_json::Value,
166 /// Edges where the subject is the source.
167 pub outgoing: Vec<EdgeRef>,
168 /// Edges where the subject is the destination.
169 pub incoming: Vec<EdgeRef>,
170}
171
172/// A listing of all nodes of one kind.
173#[derive(Debug, Clone, PartialEq, Serialize)]
174pub struct Listing {
175 /// Stable schema tag ([`SCHEMA`]).
176 pub schema: &'static str,
177 /// The kind that was listed.
178 pub kind: String,
179 /// Matching nodes, ordered by key.
180 pub nodes: Vec<NodeSummary>,
181}
182
183/// One intent-debt finding in a [`DebtReport`].
184#[derive(Debug, Clone, PartialEq, Serialize)]
185pub struct DebtItem {
186 /// Natural key of the marker node (`marker:<path>#<line>`).
187 pub key: String,
188 /// Category token (`todo` | `fixme` | `hack` | `stub` | `deferred`).
189 pub category: String,
190 /// The marker text (the trimmed source line).
191 pub text: String,
192 /// Repository-relative path of the source file, if any.
193 pub path: Option<String>,
194 /// 1-based line number, if recorded.
195 pub line: Option<u32>,
196}
197
198/// The intent-debt inventory: every `marker` node, grouped and listed. A
199/// deterministic, provenance-`derived` view of what is incomplete or postponed.
200#[derive(Debug, Clone, PartialEq, Serialize)]
201pub struct DebtReport {
202 /// Stable schema tag ([`SCHEMA`]).
203 pub schema: &'static str,
204 /// Total markers in the report (after any category filter).
205 pub total: usize,
206 /// Count per category, ordered by category token.
207 pub by_category: BTreeMap<String, usize>,
208 /// The markers, ordered by `(path, line, key)`.
209 pub items: Vec<DebtItem>,
210}
211
212/// Inventory intent-debt markers in the graph, optionally restricted to the
213/// given `categories` (empty means all) and excluding markers whose file path
214/// matches any `ignore` glob (config `[debt] ignore` — empty means keep all).
215/// Ordered by `(path, line)` so output is stable and reads top-to-bottom per
216/// file; `total` and `by_category` reflect the retained markers only.
217///
218/// # Errors
219/// Returns [`StoreError`] on query failure.
220pub fn debt(
221 store: &Store,
222 categories: &[String],
223 ignore: &[String],
224) -> Result<DebtReport, StoreError> {
225 let filter: std::collections::BTreeSet<&str> = categories.iter().map(String::as_str).collect();
226 let mut items = Vec::new();
227 let mut by_category: BTreeMap<String, usize> = BTreeMap::new();
228 for node in store.nodes_by_kind(&NodeKind::Marker)? {
229 let category = node
230 .meta
231 .get("category")
232 .and_then(serde_json::Value::as_str)
233 .unwrap_or("other")
234 .to_owned();
235 if !filter.is_empty() && !filter.contains(category.as_str()) {
236 continue;
237 }
238 // Drop markers under an ignored path (e.g. `vendor/**`) before counting.
239 if let Some(path) = node.path.as_deref()
240 && ignore.iter().any(|glob| glob_match(glob, path))
241 {
242 continue;
243 }
244 let text = node
245 .meta
246 .get("text")
247 .and_then(serde_json::Value::as_str)
248 .unwrap_or(node.name.as_str())
249 .to_owned();
250 let line = node
251 .meta
252 .get("line")
253 .and_then(serde_json::Value::as_u64)
254 .and_then(|l| u32::try_from(l).ok());
255 *by_category.entry(category.clone()).or_default() += 1;
256 items.push(DebtItem {
257 key: node.key.clone(),
258 category,
259 text,
260 path: node.path.clone(),
261 line,
262 });
263 }
264 items.sort_by(|a, b| (&a.path, a.line, &a.key).cmp(&(&b.path, b.line, &b.key)));
265 Ok(DebtReport {
266 schema: SCHEMA,
267 total: items.len(),
268 by_category,
269 items,
270 })
271}
272
273/// How a [`DebtDensityReport`]'s files are ranked.
274#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
275pub enum DensityOrder {
276 /// By `per_kloc` — markers relative to file length. The lens's own question.
277 #[default]
278 Density,
279 /// By `markers` — the raw count, which [`debt`] already reports per marker.
280 /// Offered so the two rankings can be compared on one report rather than the
281 /// reader being asked to trust that they differ.
282 Markers,
283 /// By `lines` — longest file first. Not a debt ranking; the control that
284 /// shows *which* files the denominator is large for.
285 Lines,
286}
287
288impl DensityOrder {
289 /// The stable token for this order, as accepted by [`from_token`](Self::from_token).
290 #[must_use]
291 pub fn as_str(self) -> &'static str {
292 match self {
293 Self::Density => "density",
294 Self::Markers => "markers",
295 Self::Lines => "lines",
296 }
297 }
298
299 /// Parse an order token. `None` for anything else — callers surface an error
300 /// rather than silently ranking by something the caller did not ask for.
301 #[must_use]
302 pub fn from_token(s: &str) -> Option<Self> {
303 match s {
304 "density" => Some(Self::Density),
305 "markers" => Some(Self::Markers),
306 "lines" => Some(Self::Lines),
307 _ => None,
308 }
309 }
310
311 /// The tokens [`from_token`](Self::from_token) accepts, for error messages
312 /// and argument schemas — so the accepted set is stated in exactly one place.
313 #[must_use]
314 pub fn tokens() -> [&'static str; 3] {
315 [
316 Self::Density.as_str(),
317 Self::Markers.as_str(),
318 Self::Lines.as_str(),
319 ]
320 }
321}
322
323/// The default `min_lines` floor for [`debt_density`]: files shorter than this
324/// are counted but not ranked (see [`DebtDensityReport::min_lines`] for why the
325/// floor exists at all).
326///
327/// 50 because that is where one marker stops dominating: a single marker in a
328/// 50-line file scores 20 per kloc, which is already near the top of this
329/// repository's real ranking, so any shorter file with a marker is guaranteed a
330/// high placement by its length alone. It is a default, not a rule — `0` ranks
331/// every file.
332pub const DEFAULT_MIN_LINES: u32 = 50;
333
334/// One file's intent-debt density in a [`DebtDensityReport`].
335#[derive(Debug, Clone, PartialEq, Serialize)]
336pub struct DensityItem {
337 /// Repository-relative path of the file.
338 pub path: String,
339 /// Retained markers in this file (after category and `ignore` filtering).
340 pub markers: u32,
341 /// The file's length in lines — the denominator. See [`debt_density`] for
342 /// exactly what this counts and what it does not.
343 pub lines: u32,
344 /// Markers per 1,000 lines, rounded to two decimals. Per *kilo*-line rather
345 /// than per line because per-line densities are all leading zeroes: this
346 /// repository's worst file is 0.06 markers per line, and `60.0` per kloc is
347 /// a number a reader can hold.
348 pub per_kloc: f64,
349 /// Count per category within this file, ordered by category token — so a
350 /// dense file can be read as "twelve `todo`" or "twelve `stub`", which are
351 /// not the same finding.
352 pub by_category: BTreeMap<String, usize>,
353}
354
355/// Intent-debt **density**: markers per file normalised by file length, ranked.
356/// The counterpart to [`debt`], which reports markers and therefore ranks large
357/// files first by construction.
358#[derive(Debug, Clone, PartialEq, Serialize)]
359pub struct DebtDensityReport {
360 /// Stable schema tag ([`SCHEMA`]).
361 pub schema: &'static str,
362 /// The ranking that produced `items` ([`DensityOrder::as_str`]).
363 pub order: &'static str,
364 /// The requested cap on `items`; `0` means unlimited.
365 pub limit: usize,
366 /// The `min_lines` floor applied. Files shorter than this are excluded from
367 /// the ranking and counted in `short_files`; `0` disables the floor.
368 ///
369 /// The floor exists because density is unstable in the denominator's tail: a
370 /// 3-line stub file with one marker scores 333 per kloc, which is true and
371 /// tells the reader nothing. Excluding those files is a ranking decision,
372 /// not a suppression — they stay in `files_with_markers` and `total_markers`.
373 pub min_lines: u32,
374 /// Distinct files carrying at least one retained marker, before the
375 /// `min_lines` floor. The population `items` is drawn from.
376 pub files_with_markers: usize,
377 /// Files that passed the `min_lines` floor and were therefore ranked.
378 /// `ranked_files > items.len()` means `limit` truncated the list.
379 pub ranked_files: usize,
380 /// Files excluded from the ranking by the `min_lines` floor — reported, not
381 /// silently dropped, so a short-file-heavy repository cannot read as a clean one.
382 pub short_files: usize,
383 /// Files whose marker count is known but whose length is **not**: no `file`
384 /// node, or one carrying no `meta.lines`. Excluded from the ranking, because
385 /// a density with no denominator is not a number — and reported, because
386 /// silently omitting them would understate the inventory.
387 pub unknown_length_files: usize,
388 /// Retained markers across every file in `files_with_markers`, including
389 /// those the floor excluded. Matches [`DebtReport::total`] for the same
390 /// filters, minus any marker with no `path`.
391 pub total_markers: usize,
392 /// Summed `lines` of the ranked files. The denominator behind
393 /// `overall_per_kloc`.
394 pub total_lines: u64,
395 /// Markers per 1,000 lines across the **ranked** files taken together — the
396 /// baseline a single file's `per_kloc` should be read against. `0.0` when
397 /// nothing was ranked.
398 pub overall_per_kloc: f64,
399 /// The ranked files: by `order` descending, ties broken by `path` ascending.
400 pub items: Vec<DensityItem>,
401}
402
403/// Rank files by intent-debt **density** — retained markers per 1,000 lines —
404/// most-dense first by `order`, capped at `limit` (`0` = unlimited). `categories`
405/// and `ignore` filter markers exactly as [`debt`] does, so the two lenses always
406/// agree about which markers exist.
407///
408/// # Why this is not [`debt`] with a division
409///
410/// A raw marker count ranks by file size: the biggest file wins because it has
411/// the most lines to put a marker on. Density asks the different question — *how
412/// concentrated is the debt* — and a 40-marker file of 4,000 lines and a
413/// 40-marker file of 200 lines separate by a factor of twenty under it while
414/// being indistinguishable under [`debt`].
415///
416/// # The denominator, and why it is this one
417///
418/// **`lines` is the `file` node's `meta.lines`**, recorded at extraction time as
419/// the count of `\n` bytes in the blob. It is read straight from the graph, so
420/// this lens adds **no extraction metadata and needs no `EXTRACT_VERSION` bump**.
421///
422/// It is deliberately *not* derived from [`crate::Span`]: a node's span is a pair
423/// of **byte offsets**, not line numbers, and there is no line index in the store
424/// to convert one to the other. Anything span-derived would be a byte density,
425/// which is not the quantity anyone means by "debt density".
426///
427/// Three alternatives were rejected, each for the same reason:
428///
429/// - **Source lines of code** (blank and comment lines removed) is the denominator
430/// a reader probably imagines. It does not exist in the graph and cannot be
431/// computed from it: producing it means counting lines per language at
432/// extraction, which is net-new derived metadata and would move this lens into
433/// the batch that pays for an `EXTRACT_VERSION` bump.
434/// - **Per symbol** rather than per file would be the finer-grained view — markers
435/// already attach to their innermost enclosing symbol. But a symbol's length in
436/// *lines* is exactly what `Span`'s byte offsets cannot give.
437/// - **The highest marker line in the file** is available (`meta.line`), and is a
438/// lower bound on the file's length rather than the length: a file whose only
439/// marker is on line 3 would score 333 per kloc however long it is.
440///
441/// So `lines` is what the graph honestly has. What it counts, stated plainly
442/// because the name invites over-reading:
443///
444/// - **Every line, including blanks, comments, imports and licence headers.** It
445/// is *file length*, not "lines of code". Density figures here are therefore
446/// systematically lower than an SLOC-based tool's, and by a different factor
447/// per language and per file.
448/// - **Newline bytes.** A file not ending in a newline is counted one line short,
449/// and a file of a single unterminated line counts as `0` lines and is reported
450/// under `unknown_length_files` rather than divided by zero.
451/// - **The whole blob, vendored code included.** A minified bundle is one enormous
452/// line and will look flawless. Use `ignore` (the shared `[debt] ignore` globs)
453/// rather than a second exclusion vocabulary.
454///
455/// # Confidence, and why there is no CI gate
456///
457/// Density inherits every false positive of the marker scan beneath it — the
458/// prose rules (`for now`, `deferred`, `tbd`) fire on ordinary writing, so a
459/// design document rich in the word "deferred" ranks as dense debt. It then adds
460/// one of its own: the denominator is file length, so a language or a file with
461/// low information per line (verbose config, generated code, wide indentation) is
462/// systematically flattered, and a dense language is systematically penalised.
463/// Neither is a defect being reported; both move the number. A gate would fail
464/// builds on prose and on formatting. So this lens **offers no CI gate**, and its
465/// suppression story is the one that already exists: `[debt] ignore` globs and
466/// the `roteiro:ignore` / `roteiro:ignore-file` source directives, both applied
467/// before anything is counted here.
468///
469/// Ordering is total and deterministic: by the chosen metric descending, then by
470/// `path` ascending, so identical input yields byte-identical output.
471///
472/// # Errors
473/// Returns [`StoreError`] on query failure.
474pub fn debt_density(
475 store: &Store,
476 categories: &[String],
477 ignore: &[String],
478 order: DensityOrder,
479 limit: usize,
480 min_lines: u32,
481) -> Result<DebtDensityReport, StoreError> {
482 // Reuse `debt` rather than re-walking the markers: the two lenses must never
483 // disagree about which markers exist, and the only way to guarantee that is
484 // for one to be built from the other's output.
485 let inventory = debt(store, categories, ignore)?;
486 let mut per_file: BTreeMap<String, BTreeMap<String, usize>> = BTreeMap::new();
487 let mut total_markers = 0usize;
488 for item in &inventory.items {
489 // A marker with no `path` cannot be attributed to a file, so it cannot
490 // have a density. Extraction always records one; this is defence in
491 // depth, and such a marker is left out of `total_markers` too so the
492 // report's own arithmetic stays consistent.
493 let Some(path) = item.path.as_deref() else {
494 continue;
495 };
496 *per_file
497 .entry(path.to_owned())
498 .or_default()
499 .entry(item.category.clone())
500 .or_default() += 1;
501 total_markers += 1;
502 }
503 let files_with_markers = per_file.len();
504
505 // Only files that actually carry a marker are read back, so the denominator
506 // costs one node lookup per such file rather than a whole-graph file scan —
507 // which matters because `file` nodes carry captured `meta.content`.
508 let mut ranked: Vec<(String, u32, u32, BTreeMap<String, usize>)> = Vec::new();
509 let mut short_files = 0usize;
510 let mut unknown_length_files = 0usize;
511 for (path, by_category) in per_file {
512 let markers = u32::try_from(by_category.values().sum::<usize>()).unwrap_or(u32::MAX);
513 let Some(lines) = file_lines(store, &path)? else {
514 unknown_length_files += 1;
515 continue;
516 };
517 if lines < min_lines {
518 short_files += 1;
519 continue;
520 }
521 ranked.push((path, markers, lines, by_category));
522 }
523 let ranked_files = ranked.len();
524 let total_lines: u64 = ranked
525 .iter()
526 .map(|(_, _, lines, _)| u64::from(*lines))
527 .sum();
528 let ranked_markers: u64 = ranked.iter().map(|(_, m, _, _)| u64::from(*m)).sum();
529
530 ranked.sort_by(|a, b| {
531 // Each order yields the metric as an exact `numerator / denominator`, so
532 // the comparison can cross-multiply. Ranking on the rounded `per_kloc`
533 // instead would make genuinely different densities tie and then break on
534 // `path`, silently reordering the ranking the caller asked for; ranking
535 // on `f64` at full precision would make the order depend on the last bit
536 // of a division. `u128` so the cross-product cannot overflow whatever the
537 // file lengths are, rather than relying on repositories staying small.
538 let metric =
539 |&(_, markers, lines, _): &(String, u32, u32, BTreeMap<String, usize>)| match order {
540 DensityOrder::Density => (u128::from(markers) * 1000, u128::from(lines)),
541 DensityOrder::Markers => (u128::from(markers), 1),
542 DensityOrder::Lines => (u128::from(lines), 1),
543 };
544 let (an, ad) = metric(a);
545 let (bn, bd) = metric(b);
546 (bn * ad).cmp(&(an * bd)).then_with(|| a.0.cmp(&b.0))
547 });
548 window(&mut ranked, 0, limit);
549
550 let items = ranked
551 .into_iter()
552 .map(|(path, markers, lines, by_category)| DensityItem {
553 path,
554 markers,
555 lines,
556 per_kloc: per_kloc(u64::from(markers), u64::from(lines)),
557 by_category,
558 })
559 .collect();
560
561 Ok(DebtDensityReport {
562 schema: SCHEMA,
563 order: order.as_str(),
564 limit,
565 min_lines,
566 files_with_markers,
567 ranked_files,
568 short_files,
569 unknown_length_files,
570 total_markers,
571 total_lines,
572 overall_per_kloc: per_kloc(ranked_markers, total_lines),
573 items,
574 })
575}
576
577/// A file's length in lines from its `file` node's `meta.lines`, or `None` when
578/// the node is absent, carries no `lines`, or reports **zero** lines.
579///
580/// Zero is folded into `None` deliberately: it is not a length that can be
581/// divided by, and the two cases a reader would want distinguished — a genuinely
582/// empty file and a single line with no terminating newline — are
583/// indistinguishable in a newline count. Reporting both as "length unknown" is
584/// the honest reading; reporting either as a density is not.
585fn file_lines(store: &Store, path: &str) -> Result<Option<u32>, StoreError> {
586 let Some(node) = store.get_node(&format!("file:{path}"))? else {
587 return Ok(None);
588 };
589 Ok(node
590 .meta
591 .get("lines")
592 .and_then(serde_json::Value::as_u64)
593 .and_then(|n| u32::try_from(n).ok())
594 .filter(|&n| n > 0))
595}
596
597/// Markers per 1,000 lines, rounded to two decimals; `0.0` for a zero
598/// denominator, which callers have already excluded from any ranking.
599fn per_kloc(markers: u64, lines: u64) -> f64 {
600 if lines == 0 {
601 return 0.0;
602 }
603 // `u64 as f64` is lossy above 2^53; a line count or marker count that large
604 // is not reachable from a repository on disk, and the precision lost would be
605 // below the two decimals this rounds to anyway.
606 #[expect(clippy::cast_precision_loss, reason = "counts are far below 2^53")]
607 let ratio = (markers as f64) * 1000.0 / (lines as f64);
608 round2(ratio)
609}
610
611/// The redaction state of one config key in a [`ConfigSecretReport`]. Three
612/// states, because collapsing them would misreport two of them: "declared in
613/// code" is not a redaction, and "value present" is not a safe one.
614#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
615#[serde(rename_all = "snake_case")]
616pub enum RedactionState {
617 /// The value was read from a source file and **replaced** with the redaction
618 /// placeholder before anything was persisted. The expected state for a
619 /// secret-named key extracted from a config file.
620 Redacted,
621 /// The key carries **no value at all** — a struct-derived key
622 /// (`meta.source = "struct"`), a config *field* declared in Rust with no
623 /// literal in the code to redact.
624 ///
625 /// Not a redaction and not a leak: it records that a setting by this name
626 /// exists, which is the inventory's job, and nothing about any value.
627 Declared,
628 /// The key carries a value that is **not** the redaction placeholder.
629 ///
630 /// Extraction redacts every secret-named key, so this state is unreachable
631 /// from extraction alone. It is reachable through
632 /// [`Store::apply_import_layer`](crate::Store::apply_import_layer), which
633 /// upserts whatever nodes an imported factset carries — so an import produced
634 /// by another tool, or by an older Roteiro, can put an unredacted value in the
635 /// store. That is worth reporting loudly, and it is a finding about **this
636 /// store**, not about the source repository.
637 Present,
638}
639
640impl RedactionState {
641 /// The stable token for this state, as serialised.
642 #[must_use]
643 pub fn as_str(self) -> &'static str {
644 match self {
645 Self::Redacted => "redacted",
646 Self::Declared => "declared",
647 Self::Present => "present",
648 }
649 }
650}
651
652/// One secret-named config key in a [`ConfigSecretReport`].
653#[derive(Debug, Clone, PartialEq, Serialize)]
654pub struct ConfigSecretItem {
655 /// Natural key of the config-key node (`cfgkey:<path>#<dotted>`).
656 pub key: String,
657 /// Repository-relative path of the config file or Rust source it came from.
658 pub path: Option<String>,
659 /// The dotted key name (e.g. `serve.api_token`). **Names only** — no value is
660 /// carried here, by construction as much as by choice: the value in the store
661 /// is the redaction placeholder.
662 pub name: String,
663 /// Whether the value was redacted, absent, or present.
664 pub state: RedactionState,
665 /// `meta.source`, when the node records one — `struct` for a key synthesised
666 /// from a `@rto:config` Rust struct. Absent for a file-derived key.
667 pub source: Option<String>,
668}
669
670/// An inventory of **secret-named** config keys and their redaction state.
671///
672/// See [`config_secrets`] for what this is and — more importantly — what it is
673/// not.
674#[derive(Debug, Clone, PartialEq, Serialize)]
675pub struct ConfigSecretReport {
676 /// Stable schema tag ([`SCHEMA`]).
677 pub schema: &'static str,
678 /// The requested cap on `items`; `0` means unlimited.
679 pub limit: usize,
680 /// Every `config_key` node in the graph, secret-named or not — the population
681 /// the inventory was drawn from.
682 pub config_keys: usize,
683 /// Config keys whose **name** matched the secret-name heuristic. `items` is
684 /// the first `limit` of these, so `secret_named > items.len()` means truncation.
685 pub secret_named: usize,
686 /// Of `secret_named`: how many carry the redaction placeholder.
687 pub redacted: usize,
688 /// Of `secret_named`: how many carry no value at all (struct-derived).
689 pub declared: usize,
690 /// Of `secret_named`: how many carry a value that is **not** the placeholder.
691 ///
692 /// **Expected to be zero.** A non-zero count is a finding about this store —
693 /// see [`RedactionState::Present`] for the one path that reaches it.
694 pub unredacted: usize,
695 /// Config keys that are redacted but whose name is **not** secret-looking: a
696 /// Kubernetes `Secret`'s `data`, redacted because of where it lives rather
697 /// than what it is called.
698 ///
699 /// Reported so the redaction counts reconcile against the graph: without it a
700 /// reader comparing `redacted` to the number of `<redacted>` values in the
701 /// store would find an unexplained surplus.
702 pub redacted_not_secret_named: usize,
703 /// Distinct files carrying at least one secret-named key.
704 pub files: usize,
705 /// The secret-named keys, ordered by `(path, name, key)`.
706 pub items: Vec<ConfigSecretItem>,
707}
708
709/// Inventory the **secret-named** config keys in the graph — where they are, what
710/// they are called, and whether their values were redacted before persistence —
711/// capped at `limit` (`0` = unlimited).
712///
713/// # What this reports
714///
715/// Config extraction (ADR-0009) flattens TOML/JSON/YAML/`.env` into `config_key`
716/// nodes, and **redacts the value of any secret-named key before it reaches the
717/// store** (see `crate::config_keys::REDACTED` and the redaction sites it
718/// names). This lens reads that back: *secret-named config keys are present, here
719/// are their paths and names, and here is their redaction state*. It is an
720/// **inventory with an invariant check**, and it is useful for exactly two
721/// questions: which of my config surfaces deal in credentials, and did anything
722/// unredacted get into this graph.
723///
724/// # What this CANNOT do — read this before extending it
725///
726/// **It is not a secret scanner and this architecture cannot make it one.** The
727/// lens is named for the inventory it can be, not the scanner the shortlist's
728/// original title promised.
729///
730/// - **It cannot detect a hardcoded credential in source code.** It reads
731/// `config_key` nodes, which come only from config *files* and from
732/// `@rto:config` struct declarations. An AWS key pasted into a `.rs` string
733/// literal produces no `config_key` node and is invisible here. Nothing about
734/// the node kinds this reads can change that.
735/// - **It cannot judge validity.** It never sees a value: by the time anything is
736/// in the store, a secret-named value has already been replaced. There is no
737/// entropy test, no format check, no liveness probe, and there cannot be one
738/// without persisting the very thing extraction exists to redact.
739/// - **It cannot tell a real secret from a placeholder.** `API_TOKEN=changeme` in
740/// a committed `.env.example` and a genuine token in an uncommitted `.env` are
741/// the same row here: same key name, same redacted value, same state.
742/// - **It cannot say a repository has no secrets.** An empty report means "no
743/// secret-*named* config key", which is a statement about naming. A credential
744/// under an innocuous key (`endpoint`, `dsn`, `url`) is not secret-named, is not
745/// redacted, and does not appear.
746///
747/// If you find yourself wanting to widen this toward detecting real credentials,
748/// **that instinct is what the rename exists to prevent**: the widening cannot be
749/// built on these inputs, and a tool that half-does it while being named for the
750/// whole job is worse than one that does the inventory honestly. Every surface
751/// carries this limitation in its own words, so a model calling the tool passes it
752/// on rather than reporting a security guarantee that was never offered.
753///
754/// # The heuristic, stated
755///
756/// "Secret-named" is [`crate::is_secret_key`]: the key's
757/// ASCII-alphanumerics, lowercased, containing any of `secret`, `password`,
758/// `passwd`, `passphrase`, `token`, `apikey`, `credential`, `privatekey`,
759/// `accesskey`, `pwd`. So it matches `API_TOKEN`, `db.passwordFile` and
760/// `serve.apiKey`, and misses `dsn`, `connection_string` and `auth` — and it
761/// false-positives on `token_bucket_size` and `csrf_token_header`, which are
762/// settings, not secrets. Both directions of error are inherent to matching on
763/// names; neither is reported as a finding.
764///
765/// # Ordering
766///
767/// By `(path, name, key)` ascending — an inventory, like [`debt`], not a ranking.
768/// There is deliberately no ordering knob: nothing here is a magnitude worth
769/// sorting by, and offering one would suggest some keys are more secret than
770/// others. Identical input yields byte-identical output.
771///
772/// # Errors
773/// Returns [`StoreError`] on query failure.
774pub fn config_secrets(store: &Store, limit: usize) -> Result<ConfigSecretReport, StoreError> {
775 let nodes = store.nodes_by_kind(&NodeKind::Other(crate::config_keys::KIND.to_owned()))?;
776 let config_keys = nodes.len();
777 let mut items = Vec::new();
778 let mut redacted = 0usize;
779 let mut declared = 0usize;
780 let mut unredacted = 0usize;
781 let mut redacted_not_secret_named = 0usize;
782 let mut files: BTreeSet<String> = BTreeSet::new();
783 for node in nodes {
784 // Prefer `meta.key` — the dotted key as the source spelled it — and fall
785 // back to the node's name, which extraction sets to the same string.
786 let name = node
787 .meta
788 .get("key")
789 .and_then(serde_json::Value::as_str)
790 .unwrap_or(node.name.as_str())
791 .to_owned();
792 let value = node.meta.get("value").and_then(serde_json::Value::as_str);
793 if !crate::config_keys::is_secret_key(&name) {
794 // A redacted value under a non-secret name is a k8s `Secret`'s data:
795 // counted so the report's redaction figures reconcile with the graph,
796 // but not listed — this lens's subject is secret-*named* keys.
797 if value == Some(crate::config_keys::REDACTED) {
798 redacted_not_secret_named += 1;
799 }
800 continue;
801 }
802 let state = match value {
803 Some(v) if v == crate::config_keys::REDACTED => {
804 redacted += 1;
805 RedactionState::Redacted
806 }
807 // A struct-derived key omits `meta.value` entirely: there is no
808 // literal in the code to redact, so "absent" is the honest state
809 // rather than folding it in with a successful redaction.
810 None => {
811 declared += 1;
812 RedactionState::Declared
813 }
814 Some(_) => {
815 unredacted += 1;
816 RedactionState::Present
817 }
818 };
819 if let Some(path) = node.path.as_deref() {
820 files.insert(path.to_owned());
821 }
822 items.push(ConfigSecretItem {
823 key: node.key,
824 path: node.path,
825 name,
826 state,
827 source: node
828 .meta
829 .get("source")
830 .and_then(serde_json::Value::as_str)
831 .map(str::to_owned),
832 });
833 }
834 let secret_named = items.len();
835 items.sort_by(|a, b| (&a.path, &a.name, &a.key).cmp(&(&b.path, &b.name, &b.key)));
836 window(&mut items, 0, limit);
837
838 Ok(ConfigSecretReport {
839 schema: SCHEMA,
840 limit,
841 config_keys,
842 secret_named,
843 redacted,
844 declared,
845 unredacted,
846 redacted_not_secret_named,
847 files: files.len(),
848 items,
849 })
850}
851
852// `glob_match` — the matcher `[debt] ignore` filters with — now lives in
853// [`crate::paths`], imported at the top of this file. `[paths]` exclusion needs
854// the identical semantics, and a user writing `vendor/**` in either place is
855// entitled to have it mean one thing. The tests below stay: they pin what this
856// *consumer* requires of it, which is not the same claim as the matcher's own.
857
858/// How a [`CouplingReport`]'s items are ranked. The three orders answer three
859/// different questions, which a single undirected degree cannot tell apart.
860#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
861pub enum CouplingOrder {
862 /// By `fan_in + fan_out` — overall call coupling.
863 #[default]
864 Total,
865 /// By `fan_in` — the most depended-on symbols ("what calls this?").
866 FanIn,
867 /// By `fan_out` — the symbols that reach furthest ("what does this call?").
868 FanOut,
869}
870
871impl CouplingOrder {
872 /// The stable token for this order, as accepted by [`from_token`](Self::from_token).
873 #[must_use]
874 pub fn as_str(self) -> &'static str {
875 match self {
876 Self::Total => "total",
877 Self::FanIn => "fan_in",
878 Self::FanOut => "fan_out",
879 }
880 }
881
882 /// Parse an order token. `None` for anything else — callers surface an error
883 /// rather than silently ranking by something the caller did not ask for.
884 #[must_use]
885 pub fn from_token(s: &str) -> Option<Self> {
886 match s {
887 "total" => Some(Self::Total),
888 "fan_in" => Some(Self::FanIn),
889 "fan_out" => Some(Self::FanOut),
890 _ => None,
891 }
892 }
893
894 /// The tokens [`from_token`](Self::from_token) accepts, for error messages
895 /// and argument schemas — so the accepted set is stated in exactly one place.
896 #[must_use]
897 pub fn tokens() -> [&'static str; 3] {
898 [
899 Self::Total.as_str(),
900 Self::FanIn.as_str(),
901 Self::FanOut.as_str(),
902 ]
903 }
904}
905
906/// One node's **directed** call coupling in a [`CouplingReport`].
907#[derive(Debug, Clone, PartialEq, Serialize)]
908pub struct CouplingItem {
909 /// Natural key of the node.
910 pub key: String,
911 /// Kind token (e.g. `fn`).
912 pub kind: String,
913 /// Human-facing name.
914 pub name: String,
915 /// Repository-relative path, if any.
916 pub path: Option<String>,
917 /// How many **distinct** other nodes call this one.
918 pub fan_in: u32,
919 /// How many **distinct** other nodes this one calls.
920 pub fan_out: u32,
921 /// `fan_in + fan_out` — the directed equivalent of the undirected degree.
922 pub total: u32,
923 /// Martin's instability, `fan_out / (fan_in + fan_out)`, rounded to two
924 /// decimals. `0.0` = purely depended-on (stable); `1.0` = purely depending
925 /// (unstable). The denominator is never zero: an item exists only when it
926 /// has at least one non-self call edge.
927 pub instability: f64,
928}
929
930/// Directed call coupling: per-node fan-in and fan-out over `Calls` edges,
931/// ranked. The counterpart to an undirected degree ranking, which cannot tell
932/// "everything calls this" from "this calls everything".
933#[derive(Debug, Clone, PartialEq, Serialize)]
934pub struct CouplingReport {
935 /// Stable schema tag ([`SCHEMA`]).
936 pub schema: &'static str,
937 /// The edge kind measured. Always `calls` — the only edge kind whose
938 /// direction carries a caller/callee meaning.
939 pub edge_kind: &'static str,
940 /// The ranking that produced `items` ([`CouplingOrder::as_str`]).
941 pub order: &'static str,
942 /// The requested cap on `items`; `0` means unlimited.
943 pub limit: usize,
944 /// Total `Calls` edges scanned, including duplicates and self-calls.
945 pub call_edges: usize,
946 /// Self-referential `Calls` edges (recursion), counted in `call_edges` but
947 /// excluded from every fan — see [`coupling`].
948 pub self_calls: usize,
949 /// `Calls` edges whose endpoints are in two different languages: name
950 /// collisions from simple-name call resolution, not calls. Counted in
951 /// `call_edges` but excluded from every fan — see [`coupling`].
952 pub cross_language_calls: usize,
953 /// Distinct nodes with at least one non-self `Calls` edge. `items` is the
954 /// top `limit` of these, so `coupled_nodes > items.len()` means truncation.
955 pub coupled_nodes: usize,
956 /// The ranked nodes: by `order` descending, ties broken by `key` ascending.
957 pub items: Vec<CouplingItem>,
958}
959
960/// Rank nodes by **directed** call coupling — fan-in (distinct callers) and
961/// fan-out (distinct callees) over `Calls` edges — most-coupled first by
962/// `order`, capped at `limit` (`0` = unlimited).
963///
964/// Three deliberate counting rules, all of which change the numbers:
965///
966/// - **Distinct counterparts, not edges.** Edges are a set per `(src, dst, kind,
967/// provenance)`, which still admits *parallel* `Calls` edges between one pair
968/// at different provenances — a `derived` extraction and an `inferred`
969/// suggestion of the same call. Counting distinct counterpart keys makes
970/// `fan_in` mean "how many things depend on this", which is the coupling
971/// question, rather than "how many layers asserted the dependency".
972/// - **Self-calls are excluded from both fans.** Recursion is a real edge but
973/// couples a node to nothing outside itself, and counting it would inflate
974/// `fan_in` *and* `fan_out` for the same node. It is reported separately as
975/// `self_calls` rather than silently dropped.
976/// - **Cross-language call edges are excluded.** Roteiro extracts no FFI, so a
977/// `Calls` edge between two languages is never a call — see
978/// `same_language`. Reported as `cross_language_calls`.
979///
980/// Ordering is total and deterministic: by the chosen metric descending, then by
981/// `key` ascending, so identical input yields byte-identical output.
982///
983/// # Precision
984///
985/// `fan_in` is exactly as precise as the `Calls` edges beneath it, and those are
986/// resolved by **simple name**: a callee that is unique by bare name anywhere in
987/// the repository binds to that definition, wherever it lives. So a single
988/// same-language helper with a very common name absorbs every call to that name,
989/// and its `fan_in` reads high for a reason that has nothing to do with design.
990/// Excluding cross-language edges removes the worst of this, but not all of it.
991/// Treat a large `fan_in` on a short, generically-named function as a question,
992/// not a finding — which is also why this lens offers no CI gate.
993///
994/// # Errors
995/// Returns [`StoreError`] on query failure.
996pub fn coupling(
997 store: &Store,
998 order: CouplingOrder,
999 limit: usize,
1000) -> Result<CouplingReport, StoreError> {
1001 // `inbound`: dst key -> distinct src keys. `outbound`: src key -> distinct dst
1002 // keys. Named for the direction rather than caller/callee, which read alike.
1003 let mut inbound: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
1004 let mut outbound: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
1005 let mut call_edges = 0usize;
1006 let mut self_calls = 0usize;
1007 let mut cross_language_calls = 0usize;
1008 for edge in store.all_edges()? {
1009 if edge.kind != EdgeKind::Calls {
1010 continue;
1011 }
1012 call_edges += 1;
1013 if edge.src == edge.dst {
1014 self_calls += 1;
1015 continue;
1016 }
1017 if !same_language(&edge.src, &edge.dst) {
1018 cross_language_calls += 1;
1019 continue;
1020 }
1021 inbound
1022 .entry(edge.dst.clone())
1023 .or_default()
1024 .insert(edge.src.clone());
1025 outbound.entry(edge.src).or_default().insert(edge.dst);
1026 }
1027
1028 // Rank on the counts alone, so only the nodes that survive the cap are read
1029 // back from the store — a whole-graph node scan is not needed to answer a
1030 // top-N question.
1031 let keys: BTreeSet<&String> = inbound.keys().chain(outbound.keys()).collect();
1032 let coupled_nodes = keys.len();
1033 let mut ranked: Vec<(u32, u32, &String)> = keys
1034 .into_iter()
1035 .map(|key| {
1036 let fan_in = count_of(&inbound, key);
1037 let fan_out = count_of(&outbound, key);
1038 (fan_in, fan_out, key)
1039 })
1040 .collect();
1041 ranked.sort_by(|a, b| {
1042 let metric = |&(fan_in, fan_out, _): &(u32, u32, &String)| match order {
1043 CouplingOrder::Total => fan_in + fan_out,
1044 CouplingOrder::FanIn => fan_in,
1045 CouplingOrder::FanOut => fan_out,
1046 };
1047 metric(b).cmp(&metric(a)).then_with(|| a.2.cmp(b.2))
1048 });
1049 window(&mut ranked, 0, limit);
1050
1051 let mut items = Vec::with_capacity(ranked.len());
1052 for (fan_in, fan_out, key) in ranked {
1053 // `edges.src`/`edges.dst` are foreign keys into `nodes`, so a node behind
1054 // a call edge always exists; the guard is defence in depth, not a case.
1055 let Some(node) = store.get_node(key)? else {
1056 continue;
1057 };
1058 let total = fan_in + fan_out;
1059 items.push(CouplingItem {
1060 key: node.key,
1061 kind: node.kind.as_str().to_owned(),
1062 name: node.name,
1063 path: node.path,
1064 fan_in,
1065 fan_out,
1066 total,
1067 instability: round2(f64::from(fan_out) / f64::from(total)),
1068 });
1069 }
1070
1071 Ok(CouplingReport {
1072 schema: SCHEMA,
1073 edge_kind: EdgeKind::Calls.as_str(),
1074 order: order.as_str(),
1075 limit,
1076 call_edges,
1077 self_calls,
1078 cross_language_calls,
1079 coupled_nodes,
1080 items,
1081 })
1082}
1083
1084/// The language token of a symbol key (`sym:<lang>:<path>#<name>` → `<lang>`),
1085/// or `None` for any other key shape.
1086fn sym_lang(key: &str) -> Option<&str> {
1087 let rest = key.strip_prefix("sym:")?;
1088 let (lang, _) = rest.split_once(':')?;
1089 (!lang.is_empty()).then_some(lang)
1090}
1091
1092/// Whether a call edge's two endpoints are in the same language — `true` unless
1093/// both keys carry a language token and the tokens differ.
1094///
1095/// Cross-file call resolution binds a callee by **simple name** across every
1096/// `Fn` node in the repository, language included. Roteiro extracts no FFI, so
1097/// nothing in the graph can legitimately record a JavaScript function calling a
1098/// Rust one; such an edge is a name collision — a lone Rust `join` helper
1099/// absorbing every JavaScript `.join(…)` in the tree. Excluding them keeps a
1100/// language's coupling figures about that language.
1101///
1102/// Unknown-shaped keys (anything that is not `sym:<lang>:…`) are **kept**: this
1103/// filter removes edges it can prove span two languages, and never guesses.
1104fn same_language(src: &str, dst: &str) -> bool {
1105 match (sym_lang(src), sym_lang(dst)) {
1106 (Some(a), Some(b)) => a == b,
1107 _ => true,
1108 }
1109}
1110
1111/// The size of `key`'s counterpart set, as a `u32` (a node cannot have more
1112/// distinct counterparts than there are nodes, so the cast cannot realistically
1113/// saturate; saturating beats wrapping if it ever did).
1114fn count_of(map: &BTreeMap<String, BTreeSet<String>>, key: &str) -> u32 {
1115 map.get(key)
1116 .map_or(0, |set| u32::try_from(set.len()).unwrap_or(u32::MAX))
1117}
1118
1119/// Round to two decimals, so the serialised ratio is short and stable rather
1120/// than carrying the full binary expansion of a division.
1121fn round2(v: f64) -> f64 {
1122 (v * 100.0).round() / 100.0
1123}
1124
1125/// One step along a [`Path`]: the edge traversed and the node it leads to.
1126#[derive(Debug, Clone, PartialEq, Serialize)]
1127pub struct PathHop {
1128 /// Edge kind token (e.g. `calls`, `contains`).
1129 pub kind: String,
1130 /// How the edge was produced.
1131 pub provenance: &'static str,
1132 /// Confidence score, present only for inferred edges.
1133 pub confidence: Option<f64>,
1134 /// The direction the edge was traversed relative to the previous node
1135 /// (`outgoing` = along the edge, `incoming` = against it).
1136 pub direction: &'static str,
1137 /// The natural key of the node this hop arrives at.
1138 pub node: String,
1139}
1140
1141/// A shortest path between two nodes. Edges are followed in either direction
1142/// (the graph is treated as undirected for reachability), and each hop records
1143/// the actual direction and provenance of the edge used.
1144#[derive(Debug, Clone, PartialEq, Serialize)]
1145pub struct Path {
1146 /// Stable schema tag ([`SCHEMA`]).
1147 pub schema: &'static str,
1148 /// Natural key of the start node.
1149 pub from: String,
1150 /// Natural key of the goal node.
1151 pub to: String,
1152 /// Whether a path (including the trivial empty one) was found.
1153 pub found: bool,
1154 /// Number of hops (edges) in the path; `0` when `from == to`.
1155 pub length: usize,
1156 /// The hops from `from` to `to`, in order.
1157 pub hops: Vec<PathHop>,
1158}
1159
1160fn out_ref(edge: &Edge) -> EdgeRef {
1161 EdgeRef {
1162 kind: edge.kind.as_str().to_owned(),
1163 provenance: edge.provenance.as_str(),
1164 confidence: edge.confidence,
1165 node: edge.dst.clone(),
1166 }
1167}
1168
1169fn in_ref(edge: &Edge) -> EdgeRef {
1170 EdgeRef {
1171 kind: edge.kind.as_str().to_owned(),
1172 provenance: edge.provenance.as_str(),
1173 confidence: edge.confidence,
1174 node: edge.src.clone(),
1175 }
1176}
1177
1178fn sort_refs(refs: &mut [EdgeRef]) {
1179 // Include provenance so edges differing only in provenance have a total,
1180 // stable order; with the edge-uniqueness constraint this key is unique.
1181 refs.sort_by(|a, b| (&a.kind, &a.node, a.provenance).cmp(&(&b.kind, &b.node, b.provenance)));
1182}
1183
1184/// Explain a node: its record plus every incoming and outgoing edge, each
1185/// labelled with provenance. Returns `None` if no node has that key.
1186///
1187/// # Errors
1188/// Returns [`StoreError`] on query failure.
1189pub fn explain(store: &Store, key: &str) -> Result<Option<Explanation>, StoreError> {
1190 let Some(node) = store.get_node(key)? else {
1191 return Ok(None);
1192 };
1193 let mut outgoing: Vec<EdgeRef> = store.edges_from(key)?.iter().map(out_ref).collect();
1194 let mut incoming: Vec<EdgeRef> = store.edges_to(key)?.iter().map(in_ref).collect();
1195 sort_refs(&mut outgoing);
1196 sort_refs(&mut incoming);
1197 Ok(Some(Explanation {
1198 schema: SCHEMA,
1199 node: NodeSummary::from_node(&node),
1200 meta: node.meta,
1201 outgoing,
1202 incoming,
1203 }))
1204}
1205
1206/// List every node of the given `kind`, ordered by key.
1207///
1208/// # Errors
1209/// Returns [`StoreError`] on query failure.
1210pub fn list_kind(store: &Store, kind: &NodeKind) -> Result<Listing, StoreError> {
1211 let nodes = store
1212 .nodes_by_kind(kind)?
1213 .iter()
1214 .map(NodeSummary::from_node)
1215 .collect();
1216 Ok(Listing {
1217 schema: SCHEMA,
1218 kind: kind.as_str().to_owned(),
1219 nodes,
1220 })
1221}
1222
1223/// A relevance-ranked search hit: a node summary plus its score.
1224#[derive(Debug, Clone, PartialEq, Serialize)]
1225pub struct SearchHit {
1226 /// Relevance score (higher is better); see [`search`] for how it is derived.
1227 pub score: u32,
1228 /// The matching node.
1229 #[serde(flatten)]
1230 pub node: NodeSummary,
1231 /// A short, whitespace-collapsed excerpt of the node's captured
1232 /// `meta.content` (see `content_snippet`), so a model that never calls
1233 /// [`explain`] still has real grounding text. `None` for pure symbol/config
1234 /// nodes with no content — the summary (name/kind/path) is the grounding then.
1235 #[serde(skip_serializing_if = "Option::is_none")]
1236 pub snippet: Option<String>,
1237}
1238
1239/// Max **chars** of a search-hit content snippet, counting the trailing ellipsis
1240/// when truncated (so the total length never exceeds this). Bounded so many hits
1241/// cannot bloat the tool response or blow the served model's context window.
1242const SNIPPET_MAX: usize = 300;
1243
1244/// Build a bounded, whitespace-collapsed snippet from a node's captured
1245/// `meta.content`, or `None` when the node has no textual content (pure symbol/
1246/// config nodes). Runs of whitespace collapse to single spaces, and the result is
1247/// at most [`SNIPPET_MAX`] chars *including* a trailing `…` when the content was
1248/// truncated, so a search hit carries grounding text even when the model never
1249/// calls [`explain`].
1250///
1251/// Processes `content` **lazily**: it collapses whitespace on the fly and stops
1252/// after ~`SNIPPET_MAX` chars, so a large content-bearing node never materialises
1253/// more than the bound regardless of how big its content is.
1254fn content_snippet(meta: &serde_json::Value) -> Option<String> {
1255 let content = meta.get("content").and_then(|v| v.as_str())?;
1256
1257 // Collect at most SNIPPET_MAX + 1 collapsed chars: the one extra char only
1258 // tells us whether the content overflowed the bound (→ needs an ellipsis);
1259 // we never buffer more than that, however large `content` is.
1260 let mut collapsed: Vec<char> = Vec::with_capacity(SNIPPET_MAX + 1);
1261 let mut pending_space = false;
1262 for ch in content.chars() {
1263 if ch.is_whitespace() {
1264 // A run of whitespace becomes a single separator, but only once a
1265 // real char has been emitted (this also drops any leading whitespace).
1266 pending_space = !collapsed.is_empty();
1267 continue;
1268 }
1269 if pending_space {
1270 collapsed.push(' ');
1271 pending_space = false;
1272 if collapsed.len() > SNIPPET_MAX {
1273 break;
1274 }
1275 }
1276 collapsed.push(ch);
1277 if collapsed.len() > SNIPPET_MAX {
1278 break;
1279 }
1280 }
1281
1282 if collapsed.is_empty() {
1283 return None;
1284 }
1285 // Overflowed the bound: truncate to SNIPPET_MAX - 1 chars and append the
1286 // ellipsis, so the total length (ellipsis included) is exactly SNIPPET_MAX.
1287 if collapsed.len() > SNIPPET_MAX {
1288 let snippet: String = collapsed[..SNIPPET_MAX - 1].iter().collect();
1289 Some(format!("{snippet}…"))
1290 } else {
1291 Some(collapsed.into_iter().collect())
1292 }
1293}
1294
1295/// Deterministically search nodes for `query`, ranked by relevance, returning at
1296/// most `limit` hits — or **every match when `limit == 0`**, which is [`window`]'s
1297/// rule and the one every list lens follows (issue #393). An empty result
1298/// therefore always means "nothing matched", never "you asked for nothing".
1299///
1300/// Case-insensitive; every whitespace/`::`-separated token must appear
1301/// somewhere in the node's **name, key, path, or captured `meta.content`**
1302/// (so a question's words find the *description*, e.g. a README/ADR, not only a
1303/// same-named symbol). Scoring favours an exact name match, then a name/content
1304/// substring, then per-token hits; it then **boosts curated intent** (`authored`
1305/// ADRs/blueprints) and READMEs/overviews and **penalises test scaffolding**, so
1306/// "what/why" questions land on the real answer rather than a same-named test
1307/// helper. Ties break by key so results are stable.
1308///
1309/// # Errors
1310/// Returns [`StoreError`] on query failure.
1311pub fn search(store: &Store, query: &str, limit: usize) -> Result<Vec<SearchHit>, StoreError> {
1312 let q = query.trim().to_lowercase();
1313 // Tokens are separated by whitespace or the `::` path separator; a lone `:`
1314 // (as in a `sym:rust:…` key) does not split a token.
1315 let tokens: Vec<&str> = q.split("::").flat_map(str::split_whitespace).collect();
1316 if tokens.is_empty() {
1317 return Ok(Vec::new());
1318 }
1319
1320 let mut hits: Vec<SearchHit> = Vec::new();
1321 for node in store.all_nodes()? {
1322 let name = node.name.to_lowercase();
1323 let key = node.key.to_lowercase();
1324 let path = node.path.as_deref().unwrap_or("").to_lowercase();
1325 // The captured knowledge base (doc comments, prose, ADR/README/blueprint
1326 // text) is searchable too, so a question's words find the *description*,
1327 // not just a same-named symbol. Only lowercase when a node actually has
1328 // content — most nodes (code symbols) don't, so skip the allocation.
1329 let content = node
1330 .meta
1331 .get("content")
1332 .and_then(|v| v.as_str())
1333 .map(str::to_lowercase);
1334 let content = content.as_deref().unwrap_or("");
1335 // Require every token to appear somewhere (including content), so a
1336 // multi-word query narrows.
1337 if !tokens
1338 .iter()
1339 .all(|t| name.contains(t) || key.contains(t) || path.contains(t) || content.contains(t))
1340 {
1341 continue;
1342 }
1343 let mut relevance: i32 = 0;
1344 if name == q {
1345 relevance += 100;
1346 } else if name.contains(&q) {
1347 relevance += 60;
1348 } else if content.contains(&q) {
1349 relevance += 25;
1350 }
1351 for t in &tokens {
1352 if name.contains(t) {
1353 relevance += 12;
1354 } else if key.contains(t) {
1355 relevance += 6;
1356 } else if content.contains(t) {
1357 relevance += 8;
1358 } else if path.contains(t) {
1359 relevance += 3;
1360 }
1361 }
1362 // Curated intent (ADRs/blueprints — `authored`) is the best answer to a
1363 // "what/why" question; a README/overview is the natural landing page; and
1364 // test scaffolding should not outrank the real thing when it shares a name.
1365 //
1366 // A **peer's** curated prose earns half of it. What the boost measures is
1367 // "somebody wrote this deliberately", and an imported ADR passes that
1368 // test as squarely as a local one — so scoring it like a code symbol
1369 // would throw away the tier the import went to the trouble of carrying.
1370 // But a question asked in this repository is a question about this
1371 // repository, so our own decisions must stay ahead of another repo's when
1372 // both match: the boost is halved rather than shared or withheld.
1373 if node.provenance.tier() == Provenance::Authored {
1374 relevance += if node.provenance.is_external() {
1375 20
1376 } else {
1377 40
1378 };
1379 }
1380 if is_overview_path(&path) {
1381 relevance += 30;
1382 }
1383 if is_test_path(&path) {
1384 relevance -= 60;
1385 }
1386 hits.push(SearchHit {
1387 score: u32::try_from(relevance.max(0)).unwrap_or(0),
1388 snippet: content_snippet(&node.meta),
1389 node: NodeSummary::from_node(&node),
1390 });
1391 }
1392 // Highest score first; ties by key for a stable, deterministic order.
1393 hits.sort_by(|a, b| {
1394 b.score
1395 .cmp(&a.score)
1396 .then_with(|| a.node.key.cmp(&b.node.key))
1397 });
1398 // `window`, not `truncate`: `0` is unlimited here as it is everywhere else.
1399 // The scan above is full-population at every limit, so an unbounded search
1400 // costs the same as a bounded one — only the printing differs.
1401 window(&mut hits, 0, limit);
1402 Ok(hits)
1403}
1404
1405/// A hit in the **generated** channel: text a model produced about a media blob,
1406/// never a graph fact.
1407///
1408/// It is deliberately *not* a [`SearchHit`]. A generated hit has no node, no
1409/// provenance and no key, and giving it a [`NodeSummary`] would be the first step
1410/// towards it being treated like one — the exact mistake ADR-0015 exists to
1411/// correct. Everything a consumer needs to label it is on the struct, including
1412/// the literal `generated: true`, so a caller that reads nothing else still
1413/// cannot mistake it for extracted text.
1414#[derive(Debug, Clone, PartialEq, Serialize)]
1415pub struct GeneratedHit {
1416 /// Relevance within the generated channel. Not comparable with a
1417 /// [`SearchHit::score`]: the two are ranked by different scorers, in
1418 /// different channels, on purpose.
1419 pub score: u32,
1420 /// Always `true`. A marker a consumer cannot miss or forget to check.
1421 pub generated: bool,
1422 /// The producer identity that wrote the text — which model, at which
1423 /// quantisation, under which prompt (see [`crate::Producer::id`]).
1424 pub producer: String,
1425 /// The model's registry name, repeated for legibility.
1426 pub model: String,
1427 /// The modality (`audio` | `vision`).
1428 pub kind: &'static str,
1429 /// Git blob id of the source media.
1430 pub blob: String,
1431 /// Repository path the blob was seen at.
1432 pub path: String,
1433 /// A bounded, whitespace-collapsed excerpt of the generated text, on the same
1434 /// terms as [`SearchHit::snippet`].
1435 pub snippet: Option<String>,
1436}
1437
1438/// A hit in the **memory** channel: something a session learned, never a graph
1439/// fact and never a re-derivable one.
1440///
1441/// Deliberately *not* a [`SearchHit`], for the reason [`GeneratedHit`] is not: a
1442/// memory record has no node, no provenance and no key, and giving it a
1443/// [`NodeSummary`] would be the first step towards its being treated like one.
1444/// Unlike either of the other channels, it also carries **what the tree thinks of
1445/// it** — [`MemoryHit::applies`] and [`MemoryHit::anchor_state`] — because a
1446/// lesson about code that has since moved is worth reading and worth labelling,
1447/// and returning it unlabelled would be the worse of the two mistakes.
1448#[derive(Debug, Clone, PartialEq, Serialize)]
1449pub struct MemoryHit {
1450 /// Relevance within the memory channel. Not comparable with a
1451 /// [`SearchHit::score`] or a [`GeneratedHit::score`]: three channels, three
1452 /// scorers, on purpose.
1453 pub score: u32,
1454 /// Always `true`. A marker a consumer cannot miss or forget to check.
1455 pub memory: bool,
1456 /// The record's id — its generation, and what `roteiro memory forget` takes.
1457 pub id: i64,
1458 /// What kind of knowledge it is (`lesson` | `attempt` | …).
1459 pub kind: &'static str,
1460 /// The namespace it was recorded in. **Not a branch label.**
1461 pub scope: String,
1462 /// The node key it is anchored to, if any.
1463 #[serde(skip_serializing_if = "Option::is_none")]
1464 pub anchor: Option<String>,
1465 /// What that anchor is worth against the current tree (`valid` | `drifted` |
1466 /// `vanished` | `unverifiable` | `unanchored`).
1467 pub anchor_state: &'static str,
1468 /// **Whether this record applies to the tree being searched.** A `false` here
1469 /// is a label, never a reason to have withheld the hit.
1470 pub applies: bool,
1471 /// The evidence multiplier the record's own ranking gave it
1472 /// (`base_confidence × anchor_penalty`), reported so the channel's score can
1473 /// be taken apart.
1474 pub evidence: f64,
1475 /// A bounded, whitespace-collapsed excerpt of the body, on the same terms as
1476 /// [`SearchHit::snippet`].
1477 pub snippet: Option<String>,
1478}
1479
1480/// The three channels a search returns.
1481///
1482/// They are separate fields rather than one merged list because merging is
1483/// precisely what must not happen: generated text and remembered prose may both
1484/// be *retrievable*, but neither may ever be *indistinguishable* from a derived or
1485/// authored fact, and a single ranked list would make the distinction a matter of
1486/// reading each element carefully.
1487#[derive(Debug, Clone, PartialEq, Serialize)]
1488pub struct SearchResults {
1489 /// Stable schema tag ([`SCHEMA`]).
1490 pub schema: &'static str,
1491 /// The graph channel: ranked nodes, exactly what [`search`] returns.
1492 pub hits: Vec<SearchHit>,
1493 /// The generated channel. **Empty unless
1494 /// [`SearchOptions::include_generated`] was set** — off by default, so a
1495 /// silent clip's confabulated prose cannot reach a default search.
1496 pub generated: Vec<GeneratedHit>,
1497 /// The memory channel. **Empty unless [`SearchOptions::include_memory`] was
1498 /// set** — off by default, so unreviewed accumulated prose cannot reach a
1499 /// default search either.
1500 pub memory: Vec<MemoryHit>,
1501}
1502
1503/// How to search.
1504#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1505pub struct SearchOptions {
1506 /// Maximum hits **per channel**, where `0` is unlimited ([`window`]'s rule,
1507 /// applied per channel). Each channel is ranked and windowed independently,
1508 /// so opting in to another one never displaces a graph hit, and never
1509 /// silently returns fewer of them — and `0` is "all of each channel asked
1510 /// for", not "all of them merged and then cut".
1511 pub limit: usize,
1512 /// Fold in the generated channel. Off by default (see
1513 /// [`SearchOptions::default`]).
1514 pub include_generated: bool,
1515 /// Fold in the memory channel. Off by default, for the same reason: what an
1516 /// agent remembers is unreviewed, unredacted and accumulated, so it is
1517 /// something a caller asks for rather than something that arrives.
1518 pub include_memory: bool,
1519}
1520
1521impl Default for SearchOptions {
1522 /// Ten hits, graph channel only. The default is the safe answer: everything
1523 /// that is not an extracted or authored fact is opt-in, always.
1524 fn default() -> Self {
1525 Self {
1526 limit: 10,
1527 include_generated: false,
1528 include_memory: false,
1529 }
1530 }
1531}
1532
1533/// Search every channel: the graph, and — each only when asked for —
1534/// model-generated media content and episodic agent memory.
1535///
1536/// The graph channel is exactly [`search`]. The other two are ranked by scorers of
1537/// their own (`generated_score`, `memory_score`) which have **no provenance
1538/// term at all**, so neither can acquire the `authored` boost that curated intent
1539/// gets. Neither could do so even by accident: neither record is a node, so
1540/// neither ever reaches the code that applies that boost.
1541///
1542/// The memory channel is scored with **no decay** regardless of what a caller
1543/// might prefer elsewhere, so a search is reproducible for a fixed store and a
1544/// fixed tree.
1545///
1546/// # Errors
1547/// Returns [`StoreError`] on query failure.
1548pub fn search_channels(
1549 store: &Store,
1550 query: &str,
1551 opts: SearchOptions,
1552) -> Result<SearchResults, StoreError> {
1553 let hits = search(store, query, opts.limit)?;
1554 let generated = if opts.include_generated {
1555 search_generated(store, query, opts.limit)?
1556 } else {
1557 Vec::new()
1558 };
1559 let memory = if opts.include_memory {
1560 search_memory(store, query, opts.limit)?
1561 } else {
1562 Vec::new()
1563 };
1564 Ok(SearchResults {
1565 schema: SCHEMA,
1566 hits,
1567 generated,
1568 memory,
1569 })
1570}
1571
1572/// Rank the memory channel alone.
1573///
1574/// Built on [`Store::recall_memory`] rather than on a query of its own, so the
1575/// channel inherits every promise recall makes without restating any of them: a
1576/// superseded record is already gone, an unanchored one is already labelled, and
1577/// nothing here writes anything. Decay is fixed at [`crate::Decay::None`] so a
1578/// search over an unchanged store and tree is reproducible.
1579///
1580/// Ties break by newest generation, so the order is total. `limit` follows
1581/// [`window`]: `0` is every matching record, not none of them.
1582fn search_memory(store: &Store, query: &str, limit: usize) -> Result<Vec<MemoryHit>, StoreError> {
1583 let q = query.trim().to_lowercase();
1584 let tokens: Vec<&str> = q.split("::").flat_map(str::split_whitespace).collect();
1585 if tokens.is_empty() {
1586 return Ok(Vec::new());
1587 }
1588 // Recall does the filtering, the anchor resolution and the evidence
1589 // weighting; this function only adds the lexical relevance a search wants.
1590 let recalled = store.recall_memory(&crate::RecallOptions {
1591 query: Some(query),
1592 decay: crate::Decay::None,
1593 ..crate::RecallOptions::default()
1594 })?;
1595
1596 let mut hits: Vec<MemoryHit> = recalled
1597 .results
1598 .into_iter()
1599 .map(|r| {
1600 let body = r.record.body.to_lowercase();
1601 let anchor = r
1602 .record
1603 .anchor
1604 .as_ref()
1605 .map(|a| a.key.to_lowercase())
1606 .unwrap_or_default();
1607 MemoryHit {
1608 score: memory_score(&q, &tokens, &body, &anchor, r.score),
1609 memory: true,
1610 id: r.record.id,
1611 kind: r.record.kind.as_str(),
1612 scope: r.record.scope.clone(),
1613 anchor: r.record.anchor.as_ref().map(|a| a.key.clone()),
1614 anchor_state: r.record.anchor_state.as_str(),
1615 applies: r.record.applies,
1616 evidence: r.score,
1617 snippet: content_snippet(&serde_json::json!({ "content": r.record.body })),
1618 }
1619 })
1620 .collect();
1621 hits.sort_by(|a, b| b.score.cmp(&a.score).then_with(|| b.id.cmp(&a.id)));
1622 window(&mut hits, 0, limit);
1623 Ok(hits)
1624}
1625
1626/// Relevance of one memory record: **lexical match, weighted by the record's own
1627/// evidence**, and nothing else.
1628///
1629/// The `evidence` factor is `base_confidence × anchor_penalty` from
1630/// [`crate::Store::recall_memory`] — so a lesson whose anchor still resolves in
1631/// this tree outranks an equally-worded one whose code has moved on, which is the
1632/// whole depreciation model showing up in search.
1633///
1634/// # The weight is in `[0, 1]`, and zero is reachable — deliberately
1635///
1636/// An earlier version of this comment said `(0, 1]`. That was wrong, and the
1637/// half-open interval hid a decision rather than describing one. The two factors
1638/// are not alike and the difference is the point:
1639///
1640/// - **[`crate::anchor_penalty`] can never be zero.** Its floor is `0.25`
1641/// ([`crate::AnchorState::Drifted`]), and
1642/// `memory::tests::anchor_penalty_demotes_without_ever_silencing` pins that
1643/// every state is `> 0`. So **drift can never drive evidence to zero** — which
1644/// is ADR-0013's "demote, never delete" rule holding *structurally*, not by
1645/// convention. Roteiro's own inference about a record is never allowed to
1646/// reduce it to nothing.
1647/// - **`base_confidence` can be exactly `0.0`**, because the writer can say so.
1648/// `roteiro memory add --confidence 0` is an operator stating "I am recording
1649/// this and I give it no credence." Flooring that would silently overrule an
1650/// explicit statement — and the value is a probability, where `0.0` is
1651/// legitimate rather than a boundary error.
1652///
1653/// So the asymmetry is exactly the right way round: **what Roteiro infers never
1654/// silences a record; what the operator explicitly states is honoured.**
1655///
1656/// # Zero relevance is not zero visibility
1657///
1658/// A zero score does **not** remove a hit. Nothing in this module or in
1659/// [`crate::Store::recall_memory`] filters on the score — it orders, and the
1660/// record comes back, is printed, and is labelled exactly as any other.
1661/// `a_zero_confidence_memory_is_ranked_last_and_still_returned` enforces that in
1662/// both surfaces, so the claim is a tested property rather than something this
1663/// comment asserts and nothing checks. (A limit can still truncate a
1664/// bottom-ranked hit — that is what a limit means, and it applies to every hit
1665/// regardless of score.)
1666///
1667/// The omissions are the point, and each is deliberate:
1668///
1669/// - **no `authored` boost** — this is the whole reason the channel exists. That
1670/// +40 is for intent a human deliberately wrote into a reviewed file;
1671/// accumulated, unreviewed, unredacted prose riding it would be trust-model
1672/// contamination by construction.
1673/// - **no overview boost** — a README's landing-page privilege is about authored
1674/// documentation.
1675/// - **no name or key term** — a memory record has neither.
1676///
1677/// Because this scorer shares no branch with the node scorer, "memory never
1678/// acquires the authored boost" is a structural fact rather than a condition to be
1679/// maintained.
1680fn memory_score(q: &str, tokens: &[&str], body: &str, anchor: &str, evidence: f64) -> u32 {
1681 let mut relevance: i32 = 0;
1682 if body.contains(q) {
1683 relevance += 25;
1684 }
1685 for t in tokens {
1686 if body.contains(t) {
1687 relevance += 8;
1688 } else if anchor.contains(t) {
1689 relevance += 3;
1690 }
1691 }
1692 // `[0.0, 1.0]`, closed at both ends: zero is reachable, and only ever because
1693 // a writer stated it. See the header — `anchor_penalty` cannot contribute a
1694 // zero, so drift can never land here.
1695 let weighted = f64::from(relevance.max(0)) * evidence.clamp(0.0, 1.0);
1696 #[expect(
1697 clippy::cast_possible_truncation,
1698 clippy::cast_sign_loss,
1699 reason = "the product of a small non-negative relevance and a weight in [0, 1]"
1700 )]
1701 let score = weighted.round() as u32;
1702 score
1703}
1704
1705/// Rank the generated channel alone. Ties break by `(producer, blob)` so results
1706/// are stable. `limit` follows [`window`]: `0` is every matching record, not none
1707/// of them.
1708fn search_generated(
1709 store: &Store,
1710 query: &str,
1711 limit: usize,
1712) -> Result<Vec<GeneratedHit>, StoreError> {
1713 let q = query.trim().to_lowercase();
1714 let tokens: Vec<&str> = q.split("::").flat_map(str::split_whitespace).collect();
1715 if tokens.is_empty() {
1716 return Ok(Vec::new());
1717 }
1718 let mut hits: Vec<GeneratedHit> = Vec::new();
1719 for record in store.media_records(&crate::MediaFilter::default())? {
1720 // A record the pre-generation gate refused holds a measurement, not text.
1721 // It is deliberately unsearchable: it has nothing to match on, and the
1722 // path *would* match — which would put a silent clip back into search
1723 // results as a hit with an empty snippet, which is the shape of the very
1724 // bug ADR-0015 exists to correct.
1725 let Some(generated_text) = record.outcome.text() else {
1726 continue;
1727 };
1728 let text = generated_text.to_lowercase();
1729 let path = record.path.to_lowercase();
1730 if !tokens.iter().all(|t| text.contains(t) || path.contains(t)) {
1731 continue;
1732 }
1733 hits.push(GeneratedHit {
1734 score: generated_score(&q, &tokens, &text, &path),
1735 generated: true,
1736 producer: record.producer_id.to_string(),
1737 model: record.producer.model.clone(),
1738 kind: record.producer.kind.as_str(),
1739 blob: record.blob_id.clone(),
1740 path: record.path.clone(),
1741 snippet: content_snippet(&serde_json::json!({ "content": generated_text })),
1742 });
1743 }
1744 hits.sort_by(|a, b| {
1745 b.score
1746 .cmp(&a.score)
1747 .then_with(|| (&a.producer, &a.blob).cmp(&(&b.producer, &b.blob)))
1748 });
1749 window(&mut hits, 0, limit);
1750 Ok(hits)
1751}
1752
1753/// Relevance of one generated record: whole-query and per-token matches over its
1754/// text and path, and **nothing else**.
1755///
1756/// The omissions are the point, and each is deliberate:
1757///
1758/// - **no `authored` boost** — generated text is not curated intent, and the
1759/// graph's +40 for an ADR must never land on a transcript;
1760/// - **no overview boost** — a README's landing-page privilege is about authored
1761/// documentation;
1762/// - **no name or key term** — a generated record has neither.
1763///
1764/// Because this scorer shares no branch with the node scorer, "generated content
1765/// never acquires the authored boost" is a structural fact rather than a
1766/// condition to be maintained.
1767fn generated_score(q: &str, tokens: &[&str], text: &str, path: &str) -> u32 {
1768 let mut relevance: i32 = 0;
1769 if text.contains(q) {
1770 relevance += 25;
1771 }
1772 for t in tokens {
1773 if text.contains(t) {
1774 relevance += 8;
1775 } else if path.contains(t) {
1776 relevance += 3;
1777 }
1778 }
1779 u32::try_from(relevance.max(0)).unwrap_or(0)
1780}
1781
1782/// Whether `path` (already lowercased) is a README/overview doc — the natural
1783/// landing for "what is this project" questions, so it is ranked up. Matches a
1784/// `readme*` or `overview*` basename (blueprints, the other overview docs, are
1785/// already boosted via their `authored` provenance).
1786fn is_overview_path(path: &str) -> bool {
1787 path.rsplit('/')
1788 .next()
1789 .is_some_and(|base| base.starts_with("readme") || base.starts_with("overview"))
1790}
1791
1792/// Whether `path` (already lowercased) is test scaffolding, which should not
1793/// outrank real content that happens to share a name.
1794fn is_test_path(path: &str) -> bool {
1795 path.contains("/tests/") || path.contains("/test/")
1796}
1797
1798/// A candidate step out of a node during traversal: the edge used and the node
1799/// on the other end. Ordered so BFS expansion is deterministic.
1800struct Step {
1801 node: String,
1802 hop: PathHop,
1803}
1804
1805/// All one-hop steps out of `key`, following edges in either direction, sorted
1806/// for deterministic traversal.
1807fn steps_from(store: &Store, key: &str) -> Result<Vec<Step>, StoreError> {
1808 let mut steps = Vec::new();
1809 for edge in store.edges_from(key)? {
1810 steps.push(Step {
1811 node: edge.dst.clone(),
1812 hop: hop(&edge, "outgoing", edge.dst.clone()),
1813 });
1814 }
1815 for edge in store.edges_to(key)? {
1816 steps.push(Step {
1817 node: edge.src.clone(),
1818 hop: hop(&edge, "incoming", edge.src.clone()),
1819 });
1820 }
1821 steps.sort_by(|a, b| {
1822 (&a.node, &a.hop.kind, a.hop.provenance, a.hop.direction).cmp(&(
1823 &b.node,
1824 &b.hop.kind,
1825 b.hop.provenance,
1826 b.hop.direction,
1827 ))
1828 });
1829 Ok(steps)
1830}
1831
1832fn hop(edge: &Edge, direction: &'static str, node: String) -> PathHop {
1833 PathHop {
1834 kind: edge.kind.as_str().to_owned(),
1835 provenance: edge.provenance.as_str(),
1836 confidence: edge.confidence,
1837 direction,
1838 node,
1839 }
1840}
1841
1842/// Find a shortest path from `from` to `to`, following edges in either
1843/// direction. Returns a [`Path`] with `found = false` (and no hops) if either
1844/// endpoint is absent or `to` is unreachable; `from == to` yields the trivial
1845/// zero-length path.
1846///
1847/// The search is breadth-first with deterministic neighbour ordering, so the
1848/// returned path is stable for a given graph.
1849///
1850/// # Errors
1851/// Returns [`StoreError`] on query failure.
1852pub fn path(store: &Store, from: &str, to: &str) -> Result<Path, StoreError> {
1853 let not_found = |found: bool, hops: Vec<PathHop>| Path {
1854 schema: SCHEMA,
1855 from: from.to_owned(),
1856 to: to.to_owned(),
1857 found,
1858 length: hops.len(),
1859 hops,
1860 };
1861
1862 // Both endpoints must exist in the graph.
1863 if store.get_node(from)?.is_none() || store.get_node(to)?.is_none() {
1864 return Ok(not_found(false, Vec::new()));
1865 }
1866 if from == to {
1867 return Ok(not_found(true, Vec::new()));
1868 }
1869
1870 // BFS, recording for each visited node the (predecessor, hop) that reached
1871 // it so the path can be reconstructed.
1872 let mut came_from: BTreeMap<String, (String, PathHop)> = BTreeMap::new();
1873 let mut queue: VecDeque<String> = VecDeque::new();
1874 queue.push_back(from.to_owned());
1875 came_from.insert(from.to_owned(), (String::new(), placeholder_hop()));
1876
1877 while let Some(current) = queue.pop_front() {
1878 if current == to {
1879 break;
1880 }
1881 for step in steps_from(store, ¤t)? {
1882 if came_from.contains_key(&step.node) {
1883 continue;
1884 }
1885 came_from.insert(step.node.clone(), (current.clone(), step.hop));
1886 queue.push_back(step.node);
1887 }
1888 }
1889
1890 // Walk predecessors back from `to` to `from`, then reverse. Every node in
1891 // `came_from` other than `from` has a real predecessor, so this terminates
1892 // at `from`. If the chain is ever broken (an invariant violation), treat it
1893 // as no path rather than silently returning a partial one.
1894 let mut hops = Vec::new();
1895 let mut cursor = to.to_owned();
1896 while cursor != from {
1897 let Some((prev, hop)) = came_from.get(&cursor) else {
1898 return Ok(not_found(false, Vec::new()));
1899 };
1900 hops.push(hop.clone());
1901 cursor = prev.clone();
1902 }
1903 hops.reverse();
1904 Ok(not_found(true, hops))
1905}
1906
1907/// A sentinel hop for the BFS start node (never emitted in a result).
1908fn placeholder_hop() -> PathHop {
1909 PathHop {
1910 kind: String::new(),
1911 provenance: "derived",
1912 confidence: None,
1913 direction: "outgoing",
1914 node: String::new(),
1915 }
1916}
1917
1918#[cfg(test)]
1919mod tests {
1920 use super::{
1921 ConfigSecretReport, CouplingItem, CouplingOrder, CouplingReport, DebtDensityReport,
1922 DensityItem, DensityOrder, RedactionState, SCHEMA, SNIPPET_MAX, SearchOptions,
1923 SearchResults, config_secrets, coupling, debt_density, explain, glob_match, list_kind,
1924 memory_score, path, search, search_channels, window,
1925 };
1926 use crate::{AnchorState, Edge, EdgeKind, FactSet, Node, NodeKind, Store};
1927
1928 fn seeded() -> Store {
1929 let mut store = Store::open_in_memory().expect("store");
1930 let facts = FactSet::new()
1931 .with_node(Node::new("sym:rust:a.rs#main", NodeKind::Fn, "main"))
1932 .with_node(Node::new("sym:rust:a.rs#helper", NodeKind::Fn, "helper"))
1933 .with_node(Node::new("adr:0001", NodeKind::Adr, "Build Roteiro"))
1934 .with_edge(Edge::derived(
1935 "sym:rust:a.rs#main",
1936 "sym:rust:a.rs#helper",
1937 EdgeKind::Calls,
1938 ))
1939 .with_edge(Edge::authored(
1940 "adr:0001",
1941 "sym:rust:a.rs#main",
1942 EdgeKind::References,
1943 ));
1944 store.apply_factset(&facts).expect("apply");
1945 store
1946 }
1947
1948 /// [`window`] is the single definition of `limit`/`offset` for every list
1949 /// lens, so its contract is pinned here rather than only through the lenses
1950 /// that call it.
1951 #[test]
1952 fn window_reads_zero_as_unlimited_and_offsets_before_limiting() {
1953 let ten = || (0..10).collect::<Vec<u8>>();
1954
1955 // `0` is unlimited, not empty — the whole point of #375.
1956 let mut all = ten();
1957 window(&mut all, 0, 0);
1958 assert_eq!(all, ten(), "limit 0 keeps everything");
1959
1960 // A non-zero limit cuts from the end, keeping the caller's order.
1961 let mut top = ten();
1962 window(&mut top, 0, 3);
1963 assert_eq!(top, vec![0, 1, 2]);
1964
1965 // A limit at or beyond the population is a no-op, so the boundary
1966 // between "bounded" and "unbounded" has no step in it.
1967 let mut exact = ten();
1968 window(&mut exact, 0, 10);
1969 assert_eq!(exact, ten());
1970 let mut over = ten();
1971 window(&mut over, 0, 99);
1972 assert_eq!(over, ten());
1973
1974 // `offset` applies first and `limit` to what remains.
1975 let mut paged = ten();
1976 window(&mut paged, 4, 3);
1977 assert_eq!(paged, vec![4, 5, 6]);
1978
1979 // The decision this fix had to make: offset with an unlimited limit is
1980 // "skip N, then every remaining item" — not "skip N, then nothing".
1981 let mut rest = ten();
1982 window(&mut rest, 7, 0);
1983 assert_eq!(rest, vec![7, 8, 9], "offset then unlimited");
1984
1985 // An offset at or past the end is an empty page, not a panic and not a
1986 // wrapped-around one.
1987 let mut at_end = ten();
1988 window(&mut at_end, 10, 0);
1989 assert!(at_end.is_empty());
1990 let mut past_end = ten();
1991 window(&mut past_end, 500, 0);
1992 assert!(past_end.is_empty());
1993 let mut past_end_limited = ten();
1994 window(&mut past_end_limited, 500, 5);
1995 assert!(past_end_limited.is_empty());
1996
1997 // An empty input stays empty under every combination.
1998 let mut empty: Vec<u8> = Vec::new();
1999 window(&mut empty, 0, 0);
2000 window(&mut empty, 3, 0);
2001 window(&mut empty, 0, 3);
2002 assert!(empty.is_empty());
2003 }
2004
2005 #[test]
2006 fn search_ranks_by_relevance_and_is_bounded() {
2007 let store = seeded();
2008 // An exact name match outranks a substring match.
2009 let hits = search(&store, "helper", 10).expect("search");
2010 assert_eq!(hits[0].node.key, "sym:rust:a.rs#helper");
2011 assert!(hits[0].score >= 100, "exact name match scores high");
2012
2013 // Every token must appear: "main roteiro" matches nothing (no node has both).
2014 assert!(
2015 search(&store, "main roteiro", 10)
2016 .expect("search")
2017 .is_empty()
2018 );
2019
2020 // A lone `:` does not split a token: `sym:rust` is one token matching the
2021 // code-symbol keys but not `adr:0001`.
2022 let by_prefix = search(&store, "sym:rust", 10).expect("search");
2023 assert!(!by_prefix.is_empty());
2024 assert!(
2025 by_prefix
2026 .iter()
2027 .all(|h| h.node.key.starts_with("sym:rust:"))
2028 );
2029
2030 // A blank query yields nothing; the limit is respected.
2031 assert!(search(&store, " ", 10).expect("search").is_empty());
2032 assert!(search(&store, "a.rs", 1).expect("search").len() <= 1);
2033 }
2034
2035 /// The population every issue-#393 test below works over: 12 in each of the
2036 /// three channels, each matching a term only its own channel carries.
2037 ///
2038 /// 12 is deliberately above the default of 10, so a `limit` of `0` that had
2039 /// quietly fallen back to the default could not pass for "unlimited".
2040 fn three_channels(population: usize) -> Store {
2041 use crate::{
2042 GeneratedContent, MediaKind, MediaOutcome, MediaWrite, MemoryKind, MemoryWrite,
2043 Producer,
2044 };
2045
2046 let mut store = Store::open_in_memory().expect("store");
2047 let mut facts = FactSet::new();
2048 for i in 0..population {
2049 facts = facts.with_node(Node::new(
2050 format!("sym:rust:a.rs#quokka{i}"),
2051 NodeKind::Fn,
2052 format!("quokka{i}"),
2053 ));
2054 }
2055 store.apply_factset(&facts).expect("apply");
2056
2057 let producer = Producer {
2058 kind: MediaKind::Audio,
2059 model: "voxtral-mini-3b".to_owned(),
2060 model_digest: "4705be8e".to_owned(),
2061 quantisation: "Q4_K_M".to_owned(),
2062 mmproj_digest: "4f24c4ef".to_owned(),
2063 prompt: "Transcribe this audio recording.".to_owned(),
2064 temperature: 0.0,
2065 max_tokens: 512,
2066 };
2067 for i in 0..population {
2068 store
2069 .record_memory(&MemoryWrite {
2070 scope: crate::DEFAULT_MEMORY_SCOPE,
2071 kind: MemoryKind::Lesson,
2072 anchor: None,
2073 body: &format!("wombat lesson number {i}"),
2074 confidence: None,
2075 supersedes: None,
2076 })
2077 .expect("memory write");
2078 assert!(
2079 store
2080 .record_media_content(&MediaWrite {
2081 blob_id: &format!("blob-{i}"),
2082 path: &format!("assets/clip{i}.wav"),
2083 producer: &producer,
2084 tool_version: "0.0.0",
2085 outcome: &MediaOutcome::Generated(GeneratedContent {
2086 text: format!("narwhal transcript number {i}"),
2087 confidence: None,
2088 }),
2089 replace: false,
2090 })
2091 .expect("media write"),
2092 "each clip is a fresh record",
2093 );
2094 }
2095 store
2096 }
2097
2098 /// Every channel asked for, at `limit`.
2099 fn all_channels(store: &Store, query: &str, limit: usize) -> SearchResults {
2100 search_channels(
2101 store,
2102 query,
2103 SearchOptions {
2104 limit,
2105 include_generated: true,
2106 include_memory: true,
2107 },
2108 )
2109 .expect("search")
2110 }
2111
2112 /// Issue #393: `limit == 0` reads as **unlimited on the graph channel**, and
2113 /// it is [`window`] that says so rather than a rule of `search`'s own — the
2114 /// third reading of one parameter name is gone, not relocated.
2115 #[test]
2116 fn search_reads_zero_as_unlimited_and_only_removes_the_cut() {
2117 const POPULATION: usize = 12;
2118 let store = three_channels(POPULATION);
2119
2120 let bounded = search(&store, "quokka", 10).expect("search");
2121 assert_eq!(bounded.len(), 10, "a positive limit still cuts");
2122
2123 let unlimited = search(&store, "quokka", 0).expect("search");
2124 assert_eq!(unlimited.len(), POPULATION, "0 is every match");
2125
2126 // An unlimited search is the same ranking uncut, not a different one:
2127 // the bounded page is the prefix of the unbounded one.
2128 assert_eq!(
2129 unlimited[..10]
2130 .iter()
2131 .map(|h| h.node.key.as_str())
2132 .collect::<Vec<_>>(),
2133 bounded
2134 .iter()
2135 .map(|h| h.node.key.as_str())
2136 .collect::<Vec<_>>(),
2137 "unlimited only removes the cut",
2138 );
2139 }
2140
2141 /// The unit is **per channel**: `0` is "all of each channel that was asked
2142 /// for", not "all of them merged and then cut". Each channel here matches a
2143 /// term the other two do not, so the three populations stay separable.
2144 #[test]
2145 fn each_search_channel_reads_zero_as_unlimited_over_its_own_population() {
2146 const POPULATION: usize = 12;
2147 let store = three_channels(POPULATION);
2148
2149 // Each channel matches a term the other two do not, so a bounded and an
2150 // unbounded read of one says nothing about the others.
2151 assert_eq!(all_channels(&store, "quokka", 10).hits.len(), 10);
2152 assert_eq!(
2153 all_channels(&store, "quokka", 0).hits.len(),
2154 POPULATION,
2155 "graph channel: 0 is unlimited",
2156 );
2157 assert_eq!(all_channels(&store, "wombat", 10).memory.len(), 10);
2158 assert_eq!(
2159 all_channels(&store, "wombat", 0).memory.len(),
2160 POPULATION,
2161 "memory channel: 0 is unlimited",
2162 );
2163 assert_eq!(all_channels(&store, "narwhal", 10).generated.len(), 10);
2164 assert_eq!(
2165 all_channels(&store, "narwhal", 0).generated.len(),
2166 POPULATION,
2167 "generated channel: 0 is unlimited",
2168 );
2169
2170 // And the unit really is per channel: an unbounded search of one term
2171 // leaves the channels it does not match empty rather than filling them.
2172 let graph_only = all_channels(&store, "quokka", 0);
2173 assert!(
2174 graph_only.memory.is_empty() && graph_only.generated.is_empty(),
2175 "unlimited is per channel, not a merged population",
2176 );
2177 }
2178
2179 /// What keeps "unlimited" from meaning "the whole store": a query with no
2180 /// tokens matches nothing, at `0` exactly as at any other limit. `--limit 0`
2181 /// is bounded by what was asked for, not by the population.
2182 #[test]
2183 fn a_tokenless_query_is_nothing_in_every_channel_at_every_limit() {
2184 let store = three_channels(12);
2185 for blank in ["", " ", "\t\n"] {
2186 for limit in [0, 10] {
2187 let nothing = all_channels(&store, blank, limit);
2188 assert!(
2189 nothing.hits.is_empty()
2190 && nothing.generated.is_empty()
2191 && nothing.memory.is_empty(),
2192 "a query with no tokens is nothing, not everything ({blank:?}, limit {limit})",
2193 );
2194 }
2195 }
2196 }
2197
2198 #[test]
2199 fn search_prefers_curated_content_over_same_named_test_symbols() {
2200 use crate::Provenance;
2201 let mut store = Store::open_in_memory().expect("store");
2202 // A same-named test helper (exact name, but test scaffolding)…
2203 let mut test_fn = Node::new(
2204 "sym:rust:crates/x/tests/cli.rs#roteiro",
2205 NodeKind::Fn,
2206 "roteiro",
2207 );
2208 test_fn.path = Some("crates/x/tests/cli.rs".into());
2209 // …the authored ADR that actually answers "what is roteiro"…
2210 let mut adr = Node::new("adr:0001", NodeKind::Adr, "Build Roteiro")
2211 .with_provenance(Provenance::Authored);
2212 adr.path = Some("docs/adr/0001.md".into());
2213 adr.meta = serde_json::json!({ "content": "Roteiro is a provenance-tagged codebase knowledge graph." });
2214 // …and a README whose *content* (not its name) describes the project.
2215 let mut readme = Node::new("file:README.md", NodeKind::File, "README.md");
2216 readme.path = Some("README.md".into());
2217 readme.meta =
2218 serde_json::json!({ "content": "Roteiro turns a repo into one knowledge graph." });
2219 store
2220 .apply_factset(
2221 &FactSet::new()
2222 .with_node(test_fn)
2223 .with_node(adr)
2224 .with_node(readme),
2225 )
2226 .expect("apply");
2227
2228 let hits = search(&store, "roteiro", 10).expect("search");
2229 let keys: Vec<&str> = hits.iter().map(|h| h.node.key.as_str()).collect();
2230 let idx = |k: &str| keys.iter().position(|x| *x == k).expect("present");
2231 // The authored ADR and the README (found *by content*) both outrank the
2232 // same-named test helper.
2233 assert!(
2234 idx("adr:0001") < idx("sym:rust:crates/x/tests/cli.rs#roteiro"),
2235 "authored ADR outranks the test symbol: {keys:?}"
2236 );
2237 assert!(
2238 idx("file:README.md") < idx("sym:rust:crates/x/tests/cli.rs#roteiro"),
2239 "README (matched via content) outranks the test symbol: {keys:?}"
2240 );
2241
2242 // A content-only term finds the node even though no name/key/path has it.
2243 let by_content = search(&store, "provenance-tagged", 10).expect("search");
2244 assert_eq!(
2245 by_content.first().map(|h| h.node.key.as_str()),
2246 Some("adr:0001"),
2247 "content search matches the ADR by its captured text"
2248 );
2249 }
2250
2251 #[test]
2252 fn search_hit_carries_a_bounded_content_snippet() {
2253 use crate::Provenance;
2254 let mut store = Store::open_in_memory().expect("store");
2255 // A content-bearing node whose content is longer than the cap and has
2256 // messy whitespace to collapse.
2257 let long = "word ".repeat(200);
2258 let mut adr =
2259 Node::new("adr:0001", NodeKind::Adr, "Overview").with_provenance(Provenance::Authored);
2260 adr.meta = serde_json::json!({ "content": format!("Roteiro is\n\na graph. {long}") });
2261 // A pure symbol node with no captured content.
2262 let sym = Node::new("sym:rust:a.rs#main", NodeKind::Fn, "main");
2263 store
2264 .apply_factset(&FactSet::new().with_node(adr).with_node(sym))
2265 .expect("apply");
2266
2267 let hits = search(&store, "roteiro", 10).expect("search");
2268 let adr_hit = hits
2269 .iter()
2270 .find(|h| h.node.key == "adr:0001")
2271 .expect("adr hit");
2272 let snippet = adr_hit
2273 .snippet
2274 .as_deref()
2275 .expect("a content-bearing node yields a snippet");
2276 // Whitespace is collapsed to single spaces (no runs, no newlines)…
2277 assert!(snippet.starts_with("Roteiro is a graph."), "got: {snippet}");
2278 assert!(!snippet.contains(" "));
2279 assert!(!snippet.contains('\n'));
2280 // …and the snippet is bounded to SNIPPET_MAX chars *including* the ellipsis.
2281 assert!(
2282 snippet.chars().count() <= SNIPPET_MAX,
2283 "snippet is bounded: {} chars",
2284 snippet.chars().count()
2285 );
2286 assert!(
2287 snippet.ends_with('…'),
2288 "over-long content is truncated with an ellipsis"
2289 );
2290
2291 // A node without content falls back cleanly: no snippet, so the summary
2292 // (name/kind/path) is the grounding.
2293 let hits = search(&store, "main", 10).expect("search");
2294 let sym_hit = hits
2295 .iter()
2296 .find(|h| h.node.key == "sym:rust:a.rs#main")
2297 .expect("sym hit");
2298 assert!(
2299 sym_hit.snippet.is_none(),
2300 "a node with no content has no snippet"
2301 );
2302 }
2303
2304 #[test]
2305 fn explain_reports_labelled_neighbourhood() {
2306 let store = seeded();
2307 let ex = explain(&store, "sym:rust:a.rs#main")
2308 .expect("query")
2309 .expect("present");
2310 assert_eq!(ex.schema, SCHEMA);
2311 assert_eq!(ex.node.kind, "fn");
2312
2313 // Outgoing: derived call to helper.
2314 assert_eq!(ex.outgoing.len(), 1);
2315 assert_eq!(ex.outgoing[0].kind, "calls");
2316 assert_eq!(ex.outgoing[0].provenance, "derived");
2317 assert_eq!(ex.outgoing[0].node, "sym:rust:a.rs#helper");
2318
2319 // Incoming: authored reference from the ADR.
2320 assert_eq!(ex.incoming.len(), 1);
2321 assert_eq!(ex.incoming[0].provenance, "authored");
2322 assert_eq!(ex.incoming[0].node, "adr:0001");
2323 }
2324
2325 #[test]
2326 fn explain_missing_node_is_none() {
2327 let store = seeded();
2328 assert!(explain(&store, "sym:rust:a.rs#ghost").expect("q").is_none());
2329 }
2330
2331 #[test]
2332 fn edges_differing_only_in_provenance_are_ordered() {
2333 // Two edges A->B with the same kind but different provenance must sort
2334 // into a stable, deterministic order (authored before derived).
2335 let mut store = Store::open_in_memory().expect("store");
2336 let facts = FactSet::new()
2337 .with_node(Node::new("a", NodeKind::Fn, "a"))
2338 .with_node(Node::new("b", NodeKind::Fn, "b"))
2339 .with_edge(Edge::derived("a", "b", EdgeKind::References))
2340 .with_edge(Edge::authored("a", "b", EdgeKind::References));
2341 store.apply_factset(&facts).expect("apply");
2342
2343 let ex = explain(&store, "a").expect("q").expect("present");
2344 let provs: Vec<_> = ex.outgoing.iter().map(|e| e.provenance).collect();
2345 assert_eq!(provs, ["authored", "derived"]);
2346 }
2347
2348 #[test]
2349 fn list_kind_is_ordered() {
2350 let store = seeded();
2351 let listing = list_kind(&store, &NodeKind::Fn).expect("list");
2352 let keys: Vec<_> = listing.nodes.iter().map(|n| n.key.as_str()).collect();
2353 assert_eq!(keys, ["sym:rust:a.rs#helper", "sym:rust:a.rs#main"]);
2354 }
2355
2356 #[test]
2357 fn json_schema_is_stable() {
2358 let store = seeded();
2359 let ex = explain(&store, "adr:0001").expect("q").expect("present");
2360 let json = serde_json::to_value(&ex).expect("json");
2361 assert_eq!(json["schema"], SCHEMA);
2362 assert_eq!(json["node"]["key"], "adr:0001");
2363 assert_eq!(json["node"]["kind"], "adr");
2364 // Outgoing authored reference is present with its provenance label.
2365 assert_eq!(json["outgoing"][0]["kind"], "references");
2366 assert_eq!(json["outgoing"][0]["provenance"], "authored");
2367 assert_eq!(json["outgoing"][0]["node"], "sym:rust:a.rs#main");
2368 assert!(json["outgoing"][0]["confidence"].is_null());
2369 }
2370
2371 #[test]
2372 fn path_crosses_provenance_and_direction() {
2373 // adr:0001 --authored/references--> main --derived/calls--> helper.
2374 // A path from the ADR to helper must traverse both, each hop labelled.
2375 let store = seeded();
2376 let p = path(&store, "adr:0001", "sym:rust:a.rs#helper").expect("path");
2377 assert!(p.found);
2378 assert_eq!(p.length, 2);
2379 assert_eq!(p.schema, SCHEMA);
2380
2381 assert_eq!(p.hops[0].kind, "references");
2382 assert_eq!(p.hops[0].provenance, "authored");
2383 assert_eq!(p.hops[0].direction, "outgoing");
2384 assert_eq!(p.hops[0].node, "sym:rust:a.rs#main");
2385
2386 assert_eq!(p.hops[1].kind, "calls");
2387 assert_eq!(p.hops[1].provenance, "derived");
2388 assert_eq!(p.hops[1].node, "sym:rust:a.rs#helper");
2389 }
2390
2391 #[test]
2392 fn path_follows_edges_against_direction() {
2393 // From helper back to the ADR: both edges are traversed against their
2394 // stored direction, so each hop is `incoming`.
2395 let store = seeded();
2396 let p = path(&store, "sym:rust:a.rs#helper", "adr:0001").expect("path");
2397 assert!(p.found);
2398 assert_eq!(p.length, 2);
2399 assert!(p.hops.iter().all(|h| h.direction == "incoming"));
2400 assert_eq!(p.hops.last().unwrap().node, "adr:0001");
2401 }
2402
2403 #[test]
2404 fn path_same_node_is_trivial() {
2405 let store = seeded();
2406 let p = path(&store, "adr:0001", "adr:0001").expect("path");
2407 assert!(p.found);
2408 assert_eq!(p.length, 0);
2409 assert!(p.hops.is_empty());
2410 }
2411
2412 #[test]
2413 fn path_missing_endpoint_or_unreachable_is_not_found() {
2414 let mut store = Store::open_in_memory().expect("store");
2415 // Two disconnected components: a-b and an isolated island.
2416 let facts = FactSet::new()
2417 .with_node(Node::new("a", NodeKind::Fn, "a"))
2418 .with_node(Node::new("b", NodeKind::Fn, "b"))
2419 .with_node(Node::new("island", NodeKind::Fn, "island"))
2420 .with_edge(Edge::derived("a", "b", EdgeKind::Calls));
2421 store.apply_factset(&facts).expect("apply");
2422
2423 // Absent endpoint.
2424 let missing = path(&store, "a", "ghost").expect("path");
2425 assert!(!missing.found);
2426 assert!(missing.hops.is_empty());
2427
2428 // Present but unreachable.
2429 let unreachable = path(&store, "a", "island").expect("path");
2430 assert!(!unreachable.found);
2431 assert!(unreachable.hops.is_empty());
2432 }
2433
2434 #[test]
2435 fn path_is_shortest() {
2436 // a-b-c-d chain plus a direct a-d edge: the path must take the shortcut.
2437 let mut store = Store::open_in_memory().expect("store");
2438 let facts = FactSet::new()
2439 .with_node(Node::new("a", NodeKind::Fn, "a"))
2440 .with_node(Node::new("b", NodeKind::Fn, "b"))
2441 .with_node(Node::new("c", NodeKind::Fn, "c"))
2442 .with_node(Node::new("d", NodeKind::Fn, "d"))
2443 .with_edge(Edge::derived("a", "b", EdgeKind::Calls))
2444 .with_edge(Edge::derived("b", "c", EdgeKind::Calls))
2445 .with_edge(Edge::derived("c", "d", EdgeKind::Calls))
2446 .with_edge(Edge::derived("a", "d", EdgeKind::Calls));
2447 store.apply_factset(&facts).expect("apply");
2448
2449 let p = path(&store, "a", "d").expect("path");
2450 assert!(p.found);
2451 assert_eq!(p.length, 1, "the direct a->d edge is the shortest path");
2452 assert_eq!(p.hops[0].node, "d");
2453 }
2454
2455 #[test]
2456 fn glob_matches_segments_and_wildcards() {
2457 // `**` spans segments (including zero) and anchors both ends.
2458 assert!(glob_match("vendor/**", "vendor/lib/a.rs"));
2459 assert!(glob_match("vendor/**", "vendor")); // zero trailing segments
2460 assert!(glob_match("**/generated/*", "src/gen/generated/x.rs"));
2461 assert!(glob_match("**/*.rs", "a/b/c.rs"));
2462 // `*` and `?` stay within one segment.
2463 assert!(glob_match("src/*.rs", "src/main.rs"));
2464 assert!(!glob_match("src/*.rs", "src/sub/main.rs"));
2465 assert!(glob_match("a?c.rs", "abc.rs"));
2466 assert!(!glob_match("a?c.rs", "ac.rs"));
2467 // Anchored: a bare name does not match a nested path.
2468 assert!(!glob_match("generated", "src/generated"));
2469 assert!(!glob_match("vendor/**", "third_party/vendor/a.rs"));
2470 }
2471
2472 /// **The evidence weight is closed at both ends**, and the two ends mean
2473 /// different things.
2474 ///
2475 /// The boundary the doc comment used to get wrong: it claimed `(0, 1]`, which
2476 /// would have made a zero unreachable. It is reachable, from a writer stating
2477 /// `--confidence 0` and from nowhere else — the lowest weight Roteiro can
2478 /// *infer* is `anchor_penalty(Drifted)`, and that still leaves a score
2479 /// standing, which is checked here against the real constant rather than a
2480 /// number copied from it.
2481 #[test]
2482 fn the_evidence_weight_is_closed_at_both_ends() {
2483 let score = |evidence| memory_score("batch", &["batch"], "a batch cursor", "", evidence);
2484 let full = score(1.0);
2485 assert!(full > 0, "a fully-evidenced hit scores");
2486 assert_eq!(score(0.0), 0, "and a zero weight takes it to zero");
2487 assert!(
2488 score(0.5) < full && score(0.5) > 0,
2489 "in between, in between"
2490 );
2491
2492 // The worst Roteiro can infer about a record still leaves it scoring —
2493 // "demote, never delete", holding as arithmetic.
2494 let worst_inferable = [
2495 AnchorState::Valid,
2496 AnchorState::Unanchored,
2497 AnchorState::Unverifiable,
2498 AnchorState::Vanished,
2499 AnchorState::Drifted,
2500 ]
2501 .into_iter()
2502 .map(crate::anchor_penalty)
2503 .fold(f64::INFINITY, f64::min);
2504 assert!(
2505 score(worst_inferable) > 0,
2506 "the most demoted anchor state ({worst_inferable}) must not silence a hit",
2507 );
2508
2509 // Out-of-range input is clamped rather than trusted, so a corrupt stored
2510 // confidence cannot manufacture a score above the honest ceiling.
2511 assert_eq!(score(2.0), full, "clamped at the top");
2512 assert_eq!(score(-1.0), 0, "and at the bottom");
2513 }
2514
2515 // -- coupling (Q3) -----------------------------------------------------
2516
2517 /// A graph whose two most-coupled nodes have the **same undirected degree**
2518 /// but opposite direction: `hub` is called by two callers and calls nothing;
2519 /// `spread` calls two callees and is called by nothing. An undirected degree
2520 /// ranking cannot tell them apart, which is the whole point of this lens.
2521 fn coupled() -> Store {
2522 let mut store = Store::open_in_memory().expect("store");
2523 let mut facts = FactSet::new();
2524 for name in ["hub", "spread", "a", "b", "x", "y"] {
2525 facts = facts.with_node(Node::new(
2526 format!("sym:rust:a.rs#{name}"),
2527 NodeKind::Fn,
2528 name,
2529 ));
2530 }
2531 for (src, dst) in [("a", "hub"), ("b", "hub"), ("spread", "x"), ("spread", "y")] {
2532 facts = facts.with_edge(Edge::derived(
2533 format!("sym:rust:a.rs#{src}"),
2534 format!("sym:rust:a.rs#{dst}"),
2535 EdgeKind::Calls,
2536 ));
2537 }
2538 store.apply_factset(&facts).expect("apply");
2539 store
2540 }
2541
2542 /// Find an item by symbol name, so assertions read by name not by index.
2543 fn item<'a>(report: &'a CouplingReport, name: &str) -> &'a CouplingItem {
2544 report
2545 .items
2546 .iter()
2547 .find(|i| i.name == name)
2548 .unwrap_or_else(|| panic!("`{name}` missing from {:?}", report.items))
2549 }
2550
2551 #[test]
2552 fn coupling_keeps_the_direction_an_undirected_degree_discards() {
2553 let report = coupling(&coupled(), CouplingOrder::Total, 0).expect("coupling");
2554 let hub = item(&report, "hub");
2555 let spread = item(&report, "spread");
2556
2557 // Identical undirected degree — what a both-ends-incremented ranking sees.
2558 assert_eq!(hub.total, spread.total, "same total coupling");
2559
2560 // …and opposite direction, which is what this lens exists to report.
2561 assert_eq!((hub.fan_in, hub.fan_out), (2, 0), "hub is depended upon");
2562 assert_eq!(
2563 (spread.fan_in, spread.fan_out),
2564 (0, 2),
2565 "spread depends on others"
2566 );
2567 assert!(
2568 (hub.instability - 0.0).abs() < f64::EPSILON,
2569 "a purely called node is maximally stable: {}",
2570 hub.instability
2571 );
2572 assert!(
2573 (spread.instability - 1.0).abs() < f64::EPSILON,
2574 "a purely calling node is maximally unstable: {}",
2575 spread.instability
2576 );
2577
2578 assert_eq!(report.edge_kind, "calls");
2579 assert_eq!(report.coupled_nodes, 6);
2580 assert_eq!(report.call_edges, 4);
2581 assert_eq!(report.self_calls, 0);
2582 assert_eq!(report.cross_language_calls, 0);
2583 }
2584
2585 #[test]
2586 fn coupling_excludes_cross_language_name_collisions() {
2587 // Cross-file call resolution binds a callee by simple name across every
2588 // `Fn` node regardless of language, and Roteiro extracts no FFI — so a
2589 // JavaScript function "calling" a Rust one is a name collision. On this
2590 // repository that single rule is the difference between a Rust helper
2591 // reading as the most depended-on symbol in the tree and not appearing
2592 // at all.
2593 let mut store = coupled();
2594 let mut facts = FactSet::new().with_node(Node::new(
2595 "sym:javascript:app.js#render",
2596 NodeKind::Fn,
2597 "render",
2598 ));
2599 facts = facts.with_edge(Edge::derived(
2600 "sym:javascript:app.js#render",
2601 "sym:rust:a.rs#hub",
2602 EdgeKind::Calls,
2603 ));
2604 store.apply_factset(&facts).expect("apply");
2605
2606 let report = coupling(&store, CouplingOrder::Total, 0).expect("coupling");
2607 assert_eq!(
2608 item(&report, "hub").fan_in,
2609 2,
2610 "a JavaScript caller is not a dependant of a Rust function"
2611 );
2612 assert_eq!(
2613 report.cross_language_calls, 1,
2614 "the excluded edge is reported, not silently dropped"
2615 );
2616 assert_eq!(report.call_edges, 5, "and still counted as scanned");
2617 }
2618
2619 #[test]
2620 fn same_language_never_guesses_about_unknown_key_shapes() {
2621 assert!(super::same_language("sym:rust:a.rs#f", "sym:rust:b.rs#g"));
2622 assert!(!super::same_language(
2623 "sym:javascript:a.js#f",
2624 "sym:rust:b.rs#g"
2625 ));
2626 // A key that is not `sym:<lang>:…` carries no language to compare, so the
2627 // edge is kept: this filter drops only what it can prove spans languages.
2628 assert!(super::same_language("file:a.md", "sym:rust:b.rs#g"));
2629 assert!(super::same_language("sym:", "sym:rust:b.rs#g"));
2630 assert_eq!(super::sym_lang("sym:rust:a.rs#f"), Some("rust"));
2631 assert_eq!(
2632 super::sym_lang("sym::a.rs#f"),
2633 None,
2634 "empty lang is no lang"
2635 );
2636 assert_eq!(super::sym_lang("marker:a.rs#7"), None);
2637 }
2638
2639 #[test]
2640 fn coupling_counts_distinct_callers_not_parallel_edges() {
2641 // Migration 3 makes edges a set per `(src, dst, kind, provenance)` — so
2642 // the way one caller contributes two `Calls` rows is by **provenance**:
2643 // an extractor's `derived` call and an inference layer's `inferred` one.
2644 // Two rows, one dependant.
2645 let mut store = coupled();
2646 let inferred = Edge::inferred("sym:rust:a.rs#a", "sym:rust:a.rs#hub", EdgeKind::Calls, 0.9);
2647 store
2648 .apply_factset(&FactSet::new().with_edge(inferred))
2649 .expect("apply");
2650
2651 let report = coupling(&store, CouplingOrder::Total, 0).expect("coupling");
2652 assert_eq!(
2653 item(&report, "hub").fan_in,
2654 2,
2655 "the same caller at two provenances is one dependant, not two"
2656 );
2657 // The raw edge is still counted, so the parallel edge stays visible
2658 // rather than being silently normalised away.
2659 assert_eq!(
2660 report.call_edges, 5,
2661 "the extra edge is reported as scanned"
2662 );
2663 }
2664
2665 #[test]
2666 fn coupling_excludes_self_calls_from_both_fans() {
2667 // Recursion is a real edge that couples a node to nothing outside itself;
2668 // counting it would inflate `fan_in` AND `fan_out` for the same node.
2669 let mut store = coupled();
2670 let recursive = Edge::derived("sym:rust:a.rs#hub", "sym:rust:a.rs#hub", EdgeKind::Calls);
2671 store
2672 .apply_factset(&FactSet::new().with_edge(recursive))
2673 .expect("apply");
2674
2675 let report = coupling(&store, CouplingOrder::Total, 0).expect("coupling");
2676 let hub = item(&report, "hub");
2677 assert_eq!(
2678 (hub.fan_in, hub.fan_out),
2679 (2, 0),
2680 "recursion changes neither fan"
2681 );
2682 assert_eq!(report.self_calls, 1, "but it is reported, not dropped");
2683 }
2684
2685 #[test]
2686 fn coupling_order_picks_the_question_being_asked() {
2687 let store = coupled();
2688 let top = |order| {
2689 coupling(&store, order, 1).expect("coupling").items[0]
2690 .name
2691 .clone()
2692 };
2693 assert_eq!(top(CouplingOrder::FanIn), "hub", "most depended-on");
2694 assert_eq!(top(CouplingOrder::FanOut), "spread", "reaches furthest");
2695
2696 // `total` cannot separate the two, so the tie must break on `key` —
2697 // a stable order rather than whatever the map iteration yields.
2698 let by_total = coupling(&store, CouplingOrder::Total, 2).expect("coupling");
2699 assert_eq!(
2700 by_total.items.iter().map(|i| &i.name).collect::<Vec<_>>(),
2701 ["hub", "spread"],
2702 "ties break by key ascending"
2703 );
2704 }
2705
2706 #[test]
2707 fn coupling_reports_truncation_and_is_deterministic() {
2708 let store = coupled();
2709 let capped = coupling(&store, CouplingOrder::Total, 2).expect("coupling");
2710 assert_eq!(capped.items.len(), 2);
2711 assert_eq!(
2712 capped.coupled_nodes, 6,
2713 "the population is reported, so a capped list cannot read as the whole graph"
2714 );
2715 assert_eq!(capped.limit, 2);
2716
2717 // Identical input → byte-identical output, including the ratio's rendering.
2718 let a = serde_json::to_string(&capped).expect("json");
2719 let b =
2720 serde_json::to_string(&coupling(&store, CouplingOrder::Total, 2).expect("coupling"))
2721 .expect("json");
2722 assert_eq!(a, b, "deterministic serialisation");
2723 }
2724
2725 #[test]
2726 fn coupling_ignores_edge_kinds_whose_direction_is_not_a_call() {
2727 // `references` is directed too, but an ADR referencing a symbol is not a
2728 // caller. Only `Calls` may move these numbers.
2729 let mut store = coupled();
2730 let mut facts = FactSet::new().with_node(Node::new("adr:0001", NodeKind::Adr, "A"));
2731 facts = facts.with_edge(Edge::authored(
2732 "adr:0001",
2733 "sym:rust:a.rs#hub",
2734 EdgeKind::References,
2735 ));
2736 store.apply_factset(&facts).expect("apply");
2737
2738 let report = coupling(&store, CouplingOrder::Total, 0).expect("coupling");
2739 assert_eq!(item(&report, "hub").fan_in, 2, "a reference is not a call");
2740 assert!(
2741 !report.items.iter().any(|i| i.key == "adr:0001"),
2742 "a node with no call edges is not in the population: {:?}",
2743 report.items
2744 );
2745 assert_eq!(report.call_edges, 4);
2746 }
2747
2748 #[test]
2749 fn coupling_order_tokens_round_trip() {
2750 for token in CouplingOrder::tokens() {
2751 let order = CouplingOrder::from_token(token)
2752 .unwrap_or_else(|| panic!("`{token}` is advertised but not accepted"));
2753 assert_eq!(order.as_str(), token);
2754 }
2755 assert!(
2756 CouplingOrder::from_token("degree").is_none(),
2757 "an unknown order is rejected, not silently defaulted"
2758 );
2759 }
2760
2761 // -- debt density (Q1) -------------------------------------------------
2762
2763 /// A `file` node carrying the `meta.lines` this lens divides by — the shape
2764 /// `extract::file_node` emits for every blob.
2765 fn file_of(path: &str, lines: u64) -> Node {
2766 let mut node = Node::new(format!("file:{path}"), NodeKind::File, path);
2767 node.path = Some(path.to_owned());
2768 node.meta = serde_json::json!({ "bytes": lines * 30, "lines": lines });
2769 node
2770 }
2771
2772 /// A marker node as `markers::augment` emits it.
2773 fn marker_of(path: &str, line: u32, category: &str) -> Node {
2774 let mut node = Node::new(
2775 format!("marker:{path}#{line}"),
2776 NodeKind::Marker,
2777 format!("TODO {line}"), // roteiro:ignore
2778 );
2779 node.path = Some(path.to_owned());
2780 node.meta = serde_json::json!({
2781 "category": category,
2782 "text": format!("TODO {line}"), // roteiro:ignore
2783 "line": line,
2784 });
2785 node
2786 }
2787
2788 /// Two files with the **same marker count** and very different lengths —
2789 /// indistinguishable under `debt`, twenty-fold apart under density. Plus a
2790 /// third, short file whose single marker would top the ranking on arithmetic
2791 /// alone.
2792 fn marked() -> Store {
2793 let mut store = Store::open_in_memory().expect("store");
2794 let mut facts = FactSet::new()
2795 .with_node(file_of("big.rs", 4000))
2796 .with_node(file_of("small.rs", 200))
2797 .with_node(file_of("tiny.rs", 10));
2798 for line in 1..=40 {
2799 facts = facts.with_node(marker_of("big.rs", line, "todo")); // roteiro:ignore
2800 facts = facts.with_node(marker_of("small.rs", line, "todo")); // roteiro:ignore
2801 }
2802 facts = facts.with_node(marker_of("tiny.rs", 3, "stub"));
2803 store.apply_factset(&facts).expect("apply");
2804 store
2805 }
2806
2807 /// Every default: no category filter, no ignore globs, unlimited, floored at
2808 /// [`super::DEFAULT_MIN_LINES`].
2809 fn density(store: &Store, order: DensityOrder) -> DebtDensityReport {
2810 debt_density(store, &[], &[], order, 0, super::DEFAULT_MIN_LINES).expect("density")
2811 }
2812
2813 /// Find an item by path, so assertions read by file not by index.
2814 fn at<'a>(report: &'a DebtDensityReport, path: &str) -> &'a DensityItem {
2815 report
2816 .items
2817 .iter()
2818 .find(|i| i.path == path)
2819 .unwrap_or_else(|| panic!("`{path}` missing from {:?}", report.items))
2820 }
2821
2822 #[test]
2823 fn density_separates_files_a_raw_marker_count_cannot() {
2824 let report = density(&marked(), DensityOrder::Density);
2825 let big = at(&report, "big.rs");
2826 let small = at(&report, "small.rs");
2827
2828 // Identical under `debt` — the same forty markers each.
2829 assert_eq!(big.markers, small.markers, "same raw count");
2830
2831 // …and twenty-fold apart under density, which is the whole lens.
2832 assert!(
2833 (big.per_kloc - 10.0).abs() < f64::EPSILON,
2834 "40 markers in 4000 lines is 10 per kloc, was {}",
2835 big.per_kloc
2836 );
2837 assert!(
2838 (small.per_kloc - 200.0).abs() < f64::EPSILON,
2839 "40 markers in 200 lines is 200 per kloc, was {}",
2840 small.per_kloc
2841 );
2842 assert_eq!(
2843 report.items.first().map(|i| i.path.as_str()),
2844 Some("small.rs"),
2845 "the dense file ranks first: {:?}",
2846 report.items
2847 );
2848
2849 // The per-file category split, so "forty todo" and "forty stub" stay
2850 // distinguishable in a report that otherwise shows one number per file.
2851 assert_eq!(small.by_category.get("todo"), Some(&40)); // roteiro:ignore
2852 assert_eq!(report.schema, SCHEMA);
2853 }
2854
2855 #[test]
2856 fn markers_order_ranks_the_way_debt_already_does() {
2857 // The control: on `markers` the two forty-marker files tie and break on
2858 // path, so density is demonstrably the thing that separated them — not
2859 // some other difference in the fixture.
2860 let report = density(&marked(), DensityOrder::Markers);
2861 assert_eq!(
2862 report.items.iter().map(|i| &i.path).collect::<Vec<_>>(),
2863 ["big.rs", "small.rs"],
2864 "equal counts tie and break on path ascending"
2865 );
2866 }
2867
2868 #[test]
2869 fn the_short_file_floor_excludes_without_hiding() {
2870 // `tiny.rs` is 1 marker in 10 lines = 100 per kloc, which would place it
2871 // second on arithmetic alone. The floor keeps it out of the *ranking*
2872 // while leaving it in the population and the totals.
2873 let report = density(&marked(), DensityOrder::Density);
2874 assert!(
2875 !report.items.iter().any(|i| i.path == "tiny.rs"),
2876 "a 10-line file is not ranked: {:?}",
2877 report.items
2878 );
2879 assert_eq!(report.short_files, 1, "and its exclusion is reported");
2880 assert_eq!(
2881 report.files_with_markers, 3,
2882 "the population still counts it"
2883 );
2884 assert_eq!(report.ranked_files, 2);
2885 assert_eq!(
2886 report.total_markers, 81,
2887 "and so do the totals: 40 + 40 + 1"
2888 );
2889
2890 // `min_lines = 0` disables the floor rather than merely lowering it.
2891 let unfloored =
2892 debt_density(&marked(), &[], &[], DensityOrder::Density, 0, 0).expect("density");
2893 assert_eq!(unfloored.short_files, 0);
2894 assert_eq!(unfloored.ranked_files, 3);
2895 let tiny = at(&unfloored, "tiny.rs").per_kloc;
2896 assert!(
2897 (tiny - 100.0).abs() < f64::EPSILON,
2898 "the arithmetic the floor exists to keep out of the ranking, was {tiny}"
2899 );
2900 }
2901
2902 #[test]
2903 fn a_file_with_no_recorded_length_is_reported_not_divided_by() {
2904 // Three ways a denominator goes missing, all of which must land in
2905 // `unknown_length_files` rather than in the ranking with a fabricated
2906 // density: no `file` node at all, a `file` node with no `meta.lines`, and
2907 // a `lines` of zero (an empty file, or one unterminated line — a newline
2908 // count cannot tell those apart, so neither does this).
2909 let mut store = Store::open_in_memory().expect("store");
2910 let mut no_lines = Node::new("file:b.rs", NodeKind::File, "b.rs");
2911 no_lines.path = Some("b.rs".into());
2912 no_lines.meta = serde_json::json!({ "bytes": 90 });
2913 let facts = FactSet::new()
2914 .with_node(marker_of("orphan.rs", 1, "todo")) // roteiro:ignore
2915 .with_node(no_lines)
2916 .with_node(marker_of("b.rs", 1, "todo")) // roteiro:ignore
2917 .with_node(file_of("empty.rs", 0))
2918 .with_node(marker_of("empty.rs", 1, "todo")); // roteiro:ignore
2919 store.apply_factset(&facts).expect("apply");
2920
2921 let report = density(&store, DensityOrder::Density);
2922 assert!(report.items.is_empty(), "nothing rankable: {report:?}");
2923 assert_eq!(report.unknown_length_files, 3);
2924 assert_eq!(
2925 report.total_markers, 3,
2926 "the markers are still inventoried, so the file cannot vanish silently"
2927 );
2928 assert!(
2929 (report.overall_per_kloc - 0.0).abs() < f64::EPSILON,
2930 "and no density is invented from a zero denominator"
2931 );
2932 }
2933
2934 #[test]
2935 fn density_shares_debt_s_filters_rather_than_adding_a_second_vocabulary() {
2936 let store = marked();
2937 // The `[debt] ignore` globs `debt` already honours.
2938 let ignored = debt_density(
2939 &store,
2940 &[],
2941 &["small.rs".into()],
2942 DensityOrder::Density,
2943 0,
2944 super::DEFAULT_MIN_LINES,
2945 )
2946 .expect("density");
2947 assert!(
2948 !ignored.items.iter().any(|i| i.path == "small.rs"),
2949 "an ignored path leaves the report entirely: {:?}",
2950 ignored.items
2951 );
2952 assert_eq!(
2953 ignored.files_with_markers, 2,
2954 "not merely unranked — it is not in the population either"
2955 );
2956
2957 // And the same category filter, so a `--kind stub` density is the density
2958 // of stubs and not of everything.
2959 let stubs = debt_density(&store, &["stub".into()], &[], DensityOrder::Density, 0, 0)
2960 .expect("density");
2961 assert_eq!(stubs.total_markers, 1);
2962 assert_eq!(
2963 stubs.items.iter().map(|i| &i.path).collect::<Vec<_>>(),
2964 ["tiny.rs"]
2965 );
2966 }
2967
2968 #[test]
2969 fn density_ranks_on_the_exact_ratio_not_the_rounded_one() {
2970 // Two files whose densities differ in the fourth decimal: 1/3000 is
2971 // 0.3333 per kloc and 1/3001 is 0.3332. Both round to 0.33, so a ranking
2972 // built on `per_kloc` would tie them and break on path — putting the
2973 // *less* dense file first, since `a.rs` sorts before `b.rs`.
2974 let mut store = Store::open_in_memory().expect("store");
2975 let facts = FactSet::new()
2976 .with_node(file_of("a.rs", 3001))
2977 .with_node(marker_of("a.rs", 1, "todo")) // roteiro:ignore
2978 .with_node(file_of("b.rs", 3000))
2979 .with_node(marker_of("b.rs", 1, "todo")); // roteiro:ignore
2980 store.apply_factset(&facts).expect("apply");
2981
2982 let report = density(&store, DensityOrder::Density);
2983 assert_eq!(
2984 report.items.iter().map(|i| &i.path).collect::<Vec<_>>(),
2985 ["b.rs", "a.rs"],
2986 "the shorter file is denser, however the figures round"
2987 );
2988 assert_eq!(
2989 (report.items[0].per_kloc, report.items[1].per_kloc),
2990 (0.33, 0.33),
2991 "and the rendered figures really are equal, so the order came from elsewhere"
2992 );
2993 }
2994
2995 #[test]
2996 fn density_reports_truncation_and_is_deterministic() {
2997 let store = marked();
2998 let capped = debt_density(&store, &[], &[], DensityOrder::Density, 1, 0).expect("density");
2999 assert_eq!(capped.items.len(), 1);
3000 assert_eq!(capped.limit, 1);
3001 assert_eq!(
3002 capped.ranked_files, 3,
3003 "the population is reported, so a capped list cannot read as the whole repository"
3004 );
3005 // `overall_per_kloc` is the baseline across every ranked file, not across
3006 // the ones that survived the cap — otherwise the top file's own density
3007 // would be its own baseline.
3008 assert_eq!(capped.total_lines, 4210);
3009 assert!(
3010 (capped.overall_per_kloc - 19.24).abs() < f64::EPSILON,
3011 "81 markers over 4210 lines, was {}",
3012 capped.overall_per_kloc
3013 );
3014
3015 let a = serde_json::to_string(&capped).expect("json");
3016 let b = serde_json::to_string(
3017 &debt_density(&store, &[], &[], DensityOrder::Density, 1, 0).expect("density"),
3018 )
3019 .expect("json");
3020 assert_eq!(a, b, "deterministic serialisation");
3021 }
3022
3023 #[test]
3024 fn density_order_tokens_round_trip() {
3025 for token in DensityOrder::tokens() {
3026 let order = DensityOrder::from_token(token)
3027 .unwrap_or_else(|| panic!("`{token}` is advertised but not accepted"));
3028 assert_eq!(order.as_str(), token);
3029 }
3030 assert!(
3031 DensityOrder::from_token("count").is_none(),
3032 "an unknown order is rejected, not silently defaulted"
3033 );
3034 }
3035
3036 // -- config-secret inventory (S1) --------------------------------------
3037
3038 /// A `config_key` node as `extract::config_facts` emits it: `meta.value`
3039 /// present (already redacted, if the key name called for it).
3040 fn cfgkey(path: &str, dotted: &str, value: &str) -> Node {
3041 let mut node = Node::new(
3042 format!("cfgkey:{path}#{dotted}"),
3043 NodeKind::Other("config_key".to_owned()),
3044 dotted,
3045 );
3046 node.path = Some(path.to_owned());
3047 node.meta = serde_json::json!({ "key": dotted, "value": value });
3048 node
3049 }
3050
3051 /// A **struct-derived** `config_key` node as `synthesize_config_keys` emits
3052 /// it: `meta.value` OMITTED, because a Rust field declares no literal value.
3053 fn struct_cfgkey(path: &str, dotted: &str) -> Node {
3054 let mut node = Node::new(
3055 format!("cfgkey:{path}#{dotted}"),
3056 NodeKind::Other("config_key".to_owned()),
3057 dotted,
3058 );
3059 node.path = Some(path.to_owned());
3060 node.meta = serde_json::json!({
3061 "key": dotted,
3062 "source": "struct",
3063 "struct": "AppConfig",
3064 });
3065 node
3066 }
3067
3068 /// One of each state extraction can produce, plus a non-secret key and a
3069 /// k8s-`Secret`-style redaction under an innocuous name.
3070 fn configured() -> Store {
3071 let mut store = Store::open_in_memory().expect("store");
3072 let facts = FactSet::new()
3073 // Secret-named, redacted by extraction — the expected state.
3074 .with_node(cfgkey(".env", "API_TOKEN", "<redacted>"))
3075 .with_node(cfgkey("config.toml", "db.password", "<redacted>"))
3076 // Secret-named, struct-derived — no value to redact.
3077 .with_node(struct_cfgkey("src/config.rs", "serve.api_key"))
3078 // Not secret-named — not this lens's subject at all.
3079 .with_node(cfgkey("config.toml", "serve.addr", "127.0.0.1:8017"))
3080 // A k8s `Secret`'s data: redacted for where it lives, not what it is
3081 // called, so it is counted but not listed.
3082 .with_node(cfgkey("k8s/secret.yaml", "database-url", "<redacted>"));
3083 store.apply_factset(&facts).expect("apply");
3084 store
3085 }
3086
3087 /// Find an item by dotted name.
3088 fn secret<'a>(report: &'a ConfigSecretReport, name: &str) -> &'a super::ConfigSecretItem {
3089 report
3090 .items
3091 .iter()
3092 .find(|i| i.name == name)
3093 .unwrap_or_else(|| panic!("`{name}` missing from {:?}", report.items))
3094 }
3095
3096 #[test]
3097 fn the_inventory_reports_presence_and_redaction_not_values() {
3098 let report = config_secrets(&configured(), 0).expect("config_secrets");
3099
3100 assert_eq!(report.config_keys, 5, "the population it drew from");
3101 assert_eq!(report.secret_named, 3, "{:?}", report.items);
3102 assert_eq!(report.files, 3);
3103 assert_eq!(report.schema, SCHEMA);
3104
3105 // Paths, key names and state, which is what the lens is for.
3106 assert_eq!(secret(&report, "API_TOKEN").path.as_deref(), Some(".env"));
3107 assert_eq!(
3108 secret(&report, "db.password").key,
3109 "cfgkey:config.toml#db.password"
3110 );
3111 // The state comes from comparing the stored value against the redactor's
3112 // own constant, so asserting it is what keeps reader and writer from
3113 // drifting apart on a spelling.
3114 assert_eq!(
3115 secret(&report, "API_TOKEN").state,
3116 RedactionState::Redacted,
3117 "the placeholder extraction wrote is recognised as a redaction"
3118 );
3119 assert_eq!(report.redacted, 2, "{report:?}");
3120
3121 // No value is carried on any item — there is no field for one. The
3122 // serialised shape is the contract, so assert against that, not the type.
3123 let json = serde_json::to_value(&report).expect("json");
3124 let text = serde_json::to_string(&report).expect("json");
3125 assert!(
3126 json["items"][0].get("value").is_none(),
3127 "an item carries no value field: {text}"
3128 );
3129 assert!(
3130 !text.contains("<redacted>"),
3131 "not even the placeholder is echoed back: {text}"
3132 );
3133
3134 // Ordering is `(path, name, key)` — an inventory, not a ranking.
3135 assert_eq!(
3136 report.items.iter().map(|i| &i.name).collect::<Vec<_>>(),
3137 ["API_TOKEN", "db.password", "serve.api_key"]
3138 );
3139 }
3140
3141 #[test]
3142 fn a_struct_declared_key_is_neither_redacted_nor_a_leak() {
3143 // A `@rto:config` struct field has no literal value in code, so extraction
3144 // omits `meta.value` entirely. Folding that in with a successful redaction
3145 // would claim a redaction that never happened; calling it unredacted would
3146 // report a leak that does not exist.
3147 let report = config_secrets(&configured(), 0).expect("config_secrets");
3148 let declared = secret(&report, "serve.api_key");
3149 assert_eq!(declared.state, RedactionState::Declared);
3150 assert_eq!(declared.source.as_deref(), Some("struct"));
3151
3152 assert_eq!(report.redacted, 2, "the two file-derived keys");
3153 assert_eq!(report.declared, 1);
3154 assert_eq!(
3155 report.unredacted, 0,
3156 "the invariant extraction maintains: {report:?}"
3157 );
3158 }
3159
3160 #[test]
3161 fn an_unredacted_secret_named_value_is_reported_as_a_finding() {
3162 // Extraction redacts every secret-named key, so this state is unreachable
3163 // from extraction — but `apply_import_layer` upserts whatever nodes an
3164 // imported factset carries, so another tool's import can put an unredacted
3165 // value in the store. That is the one path worth reporting, and it is a
3166 // finding about THIS STORE, not about the source repository.
3167 let mut store = configured();
3168 store
3169 .apply_import_layer(
3170 "other-tool",
3171 &FactSet::new().with_node(cfgkey("imported.env", "AWS_SECRET", "AKIAnot-redacted")),
3172 )
3173 .expect("import");
3174
3175 let report = config_secrets(&store, 0).expect("config_secrets");
3176 assert_eq!(report.unredacted, 1, "{report:?}");
3177 assert_eq!(secret(&report, "AWS_SECRET").state, RedactionState::Present);
3178 // And still no value in the report: the lens says *that* something is
3179 // unredacted, and never repeats it.
3180 let text = serde_json::to_string(&report).expect("json");
3181 assert!(
3182 !text.contains("AKIA"),
3183 "the value is not echoed back: {text}"
3184 );
3185 }
3186
3187 #[test]
3188 fn a_redaction_under_an_innocuous_name_is_counted_but_not_listed() {
3189 // A k8s `Secret`'s `data` is redacted because of where it lives, whatever
3190 // the key is called. It is not secret-*named*, so it is not this lens's
3191 // subject — but it is counted, so a reader comparing `redacted` against the
3192 // number of `<redacted>` values in the graph does not find a surplus they
3193 // cannot explain.
3194 let report = config_secrets(&configured(), 0).expect("config_secrets");
3195 assert_eq!(report.redacted_not_secret_named, 1);
3196 assert!(
3197 !report.items.iter().any(|i| i.name == "database-url"),
3198 "not listed: {:?}",
3199 report.items
3200 );
3201 assert_eq!(
3202 report.redacted + report.redacted_not_secret_named,
3203 3,
3204 "and the two figures together account for every redacted value"
3205 );
3206 }
3207
3208 #[test]
3209 fn the_inventory_cannot_see_a_credential_that_is_not_a_config_key() {
3210 // The load-bearing limitation, asserted rather than only documented: a
3211 // credential in a Rust string literal produces no `config_key` node, so it
3212 // is invisible here. No extension of this lens can change that — which is
3213 // why it is named for the inventory it is, not the scanner it is not.
3214 let mut store = configured();
3215 let mut hardcoded = Node::new("sym:rust:src/main.rs#connect", NodeKind::Fn, "connect");
3216 hardcoded.path = Some("src/main.rs".into());
3217 // Split at the prefix for the same reason as `FAKE_TOKEN` in
3218 // `roteiro/tests/config_secrets_cli.rs`: assembled, this is AWS's own
3219 // documentation placeholder, but it matches the canonical access-key-id
3220 // rule exactly and a regex-rule scanner cannot know the difference. The
3221 // assembled value is unchanged; no assertion here matches on its text.
3222 hardcoded.meta = serde_json::json!({
3223 "content": concat!("let token = \"AKIA", "IOSFODNN7EXAMPLE\";"),
3224 });
3225 store
3226 .apply_factset(&FactSet::new().with_node(hardcoded))
3227 .expect("apply");
3228
3229 let report = config_secrets(&store, 0).expect("config_secrets");
3230 assert_eq!(
3231 report.secret_named, 3,
3232 "a hardcoded credential does not appear: {:?}",
3233 report.items
3234 );
3235 assert_eq!(report.config_keys, 5, "and is not a config key at all");
3236 }
3237
3238 #[test]
3239 fn the_inventory_reports_truncation_and_is_deterministic() {
3240 let store = configured();
3241 let capped = config_secrets(&store, 1).expect("config_secrets");
3242 assert_eq!(capped.items.len(), 1);
3243 assert_eq!(capped.limit, 1);
3244 assert_eq!(
3245 capped.secret_named, 3,
3246 "the population is reported, so a capped list cannot read as a clean repository"
3247 );
3248 // The state counts are over the whole population too, not the shown rows —
3249 // otherwise a cap could hide an `unredacted` finding.
3250 assert_eq!((capped.redacted, capped.declared), (2, 1));
3251
3252 let a = serde_json::to_string(&capped).expect("json");
3253 let b = serde_json::to_string(&config_secrets(&store, 1).expect("config_secrets"))
3254 .expect("json");
3255 assert_eq!(a, b, "deterministic serialisation");
3256 }
3257
3258 #[test]
3259 fn an_empty_report_means_no_secret_named_key_not_no_secret() {
3260 // The distinction the lens must never blur: a credential under an
3261 // innocuous key name (`dsn`) is not secret-named, is not redacted, and does
3262 // not appear. So "nothing found" is a statement about naming.
3263 let mut store = Store::open_in_memory().expect("store");
3264 store
3265 .apply_factset(&FactSet::new().with_node(cfgkey(
3266 ".env",
3267 "DSN",
3268 "postgres://u:pw@host/db",
3269 )))
3270 .expect("apply");
3271
3272 let report = config_secrets(&store, 0).expect("config_secrets");
3273 assert_eq!(report.secret_named, 0, "nothing is secret-*named*");
3274 assert_eq!(report.redacted_not_secret_named, 0);
3275 assert_eq!(
3276 report.config_keys, 1,
3277 "while the graph does hold a config key with a credential in it"
3278 );
3279 }
3280
3281 #[test]
3282 fn redaction_state_tokens_match_their_serialisation() {
3283 // The token and the wire form are the same string, so a caller matching on
3284 // the JSON and a caller matching on `as_str` cannot disagree.
3285 for state in [
3286 RedactionState::Redacted,
3287 RedactionState::Declared,
3288 RedactionState::Present,
3289 ] {
3290 let json = serde_json::to_string(&state).expect("json");
3291 assert_eq!(json, format!("\"{}\"", state.as_str()));
3292 }
3293 }
3294}