rto_graph/query.rs
1//! The agent- and human-facing query surface over the graph.
2//!
3//! Everything here is a read-only view built from the store's typed queries,
4//! serialised under a **stable, versioned** JSON schema ([`SCHEMA`]) so agents
5//! can depend on the shape. The primitives are [`explain`] (a node and its
6//! provenance-labelled neighbourhood), [`list_kind`] (all nodes of a kind),
7//! [`path`] (a shortest path between two nodes), [`debt`] (the intent-debt marker
8//! inventory), [`debt_density`] (that inventory per file, normalised by file
9//! length), [`coupling`] (directed fan-in/fan-out over `Calls` edges),
10//! [`config_secrets`] (secret-named config keys and their redaction state), and
11//! [`search`] (relevance-ranked node search). All return
12//! mixed-provenance results — the "one query surface" from ADR-0001 — with every
13//! edge carrying its `provenance`.
14
15use std::collections::{BTreeMap, BTreeSet, VecDeque};
16
17use serde::Serialize;
18
19use crate::store::{Store, StoreError};
20use crate::{Edge, EdgeKind, NodeKind, Provenance};
21
22/// The versioned schema tag emitted on every query result. Bump the version on
23/// any breaking change to the shape.
24pub const SCHEMA: &str = "roteiro.query/v1";
25
26/// Cut an already-ordered, already-materialised list down to the window a caller
27/// asked for: skip `offset` items from the front, then keep at most `limit`.
28///
29/// **This is the one place that decides what `limit` and `offset` mean** for the
30/// graph's list lenses — [`debt_density`], [`config_secrets`] and [`coupling`]
31/// here, and the `/nodes` and `/hotspots` endpoints in the `roteiro` binary. It
32/// exists because the parameter previously had two implementations: three lenses
33/// truncated here and treated `0` as "no limit", while two HTTP handlers used
34/// [`Iterator::take`] and so returned nothing for `0` — the same parameter name
35/// with opposite meanings, and nothing that could make the disagreement visible
36/// (issue #375). A sixth list lens should call this rather than write a third.
37/// One did write a third — see *Episodic recall* below — and the warning is left
38/// standing because the next one will too.
39///
40/// The contract:
41///
42/// - **`limit == 0` means unlimited** — every item that survives `offset` is
43/// kept. This is the reading the CLI already documents (`roteiro
44/// config-secrets --help`: *"0 shows every secret-named key"*), so no
45/// published promise is withdrawn by making it universal; and it is the safer
46/// of the two, because a caller who passes an unset variable then gets more
47/// data than they meant to ask for rather than an empty page that reads like a
48/// truthful "nothing found".
49/// - **`offset` applies first, and `limit` to what remains.** So `offset = 20,
50/// limit = 0` is "skip the first 20, then every remaining item", not "skip 20,
51/// then nothing". An `offset` past the end yields an empty window rather than
52/// panicking — a page beyond the last one is empty, not an error.
53/// - A caller's reported `total` must be taken **before** this runs: every
54/// surface reports the pre-windowing population, so a cut page still says what
55/// it was cut from.
56///
57/// Ordering is the caller's job; this only removes, and only from the ends, so a
58/// deterministically-ordered input yields a deterministic window.
59///
60/// # The search channels call this too, and the unit there is *per channel*
61///
62/// [`search`] and the generated/memory channels behind [`search_channels`] used
63/// to keep a `limit == 0 => no hits` guard of their own — a third reading of one
64/// parameter name, in the same crate as the two #375 reconciled (issue #393).
65/// They now window here like every other lens, so `limit` has one definition and
66/// not a second implementation of it, which is exactly how the first two drifted.
67///
68/// What differs is the **unit**, not the rule: a search `limit` bounds *each
69/// channel* independently, so `0` is "every match, in every channel that was
70/// asked for", not "every match overall". That is a bounded request rather than
71/// "dump the graph": a channel's ranking only *orders* a set the query has
72/// already filtered — every token must appear in a hit — and a query with no
73/// tokens returns nothing at any limit, `0` included. Measured on this
74/// repository at 6,685 nodes: an unbounded one-token search returned ~2.7k hits
75/// in 0.24 s and a two-token one returned 3, against a full-population scan that
76/// every limit pays anyway, so unlimited costs no more than the default does.
77///
78/// The **MCP tools are the one surface that cannot ask for it**, deliberately:
79/// they clamp `limit` into `1..=25` and advertise `"minimum": 1`, because their
80/// results are spent against a model's context window and `0` would be the one
81/// value that escaped the ceiling those clamps exist to impose. That is a
82/// surface declining to offer a value, not a second meaning for it — a model
83/// that sends `0` anyway gets the smallest page, never the silent empty answer
84/// this issue is about. The reasoning is restated where each clamp lives, in
85/// `rto_render::mcp::GraphServer::search` and the served-chat `search` arm in
86/// the `roteiro` binary; if this rule changes, those two must change with it.
87///
88/// # Episodic recall — the third implementation this doc predicted (issue #447)
89///
90/// [`crate::Store::recall_memory`] ranked, then called
91/// [`Vec::truncate`] directly, so `limit = 0` emptied the result: `recall
92/// --limit 0` returned nothing on a store where five other surfaces returned
93/// everything, and the JSON said `"live": 8000` beside `"results": []` — not a
94/// lie, and no help at all. It calls this now. Two details are worth keeping:
95///
96/// - **`None` and `Some(0)` had to collapse onto one meaning, not two.**
97/// `RecallOptions::limit` is an `Option`, so "unlimited" was already sayable
98/// twice; the fix maps `None` to `0` rather than adding a branch, because two
99/// spellings of one request are how the first divergence started.
100/// - **`memory list` cuts in SQL and so cannot call this.** `LIMIT 0` in SQL
101/// means the *opposite* of what this function means, so `memory::records`
102/// omits the clause entirely for `0`. That is the contract translated, and it
103/// is the only place in the crate where the rule is re-expressed rather than
104/// called — worth knowing if it ever has to change.
105pub fn window<T>(items: &mut Vec<T>, offset: usize, limit: usize) {
106 // `min(len)` rather than a bounds check: `drain(..offset)` panics past the
107 // end of the vector, and "page 900 of 3" is an empty page, not a 500.
108 items.drain(..offset.min(items.len()));
109 if limit > 0 {
110 items.truncate(limit);
111 }
112}
113
114/// A compact node summary (used in listings and as the subject of an
115/// [`Explanation`]).
116#[derive(Debug, Clone, PartialEq, Serialize)]
117pub struct NodeSummary {
118 /// Natural key.
119 pub key: String,
120 /// Kind token (e.g. `fn`, `adr`).
121 pub kind: String,
122 /// Human-facing name.
123 pub name: String,
124 /// Repository-relative path, if any.
125 pub path: Option<String>,
126 /// Language token, if any.
127 pub lang: Option<String>,
128}
129
130impl NodeSummary {
131 fn from_node(node: &crate::Node) -> Self {
132 Self {
133 key: node.key.clone(),
134 kind: node.kind.as_str().to_owned(),
135 name: node.name.clone(),
136 path: node.path.clone(),
137 lang: node.lang.clone(),
138 }
139 }
140}
141
142/// One end of an edge as seen from a subject node: the relationship, how it was
143/// produced, and the node on the other end.
144#[derive(Debug, Clone, PartialEq, Serialize)]
145pub struct EdgeRef {
146 /// Edge kind token (e.g. `calls`, `references`).
147 pub kind: String,
148 /// How the edge was produced (`derived` | `authored` | `inferred`).
149 pub provenance: &'static str,
150 /// Confidence score, present only for inferred edges.
151 pub confidence: Option<f64>,
152 /// The natural key of the node at the other end.
153 pub node: String,
154}
155
156/// A node together with its provenance-labelled neighbourhood.
157#[derive(Debug, Clone, PartialEq, Serialize)]
158pub struct Explanation {
159 /// Stable schema tag ([`SCHEMA`]).
160 pub schema: &'static str,
161 /// The subject node.
162 pub node: NodeSummary,
163 /// Structured metadata attached to the node.
164 pub meta: serde_json::Value,
165 /// Edges where the subject is the source.
166 pub outgoing: Vec<EdgeRef>,
167 /// Edges where the subject is the destination.
168 pub incoming: Vec<EdgeRef>,
169}
170
171/// A listing of all nodes of one kind.
172#[derive(Debug, Clone, PartialEq, Serialize)]
173pub struct Listing {
174 /// Stable schema tag ([`SCHEMA`]).
175 pub schema: &'static str,
176 /// The kind that was listed.
177 pub kind: String,
178 /// Matching nodes, ordered by key.
179 pub nodes: Vec<NodeSummary>,
180}
181
182/// One intent-debt finding in a [`DebtReport`].
183#[derive(Debug, Clone, PartialEq, Serialize)]
184pub struct DebtItem {
185 /// Natural key of the marker node (`marker:<path>#<line>`).
186 pub key: String,
187 /// Category token (`todo` | `fixme` | `hack` | `stub` | `deferred`).
188 pub category: String,
189 /// The marker text (the trimmed source line).
190 pub text: String,
191 /// Repository-relative path of the source file, if any.
192 pub path: Option<String>,
193 /// 1-based line number, if recorded.
194 pub line: Option<u32>,
195}
196
197/// The intent-debt inventory: every `marker` node, grouped and listed. A
198/// deterministic, provenance-`derived` view of what is incomplete or postponed.
199#[derive(Debug, Clone, PartialEq, Serialize)]
200pub struct DebtReport {
201 /// Stable schema tag ([`SCHEMA`]).
202 pub schema: &'static str,
203 /// Total markers in the report (after any category filter).
204 pub total: usize,
205 /// Count per category, ordered by category token.
206 pub by_category: BTreeMap<String, usize>,
207 /// The markers, ordered by `(path, line, key)`.
208 pub items: Vec<DebtItem>,
209}
210
211/// Inventory intent-debt markers in the graph, optionally restricted to the
212/// given `categories` (empty means all) and excluding markers whose file path
213/// matches any `ignore` glob (config `[debt] ignore` — empty means keep all).
214/// Ordered by `(path, line)` so output is stable and reads top-to-bottom per
215/// file; `total` and `by_category` reflect the retained markers only.
216///
217/// # Errors
218/// Returns [`StoreError`] on query failure.
219pub fn debt(
220 store: &Store,
221 categories: &[String],
222 ignore: &[String],
223) -> Result<DebtReport, StoreError> {
224 let filter: std::collections::BTreeSet<&str> = categories.iter().map(String::as_str).collect();
225 let mut items = Vec::new();
226 let mut by_category: BTreeMap<String, usize> = BTreeMap::new();
227 for node in store.nodes_by_kind(&NodeKind::Marker)? {
228 let category = node
229 .meta
230 .get("category")
231 .and_then(serde_json::Value::as_str)
232 .unwrap_or("other")
233 .to_owned();
234 if !filter.is_empty() && !filter.contains(category.as_str()) {
235 continue;
236 }
237 // Drop markers under an ignored path (e.g. `vendor/**`) before counting.
238 if let Some(path) = node.path.as_deref()
239 && ignore.iter().any(|glob| glob_match(glob, path))
240 {
241 continue;
242 }
243 let text = node
244 .meta
245 .get("text")
246 .and_then(serde_json::Value::as_str)
247 .unwrap_or(node.name.as_str())
248 .to_owned();
249 let line = node
250 .meta
251 .get("line")
252 .and_then(serde_json::Value::as_u64)
253 .and_then(|l| u32::try_from(l).ok());
254 *by_category.entry(category.clone()).or_default() += 1;
255 items.push(DebtItem {
256 key: node.key.clone(),
257 category,
258 text,
259 path: node.path.clone(),
260 line,
261 });
262 }
263 items.sort_by(|a, b| (&a.path, a.line, &a.key).cmp(&(&b.path, b.line, &b.key)));
264 Ok(DebtReport {
265 schema: SCHEMA,
266 total: items.len(),
267 by_category,
268 items,
269 })
270}
271
272/// How a [`DebtDensityReport`]'s files are ranked.
273#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
274pub enum DensityOrder {
275 /// By `per_kloc` — markers relative to file length. The lens's own question.
276 #[default]
277 Density,
278 /// By `markers` — the raw count, which [`debt`] already reports per marker.
279 /// Offered so the two rankings can be compared on one report rather than the
280 /// reader being asked to trust that they differ.
281 Markers,
282 /// By `lines` — longest file first. Not a debt ranking; the control that
283 /// shows *which* files the denominator is large for.
284 Lines,
285}
286
287impl DensityOrder {
288 /// The stable token for this order, as accepted by [`from_token`](Self::from_token).
289 #[must_use]
290 pub fn as_str(self) -> &'static str {
291 match self {
292 Self::Density => "density",
293 Self::Markers => "markers",
294 Self::Lines => "lines",
295 }
296 }
297
298 /// Parse an order token. `None` for anything else — callers surface an error
299 /// rather than silently ranking by something the caller did not ask for.
300 #[must_use]
301 pub fn from_token(s: &str) -> Option<Self> {
302 match s {
303 "density" => Some(Self::Density),
304 "markers" => Some(Self::Markers),
305 "lines" => Some(Self::Lines),
306 _ => None,
307 }
308 }
309
310 /// The tokens [`from_token`](Self::from_token) accepts, for error messages
311 /// and argument schemas — so the accepted set is stated in exactly one place.
312 #[must_use]
313 pub fn tokens() -> [&'static str; 3] {
314 [
315 Self::Density.as_str(),
316 Self::Markers.as_str(),
317 Self::Lines.as_str(),
318 ]
319 }
320}
321
322/// The default `min_lines` floor for [`debt_density`]: files shorter than this
323/// are counted but not ranked (see [`DebtDensityReport::min_lines`] for why the
324/// floor exists at all).
325///
326/// 50 because that is where one marker stops dominating: a single marker in a
327/// 50-line file scores 20 per kloc, which is already near the top of this
328/// repository's real ranking, so any shorter file with a marker is guaranteed a
329/// high placement by its length alone. It is a default, not a rule — `0` ranks
330/// every file.
331pub const DEFAULT_MIN_LINES: u32 = 50;
332
333/// One file's intent-debt density in a [`DebtDensityReport`].
334#[derive(Debug, Clone, PartialEq, Serialize)]
335pub struct DensityItem {
336 /// Repository-relative path of the file.
337 pub path: String,
338 /// Retained markers in this file (after category and `ignore` filtering).
339 pub markers: u32,
340 /// The file's length in lines — the denominator. See [`debt_density`] for
341 /// exactly what this counts and what it does not.
342 pub lines: u32,
343 /// Markers per 1,000 lines, rounded to two decimals. Per *kilo*-line rather
344 /// than per line because per-line densities are all leading zeroes: this
345 /// repository's worst file is 0.06 markers per line, and `60.0` per kloc is
346 /// a number a reader can hold.
347 pub per_kloc: f64,
348 /// Count per category within this file, ordered by category token — so a
349 /// dense file can be read as "twelve `todo`" or "twelve `stub`", which are
350 /// not the same finding.
351 pub by_category: BTreeMap<String, usize>,
352}
353
354/// Intent-debt **density**: markers per file normalised by file length, ranked.
355/// The counterpart to [`debt`], which reports markers and therefore ranks large
356/// files first by construction.
357#[derive(Debug, Clone, PartialEq, Serialize)]
358pub struct DebtDensityReport {
359 /// Stable schema tag ([`SCHEMA`]).
360 pub schema: &'static str,
361 /// The ranking that produced `items` ([`DensityOrder::as_str`]).
362 pub order: &'static str,
363 /// The requested cap on `items`; `0` means unlimited.
364 pub limit: usize,
365 /// The `min_lines` floor applied. Files shorter than this are excluded from
366 /// the ranking and counted in `short_files`; `0` disables the floor.
367 ///
368 /// The floor exists because density is unstable in the denominator's tail: a
369 /// 3-line stub file with one marker scores 333 per kloc, which is true and
370 /// tells the reader nothing. Excluding those files is a ranking decision,
371 /// not a suppression — they stay in `files_with_markers` and `total_markers`.
372 pub min_lines: u32,
373 /// Distinct files carrying at least one retained marker, before the
374 /// `min_lines` floor. The population `items` is drawn from.
375 pub files_with_markers: usize,
376 /// Files that passed the `min_lines` floor and were therefore ranked.
377 /// `ranked_files > items.len()` means `limit` truncated the list.
378 pub ranked_files: usize,
379 /// Files excluded from the ranking by the `min_lines` floor — reported, not
380 /// silently dropped, so a short-file-heavy repository cannot read as a clean one.
381 pub short_files: usize,
382 /// Files whose marker count is known but whose length is **not**: no `file`
383 /// node, or one carrying no `meta.lines`. Excluded from the ranking, because
384 /// a density with no denominator is not a number — and reported, because
385 /// silently omitting them would understate the inventory.
386 pub unknown_length_files: usize,
387 /// Retained markers across every file in `files_with_markers`, including
388 /// those the floor excluded. Matches [`DebtReport::total`] for the same
389 /// filters, minus any marker with no `path`.
390 pub total_markers: usize,
391 /// Summed `lines` of the ranked files. The denominator behind
392 /// `overall_per_kloc`.
393 pub total_lines: u64,
394 /// Markers per 1,000 lines across the **ranked** files taken together — the
395 /// baseline a single file's `per_kloc` should be read against. `0.0` when
396 /// nothing was ranked.
397 pub overall_per_kloc: f64,
398 /// The ranked files: by `order` descending, ties broken by `path` ascending.
399 pub items: Vec<DensityItem>,
400}
401
402/// Rank files by intent-debt **density** — retained markers per 1,000 lines —
403/// most-dense first by `order`, capped at `limit` (`0` = unlimited). `categories`
404/// and `ignore` filter markers exactly as [`debt`] does, so the two lenses always
405/// agree about which markers exist.
406///
407/// # Why this is not [`debt`] with a division
408///
409/// A raw marker count ranks by file size: the biggest file wins because it has
410/// the most lines to put a marker on. Density asks the different question — *how
411/// concentrated is the debt* — and a 40-marker file of 4,000 lines and a
412/// 40-marker file of 200 lines separate by a factor of twenty under it while
413/// being indistinguishable under [`debt`].
414///
415/// # The denominator, and why it is this one
416///
417/// **`lines` is the `file` node's `meta.lines`**, recorded at extraction time as
418/// the count of `\n` bytes in the blob. It is read straight from the graph, so
419/// this lens adds **no extraction metadata and needs no `EXTRACT_VERSION` bump**.
420///
421/// It is deliberately *not* derived from [`crate::Span`]: a node's span is a pair
422/// of **byte offsets**, not line numbers, and there is no line index in the store
423/// to convert one to the other. Anything span-derived would be a byte density,
424/// which is not the quantity anyone means by "debt density".
425///
426/// Three alternatives were rejected, each for the same reason:
427///
428/// - **Source lines of code** (blank and comment lines removed) is the denominator
429/// a reader probably imagines. It does not exist in the graph and cannot be
430/// computed from it: producing it means counting lines per language at
431/// extraction, which is net-new derived metadata and would move this lens into
432/// the batch that pays for an `EXTRACT_VERSION` bump.
433/// - **Per symbol** rather than per file would be the finer-grained view — markers
434/// already attach to their innermost enclosing symbol. But a symbol's length in
435/// *lines* is exactly what `Span`'s byte offsets cannot give.
436/// - **The highest marker line in the file** is available (`meta.line`), and is a
437/// lower bound on the file's length rather than the length: a file whose only
438/// marker is on line 3 would score 333 per kloc however long it is.
439///
440/// So `lines` is what the graph honestly has. What it counts, stated plainly
441/// because the name invites over-reading:
442///
443/// - **Every line, including blanks, comments, imports and licence headers.** It
444/// is *file length*, not "lines of code". Density figures here are therefore
445/// systematically lower than an SLOC-based tool's, and by a different factor
446/// per language and per file.
447/// - **Newline bytes.** A file not ending in a newline is counted one line short,
448/// and a file of a single unterminated line counts as `0` lines and is reported
449/// under `unknown_length_files` rather than divided by zero.
450/// - **The whole blob, vendored code included.** A minified bundle is one enormous
451/// line and will look flawless. Use `ignore` (the shared `[debt] ignore` globs)
452/// rather than a second exclusion vocabulary.
453///
454/// # Confidence, and why there is no CI gate
455///
456/// Density inherits every false positive of the marker scan beneath it — the
457/// prose rules (`for now`, `deferred`, `tbd`) fire on ordinary writing, so a
458/// design document rich in the word "deferred" ranks as dense debt. It then adds
459/// one of its own: the denominator is file length, so a language or a file with
460/// low information per line (verbose config, generated code, wide indentation) is
461/// systematically flattered, and a dense language is systematically penalised.
462/// Neither is a defect being reported; both move the number. A gate would fail
463/// builds on prose and on formatting. So this lens **offers no CI gate**, and its
464/// suppression story is the one that already exists: `[debt] ignore` globs and
465/// the `roteiro:ignore` / `roteiro:ignore-file` source directives, both applied
466/// before anything is counted here.
467///
468/// Ordering is total and deterministic: by the chosen metric descending, then by
469/// `path` ascending, so identical input yields byte-identical output.
470///
471/// # Errors
472/// Returns [`StoreError`] on query failure.
473pub fn debt_density(
474 store: &Store,
475 categories: &[String],
476 ignore: &[String],
477 order: DensityOrder,
478 limit: usize,
479 min_lines: u32,
480) -> Result<DebtDensityReport, StoreError> {
481 // Reuse `debt` rather than re-walking the markers: the two lenses must never
482 // disagree about which markers exist, and the only way to guarantee that is
483 // for one to be built from the other's output.
484 let inventory = debt(store, categories, ignore)?;
485 let mut per_file: BTreeMap<String, BTreeMap<String, usize>> = BTreeMap::new();
486 let mut total_markers = 0usize;
487 for item in &inventory.items {
488 // A marker with no `path` cannot be attributed to a file, so it cannot
489 // have a density. Extraction always records one; this is defence in
490 // depth, and such a marker is left out of `total_markers` too so the
491 // report's own arithmetic stays consistent.
492 let Some(path) = item.path.as_deref() else {
493 continue;
494 };
495 *per_file
496 .entry(path.to_owned())
497 .or_default()
498 .entry(item.category.clone())
499 .or_default() += 1;
500 total_markers += 1;
501 }
502 let files_with_markers = per_file.len();
503
504 // Only files that actually carry a marker are read back, so the denominator
505 // costs one node lookup per such file rather than a whole-graph file scan —
506 // which matters because `file` nodes carry captured `meta.content`.
507 let mut ranked: Vec<(String, u32, u32, BTreeMap<String, usize>)> = Vec::new();
508 let mut short_files = 0usize;
509 let mut unknown_length_files = 0usize;
510 for (path, by_category) in per_file {
511 let markers = u32::try_from(by_category.values().sum::<usize>()).unwrap_or(u32::MAX);
512 let Some(lines) = file_lines(store, &path)? else {
513 unknown_length_files += 1;
514 continue;
515 };
516 if lines < min_lines {
517 short_files += 1;
518 continue;
519 }
520 ranked.push((path, markers, lines, by_category));
521 }
522 let ranked_files = ranked.len();
523 let total_lines: u64 = ranked
524 .iter()
525 .map(|(_, _, lines, _)| u64::from(*lines))
526 .sum();
527 let ranked_markers: u64 = ranked.iter().map(|(_, m, _, _)| u64::from(*m)).sum();
528
529 ranked.sort_by(|a, b| {
530 // Each order yields the metric as an exact `numerator / denominator`, so
531 // the comparison can cross-multiply. Ranking on the rounded `per_kloc`
532 // instead would make genuinely different densities tie and then break on
533 // `path`, silently reordering the ranking the caller asked for; ranking
534 // on `f64` at full precision would make the order depend on the last bit
535 // of a division. `u128` so the cross-product cannot overflow whatever the
536 // file lengths are, rather than relying on repositories staying small.
537 let metric =
538 |&(_, markers, lines, _): &(String, u32, u32, BTreeMap<String, usize>)| match order {
539 DensityOrder::Density => (u128::from(markers) * 1000, u128::from(lines)),
540 DensityOrder::Markers => (u128::from(markers), 1),
541 DensityOrder::Lines => (u128::from(lines), 1),
542 };
543 let (an, ad) = metric(a);
544 let (bn, bd) = metric(b);
545 (bn * ad).cmp(&(an * bd)).then_with(|| a.0.cmp(&b.0))
546 });
547 window(&mut ranked, 0, limit);
548
549 let items = ranked
550 .into_iter()
551 .map(|(path, markers, lines, by_category)| DensityItem {
552 path,
553 markers,
554 lines,
555 per_kloc: per_kloc(u64::from(markers), u64::from(lines)),
556 by_category,
557 })
558 .collect();
559
560 Ok(DebtDensityReport {
561 schema: SCHEMA,
562 order: order.as_str(),
563 limit,
564 min_lines,
565 files_with_markers,
566 ranked_files,
567 short_files,
568 unknown_length_files,
569 total_markers,
570 total_lines,
571 overall_per_kloc: per_kloc(ranked_markers, total_lines),
572 items,
573 })
574}
575
576/// A file's length in lines from its `file` node's `meta.lines`, or `None` when
577/// the node is absent, carries no `lines`, or reports **zero** lines.
578///
579/// Zero is folded into `None` deliberately: it is not a length that can be
580/// divided by, and the two cases a reader would want distinguished — a genuinely
581/// empty file and a single line with no terminating newline — are
582/// indistinguishable in a newline count. Reporting both as "length unknown" is
583/// the honest reading; reporting either as a density is not.
584fn file_lines(store: &Store, path: &str) -> Result<Option<u32>, StoreError> {
585 let Some(node) = store.get_node(&format!("file:{path}"))? else {
586 return Ok(None);
587 };
588 Ok(node
589 .meta
590 .get("lines")
591 .and_then(serde_json::Value::as_u64)
592 .and_then(|n| u32::try_from(n).ok())
593 .filter(|&n| n > 0))
594}
595
596/// Markers per 1,000 lines, rounded to two decimals; `0.0` for a zero
597/// denominator, which callers have already excluded from any ranking.
598fn per_kloc(markers: u64, lines: u64) -> f64 {
599 if lines == 0 {
600 return 0.0;
601 }
602 // `u64 as f64` is lossy above 2^53; a line count or marker count that large
603 // is not reachable from a repository on disk, and the precision lost would be
604 // below the two decimals this rounds to anyway.
605 #[expect(clippy::cast_precision_loss, reason = "counts are far below 2^53")]
606 let ratio = (markers as f64) * 1000.0 / (lines as f64);
607 round2(ratio)
608}
609
610/// The redaction state of one config key in a [`ConfigSecretReport`]. Three
611/// states, because collapsing them would misreport two of them: "declared in
612/// code" is not a redaction, and "value present" is not a safe one.
613#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
614#[serde(rename_all = "snake_case")]
615pub enum RedactionState {
616 /// The value was read from a source file and **replaced** with the redaction
617 /// placeholder before anything was persisted. The expected state for a
618 /// secret-named key extracted from a config file.
619 Redacted,
620 /// The key carries **no value at all** — a struct-derived key
621 /// (`meta.source = "struct"`), a config *field* declared in Rust with no
622 /// literal in the code to redact.
623 ///
624 /// Not a redaction and not a leak: it records that a setting by this name
625 /// exists, which is the inventory's job, and nothing about any value.
626 Declared,
627 /// The key carries a value that is **not** the redaction placeholder.
628 ///
629 /// Extraction redacts every secret-named key, so this state is unreachable
630 /// from extraction alone. It is reachable through
631 /// [`Store::apply_import_layer`](crate::Store::apply_import_layer), which
632 /// upserts whatever nodes an imported factset carries — so an import produced
633 /// by another tool, or by an older Roteiro, can put an unredacted value in the
634 /// store. That is worth reporting loudly, and it is a finding about **this
635 /// store**, not about the source repository.
636 Present,
637}
638
639impl RedactionState {
640 /// The stable token for this state, as serialised.
641 #[must_use]
642 pub fn as_str(self) -> &'static str {
643 match self {
644 Self::Redacted => "redacted",
645 Self::Declared => "declared",
646 Self::Present => "present",
647 }
648 }
649}
650
651/// One secret-named config key in a [`ConfigSecretReport`].
652#[derive(Debug, Clone, PartialEq, Serialize)]
653pub struct ConfigSecretItem {
654 /// Natural key of the config-key node (`cfgkey:<path>#<dotted>`).
655 pub key: String,
656 /// Repository-relative path of the config file or Rust source it came from.
657 pub path: Option<String>,
658 /// The dotted key name (e.g. `serve.api_token`). **Names only** — no value is
659 /// carried here, by construction as much as by choice: the value in the store
660 /// is the redaction placeholder.
661 pub name: String,
662 /// Whether the value was redacted, absent, or present.
663 pub state: RedactionState,
664 /// `meta.source`, when the node records one — `struct` for a key synthesised
665 /// from a `@rto:config` Rust struct. Absent for a file-derived key.
666 pub source: Option<String>,
667}
668
669/// An inventory of **secret-named** config keys and their redaction state.
670///
671/// See [`config_secrets`] for what this is and — more importantly — what it is
672/// not.
673#[derive(Debug, Clone, PartialEq, Serialize)]
674pub struct ConfigSecretReport {
675 /// Stable schema tag ([`SCHEMA`]).
676 pub schema: &'static str,
677 /// The requested cap on `items`; `0` means unlimited.
678 pub limit: usize,
679 /// Every `config_key` node in the graph, secret-named or not — the population
680 /// the inventory was drawn from.
681 pub config_keys: usize,
682 /// Config keys whose **name** matched the secret-name heuristic. `items` is
683 /// the first `limit` of these, so `secret_named > items.len()` means truncation.
684 pub secret_named: usize,
685 /// Of `secret_named`: how many carry the redaction placeholder.
686 pub redacted: usize,
687 /// Of `secret_named`: how many carry no value at all (struct-derived).
688 pub declared: usize,
689 /// Of `secret_named`: how many carry a value that is **not** the placeholder.
690 ///
691 /// **Expected to be zero.** A non-zero count is a finding about this store —
692 /// see [`RedactionState::Present`] for the one path that reaches it.
693 pub unredacted: usize,
694 /// Config keys that are redacted but whose name is **not** secret-looking: a
695 /// Kubernetes `Secret`'s `data`, redacted because of where it lives rather
696 /// than what it is called.
697 ///
698 /// Reported so the redaction counts reconcile against the graph: without it a
699 /// reader comparing `redacted` to the number of `<redacted>` values in the
700 /// store would find an unexplained surplus.
701 pub redacted_not_secret_named: usize,
702 /// Distinct files carrying at least one secret-named key.
703 pub files: usize,
704 /// The secret-named keys, ordered by `(path, name, key)`.
705 pub items: Vec<ConfigSecretItem>,
706}
707
708/// Inventory the **secret-named** config keys in the graph — where they are, what
709/// they are called, and whether their values were redacted before persistence —
710/// capped at `limit` (`0` = unlimited).
711///
712/// # What this reports
713///
714/// Config extraction (ADR-0009) flattens TOML/JSON/YAML/`.env` into `config_key`
715/// nodes, and **redacts the value of any secret-named key before it reaches the
716/// store** (see [`crate::config_keys::REDACTED`] and the redaction sites it
717/// names). This lens reads that back: *secret-named config keys are present, here
718/// are their paths and names, and here is their redaction state*. It is an
719/// **inventory with an invariant check**, and it is useful for exactly two
720/// questions: which of my config surfaces deal in credentials, and did anything
721/// unredacted get into this graph.
722///
723/// # What this CANNOT do — read this before extending it
724///
725/// **It is not a secret scanner and this architecture cannot make it one.** The
726/// lens is named for the inventory it can be, not the scanner the shortlist's
727/// original title promised.
728///
729/// - **It cannot detect a hardcoded credential in source code.** It reads
730/// `config_key` nodes, which come only from config *files* and from
731/// `@rto:config` struct declarations. An AWS key pasted into a `.rs` string
732/// literal produces no `config_key` node and is invisible here. Nothing about
733/// the node kinds this reads can change that.
734/// - **It cannot judge validity.** It never sees a value: by the time anything is
735/// in the store, a secret-named value has already been replaced. There is no
736/// entropy test, no format check, no liveness probe, and there cannot be one
737/// without persisting the very thing extraction exists to redact.
738/// - **It cannot tell a real secret from a placeholder.** `API_TOKEN=changeme` in
739/// a committed `.env.example` and a genuine token in an uncommitted `.env` are
740/// the same row here: same key name, same redacted value, same state.
741/// - **It cannot say a repository has no secrets.** An empty report means "no
742/// secret-*named* config key", which is a statement about naming. A credential
743/// under an innocuous key (`endpoint`, `dsn`, `url`) is not secret-named, is not
744/// redacted, and does not appear.
745///
746/// If you find yourself wanting to widen this toward detecting real credentials,
747/// **that instinct is what the rename exists to prevent**: the widening cannot be
748/// built on these inputs, and a tool that half-does it while being named for the
749/// whole job is worse than one that does the inventory honestly. Every surface
750/// carries this limitation in its own words, so a model calling the tool passes it
751/// on rather than reporting a security guarantee that was never offered.
752///
753/// # The heuristic, stated
754///
755/// "Secret-named" is [`crate::config_keys::is_secret_key`]: the key's
756/// ASCII-alphanumerics, lowercased, containing any of `secret`, `password`,
757/// `passwd`, `passphrase`, `token`, `apikey`, `credential`, `privatekey`,
758/// `accesskey`, `pwd`. So it matches `API_TOKEN`, `db.passwordFile` and
759/// `serve.apiKey`, and misses `dsn`, `connection_string` and `auth` — and it
760/// false-positives on `token_bucket_size` and `csrf_token_header`, which are
761/// settings, not secrets. Both directions of error are inherent to matching on
762/// names; neither is reported as a finding.
763///
764/// # Ordering
765///
766/// By `(path, name, key)` ascending — an inventory, like [`debt`], not a ranking.
767/// There is deliberately no ordering knob: nothing here is a magnitude worth
768/// sorting by, and offering one would suggest some keys are more secret than
769/// others. Identical input yields byte-identical output.
770///
771/// # Errors
772/// Returns [`StoreError`] on query failure.
773pub fn config_secrets(store: &Store, limit: usize) -> Result<ConfigSecretReport, StoreError> {
774 let nodes = store.nodes_by_kind(&NodeKind::Other(crate::config_keys::KIND.to_owned()))?;
775 let config_keys = nodes.len();
776 let mut items = Vec::new();
777 let mut redacted = 0usize;
778 let mut declared = 0usize;
779 let mut unredacted = 0usize;
780 let mut redacted_not_secret_named = 0usize;
781 let mut files: BTreeSet<String> = BTreeSet::new();
782 for node in nodes {
783 // Prefer `meta.key` — the dotted key as the source spelled it — and fall
784 // back to the node's name, which extraction sets to the same string.
785 let name = node
786 .meta
787 .get("key")
788 .and_then(serde_json::Value::as_str)
789 .unwrap_or(node.name.as_str())
790 .to_owned();
791 let value = node.meta.get("value").and_then(serde_json::Value::as_str);
792 if !crate::config_keys::is_secret_key(&name) {
793 // A redacted value under a non-secret name is a k8s `Secret`'s data:
794 // counted so the report's redaction figures reconcile with the graph,
795 // but not listed — this lens's subject is secret-*named* keys.
796 if value == Some(crate::config_keys::REDACTED) {
797 redacted_not_secret_named += 1;
798 }
799 continue;
800 }
801 let state = match value {
802 Some(v) if v == crate::config_keys::REDACTED => {
803 redacted += 1;
804 RedactionState::Redacted
805 }
806 // A struct-derived key omits `meta.value` entirely: there is no
807 // literal in the code to redact, so "absent" is the honest state
808 // rather than folding it in with a successful redaction.
809 None => {
810 declared += 1;
811 RedactionState::Declared
812 }
813 Some(_) => {
814 unredacted += 1;
815 RedactionState::Present
816 }
817 };
818 if let Some(path) = node.path.as_deref() {
819 files.insert(path.to_owned());
820 }
821 items.push(ConfigSecretItem {
822 key: node.key,
823 path: node.path,
824 name,
825 state,
826 source: node
827 .meta
828 .get("source")
829 .and_then(serde_json::Value::as_str)
830 .map(str::to_owned),
831 });
832 }
833 let secret_named = items.len();
834 items.sort_by(|a, b| (&a.path, &a.name, &a.key).cmp(&(&b.path, &b.name, &b.key)));
835 window(&mut items, 0, limit);
836
837 Ok(ConfigSecretReport {
838 schema: SCHEMA,
839 limit,
840 config_keys,
841 secret_named,
842 redacted,
843 declared,
844 unredacted,
845 redacted_not_secret_named,
846 files: files.len(),
847 items,
848 })
849}
850
851/// Match a slash-separated `path` against a glob `pattern`, anchored end-to-end.
852/// `?` matches one non-`/` character, `*` matches any run within a single path
853/// segment, and `**` matches zero or more whole segments. Used for config
854/// `[debt] ignore` patterns (e.g. `vendor/**`, `**/generated/*`).
855#[must_use]
856fn glob_match(pattern: &str, path: &str) -> bool {
857 let pat: Vec<&str> = pattern.split('/').collect();
858 let seg: Vec<&str> = path.split('/').collect();
859 match_segments(&pat, &seg)
860}
861
862/// Anchored match of glob segments `pat` against path segments `seg`, with `**`
863/// consuming zero or more segments.
864fn match_segments(pat: &[&str], seg: &[&str]) -> bool {
865 match pat.first() {
866 None => seg.is_empty(),
867 Some(&"**") => (0..=seg.len()).any(|i| match_segments(&pat[1..], &seg[i..])),
868 Some(token) => {
869 !seg.is_empty() && match_token(token, seg[0]) && match_segments(&pat[1..], &seg[1..])
870 }
871 }
872}
873
874/// Match a single path segment `s` against a `pattern` token containing `*`
875/// (any run, no `/`) and `?` (one char, no `/`).
876fn match_token(pattern: &str, s: &str) -> bool {
877 let pat: Vec<char> = pattern.chars().collect();
878 let chars: Vec<char> = s.chars().collect();
879 match_token_chars(&pat, &chars)
880}
881
882/// Recursive char-slice matcher backing [`match_token`].
883fn match_token_chars(pat: &[char], chars: &[char]) -> bool {
884 match pat.first() {
885 None => chars.is_empty(),
886 Some('*') => (0..=chars.len()).any(|i| match_token_chars(&pat[1..], &chars[i..])),
887 Some('?') => !chars.is_empty() && match_token_chars(&pat[1..], &chars[1..]),
888 Some(&ch) => {
889 !chars.is_empty() && chars[0] == ch && match_token_chars(&pat[1..], &chars[1..])
890 }
891 }
892}
893
894/// How a [`CouplingReport`]'s items are ranked. The three orders answer three
895/// different questions, which a single undirected degree cannot tell apart.
896#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
897pub enum CouplingOrder {
898 /// By `fan_in + fan_out` — overall call coupling.
899 #[default]
900 Total,
901 /// By `fan_in` — the most depended-on symbols ("what calls this?").
902 FanIn,
903 /// By `fan_out` — the symbols that reach furthest ("what does this call?").
904 FanOut,
905}
906
907impl CouplingOrder {
908 /// The stable token for this order, as accepted by [`from_token`](Self::from_token).
909 #[must_use]
910 pub fn as_str(self) -> &'static str {
911 match self {
912 Self::Total => "total",
913 Self::FanIn => "fan_in",
914 Self::FanOut => "fan_out",
915 }
916 }
917
918 /// Parse an order token. `None` for anything else — callers surface an error
919 /// rather than silently ranking by something the caller did not ask for.
920 #[must_use]
921 pub fn from_token(s: &str) -> Option<Self> {
922 match s {
923 "total" => Some(Self::Total),
924 "fan_in" => Some(Self::FanIn),
925 "fan_out" => Some(Self::FanOut),
926 _ => None,
927 }
928 }
929
930 /// The tokens [`from_token`](Self::from_token) accepts, for error messages
931 /// and argument schemas — so the accepted set is stated in exactly one place.
932 #[must_use]
933 pub fn tokens() -> [&'static str; 3] {
934 [
935 Self::Total.as_str(),
936 Self::FanIn.as_str(),
937 Self::FanOut.as_str(),
938 ]
939 }
940}
941
942/// One node's **directed** call coupling in a [`CouplingReport`].
943#[derive(Debug, Clone, PartialEq, Serialize)]
944pub struct CouplingItem {
945 /// Natural key of the node.
946 pub key: String,
947 /// Kind token (e.g. `fn`).
948 pub kind: String,
949 /// Human-facing name.
950 pub name: String,
951 /// Repository-relative path, if any.
952 pub path: Option<String>,
953 /// How many **distinct** other nodes call this one.
954 pub fan_in: u32,
955 /// How many **distinct** other nodes this one calls.
956 pub fan_out: u32,
957 /// `fan_in + fan_out` — the directed equivalent of the undirected degree.
958 pub total: u32,
959 /// Martin's instability, `fan_out / (fan_in + fan_out)`, rounded to two
960 /// decimals. `0.0` = purely depended-on (stable); `1.0` = purely depending
961 /// (unstable). The denominator is never zero: an item exists only when it
962 /// has at least one non-self call edge.
963 pub instability: f64,
964}
965
966/// Directed call coupling: per-node fan-in and fan-out over `Calls` edges,
967/// ranked. The counterpart to an undirected degree ranking, which cannot tell
968/// "everything calls this" from "this calls everything".
969#[derive(Debug, Clone, PartialEq, Serialize)]
970pub struct CouplingReport {
971 /// Stable schema tag ([`SCHEMA`]).
972 pub schema: &'static str,
973 /// The edge kind measured. Always `calls` — the only edge kind whose
974 /// direction carries a caller/callee meaning.
975 pub edge_kind: &'static str,
976 /// The ranking that produced `items` ([`CouplingOrder::as_str`]).
977 pub order: &'static str,
978 /// The requested cap on `items`; `0` means unlimited.
979 pub limit: usize,
980 /// Total `Calls` edges scanned, including duplicates and self-calls.
981 pub call_edges: usize,
982 /// Self-referential `Calls` edges (recursion), counted in `call_edges` but
983 /// excluded from every fan — see [`coupling`].
984 pub self_calls: usize,
985 /// `Calls` edges whose endpoints are in two different languages: name
986 /// collisions from simple-name call resolution, not calls. Counted in
987 /// `call_edges` but excluded from every fan — see [`coupling`].
988 pub cross_language_calls: usize,
989 /// Distinct nodes with at least one non-self `Calls` edge. `items` is the
990 /// top `limit` of these, so `coupled_nodes > items.len()` means truncation.
991 pub coupled_nodes: usize,
992 /// The ranked nodes: by `order` descending, ties broken by `key` ascending.
993 pub items: Vec<CouplingItem>,
994}
995
996/// Rank nodes by **directed** call coupling — fan-in (distinct callers) and
997/// fan-out (distinct callees) over `Calls` edges — most-coupled first by
998/// `order`, capped at `limit` (`0` = unlimited).
999///
1000/// Three deliberate counting rules, all of which change the numbers:
1001///
1002/// - **Distinct counterparts, not edges.** Edges are a set per `(src, dst, kind,
1003/// provenance)`, which still admits *parallel* `Calls` edges between one pair
1004/// at different provenances — a `derived` extraction and an `inferred`
1005/// suggestion of the same call. Counting distinct counterpart keys makes
1006/// `fan_in` mean "how many things depend on this", which is the coupling
1007/// question, rather than "how many layers asserted the dependency".
1008/// - **Self-calls are excluded from both fans.** Recursion is a real edge but
1009/// couples a node to nothing outside itself, and counting it would inflate
1010/// `fan_in` *and* `fan_out` for the same node. It is reported separately as
1011/// `self_calls` rather than silently dropped.
1012/// - **Cross-language call edges are excluded.** Roteiro extracts no FFI, so a
1013/// `Calls` edge between two languages is never a call — see
1014/// [`same_language`]. Reported as `cross_language_calls`.
1015///
1016/// Ordering is total and deterministic: by the chosen metric descending, then by
1017/// `key` ascending, so identical input yields byte-identical output.
1018///
1019/// # Precision
1020///
1021/// `fan_in` is exactly as precise as the `Calls` edges beneath it, and those are
1022/// resolved by **simple name**: a callee that is unique by bare name anywhere in
1023/// the repository binds to that definition, wherever it lives. So a single
1024/// same-language helper with a very common name absorbs every call to that name,
1025/// and its `fan_in` reads high for a reason that has nothing to do with design.
1026/// Excluding cross-language edges removes the worst of this, but not all of it.
1027/// Treat a large `fan_in` on a short, generically-named function as a question,
1028/// not a finding — which is also why this lens offers no CI gate.
1029///
1030/// # Errors
1031/// Returns [`StoreError`] on query failure.
1032pub fn coupling(
1033 store: &Store,
1034 order: CouplingOrder,
1035 limit: usize,
1036) -> Result<CouplingReport, StoreError> {
1037 // `inbound`: dst key -> distinct src keys. `outbound`: src key -> distinct dst
1038 // keys. Named for the direction rather than caller/callee, which read alike.
1039 let mut inbound: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
1040 let mut outbound: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
1041 let mut call_edges = 0usize;
1042 let mut self_calls = 0usize;
1043 let mut cross_language_calls = 0usize;
1044 for edge in store.all_edges()? {
1045 if edge.kind != EdgeKind::Calls {
1046 continue;
1047 }
1048 call_edges += 1;
1049 if edge.src == edge.dst {
1050 self_calls += 1;
1051 continue;
1052 }
1053 if !same_language(&edge.src, &edge.dst) {
1054 cross_language_calls += 1;
1055 continue;
1056 }
1057 inbound
1058 .entry(edge.dst.clone())
1059 .or_default()
1060 .insert(edge.src.clone());
1061 outbound.entry(edge.src).or_default().insert(edge.dst);
1062 }
1063
1064 // Rank on the counts alone, so only the nodes that survive the cap are read
1065 // back from the store — a whole-graph node scan is not needed to answer a
1066 // top-N question.
1067 let keys: BTreeSet<&String> = inbound.keys().chain(outbound.keys()).collect();
1068 let coupled_nodes = keys.len();
1069 let mut ranked: Vec<(u32, u32, &String)> = keys
1070 .into_iter()
1071 .map(|key| {
1072 let fan_in = count_of(&inbound, key);
1073 let fan_out = count_of(&outbound, key);
1074 (fan_in, fan_out, key)
1075 })
1076 .collect();
1077 ranked.sort_by(|a, b| {
1078 let metric = |&(fan_in, fan_out, _): &(u32, u32, &String)| match order {
1079 CouplingOrder::Total => fan_in + fan_out,
1080 CouplingOrder::FanIn => fan_in,
1081 CouplingOrder::FanOut => fan_out,
1082 };
1083 metric(b).cmp(&metric(a)).then_with(|| a.2.cmp(b.2))
1084 });
1085 window(&mut ranked, 0, limit);
1086
1087 let mut items = Vec::with_capacity(ranked.len());
1088 for (fan_in, fan_out, key) in ranked {
1089 // `edges.src`/`edges.dst` are foreign keys into `nodes`, so a node behind
1090 // a call edge always exists; the guard is defence in depth, not a case.
1091 let Some(node) = store.get_node(key)? else {
1092 continue;
1093 };
1094 let total = fan_in + fan_out;
1095 items.push(CouplingItem {
1096 key: node.key,
1097 kind: node.kind.as_str().to_owned(),
1098 name: node.name,
1099 path: node.path,
1100 fan_in,
1101 fan_out,
1102 total,
1103 instability: round2(f64::from(fan_out) / f64::from(total)),
1104 });
1105 }
1106
1107 Ok(CouplingReport {
1108 schema: SCHEMA,
1109 edge_kind: EdgeKind::Calls.as_str(),
1110 order: order.as_str(),
1111 limit,
1112 call_edges,
1113 self_calls,
1114 cross_language_calls,
1115 coupled_nodes,
1116 items,
1117 })
1118}
1119
1120/// The language token of a symbol key (`sym:<lang>:<path>#<name>` → `<lang>`),
1121/// or `None` for any other key shape.
1122fn sym_lang(key: &str) -> Option<&str> {
1123 let rest = key.strip_prefix("sym:")?;
1124 let (lang, _) = rest.split_once(':')?;
1125 (!lang.is_empty()).then_some(lang)
1126}
1127
1128/// Whether a call edge's two endpoints are in the same language — `true` unless
1129/// both keys carry a language token and the tokens differ.
1130///
1131/// Cross-file call resolution binds a callee by **simple name** across every
1132/// `Fn` node in the repository, language included. Roteiro extracts no FFI, so
1133/// nothing in the graph can legitimately record a JavaScript function calling a
1134/// Rust one; such an edge is a name collision — a lone Rust `join` helper
1135/// absorbing every JavaScript `.join(…)` in the tree. Excluding them keeps a
1136/// language's coupling figures about that language.
1137///
1138/// Unknown-shaped keys (anything that is not `sym:<lang>:…`) are **kept**: this
1139/// filter removes edges it can prove span two languages, and never guesses.
1140fn same_language(src: &str, dst: &str) -> bool {
1141 match (sym_lang(src), sym_lang(dst)) {
1142 (Some(a), Some(b)) => a == b,
1143 _ => true,
1144 }
1145}
1146
1147/// The size of `key`'s counterpart set, as a `u32` (a node cannot have more
1148/// distinct counterparts than there are nodes, so the cast cannot realistically
1149/// saturate; saturating beats wrapping if it ever did).
1150fn count_of(map: &BTreeMap<String, BTreeSet<String>>, key: &str) -> u32 {
1151 map.get(key)
1152 .map_or(0, |set| u32::try_from(set.len()).unwrap_or(u32::MAX))
1153}
1154
1155/// Round to two decimals, so the serialised ratio is short and stable rather
1156/// than carrying the full binary expansion of a division.
1157fn round2(v: f64) -> f64 {
1158 (v * 100.0).round() / 100.0
1159}
1160
1161/// One step along a [`Path`]: the edge traversed and the node it leads to.
1162#[derive(Debug, Clone, PartialEq, Serialize)]
1163pub struct PathHop {
1164 /// Edge kind token (e.g. `calls`, `contains`).
1165 pub kind: String,
1166 /// How the edge was produced.
1167 pub provenance: &'static str,
1168 /// Confidence score, present only for inferred edges.
1169 pub confidence: Option<f64>,
1170 /// The direction the edge was traversed relative to the previous node
1171 /// (`outgoing` = along the edge, `incoming` = against it).
1172 pub direction: &'static str,
1173 /// The natural key of the node this hop arrives at.
1174 pub node: String,
1175}
1176
1177/// A shortest path between two nodes. Edges are followed in either direction
1178/// (the graph is treated as undirected for reachability), and each hop records
1179/// the actual direction and provenance of the edge used.
1180#[derive(Debug, Clone, PartialEq, Serialize)]
1181pub struct Path {
1182 /// Stable schema tag ([`SCHEMA`]).
1183 pub schema: &'static str,
1184 /// Natural key of the start node.
1185 pub from: String,
1186 /// Natural key of the goal node.
1187 pub to: String,
1188 /// Whether a path (including the trivial empty one) was found.
1189 pub found: bool,
1190 /// Number of hops (edges) in the path; `0` when `from == to`.
1191 pub length: usize,
1192 /// The hops from `from` to `to`, in order.
1193 pub hops: Vec<PathHop>,
1194}
1195
1196fn out_ref(edge: &Edge) -> EdgeRef {
1197 EdgeRef {
1198 kind: edge.kind.as_str().to_owned(),
1199 provenance: edge.provenance.as_str(),
1200 confidence: edge.confidence,
1201 node: edge.dst.clone(),
1202 }
1203}
1204
1205fn in_ref(edge: &Edge) -> EdgeRef {
1206 EdgeRef {
1207 kind: edge.kind.as_str().to_owned(),
1208 provenance: edge.provenance.as_str(),
1209 confidence: edge.confidence,
1210 node: edge.src.clone(),
1211 }
1212}
1213
1214fn sort_refs(refs: &mut [EdgeRef]) {
1215 // Include provenance so edges differing only in provenance have a total,
1216 // stable order; with the edge-uniqueness constraint this key is unique.
1217 refs.sort_by(|a, b| (&a.kind, &a.node, a.provenance).cmp(&(&b.kind, &b.node, b.provenance)));
1218}
1219
1220/// Explain a node: its record plus every incoming and outgoing edge, each
1221/// labelled with provenance. Returns `None` if no node has that key.
1222///
1223/// # Errors
1224/// Returns [`StoreError`] on query failure.
1225pub fn explain(store: &Store, key: &str) -> Result<Option<Explanation>, StoreError> {
1226 let Some(node) = store.get_node(key)? else {
1227 return Ok(None);
1228 };
1229 let mut outgoing: Vec<EdgeRef> = store.edges_from(key)?.iter().map(out_ref).collect();
1230 let mut incoming: Vec<EdgeRef> = store.edges_to(key)?.iter().map(in_ref).collect();
1231 sort_refs(&mut outgoing);
1232 sort_refs(&mut incoming);
1233 Ok(Some(Explanation {
1234 schema: SCHEMA,
1235 node: NodeSummary::from_node(&node),
1236 meta: node.meta,
1237 outgoing,
1238 incoming,
1239 }))
1240}
1241
1242/// List every node of the given `kind`, ordered by key.
1243///
1244/// # Errors
1245/// Returns [`StoreError`] on query failure.
1246pub fn list_kind(store: &Store, kind: &NodeKind) -> Result<Listing, StoreError> {
1247 let nodes = store
1248 .nodes_by_kind(kind)?
1249 .iter()
1250 .map(NodeSummary::from_node)
1251 .collect();
1252 Ok(Listing {
1253 schema: SCHEMA,
1254 kind: kind.as_str().to_owned(),
1255 nodes,
1256 })
1257}
1258
1259/// A relevance-ranked search hit: a node summary plus its score.
1260#[derive(Debug, Clone, PartialEq, Serialize)]
1261pub struct SearchHit {
1262 /// Relevance score (higher is better); see [`search`] for how it is derived.
1263 pub score: u32,
1264 /// The matching node.
1265 #[serde(flatten)]
1266 pub node: NodeSummary,
1267 /// A short, whitespace-collapsed excerpt of the node's captured
1268 /// `meta.content` (see [`content_snippet`]), so a model that never calls
1269 /// [`explain`] still has real grounding text. `None` for pure symbol/config
1270 /// nodes with no content — the summary (name/kind/path) is the grounding then.
1271 #[serde(skip_serializing_if = "Option::is_none")]
1272 pub snippet: Option<String>,
1273}
1274
1275/// Max **chars** of a search-hit content snippet, counting the trailing ellipsis
1276/// when truncated (so the total length never exceeds this). Bounded so many hits
1277/// cannot bloat the tool response or blow the served model's context window.
1278const SNIPPET_MAX: usize = 300;
1279
1280/// Build a bounded, whitespace-collapsed snippet from a node's captured
1281/// `meta.content`, or `None` when the node has no textual content (pure symbol/
1282/// config nodes). Runs of whitespace collapse to single spaces, and the result is
1283/// at most [`SNIPPET_MAX`] chars *including* a trailing `…` when the content was
1284/// truncated, so a search hit carries grounding text even when the model never
1285/// calls [`explain`].
1286///
1287/// Processes `content` **lazily**: it collapses whitespace on the fly and stops
1288/// after ~`SNIPPET_MAX` chars, so a large content-bearing node never materialises
1289/// more than the bound regardless of how big its content is.
1290fn content_snippet(meta: &serde_json::Value) -> Option<String> {
1291 let content = meta.get("content").and_then(|v| v.as_str())?;
1292
1293 // Collect at most SNIPPET_MAX + 1 collapsed chars: the one extra char only
1294 // tells us whether the content overflowed the bound (→ needs an ellipsis);
1295 // we never buffer more than that, however large `content` is.
1296 let mut collapsed: Vec<char> = Vec::with_capacity(SNIPPET_MAX + 1);
1297 let mut pending_space = false;
1298 for ch in content.chars() {
1299 if ch.is_whitespace() {
1300 // A run of whitespace becomes a single separator, but only once a
1301 // real char has been emitted (this also drops any leading whitespace).
1302 pending_space = !collapsed.is_empty();
1303 continue;
1304 }
1305 if pending_space {
1306 collapsed.push(' ');
1307 pending_space = false;
1308 if collapsed.len() > SNIPPET_MAX {
1309 break;
1310 }
1311 }
1312 collapsed.push(ch);
1313 if collapsed.len() > SNIPPET_MAX {
1314 break;
1315 }
1316 }
1317
1318 if collapsed.is_empty() {
1319 return None;
1320 }
1321 // Overflowed the bound: truncate to SNIPPET_MAX - 1 chars and append the
1322 // ellipsis, so the total length (ellipsis included) is exactly SNIPPET_MAX.
1323 if collapsed.len() > SNIPPET_MAX {
1324 let snippet: String = collapsed[..SNIPPET_MAX - 1].iter().collect();
1325 Some(format!("{snippet}…"))
1326 } else {
1327 Some(collapsed.into_iter().collect())
1328 }
1329}
1330
1331/// Deterministically search nodes for `query`, ranked by relevance, returning at
1332/// most `limit` hits — or **every match when `limit == 0`**, which is [`window`]'s
1333/// rule and the one every list lens follows (issue #393). An empty result
1334/// therefore always means "nothing matched", never "you asked for nothing".
1335///
1336/// Case-insensitive; every whitespace/`::`-separated token must appear
1337/// somewhere in the node's **name, key, path, or captured `meta.content`**
1338/// (so a question's words find the *description*, e.g. a README/ADR, not only a
1339/// same-named symbol). Scoring favours an exact name match, then a name/content
1340/// substring, then per-token hits; it then **boosts curated intent** (`authored`
1341/// ADRs/blueprints) and READMEs/overviews and **penalises test scaffolding**, so
1342/// "what/why" questions land on the real answer rather than a same-named test
1343/// helper. Ties break by key so results are stable.
1344///
1345/// # Errors
1346/// Returns [`StoreError`] on query failure.
1347pub fn search(store: &Store, query: &str, limit: usize) -> Result<Vec<SearchHit>, StoreError> {
1348 let q = query.trim().to_lowercase();
1349 // Tokens are separated by whitespace or the `::` path separator; a lone `:`
1350 // (as in a `sym:rust:…` key) does not split a token.
1351 let tokens: Vec<&str> = q.split("::").flat_map(str::split_whitespace).collect();
1352 if tokens.is_empty() {
1353 return Ok(Vec::new());
1354 }
1355
1356 let mut hits: Vec<SearchHit> = Vec::new();
1357 for node in store.all_nodes()? {
1358 let name = node.name.to_lowercase();
1359 let key = node.key.to_lowercase();
1360 let path = node.path.as_deref().unwrap_or("").to_lowercase();
1361 // The captured knowledge base (doc comments, prose, ADR/README/blueprint
1362 // text) is searchable too, so a question's words find the *description*,
1363 // not just a same-named symbol. Only lowercase when a node actually has
1364 // content — most nodes (code symbols) don't, so skip the allocation.
1365 let content = node
1366 .meta
1367 .get("content")
1368 .and_then(|v| v.as_str())
1369 .map(str::to_lowercase);
1370 let content = content.as_deref().unwrap_or("");
1371 // Require every token to appear somewhere (including content), so a
1372 // multi-word query narrows.
1373 if !tokens
1374 .iter()
1375 .all(|t| name.contains(t) || key.contains(t) || path.contains(t) || content.contains(t))
1376 {
1377 continue;
1378 }
1379 let mut relevance: i32 = 0;
1380 if name == q {
1381 relevance += 100;
1382 } else if name.contains(&q) {
1383 relevance += 60;
1384 } else if content.contains(&q) {
1385 relevance += 25;
1386 }
1387 for t in &tokens {
1388 if name.contains(t) {
1389 relevance += 12;
1390 } else if key.contains(t) {
1391 relevance += 6;
1392 } else if content.contains(t) {
1393 relevance += 8;
1394 } else if path.contains(t) {
1395 relevance += 3;
1396 }
1397 }
1398 // Curated intent (ADRs/blueprints — `authored`) is the best answer to a
1399 // "what/why" question; a README/overview is the natural landing page; and
1400 // test scaffolding should not outrank the real thing when it shares a name.
1401 if node.provenance == Provenance::Authored {
1402 relevance += 40;
1403 }
1404 if is_overview_path(&path) {
1405 relevance += 30;
1406 }
1407 if is_test_path(&path) {
1408 relevance -= 60;
1409 }
1410 hits.push(SearchHit {
1411 score: u32::try_from(relevance.max(0)).unwrap_or(0),
1412 snippet: content_snippet(&node.meta),
1413 node: NodeSummary::from_node(&node),
1414 });
1415 }
1416 // Highest score first; ties by key for a stable, deterministic order.
1417 hits.sort_by(|a, b| {
1418 b.score
1419 .cmp(&a.score)
1420 .then_with(|| a.node.key.cmp(&b.node.key))
1421 });
1422 // `window`, not `truncate`: `0` is unlimited here as it is everywhere else.
1423 // The scan above is full-population at every limit, so an unbounded search
1424 // costs the same as a bounded one — only the printing differs.
1425 window(&mut hits, 0, limit);
1426 Ok(hits)
1427}
1428
1429/// A hit in the **generated** channel: text a model produced about a media blob,
1430/// never a graph fact.
1431///
1432/// It is deliberately *not* a [`SearchHit`]. A generated hit has no node, no
1433/// provenance and no key, and giving it a [`NodeSummary`] would be the first step
1434/// towards it being treated like one — the exact mistake ADR-0015 exists to
1435/// correct. Everything a consumer needs to label it is on the struct, including
1436/// the literal `generated: true`, so a caller that reads nothing else still
1437/// cannot mistake it for extracted text.
1438#[derive(Debug, Clone, PartialEq, Serialize)]
1439pub struct GeneratedHit {
1440 /// Relevance within the generated channel. Not comparable with a
1441 /// [`SearchHit::score`]: the two are ranked by different scorers, in
1442 /// different channels, on purpose.
1443 pub score: u32,
1444 /// Always `true`. A marker a consumer cannot miss or forget to check.
1445 pub generated: bool,
1446 /// The producer identity that wrote the text — which model, at which
1447 /// quantisation, under which prompt (see [`crate::Producer::id`]).
1448 pub producer: String,
1449 /// The model's registry name, repeated for legibility.
1450 pub model: String,
1451 /// The modality (`audio` | `vision`).
1452 pub kind: &'static str,
1453 /// Git blob id of the source media.
1454 pub blob: String,
1455 /// Repository path the blob was seen at.
1456 pub path: String,
1457 /// A bounded, whitespace-collapsed excerpt of the generated text, on the same
1458 /// terms as [`SearchHit::snippet`].
1459 pub snippet: Option<String>,
1460}
1461
1462/// A hit in the **memory** channel: something a session learned, never a graph
1463/// fact and never a re-derivable one.
1464///
1465/// Deliberately *not* a [`SearchHit`], for the reason [`GeneratedHit`] is not: a
1466/// memory record has no node, no provenance and no key, and giving it a
1467/// [`NodeSummary`] would be the first step towards its being treated like one.
1468/// Unlike either of the other channels, it also carries **what the tree thinks of
1469/// it** — [`MemoryHit::applies`] and [`MemoryHit::anchor_state`] — because a
1470/// lesson about code that has since moved is worth reading and worth labelling,
1471/// and returning it unlabelled would be the worse of the two mistakes.
1472#[derive(Debug, Clone, PartialEq, Serialize)]
1473pub struct MemoryHit {
1474 /// Relevance within the memory channel. Not comparable with a
1475 /// [`SearchHit::score`] or a [`GeneratedHit::score`]: three channels, three
1476 /// scorers, on purpose.
1477 pub score: u32,
1478 /// Always `true`. A marker a consumer cannot miss or forget to check.
1479 pub memory: bool,
1480 /// The record's id — its generation, and what `roteiro memory forget` takes.
1481 pub id: i64,
1482 /// What kind of knowledge it is (`lesson` | `attempt` | …).
1483 pub kind: &'static str,
1484 /// The namespace it was recorded in. **Not a branch label.**
1485 pub scope: String,
1486 /// The node key it is anchored to, if any.
1487 #[serde(skip_serializing_if = "Option::is_none")]
1488 pub anchor: Option<String>,
1489 /// What that anchor is worth against the current tree (`valid` | `drifted` |
1490 /// `vanished` | `unverifiable` | `unanchored`).
1491 pub anchor_state: &'static str,
1492 /// **Whether this record applies to the tree being searched.** A `false` here
1493 /// is a label, never a reason to have withheld the hit.
1494 pub applies: bool,
1495 /// The evidence multiplier the record's own ranking gave it
1496 /// (`base_confidence × anchor_penalty`), reported so the channel's score can
1497 /// be taken apart.
1498 pub evidence: f64,
1499 /// A bounded, whitespace-collapsed excerpt of the body, on the same terms as
1500 /// [`SearchHit::snippet`].
1501 pub snippet: Option<String>,
1502}
1503
1504/// The three channels a search returns.
1505///
1506/// They are separate fields rather than one merged list because merging is
1507/// precisely what must not happen: generated text and remembered prose may both
1508/// be *retrievable*, but neither may ever be *indistinguishable* from a derived or
1509/// authored fact, and a single ranked list would make the distinction a matter of
1510/// reading each element carefully.
1511#[derive(Debug, Clone, PartialEq, Serialize)]
1512pub struct SearchResults {
1513 /// Stable schema tag ([`SCHEMA`]).
1514 pub schema: &'static str,
1515 /// The graph channel: ranked nodes, exactly what [`search`] returns.
1516 pub hits: Vec<SearchHit>,
1517 /// The generated channel. **Empty unless
1518 /// [`SearchOptions::include_generated`] was set** — off by default, so a
1519 /// silent clip's confabulated prose cannot reach a default search.
1520 pub generated: Vec<GeneratedHit>,
1521 /// The memory channel. **Empty unless [`SearchOptions::include_memory`] was
1522 /// set** — off by default, so unreviewed accumulated prose cannot reach a
1523 /// default search either.
1524 pub memory: Vec<MemoryHit>,
1525}
1526
1527/// How to search.
1528#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1529pub struct SearchOptions {
1530 /// Maximum hits **per channel**, where `0` is unlimited ([`window`]'s rule,
1531 /// applied per channel). Each channel is ranked and windowed independently,
1532 /// so opting in to another one never displaces a graph hit, and never
1533 /// silently returns fewer of them — and `0` is "all of each channel asked
1534 /// for", not "all of them merged and then cut".
1535 pub limit: usize,
1536 /// Fold in the generated channel. Off by default (see
1537 /// [`SearchOptions::default`]).
1538 pub include_generated: bool,
1539 /// Fold in the memory channel. Off by default, for the same reason: what an
1540 /// agent remembers is unreviewed, unredacted and accumulated, so it is
1541 /// something a caller asks for rather than something that arrives.
1542 pub include_memory: bool,
1543}
1544
1545impl Default for SearchOptions {
1546 /// Ten hits, graph channel only. The default is the safe answer: everything
1547 /// that is not an extracted or authored fact is opt-in, always.
1548 fn default() -> Self {
1549 Self {
1550 limit: 10,
1551 include_generated: false,
1552 include_memory: false,
1553 }
1554 }
1555}
1556
1557/// Search every channel: the graph, and — each only when asked for —
1558/// model-generated media content and episodic agent memory.
1559///
1560/// The graph channel is exactly [`search`]. The other two are ranked by scorers of
1561/// their own ([`generated_score`], [`memory_score`]) which have **no provenance
1562/// term at all**, so neither can acquire the `authored` boost that curated intent
1563/// gets. Neither could do so even by accident: neither record is a node, so
1564/// neither ever reaches the code that applies that boost.
1565///
1566/// The memory channel is scored with **no decay** regardless of what a caller
1567/// might prefer elsewhere, so a search is reproducible for a fixed store and a
1568/// fixed tree.
1569///
1570/// # Errors
1571/// Returns [`StoreError`] on query failure.
1572pub fn search_channels(
1573 store: &Store,
1574 query: &str,
1575 opts: SearchOptions,
1576) -> Result<SearchResults, StoreError> {
1577 let hits = search(store, query, opts.limit)?;
1578 let generated = if opts.include_generated {
1579 search_generated(store, query, opts.limit)?
1580 } else {
1581 Vec::new()
1582 };
1583 let memory = if opts.include_memory {
1584 search_memory(store, query, opts.limit)?
1585 } else {
1586 Vec::new()
1587 };
1588 Ok(SearchResults {
1589 schema: SCHEMA,
1590 hits,
1591 generated,
1592 memory,
1593 })
1594}
1595
1596/// Rank the memory channel alone.
1597///
1598/// Built on [`Store::recall_memory`] rather than on a query of its own, so the
1599/// channel inherits every promise recall makes without restating any of them: a
1600/// superseded record is already gone, an unanchored one is already labelled, and
1601/// nothing here writes anything. Decay is fixed at [`crate::Decay::None`] so a
1602/// search over an unchanged store and tree is reproducible.
1603///
1604/// Ties break by newest generation, so the order is total. `limit` follows
1605/// [`window`]: `0` is every matching record, not none of them.
1606fn search_memory(store: &Store, query: &str, limit: usize) -> Result<Vec<MemoryHit>, StoreError> {
1607 let q = query.trim().to_lowercase();
1608 let tokens: Vec<&str> = q.split("::").flat_map(str::split_whitespace).collect();
1609 if tokens.is_empty() {
1610 return Ok(Vec::new());
1611 }
1612 // Recall does the filtering, the anchor resolution and the evidence
1613 // weighting; this function only adds the lexical relevance a search wants.
1614 let recalled = store.recall_memory(&crate::RecallOptions {
1615 query: Some(query),
1616 decay: crate::Decay::None,
1617 ..crate::RecallOptions::default()
1618 })?;
1619
1620 let mut hits: Vec<MemoryHit> = recalled
1621 .results
1622 .into_iter()
1623 .map(|r| {
1624 let body = r.record.body.to_lowercase();
1625 let anchor = r
1626 .record
1627 .anchor
1628 .as_ref()
1629 .map(|a| a.key.to_lowercase())
1630 .unwrap_or_default();
1631 MemoryHit {
1632 score: memory_score(&q, &tokens, &body, &anchor, r.score),
1633 memory: true,
1634 id: r.record.id,
1635 kind: r.record.kind.as_str(),
1636 scope: r.record.scope.clone(),
1637 anchor: r.record.anchor.as_ref().map(|a| a.key.clone()),
1638 anchor_state: r.record.anchor_state.as_str(),
1639 applies: r.record.applies,
1640 evidence: r.score,
1641 snippet: content_snippet(&serde_json::json!({ "content": r.record.body })),
1642 }
1643 })
1644 .collect();
1645 hits.sort_by(|a, b| b.score.cmp(&a.score).then_with(|| b.id.cmp(&a.id)));
1646 window(&mut hits, 0, limit);
1647 Ok(hits)
1648}
1649
1650/// Relevance of one memory record: **lexical match, weighted by the record's own
1651/// evidence**, and nothing else.
1652///
1653/// The `evidence` factor is `base_confidence × anchor_penalty` from
1654/// [`crate::Store::recall_memory`] — so a lesson whose anchor still resolves in
1655/// this tree outranks an equally-worded one whose code has moved on, which is the
1656/// whole depreciation model showing up in search.
1657///
1658/// # The weight is in `[0, 1]`, and zero is reachable — deliberately
1659///
1660/// An earlier version of this comment said `(0, 1]`. That was wrong, and the
1661/// half-open interval hid a decision rather than describing one. The two factors
1662/// are not alike and the difference is the point:
1663///
1664/// - **[`crate::anchor_penalty`] can never be zero.** Its floor is `0.25`
1665/// ([`crate::AnchorState::Drifted`]), and
1666/// `memory::tests::anchor_penalty_demotes_without_ever_silencing` pins that
1667/// every state is `> 0`. So **drift can never drive evidence to zero** — which
1668/// is ADR-0013's "demote, never delete" rule holding *structurally*, not by
1669/// convention. Roteiro's own inference about a record is never allowed to
1670/// reduce it to nothing.
1671/// - **`base_confidence` can be exactly `0.0`**, because the writer can say so.
1672/// `roteiro memory add --confidence 0` is an operator stating "I am recording
1673/// this and I give it no credence." Flooring that would silently overrule an
1674/// explicit statement — and the value is a probability, where `0.0` is
1675/// legitimate rather than a boundary error.
1676///
1677/// So the asymmetry is exactly the right way round: **what Roteiro infers never
1678/// silences a record; what the operator explicitly states is honoured.**
1679///
1680/// # Zero relevance is not zero visibility
1681///
1682/// A zero score does **not** remove a hit. Nothing in this module or in
1683/// [`crate::Store::recall_memory`] filters on the score — it orders, and the
1684/// record comes back, is printed, and is labelled exactly as any other.
1685/// `a_zero_confidence_memory_is_ranked_last_and_still_returned` enforces that in
1686/// both surfaces, so the claim is a tested property rather than something this
1687/// comment asserts and nothing checks. (A limit can still truncate a
1688/// bottom-ranked hit — that is what a limit means, and it applies to every hit
1689/// regardless of score.)
1690///
1691/// The omissions are the point, and each is deliberate:
1692///
1693/// - **no `authored` boost** — this is the whole reason the channel exists. That
1694/// +40 is for intent a human deliberately wrote into a reviewed file;
1695/// accumulated, unreviewed, unredacted prose riding it would be trust-model
1696/// contamination by construction.
1697/// - **no overview boost** — a README's landing-page privilege is about authored
1698/// documentation.
1699/// - **no name or key term** — a memory record has neither.
1700///
1701/// Because this scorer shares no branch with the node scorer, "memory never
1702/// acquires the authored boost" is a structural fact rather than a condition to be
1703/// maintained.
1704fn memory_score(q: &str, tokens: &[&str], body: &str, anchor: &str, evidence: f64) -> u32 {
1705 let mut relevance: i32 = 0;
1706 if body.contains(q) {
1707 relevance += 25;
1708 }
1709 for t in tokens {
1710 if body.contains(t) {
1711 relevance += 8;
1712 } else if anchor.contains(t) {
1713 relevance += 3;
1714 }
1715 }
1716 // `[0.0, 1.0]`, closed at both ends: zero is reachable, and only ever because
1717 // a writer stated it. See the header — `anchor_penalty` cannot contribute a
1718 // zero, so drift can never land here.
1719 let weighted = f64::from(relevance.max(0)) * evidence.clamp(0.0, 1.0);
1720 #[expect(
1721 clippy::cast_possible_truncation,
1722 clippy::cast_sign_loss,
1723 reason = "the product of a small non-negative relevance and a weight in [0, 1]"
1724 )]
1725 let score = weighted.round() as u32;
1726 score
1727}
1728
1729/// Rank the generated channel alone. Ties break by `(producer, blob)` so results
1730/// are stable. `limit` follows [`window`]: `0` is every matching record, not none
1731/// of them.
1732fn search_generated(
1733 store: &Store,
1734 query: &str,
1735 limit: usize,
1736) -> Result<Vec<GeneratedHit>, StoreError> {
1737 let q = query.trim().to_lowercase();
1738 let tokens: Vec<&str> = q.split("::").flat_map(str::split_whitespace).collect();
1739 if tokens.is_empty() {
1740 return Ok(Vec::new());
1741 }
1742 let mut hits: Vec<GeneratedHit> = Vec::new();
1743 for record in store.media_records(&crate::MediaFilter::default())? {
1744 // A record the pre-generation gate refused holds a measurement, not text.
1745 // It is deliberately unsearchable: it has nothing to match on, and the
1746 // path *would* match — which would put a silent clip back into search
1747 // results as a hit with an empty snippet, which is the shape of the very
1748 // bug ADR-0015 exists to correct.
1749 let Some(generated_text) = record.outcome.text() else {
1750 continue;
1751 };
1752 let text = generated_text.to_lowercase();
1753 let path = record.path.to_lowercase();
1754 if !tokens.iter().all(|t| text.contains(t) || path.contains(t)) {
1755 continue;
1756 }
1757 hits.push(GeneratedHit {
1758 score: generated_score(&q, &tokens, &text, &path),
1759 generated: true,
1760 producer: record.producer_id.to_string(),
1761 model: record.producer.model.clone(),
1762 kind: record.producer.kind.as_str(),
1763 blob: record.blob_id.clone(),
1764 path: record.path.clone(),
1765 snippet: content_snippet(&serde_json::json!({ "content": generated_text })),
1766 });
1767 }
1768 hits.sort_by(|a, b| {
1769 b.score
1770 .cmp(&a.score)
1771 .then_with(|| (&a.producer, &a.blob).cmp(&(&b.producer, &b.blob)))
1772 });
1773 window(&mut hits, 0, limit);
1774 Ok(hits)
1775}
1776
1777/// Relevance of one generated record: whole-query and per-token matches over its
1778/// text and path, and **nothing else**.
1779///
1780/// The omissions are the point, and each is deliberate:
1781///
1782/// - **no `authored` boost** — generated text is not curated intent, and the
1783/// graph's +40 for an ADR must never land on a transcript;
1784/// - **no overview boost** — a README's landing-page privilege is about authored
1785/// documentation;
1786/// - **no name or key term** — a generated record has neither.
1787///
1788/// Because this scorer shares no branch with the node scorer, "generated content
1789/// never acquires the authored boost" is a structural fact rather than a
1790/// condition to be maintained.
1791fn generated_score(q: &str, tokens: &[&str], text: &str, path: &str) -> u32 {
1792 let mut relevance: i32 = 0;
1793 if text.contains(q) {
1794 relevance += 25;
1795 }
1796 for t in tokens {
1797 if text.contains(t) {
1798 relevance += 8;
1799 } else if path.contains(t) {
1800 relevance += 3;
1801 }
1802 }
1803 u32::try_from(relevance.max(0)).unwrap_or(0)
1804}
1805
1806/// Whether `path` (already lowercased) is a README/overview doc — the natural
1807/// landing for "what is this project" questions, so it is ranked up. Matches a
1808/// `readme*` or `overview*` basename (blueprints, the other overview docs, are
1809/// already boosted via their `authored` provenance).
1810fn is_overview_path(path: &str) -> bool {
1811 path.rsplit('/')
1812 .next()
1813 .is_some_and(|base| base.starts_with("readme") || base.starts_with("overview"))
1814}
1815
1816/// Whether `path` (already lowercased) is test scaffolding, which should not
1817/// outrank real content that happens to share a name.
1818fn is_test_path(path: &str) -> bool {
1819 path.contains("/tests/") || path.contains("/test/")
1820}
1821
1822/// A candidate step out of a node during traversal: the edge used and the node
1823/// on the other end. Ordered so BFS expansion is deterministic.
1824struct Step {
1825 node: String,
1826 hop: PathHop,
1827}
1828
1829/// All one-hop steps out of `key`, following edges in either direction, sorted
1830/// for deterministic traversal.
1831fn steps_from(store: &Store, key: &str) -> Result<Vec<Step>, StoreError> {
1832 let mut steps = Vec::new();
1833 for edge in store.edges_from(key)? {
1834 steps.push(Step {
1835 node: edge.dst.clone(),
1836 hop: hop(&edge, "outgoing", edge.dst.clone()),
1837 });
1838 }
1839 for edge in store.edges_to(key)? {
1840 steps.push(Step {
1841 node: edge.src.clone(),
1842 hop: hop(&edge, "incoming", edge.src.clone()),
1843 });
1844 }
1845 steps.sort_by(|a, b| {
1846 (&a.node, &a.hop.kind, a.hop.provenance, a.hop.direction).cmp(&(
1847 &b.node,
1848 &b.hop.kind,
1849 b.hop.provenance,
1850 b.hop.direction,
1851 ))
1852 });
1853 Ok(steps)
1854}
1855
1856fn hop(edge: &Edge, direction: &'static str, node: String) -> PathHop {
1857 PathHop {
1858 kind: edge.kind.as_str().to_owned(),
1859 provenance: edge.provenance.as_str(),
1860 confidence: edge.confidence,
1861 direction,
1862 node,
1863 }
1864}
1865
1866/// Find a shortest path from `from` to `to`, following edges in either
1867/// direction. Returns a [`Path`] with `found = false` (and no hops) if either
1868/// endpoint is absent or `to` is unreachable; `from == to` yields the trivial
1869/// zero-length path.
1870///
1871/// The search is breadth-first with deterministic neighbour ordering, so the
1872/// returned path is stable for a given graph.
1873///
1874/// # Errors
1875/// Returns [`StoreError`] on query failure.
1876pub fn path(store: &Store, from: &str, to: &str) -> Result<Path, StoreError> {
1877 let not_found = |found: bool, hops: Vec<PathHop>| Path {
1878 schema: SCHEMA,
1879 from: from.to_owned(),
1880 to: to.to_owned(),
1881 found,
1882 length: hops.len(),
1883 hops,
1884 };
1885
1886 // Both endpoints must exist in the graph.
1887 if store.get_node(from)?.is_none() || store.get_node(to)?.is_none() {
1888 return Ok(not_found(false, Vec::new()));
1889 }
1890 if from == to {
1891 return Ok(not_found(true, Vec::new()));
1892 }
1893
1894 // BFS, recording for each visited node the (predecessor, hop) that reached
1895 // it so the path can be reconstructed.
1896 let mut came_from: BTreeMap<String, (String, PathHop)> = BTreeMap::new();
1897 let mut queue: VecDeque<String> = VecDeque::new();
1898 queue.push_back(from.to_owned());
1899 came_from.insert(from.to_owned(), (String::new(), placeholder_hop()));
1900
1901 while let Some(current) = queue.pop_front() {
1902 if current == to {
1903 break;
1904 }
1905 for step in steps_from(store, ¤t)? {
1906 if came_from.contains_key(&step.node) {
1907 continue;
1908 }
1909 came_from.insert(step.node.clone(), (current.clone(), step.hop));
1910 queue.push_back(step.node);
1911 }
1912 }
1913
1914 // Walk predecessors back from `to` to `from`, then reverse. Every node in
1915 // `came_from` other than `from` has a real predecessor, so this terminates
1916 // at `from`. If the chain is ever broken (an invariant violation), treat it
1917 // as no path rather than silently returning a partial one.
1918 let mut hops = Vec::new();
1919 let mut cursor = to.to_owned();
1920 while cursor != from {
1921 let Some((prev, hop)) = came_from.get(&cursor) else {
1922 return Ok(not_found(false, Vec::new()));
1923 };
1924 hops.push(hop.clone());
1925 cursor = prev.clone();
1926 }
1927 hops.reverse();
1928 Ok(not_found(true, hops))
1929}
1930
1931/// A sentinel hop for the BFS start node (never emitted in a result).
1932fn placeholder_hop() -> PathHop {
1933 PathHop {
1934 kind: String::new(),
1935 provenance: "derived",
1936 confidence: None,
1937 direction: "outgoing",
1938 node: String::new(),
1939 }
1940}
1941
1942#[cfg(test)]
1943mod tests {
1944 use super::{
1945 ConfigSecretReport, CouplingItem, CouplingOrder, CouplingReport, DebtDensityReport,
1946 DensityItem, DensityOrder, RedactionState, SCHEMA, SNIPPET_MAX, SearchOptions,
1947 SearchResults, config_secrets, coupling, debt_density, explain, glob_match, list_kind,
1948 memory_score, path, search, search_channels, window,
1949 };
1950 use crate::{AnchorState, Edge, EdgeKind, FactSet, Node, NodeKind, Store};
1951
1952 fn seeded() -> Store {
1953 let mut store = Store::open_in_memory().expect("store");
1954 let facts = FactSet::new()
1955 .with_node(Node::new("sym:rust:a.rs#main", NodeKind::Fn, "main"))
1956 .with_node(Node::new("sym:rust:a.rs#helper", NodeKind::Fn, "helper"))
1957 .with_node(Node::new("adr:0001", NodeKind::Adr, "Build Roteiro"))
1958 .with_edge(Edge::derived(
1959 "sym:rust:a.rs#main",
1960 "sym:rust:a.rs#helper",
1961 EdgeKind::Calls,
1962 ))
1963 .with_edge(Edge::authored(
1964 "adr:0001",
1965 "sym:rust:a.rs#main",
1966 EdgeKind::References,
1967 ));
1968 store.apply_factset(&facts).expect("apply");
1969 store
1970 }
1971
1972 /// [`window`] is the single definition of `limit`/`offset` for every list
1973 /// lens, so its contract is pinned here rather than only through the lenses
1974 /// that call it.
1975 #[test]
1976 fn window_reads_zero_as_unlimited_and_offsets_before_limiting() {
1977 let ten = || (0..10).collect::<Vec<u8>>();
1978
1979 // `0` is unlimited, not empty — the whole point of #375.
1980 let mut all = ten();
1981 window(&mut all, 0, 0);
1982 assert_eq!(all, ten(), "limit 0 keeps everything");
1983
1984 // A non-zero limit cuts from the end, keeping the caller's order.
1985 let mut top = ten();
1986 window(&mut top, 0, 3);
1987 assert_eq!(top, vec![0, 1, 2]);
1988
1989 // A limit at or beyond the population is a no-op, so the boundary
1990 // between "bounded" and "unbounded" has no step in it.
1991 let mut exact = ten();
1992 window(&mut exact, 0, 10);
1993 assert_eq!(exact, ten());
1994 let mut over = ten();
1995 window(&mut over, 0, 99);
1996 assert_eq!(over, ten());
1997
1998 // `offset` applies first and `limit` to what remains.
1999 let mut paged = ten();
2000 window(&mut paged, 4, 3);
2001 assert_eq!(paged, vec![4, 5, 6]);
2002
2003 // The decision this fix had to make: offset with an unlimited limit is
2004 // "skip N, then every remaining item" — not "skip N, then nothing".
2005 let mut rest = ten();
2006 window(&mut rest, 7, 0);
2007 assert_eq!(rest, vec![7, 8, 9], "offset then unlimited");
2008
2009 // An offset at or past the end is an empty page, not a panic and not a
2010 // wrapped-around one.
2011 let mut at_end = ten();
2012 window(&mut at_end, 10, 0);
2013 assert!(at_end.is_empty());
2014 let mut past_end = ten();
2015 window(&mut past_end, 500, 0);
2016 assert!(past_end.is_empty());
2017 let mut past_end_limited = ten();
2018 window(&mut past_end_limited, 500, 5);
2019 assert!(past_end_limited.is_empty());
2020
2021 // An empty input stays empty under every combination.
2022 let mut empty: Vec<u8> = Vec::new();
2023 window(&mut empty, 0, 0);
2024 window(&mut empty, 3, 0);
2025 window(&mut empty, 0, 3);
2026 assert!(empty.is_empty());
2027 }
2028
2029 #[test]
2030 fn search_ranks_by_relevance_and_is_bounded() {
2031 let store = seeded();
2032 // An exact name match outranks a substring match.
2033 let hits = search(&store, "helper", 10).expect("search");
2034 assert_eq!(hits[0].node.key, "sym:rust:a.rs#helper");
2035 assert!(hits[0].score >= 100, "exact name match scores high");
2036
2037 // Every token must appear: "main roteiro" matches nothing (no node has both).
2038 assert!(
2039 search(&store, "main roteiro", 10)
2040 .expect("search")
2041 .is_empty()
2042 );
2043
2044 // A lone `:` does not split a token: `sym:rust` is one token matching the
2045 // code-symbol keys but not `adr:0001`.
2046 let by_prefix = search(&store, "sym:rust", 10).expect("search");
2047 assert!(!by_prefix.is_empty());
2048 assert!(
2049 by_prefix
2050 .iter()
2051 .all(|h| h.node.key.starts_with("sym:rust:"))
2052 );
2053
2054 // A blank query yields nothing; the limit is respected.
2055 assert!(search(&store, " ", 10).expect("search").is_empty());
2056 assert!(search(&store, "a.rs", 1).expect("search").len() <= 1);
2057 }
2058
2059 /// The population every issue-#393 test below works over: 12 in each of the
2060 /// three channels, each matching a term only its own channel carries.
2061 ///
2062 /// 12 is deliberately above the default of 10, so a `limit` of `0` that had
2063 /// quietly fallen back to the default could not pass for "unlimited".
2064 fn three_channels(population: usize) -> Store {
2065 use crate::{
2066 GeneratedContent, MediaKind, MediaOutcome, MediaWrite, MemoryKind, MemoryWrite,
2067 Producer,
2068 };
2069
2070 let mut store = Store::open_in_memory().expect("store");
2071 let mut facts = FactSet::new();
2072 for i in 0..population {
2073 facts = facts.with_node(Node::new(
2074 format!("sym:rust:a.rs#quokka{i}"),
2075 NodeKind::Fn,
2076 format!("quokka{i}"),
2077 ));
2078 }
2079 store.apply_factset(&facts).expect("apply");
2080
2081 let producer = Producer {
2082 kind: MediaKind::Audio,
2083 model: "voxtral-mini-3b".to_owned(),
2084 model_digest: "4705be8e".to_owned(),
2085 quantisation: "Q4_K_M".to_owned(),
2086 mmproj_digest: "4f24c4ef".to_owned(),
2087 prompt: "Transcribe this audio recording.".to_owned(),
2088 temperature: 0.0,
2089 max_tokens: 512,
2090 };
2091 for i in 0..population {
2092 store
2093 .record_memory(&MemoryWrite {
2094 scope: crate::DEFAULT_MEMORY_SCOPE,
2095 kind: MemoryKind::Lesson,
2096 anchor: None,
2097 body: &format!("wombat lesson number {i}"),
2098 confidence: None,
2099 supersedes: None,
2100 })
2101 .expect("memory write");
2102 assert!(
2103 store
2104 .record_media_content(&MediaWrite {
2105 blob_id: &format!("blob-{i}"),
2106 path: &format!("assets/clip{i}.wav"),
2107 producer: &producer,
2108 tool_version: "0.0.0",
2109 outcome: &MediaOutcome::Generated(GeneratedContent {
2110 text: format!("narwhal transcript number {i}"),
2111 confidence: None,
2112 }),
2113 replace: false,
2114 })
2115 .expect("media write"),
2116 "each clip is a fresh record",
2117 );
2118 }
2119 store
2120 }
2121
2122 /// Every channel asked for, at `limit`.
2123 fn all_channels(store: &Store, query: &str, limit: usize) -> SearchResults {
2124 search_channels(
2125 store,
2126 query,
2127 SearchOptions {
2128 limit,
2129 include_generated: true,
2130 include_memory: true,
2131 },
2132 )
2133 .expect("search")
2134 }
2135
2136 /// Issue #393: `limit == 0` reads as **unlimited on the graph channel**, and
2137 /// it is [`window`] that says so rather than a rule of `search`'s own — the
2138 /// third reading of one parameter name is gone, not relocated.
2139 #[test]
2140 fn search_reads_zero_as_unlimited_and_only_removes_the_cut() {
2141 const POPULATION: usize = 12;
2142 let store = three_channels(POPULATION);
2143
2144 let bounded = search(&store, "quokka", 10).expect("search");
2145 assert_eq!(bounded.len(), 10, "a positive limit still cuts");
2146
2147 let unlimited = search(&store, "quokka", 0).expect("search");
2148 assert_eq!(unlimited.len(), POPULATION, "0 is every match");
2149
2150 // An unlimited search is the same ranking uncut, not a different one:
2151 // the bounded page is the prefix of the unbounded one.
2152 assert_eq!(
2153 unlimited[..10]
2154 .iter()
2155 .map(|h| h.node.key.as_str())
2156 .collect::<Vec<_>>(),
2157 bounded
2158 .iter()
2159 .map(|h| h.node.key.as_str())
2160 .collect::<Vec<_>>(),
2161 "unlimited only removes the cut",
2162 );
2163 }
2164
2165 /// The unit is **per channel**: `0` is "all of each channel that was asked
2166 /// for", not "all of them merged and then cut". Each channel here matches a
2167 /// term the other two do not, so the three populations stay separable.
2168 #[test]
2169 fn each_search_channel_reads_zero_as_unlimited_over_its_own_population() {
2170 const POPULATION: usize = 12;
2171 let store = three_channels(POPULATION);
2172
2173 // Each channel matches a term the other two do not, so a bounded and an
2174 // unbounded read of one says nothing about the others.
2175 assert_eq!(all_channels(&store, "quokka", 10).hits.len(), 10);
2176 assert_eq!(
2177 all_channels(&store, "quokka", 0).hits.len(),
2178 POPULATION,
2179 "graph channel: 0 is unlimited",
2180 );
2181 assert_eq!(all_channels(&store, "wombat", 10).memory.len(), 10);
2182 assert_eq!(
2183 all_channels(&store, "wombat", 0).memory.len(),
2184 POPULATION,
2185 "memory channel: 0 is unlimited",
2186 );
2187 assert_eq!(all_channels(&store, "narwhal", 10).generated.len(), 10);
2188 assert_eq!(
2189 all_channels(&store, "narwhal", 0).generated.len(),
2190 POPULATION,
2191 "generated channel: 0 is unlimited",
2192 );
2193
2194 // And the unit really is per channel: an unbounded search of one term
2195 // leaves the channels it does not match empty rather than filling them.
2196 let graph_only = all_channels(&store, "quokka", 0);
2197 assert!(
2198 graph_only.memory.is_empty() && graph_only.generated.is_empty(),
2199 "unlimited is per channel, not a merged population",
2200 );
2201 }
2202
2203 /// What keeps "unlimited" from meaning "the whole store": a query with no
2204 /// tokens matches nothing, at `0` exactly as at any other limit. `--limit 0`
2205 /// is bounded by what was asked for, not by the population.
2206 #[test]
2207 fn a_tokenless_query_is_nothing_in_every_channel_at_every_limit() {
2208 let store = three_channels(12);
2209 for blank in ["", " ", "\t\n"] {
2210 for limit in [0, 10] {
2211 let nothing = all_channels(&store, blank, limit);
2212 assert!(
2213 nothing.hits.is_empty()
2214 && nothing.generated.is_empty()
2215 && nothing.memory.is_empty(),
2216 "a query with no tokens is nothing, not everything ({blank:?}, limit {limit})",
2217 );
2218 }
2219 }
2220 }
2221
2222 #[test]
2223 fn search_prefers_curated_content_over_same_named_test_symbols() {
2224 use crate::Provenance;
2225 let mut store = Store::open_in_memory().expect("store");
2226 // A same-named test helper (exact name, but test scaffolding)…
2227 let mut test_fn = Node::new(
2228 "sym:rust:crates/x/tests/cli.rs#roteiro",
2229 NodeKind::Fn,
2230 "roteiro",
2231 );
2232 test_fn.path = Some("crates/x/tests/cli.rs".into());
2233 // …the authored ADR that actually answers "what is roteiro"…
2234 let mut adr = Node::new("adr:0001", NodeKind::Adr, "Build Roteiro")
2235 .with_provenance(Provenance::Authored);
2236 adr.path = Some("docs/adr/0001.md".into());
2237 adr.meta = serde_json::json!({ "content": "Roteiro is a provenance-tagged codebase knowledge graph." });
2238 // …and a README whose *content* (not its name) describes the project.
2239 let mut readme = Node::new("file:README.md", NodeKind::File, "README.md");
2240 readme.path = Some("README.md".into());
2241 readme.meta =
2242 serde_json::json!({ "content": "Roteiro turns a repo into one knowledge graph." });
2243 store
2244 .apply_factset(
2245 &FactSet::new()
2246 .with_node(test_fn)
2247 .with_node(adr)
2248 .with_node(readme),
2249 )
2250 .expect("apply");
2251
2252 let hits = search(&store, "roteiro", 10).expect("search");
2253 let keys: Vec<&str> = hits.iter().map(|h| h.node.key.as_str()).collect();
2254 let idx = |k: &str| keys.iter().position(|x| *x == k).expect("present");
2255 // The authored ADR and the README (found *by content*) both outrank the
2256 // same-named test helper.
2257 assert!(
2258 idx("adr:0001") < idx("sym:rust:crates/x/tests/cli.rs#roteiro"),
2259 "authored ADR outranks the test symbol: {keys:?}"
2260 );
2261 assert!(
2262 idx("file:README.md") < idx("sym:rust:crates/x/tests/cli.rs#roteiro"),
2263 "README (matched via content) outranks the test symbol: {keys:?}"
2264 );
2265
2266 // A content-only term finds the node even though no name/key/path has it.
2267 let by_content = search(&store, "provenance-tagged", 10).expect("search");
2268 assert_eq!(
2269 by_content.first().map(|h| h.node.key.as_str()),
2270 Some("adr:0001"),
2271 "content search matches the ADR by its captured text"
2272 );
2273 }
2274
2275 #[test]
2276 fn search_hit_carries_a_bounded_content_snippet() {
2277 use crate::Provenance;
2278 let mut store = Store::open_in_memory().expect("store");
2279 // A content-bearing node whose content is longer than the cap and has
2280 // messy whitespace to collapse.
2281 let long = "word ".repeat(200);
2282 let mut adr =
2283 Node::new("adr:0001", NodeKind::Adr, "Overview").with_provenance(Provenance::Authored);
2284 adr.meta = serde_json::json!({ "content": format!("Roteiro is\n\na graph. {long}") });
2285 // A pure symbol node with no captured content.
2286 let sym = Node::new("sym:rust:a.rs#main", NodeKind::Fn, "main");
2287 store
2288 .apply_factset(&FactSet::new().with_node(adr).with_node(sym))
2289 .expect("apply");
2290
2291 let hits = search(&store, "roteiro", 10).expect("search");
2292 let adr_hit = hits
2293 .iter()
2294 .find(|h| h.node.key == "adr:0001")
2295 .expect("adr hit");
2296 let snippet = adr_hit
2297 .snippet
2298 .as_deref()
2299 .expect("a content-bearing node yields a snippet");
2300 // Whitespace is collapsed to single spaces (no runs, no newlines)…
2301 assert!(snippet.starts_with("Roteiro is a graph."), "got: {snippet}");
2302 assert!(!snippet.contains(" "));
2303 assert!(!snippet.contains('\n'));
2304 // …and the snippet is bounded to SNIPPET_MAX chars *including* the ellipsis.
2305 assert!(
2306 snippet.chars().count() <= SNIPPET_MAX,
2307 "snippet is bounded: {} chars",
2308 snippet.chars().count()
2309 );
2310 assert!(
2311 snippet.ends_with('…'),
2312 "over-long content is truncated with an ellipsis"
2313 );
2314
2315 // A node without content falls back cleanly: no snippet, so the summary
2316 // (name/kind/path) is the grounding.
2317 let hits = search(&store, "main", 10).expect("search");
2318 let sym_hit = hits
2319 .iter()
2320 .find(|h| h.node.key == "sym:rust:a.rs#main")
2321 .expect("sym hit");
2322 assert!(
2323 sym_hit.snippet.is_none(),
2324 "a node with no content has no snippet"
2325 );
2326 }
2327
2328 #[test]
2329 fn explain_reports_labelled_neighbourhood() {
2330 let store = seeded();
2331 let ex = explain(&store, "sym:rust:a.rs#main")
2332 .expect("query")
2333 .expect("present");
2334 assert_eq!(ex.schema, SCHEMA);
2335 assert_eq!(ex.node.kind, "fn");
2336
2337 // Outgoing: derived call to helper.
2338 assert_eq!(ex.outgoing.len(), 1);
2339 assert_eq!(ex.outgoing[0].kind, "calls");
2340 assert_eq!(ex.outgoing[0].provenance, "derived");
2341 assert_eq!(ex.outgoing[0].node, "sym:rust:a.rs#helper");
2342
2343 // Incoming: authored reference from the ADR.
2344 assert_eq!(ex.incoming.len(), 1);
2345 assert_eq!(ex.incoming[0].provenance, "authored");
2346 assert_eq!(ex.incoming[0].node, "adr:0001");
2347 }
2348
2349 #[test]
2350 fn explain_missing_node_is_none() {
2351 let store = seeded();
2352 assert!(explain(&store, "sym:rust:a.rs#ghost").expect("q").is_none());
2353 }
2354
2355 #[test]
2356 fn edges_differing_only_in_provenance_are_ordered() {
2357 // Two edges A->B with the same kind but different provenance must sort
2358 // into a stable, deterministic order (authored before derived).
2359 let mut store = Store::open_in_memory().expect("store");
2360 let facts = FactSet::new()
2361 .with_node(Node::new("a", NodeKind::Fn, "a"))
2362 .with_node(Node::new("b", NodeKind::Fn, "b"))
2363 .with_edge(Edge::derived("a", "b", EdgeKind::References))
2364 .with_edge(Edge::authored("a", "b", EdgeKind::References));
2365 store.apply_factset(&facts).expect("apply");
2366
2367 let ex = explain(&store, "a").expect("q").expect("present");
2368 let provs: Vec<_> = ex.outgoing.iter().map(|e| e.provenance).collect();
2369 assert_eq!(provs, ["authored", "derived"]);
2370 }
2371
2372 #[test]
2373 fn list_kind_is_ordered() {
2374 let store = seeded();
2375 let listing = list_kind(&store, &NodeKind::Fn).expect("list");
2376 let keys: Vec<_> = listing.nodes.iter().map(|n| n.key.as_str()).collect();
2377 assert_eq!(keys, ["sym:rust:a.rs#helper", "sym:rust:a.rs#main"]);
2378 }
2379
2380 #[test]
2381 fn json_schema_is_stable() {
2382 let store = seeded();
2383 let ex = explain(&store, "adr:0001").expect("q").expect("present");
2384 let json = serde_json::to_value(&ex).expect("json");
2385 assert_eq!(json["schema"], SCHEMA);
2386 assert_eq!(json["node"]["key"], "adr:0001");
2387 assert_eq!(json["node"]["kind"], "adr");
2388 // Outgoing authored reference is present with its provenance label.
2389 assert_eq!(json["outgoing"][0]["kind"], "references");
2390 assert_eq!(json["outgoing"][0]["provenance"], "authored");
2391 assert_eq!(json["outgoing"][0]["node"], "sym:rust:a.rs#main");
2392 assert!(json["outgoing"][0]["confidence"].is_null());
2393 }
2394
2395 #[test]
2396 fn path_crosses_provenance_and_direction() {
2397 // adr:0001 --authored/references--> main --derived/calls--> helper.
2398 // A path from the ADR to helper must traverse both, each hop labelled.
2399 let store = seeded();
2400 let p = path(&store, "adr:0001", "sym:rust:a.rs#helper").expect("path");
2401 assert!(p.found);
2402 assert_eq!(p.length, 2);
2403 assert_eq!(p.schema, SCHEMA);
2404
2405 assert_eq!(p.hops[0].kind, "references");
2406 assert_eq!(p.hops[0].provenance, "authored");
2407 assert_eq!(p.hops[0].direction, "outgoing");
2408 assert_eq!(p.hops[0].node, "sym:rust:a.rs#main");
2409
2410 assert_eq!(p.hops[1].kind, "calls");
2411 assert_eq!(p.hops[1].provenance, "derived");
2412 assert_eq!(p.hops[1].node, "sym:rust:a.rs#helper");
2413 }
2414
2415 #[test]
2416 fn path_follows_edges_against_direction() {
2417 // From helper back to the ADR: both edges are traversed against their
2418 // stored direction, so each hop is `incoming`.
2419 let store = seeded();
2420 let p = path(&store, "sym:rust:a.rs#helper", "adr:0001").expect("path");
2421 assert!(p.found);
2422 assert_eq!(p.length, 2);
2423 assert!(p.hops.iter().all(|h| h.direction == "incoming"));
2424 assert_eq!(p.hops.last().unwrap().node, "adr:0001");
2425 }
2426
2427 #[test]
2428 fn path_same_node_is_trivial() {
2429 let store = seeded();
2430 let p = path(&store, "adr:0001", "adr:0001").expect("path");
2431 assert!(p.found);
2432 assert_eq!(p.length, 0);
2433 assert!(p.hops.is_empty());
2434 }
2435
2436 #[test]
2437 fn path_missing_endpoint_or_unreachable_is_not_found() {
2438 let mut store = Store::open_in_memory().expect("store");
2439 // Two disconnected components: a-b and an isolated island.
2440 let facts = FactSet::new()
2441 .with_node(Node::new("a", NodeKind::Fn, "a"))
2442 .with_node(Node::new("b", NodeKind::Fn, "b"))
2443 .with_node(Node::new("island", NodeKind::Fn, "island"))
2444 .with_edge(Edge::derived("a", "b", EdgeKind::Calls));
2445 store.apply_factset(&facts).expect("apply");
2446
2447 // Absent endpoint.
2448 let missing = path(&store, "a", "ghost").expect("path");
2449 assert!(!missing.found);
2450 assert!(missing.hops.is_empty());
2451
2452 // Present but unreachable.
2453 let unreachable = path(&store, "a", "island").expect("path");
2454 assert!(!unreachable.found);
2455 assert!(unreachable.hops.is_empty());
2456 }
2457
2458 #[test]
2459 fn path_is_shortest() {
2460 // a-b-c-d chain plus a direct a-d edge: the path must take the shortcut.
2461 let mut store = Store::open_in_memory().expect("store");
2462 let facts = FactSet::new()
2463 .with_node(Node::new("a", NodeKind::Fn, "a"))
2464 .with_node(Node::new("b", NodeKind::Fn, "b"))
2465 .with_node(Node::new("c", NodeKind::Fn, "c"))
2466 .with_node(Node::new("d", NodeKind::Fn, "d"))
2467 .with_edge(Edge::derived("a", "b", EdgeKind::Calls))
2468 .with_edge(Edge::derived("b", "c", EdgeKind::Calls))
2469 .with_edge(Edge::derived("c", "d", EdgeKind::Calls))
2470 .with_edge(Edge::derived("a", "d", EdgeKind::Calls));
2471 store.apply_factset(&facts).expect("apply");
2472
2473 let p = path(&store, "a", "d").expect("path");
2474 assert!(p.found);
2475 assert_eq!(p.length, 1, "the direct a->d edge is the shortest path");
2476 assert_eq!(p.hops[0].node, "d");
2477 }
2478
2479 #[test]
2480 fn glob_matches_segments_and_wildcards() {
2481 // `**` spans segments (including zero) and anchors both ends.
2482 assert!(glob_match("vendor/**", "vendor/lib/a.rs"));
2483 assert!(glob_match("vendor/**", "vendor")); // zero trailing segments
2484 assert!(glob_match("**/generated/*", "src/gen/generated/x.rs"));
2485 assert!(glob_match("**/*.rs", "a/b/c.rs"));
2486 // `*` and `?` stay within one segment.
2487 assert!(glob_match("src/*.rs", "src/main.rs"));
2488 assert!(!glob_match("src/*.rs", "src/sub/main.rs"));
2489 assert!(glob_match("a?c.rs", "abc.rs"));
2490 assert!(!glob_match("a?c.rs", "ac.rs"));
2491 // Anchored: a bare name does not match a nested path.
2492 assert!(!glob_match("generated", "src/generated"));
2493 assert!(!glob_match("vendor/**", "third_party/vendor/a.rs"));
2494 }
2495
2496 /// **The evidence weight is closed at both ends**, and the two ends mean
2497 /// different things.
2498 ///
2499 /// The boundary the doc comment used to get wrong: it claimed `(0, 1]`, which
2500 /// would have made a zero unreachable. It is reachable, from a writer stating
2501 /// `--confidence 0` and from nowhere else — the lowest weight Roteiro can
2502 /// *infer* is `anchor_penalty(Drifted)`, and that still leaves a score
2503 /// standing, which is checked here against the real constant rather than a
2504 /// number copied from it.
2505 #[test]
2506 fn the_evidence_weight_is_closed_at_both_ends() {
2507 let score = |evidence| memory_score("batch", &["batch"], "a batch cursor", "", evidence);
2508 let full = score(1.0);
2509 assert!(full > 0, "a fully-evidenced hit scores");
2510 assert_eq!(score(0.0), 0, "and a zero weight takes it to zero");
2511 assert!(
2512 score(0.5) < full && score(0.5) > 0,
2513 "in between, in between"
2514 );
2515
2516 // The worst Roteiro can infer about a record still leaves it scoring —
2517 // "demote, never delete", holding as arithmetic.
2518 let worst_inferable = [
2519 AnchorState::Valid,
2520 AnchorState::Unanchored,
2521 AnchorState::Unverifiable,
2522 AnchorState::Vanished,
2523 AnchorState::Drifted,
2524 ]
2525 .into_iter()
2526 .map(crate::anchor_penalty)
2527 .fold(f64::INFINITY, f64::min);
2528 assert!(
2529 score(worst_inferable) > 0,
2530 "the most demoted anchor state ({worst_inferable}) must not silence a hit",
2531 );
2532
2533 // Out-of-range input is clamped rather than trusted, so a corrupt stored
2534 // confidence cannot manufacture a score above the honest ceiling.
2535 assert_eq!(score(2.0), full, "clamped at the top");
2536 assert_eq!(score(-1.0), 0, "and at the bottom");
2537 }
2538
2539 // -- coupling (Q3) -----------------------------------------------------
2540
2541 /// A graph whose two most-coupled nodes have the **same undirected degree**
2542 /// but opposite direction: `hub` is called by two callers and calls nothing;
2543 /// `spread` calls two callees and is called by nothing. An undirected degree
2544 /// ranking cannot tell them apart, which is the whole point of this lens.
2545 fn coupled() -> Store {
2546 let mut store = Store::open_in_memory().expect("store");
2547 let mut facts = FactSet::new();
2548 for name in ["hub", "spread", "a", "b", "x", "y"] {
2549 facts = facts.with_node(Node::new(
2550 format!("sym:rust:a.rs#{name}"),
2551 NodeKind::Fn,
2552 name,
2553 ));
2554 }
2555 for (src, dst) in [("a", "hub"), ("b", "hub"), ("spread", "x"), ("spread", "y")] {
2556 facts = facts.with_edge(Edge::derived(
2557 format!("sym:rust:a.rs#{src}"),
2558 format!("sym:rust:a.rs#{dst}"),
2559 EdgeKind::Calls,
2560 ));
2561 }
2562 store.apply_factset(&facts).expect("apply");
2563 store
2564 }
2565
2566 /// Find an item by symbol name, so assertions read by name not by index.
2567 fn item<'a>(report: &'a CouplingReport, name: &str) -> &'a CouplingItem {
2568 report
2569 .items
2570 .iter()
2571 .find(|i| i.name == name)
2572 .unwrap_or_else(|| panic!("`{name}` missing from {:?}", report.items))
2573 }
2574
2575 #[test]
2576 fn coupling_keeps_the_direction_an_undirected_degree_discards() {
2577 let report = coupling(&coupled(), CouplingOrder::Total, 0).expect("coupling");
2578 let hub = item(&report, "hub");
2579 let spread = item(&report, "spread");
2580
2581 // Identical undirected degree — what a both-ends-incremented ranking sees.
2582 assert_eq!(hub.total, spread.total, "same total coupling");
2583
2584 // …and opposite direction, which is what this lens exists to report.
2585 assert_eq!((hub.fan_in, hub.fan_out), (2, 0), "hub is depended upon");
2586 assert_eq!(
2587 (spread.fan_in, spread.fan_out),
2588 (0, 2),
2589 "spread depends on others"
2590 );
2591 assert!(
2592 (hub.instability - 0.0).abs() < f64::EPSILON,
2593 "a purely called node is maximally stable: {}",
2594 hub.instability
2595 );
2596 assert!(
2597 (spread.instability - 1.0).abs() < f64::EPSILON,
2598 "a purely calling node is maximally unstable: {}",
2599 spread.instability
2600 );
2601
2602 assert_eq!(report.edge_kind, "calls");
2603 assert_eq!(report.coupled_nodes, 6);
2604 assert_eq!(report.call_edges, 4);
2605 assert_eq!(report.self_calls, 0);
2606 assert_eq!(report.cross_language_calls, 0);
2607 }
2608
2609 #[test]
2610 fn coupling_excludes_cross_language_name_collisions() {
2611 // Cross-file call resolution binds a callee by simple name across every
2612 // `Fn` node regardless of language, and Roteiro extracts no FFI — so a
2613 // JavaScript function "calling" a Rust one is a name collision. On this
2614 // repository that single rule is the difference between a Rust helper
2615 // reading as the most depended-on symbol in the tree and not appearing
2616 // at all.
2617 let mut store = coupled();
2618 let mut facts = FactSet::new().with_node(Node::new(
2619 "sym:javascript:app.js#render",
2620 NodeKind::Fn,
2621 "render",
2622 ));
2623 facts = facts.with_edge(Edge::derived(
2624 "sym:javascript:app.js#render",
2625 "sym:rust:a.rs#hub",
2626 EdgeKind::Calls,
2627 ));
2628 store.apply_factset(&facts).expect("apply");
2629
2630 let report = coupling(&store, CouplingOrder::Total, 0).expect("coupling");
2631 assert_eq!(
2632 item(&report, "hub").fan_in,
2633 2,
2634 "a JavaScript caller is not a dependant of a Rust function"
2635 );
2636 assert_eq!(
2637 report.cross_language_calls, 1,
2638 "the excluded edge is reported, not silently dropped"
2639 );
2640 assert_eq!(report.call_edges, 5, "and still counted as scanned");
2641 }
2642
2643 #[test]
2644 fn same_language_never_guesses_about_unknown_key_shapes() {
2645 assert!(super::same_language("sym:rust:a.rs#f", "sym:rust:b.rs#g"));
2646 assert!(!super::same_language(
2647 "sym:javascript:a.js#f",
2648 "sym:rust:b.rs#g"
2649 ));
2650 // A key that is not `sym:<lang>:…` carries no language to compare, so the
2651 // edge is kept: this filter drops only what it can prove spans languages.
2652 assert!(super::same_language("file:a.md", "sym:rust:b.rs#g"));
2653 assert!(super::same_language("sym:", "sym:rust:b.rs#g"));
2654 assert_eq!(super::sym_lang("sym:rust:a.rs#f"), Some("rust"));
2655 assert_eq!(
2656 super::sym_lang("sym::a.rs#f"),
2657 None,
2658 "empty lang is no lang"
2659 );
2660 assert_eq!(super::sym_lang("marker:a.rs#7"), None);
2661 }
2662
2663 #[test]
2664 fn coupling_counts_distinct_callers_not_parallel_edges() {
2665 // Migration 3 makes edges a set per `(src, dst, kind, provenance)` — so
2666 // the way one caller contributes two `Calls` rows is by **provenance**:
2667 // an extractor's `derived` call and an inference layer's `inferred` one.
2668 // Two rows, one dependant.
2669 let mut store = coupled();
2670 let inferred = Edge::inferred("sym:rust:a.rs#a", "sym:rust:a.rs#hub", EdgeKind::Calls, 0.9);
2671 store
2672 .apply_factset(&FactSet::new().with_edge(inferred))
2673 .expect("apply");
2674
2675 let report = coupling(&store, CouplingOrder::Total, 0).expect("coupling");
2676 assert_eq!(
2677 item(&report, "hub").fan_in,
2678 2,
2679 "the same caller at two provenances is one dependant, not two"
2680 );
2681 // The raw edge is still counted, so the parallel edge stays visible
2682 // rather than being silently normalised away.
2683 assert_eq!(
2684 report.call_edges, 5,
2685 "the extra edge is reported as scanned"
2686 );
2687 }
2688
2689 #[test]
2690 fn coupling_excludes_self_calls_from_both_fans() {
2691 // Recursion is a real edge that couples a node to nothing outside itself;
2692 // counting it would inflate `fan_in` AND `fan_out` for the same node.
2693 let mut store = coupled();
2694 let recursive = Edge::derived("sym:rust:a.rs#hub", "sym:rust:a.rs#hub", EdgeKind::Calls);
2695 store
2696 .apply_factset(&FactSet::new().with_edge(recursive))
2697 .expect("apply");
2698
2699 let report = coupling(&store, CouplingOrder::Total, 0).expect("coupling");
2700 let hub = item(&report, "hub");
2701 assert_eq!(
2702 (hub.fan_in, hub.fan_out),
2703 (2, 0),
2704 "recursion changes neither fan"
2705 );
2706 assert_eq!(report.self_calls, 1, "but it is reported, not dropped");
2707 }
2708
2709 #[test]
2710 fn coupling_order_picks_the_question_being_asked() {
2711 let store = coupled();
2712 let top = |order| {
2713 coupling(&store, order, 1).expect("coupling").items[0]
2714 .name
2715 .clone()
2716 };
2717 assert_eq!(top(CouplingOrder::FanIn), "hub", "most depended-on");
2718 assert_eq!(top(CouplingOrder::FanOut), "spread", "reaches furthest");
2719
2720 // `total` cannot separate the two, so the tie must break on `key` —
2721 // a stable order rather than whatever the map iteration yields.
2722 let by_total = coupling(&store, CouplingOrder::Total, 2).expect("coupling");
2723 assert_eq!(
2724 by_total.items.iter().map(|i| &i.name).collect::<Vec<_>>(),
2725 ["hub", "spread"],
2726 "ties break by key ascending"
2727 );
2728 }
2729
2730 #[test]
2731 fn coupling_reports_truncation_and_is_deterministic() {
2732 let store = coupled();
2733 let capped = coupling(&store, CouplingOrder::Total, 2).expect("coupling");
2734 assert_eq!(capped.items.len(), 2);
2735 assert_eq!(
2736 capped.coupled_nodes, 6,
2737 "the population is reported, so a capped list cannot read as the whole graph"
2738 );
2739 assert_eq!(capped.limit, 2);
2740
2741 // Identical input → byte-identical output, including the ratio's rendering.
2742 let a = serde_json::to_string(&capped).expect("json");
2743 let b =
2744 serde_json::to_string(&coupling(&store, CouplingOrder::Total, 2).expect("coupling"))
2745 .expect("json");
2746 assert_eq!(a, b, "deterministic serialisation");
2747 }
2748
2749 #[test]
2750 fn coupling_ignores_edge_kinds_whose_direction_is_not_a_call() {
2751 // `references` is directed too, but an ADR referencing a symbol is not a
2752 // caller. Only `Calls` may move these numbers.
2753 let mut store = coupled();
2754 let mut facts = FactSet::new().with_node(Node::new("adr:0001", NodeKind::Adr, "A"));
2755 facts = facts.with_edge(Edge::authored(
2756 "adr:0001",
2757 "sym:rust:a.rs#hub",
2758 EdgeKind::References,
2759 ));
2760 store.apply_factset(&facts).expect("apply");
2761
2762 let report = coupling(&store, CouplingOrder::Total, 0).expect("coupling");
2763 assert_eq!(item(&report, "hub").fan_in, 2, "a reference is not a call");
2764 assert!(
2765 !report.items.iter().any(|i| i.key == "adr:0001"),
2766 "a node with no call edges is not in the population: {:?}",
2767 report.items
2768 );
2769 assert_eq!(report.call_edges, 4);
2770 }
2771
2772 #[test]
2773 fn coupling_order_tokens_round_trip() {
2774 for token in CouplingOrder::tokens() {
2775 let order = CouplingOrder::from_token(token)
2776 .unwrap_or_else(|| panic!("`{token}` is advertised but not accepted"));
2777 assert_eq!(order.as_str(), token);
2778 }
2779 assert!(
2780 CouplingOrder::from_token("degree").is_none(),
2781 "an unknown order is rejected, not silently defaulted"
2782 );
2783 }
2784
2785 // -- debt density (Q1) -------------------------------------------------
2786
2787 /// A `file` node carrying the `meta.lines` this lens divides by — the shape
2788 /// `extract::file_node` emits for every blob.
2789 fn file_of(path: &str, lines: u64) -> Node {
2790 let mut node = Node::new(format!("file:{path}"), NodeKind::File, path);
2791 node.path = Some(path.to_owned());
2792 node.meta = serde_json::json!({ "bytes": lines * 30, "lines": lines });
2793 node
2794 }
2795
2796 /// A marker node as `markers::augment` emits it.
2797 fn marker_of(path: &str, line: u32, category: &str) -> Node {
2798 let mut node = Node::new(
2799 format!("marker:{path}#{line}"),
2800 NodeKind::Marker,
2801 format!("TODO {line}"), // roteiro:ignore
2802 );
2803 node.path = Some(path.to_owned());
2804 node.meta = serde_json::json!({
2805 "category": category,
2806 "text": format!("TODO {line}"), // roteiro:ignore
2807 "line": line,
2808 });
2809 node
2810 }
2811
2812 /// Two files with the **same marker count** and very different lengths —
2813 /// indistinguishable under `debt`, twenty-fold apart under density. Plus a
2814 /// third, short file whose single marker would top the ranking on arithmetic
2815 /// alone.
2816 fn marked() -> Store {
2817 let mut store = Store::open_in_memory().expect("store");
2818 let mut facts = FactSet::new()
2819 .with_node(file_of("big.rs", 4000))
2820 .with_node(file_of("small.rs", 200))
2821 .with_node(file_of("tiny.rs", 10));
2822 for line in 1..=40 {
2823 facts = facts.with_node(marker_of("big.rs", line, "todo")); // roteiro:ignore
2824 facts = facts.with_node(marker_of("small.rs", line, "todo")); // roteiro:ignore
2825 }
2826 facts = facts.with_node(marker_of("tiny.rs", 3, "stub"));
2827 store.apply_factset(&facts).expect("apply");
2828 store
2829 }
2830
2831 /// Every default: no category filter, no ignore globs, unlimited, floored at
2832 /// [`super::DEFAULT_MIN_LINES`].
2833 fn density(store: &Store, order: DensityOrder) -> DebtDensityReport {
2834 debt_density(store, &[], &[], order, 0, super::DEFAULT_MIN_LINES).expect("density")
2835 }
2836
2837 /// Find an item by path, so assertions read by file not by index.
2838 fn at<'a>(report: &'a DebtDensityReport, path: &str) -> &'a DensityItem {
2839 report
2840 .items
2841 .iter()
2842 .find(|i| i.path == path)
2843 .unwrap_or_else(|| panic!("`{path}` missing from {:?}", report.items))
2844 }
2845
2846 #[test]
2847 fn density_separates_files_a_raw_marker_count_cannot() {
2848 let report = density(&marked(), DensityOrder::Density);
2849 let big = at(&report, "big.rs");
2850 let small = at(&report, "small.rs");
2851
2852 // Identical under `debt` — the same forty markers each.
2853 assert_eq!(big.markers, small.markers, "same raw count");
2854
2855 // …and twenty-fold apart under density, which is the whole lens.
2856 assert!(
2857 (big.per_kloc - 10.0).abs() < f64::EPSILON,
2858 "40 markers in 4000 lines is 10 per kloc, was {}",
2859 big.per_kloc
2860 );
2861 assert!(
2862 (small.per_kloc - 200.0).abs() < f64::EPSILON,
2863 "40 markers in 200 lines is 200 per kloc, was {}",
2864 small.per_kloc
2865 );
2866 assert_eq!(
2867 report.items.first().map(|i| i.path.as_str()),
2868 Some("small.rs"),
2869 "the dense file ranks first: {:?}",
2870 report.items
2871 );
2872
2873 // The per-file category split, so "forty todo" and "forty stub" stay
2874 // distinguishable in a report that otherwise shows one number per file.
2875 assert_eq!(small.by_category.get("todo"), Some(&40)); // roteiro:ignore
2876 assert_eq!(report.schema, SCHEMA);
2877 }
2878
2879 #[test]
2880 fn markers_order_ranks_the_way_debt_already_does() {
2881 // The control: on `markers` the two forty-marker files tie and break on
2882 // path, so density is demonstrably the thing that separated them — not
2883 // some other difference in the fixture.
2884 let report = density(&marked(), DensityOrder::Markers);
2885 assert_eq!(
2886 report.items.iter().map(|i| &i.path).collect::<Vec<_>>(),
2887 ["big.rs", "small.rs"],
2888 "equal counts tie and break on path ascending"
2889 );
2890 }
2891
2892 #[test]
2893 fn the_short_file_floor_excludes_without_hiding() {
2894 // `tiny.rs` is 1 marker in 10 lines = 100 per kloc, which would place it
2895 // second on arithmetic alone. The floor keeps it out of the *ranking*
2896 // while leaving it in the population and the totals.
2897 let report = density(&marked(), DensityOrder::Density);
2898 assert!(
2899 !report.items.iter().any(|i| i.path == "tiny.rs"),
2900 "a 10-line file is not ranked: {:?}",
2901 report.items
2902 );
2903 assert_eq!(report.short_files, 1, "and its exclusion is reported");
2904 assert_eq!(
2905 report.files_with_markers, 3,
2906 "the population still counts it"
2907 );
2908 assert_eq!(report.ranked_files, 2);
2909 assert_eq!(
2910 report.total_markers, 81,
2911 "and so do the totals: 40 + 40 + 1"
2912 );
2913
2914 // `min_lines = 0` disables the floor rather than merely lowering it.
2915 let unfloored =
2916 debt_density(&marked(), &[], &[], DensityOrder::Density, 0, 0).expect("density");
2917 assert_eq!(unfloored.short_files, 0);
2918 assert_eq!(unfloored.ranked_files, 3);
2919 let tiny = at(&unfloored, "tiny.rs").per_kloc;
2920 assert!(
2921 (tiny - 100.0).abs() < f64::EPSILON,
2922 "the arithmetic the floor exists to keep out of the ranking, was {tiny}"
2923 );
2924 }
2925
2926 #[test]
2927 fn a_file_with_no_recorded_length_is_reported_not_divided_by() {
2928 // Three ways a denominator goes missing, all of which must land in
2929 // `unknown_length_files` rather than in the ranking with a fabricated
2930 // density: no `file` node at all, a `file` node with no `meta.lines`, and
2931 // a `lines` of zero (an empty file, or one unterminated line — a newline
2932 // count cannot tell those apart, so neither does this).
2933 let mut store = Store::open_in_memory().expect("store");
2934 let mut no_lines = Node::new("file:b.rs", NodeKind::File, "b.rs");
2935 no_lines.path = Some("b.rs".into());
2936 no_lines.meta = serde_json::json!({ "bytes": 90 });
2937 let facts = FactSet::new()
2938 .with_node(marker_of("orphan.rs", 1, "todo")) // roteiro:ignore
2939 .with_node(no_lines)
2940 .with_node(marker_of("b.rs", 1, "todo")) // roteiro:ignore
2941 .with_node(file_of("empty.rs", 0))
2942 .with_node(marker_of("empty.rs", 1, "todo")); // roteiro:ignore
2943 store.apply_factset(&facts).expect("apply");
2944
2945 let report = density(&store, DensityOrder::Density);
2946 assert!(report.items.is_empty(), "nothing rankable: {report:?}");
2947 assert_eq!(report.unknown_length_files, 3);
2948 assert_eq!(
2949 report.total_markers, 3,
2950 "the markers are still inventoried, so the file cannot vanish silently"
2951 );
2952 assert!(
2953 (report.overall_per_kloc - 0.0).abs() < f64::EPSILON,
2954 "and no density is invented from a zero denominator"
2955 );
2956 }
2957
2958 #[test]
2959 fn density_shares_debt_s_filters_rather_than_adding_a_second_vocabulary() {
2960 let store = marked();
2961 // The `[debt] ignore` globs `debt` already honours.
2962 let ignored = debt_density(
2963 &store,
2964 &[],
2965 &["small.rs".into()],
2966 DensityOrder::Density,
2967 0,
2968 super::DEFAULT_MIN_LINES,
2969 )
2970 .expect("density");
2971 assert!(
2972 !ignored.items.iter().any(|i| i.path == "small.rs"),
2973 "an ignored path leaves the report entirely: {:?}",
2974 ignored.items
2975 );
2976 assert_eq!(
2977 ignored.files_with_markers, 2,
2978 "not merely unranked — it is not in the population either"
2979 );
2980
2981 // And the same category filter, so a `--kind stub` density is the density
2982 // of stubs and not of everything.
2983 let stubs = debt_density(&store, &["stub".into()], &[], DensityOrder::Density, 0, 0)
2984 .expect("density");
2985 assert_eq!(stubs.total_markers, 1);
2986 assert_eq!(
2987 stubs.items.iter().map(|i| &i.path).collect::<Vec<_>>(),
2988 ["tiny.rs"]
2989 );
2990 }
2991
2992 #[test]
2993 fn density_ranks_on_the_exact_ratio_not_the_rounded_one() {
2994 // Two files whose densities differ in the fourth decimal: 1/3000 is
2995 // 0.3333 per kloc and 1/3001 is 0.3332. Both round to 0.33, so a ranking
2996 // built on `per_kloc` would tie them and break on path — putting the
2997 // *less* dense file first, since `a.rs` sorts before `b.rs`.
2998 let mut store = Store::open_in_memory().expect("store");
2999 let facts = FactSet::new()
3000 .with_node(file_of("a.rs", 3001))
3001 .with_node(marker_of("a.rs", 1, "todo")) // roteiro:ignore
3002 .with_node(file_of("b.rs", 3000))
3003 .with_node(marker_of("b.rs", 1, "todo")); // roteiro:ignore
3004 store.apply_factset(&facts).expect("apply");
3005
3006 let report = density(&store, DensityOrder::Density);
3007 assert_eq!(
3008 report.items.iter().map(|i| &i.path).collect::<Vec<_>>(),
3009 ["b.rs", "a.rs"],
3010 "the shorter file is denser, however the figures round"
3011 );
3012 assert_eq!(
3013 (report.items[0].per_kloc, report.items[1].per_kloc),
3014 (0.33, 0.33),
3015 "and the rendered figures really are equal, so the order came from elsewhere"
3016 );
3017 }
3018
3019 #[test]
3020 fn density_reports_truncation_and_is_deterministic() {
3021 let store = marked();
3022 let capped = debt_density(&store, &[], &[], DensityOrder::Density, 1, 0).expect("density");
3023 assert_eq!(capped.items.len(), 1);
3024 assert_eq!(capped.limit, 1);
3025 assert_eq!(
3026 capped.ranked_files, 3,
3027 "the population is reported, so a capped list cannot read as the whole repository"
3028 );
3029 // `overall_per_kloc` is the baseline across every ranked file, not across
3030 // the ones that survived the cap — otherwise the top file's own density
3031 // would be its own baseline.
3032 assert_eq!(capped.total_lines, 4210);
3033 assert!(
3034 (capped.overall_per_kloc - 19.24).abs() < f64::EPSILON,
3035 "81 markers over 4210 lines, was {}",
3036 capped.overall_per_kloc
3037 );
3038
3039 let a = serde_json::to_string(&capped).expect("json");
3040 let b = serde_json::to_string(
3041 &debt_density(&store, &[], &[], DensityOrder::Density, 1, 0).expect("density"),
3042 )
3043 .expect("json");
3044 assert_eq!(a, b, "deterministic serialisation");
3045 }
3046
3047 #[test]
3048 fn density_order_tokens_round_trip() {
3049 for token in DensityOrder::tokens() {
3050 let order = DensityOrder::from_token(token)
3051 .unwrap_or_else(|| panic!("`{token}` is advertised but not accepted"));
3052 assert_eq!(order.as_str(), token);
3053 }
3054 assert!(
3055 DensityOrder::from_token("count").is_none(),
3056 "an unknown order is rejected, not silently defaulted"
3057 );
3058 }
3059
3060 // -- config-secret inventory (S1) --------------------------------------
3061
3062 /// A `config_key` node as `extract::config_facts` emits it: `meta.value`
3063 /// present (already redacted, if the key name called for it).
3064 fn cfgkey(path: &str, dotted: &str, value: &str) -> Node {
3065 let mut node = Node::new(
3066 format!("cfgkey:{path}#{dotted}"),
3067 NodeKind::Other("config_key".to_owned()),
3068 dotted,
3069 );
3070 node.path = Some(path.to_owned());
3071 node.meta = serde_json::json!({ "key": dotted, "value": value });
3072 node
3073 }
3074
3075 /// A **struct-derived** `config_key` node as `synthesize_config_keys` emits
3076 /// it: `meta.value` OMITTED, because a Rust field declares no literal value.
3077 fn struct_cfgkey(path: &str, dotted: &str) -> Node {
3078 let mut node = Node::new(
3079 format!("cfgkey:{path}#{dotted}"),
3080 NodeKind::Other("config_key".to_owned()),
3081 dotted,
3082 );
3083 node.path = Some(path.to_owned());
3084 node.meta = serde_json::json!({
3085 "key": dotted,
3086 "source": "struct",
3087 "struct": "AppConfig",
3088 });
3089 node
3090 }
3091
3092 /// One of each state extraction can produce, plus a non-secret key and a
3093 /// k8s-`Secret`-style redaction under an innocuous name.
3094 fn configured() -> Store {
3095 let mut store = Store::open_in_memory().expect("store");
3096 let facts = FactSet::new()
3097 // Secret-named, redacted by extraction — the expected state.
3098 .with_node(cfgkey(".env", "API_TOKEN", "<redacted>"))
3099 .with_node(cfgkey("config.toml", "db.password", "<redacted>"))
3100 // Secret-named, struct-derived — no value to redact.
3101 .with_node(struct_cfgkey("src/config.rs", "serve.api_key"))
3102 // Not secret-named — not this lens's subject at all.
3103 .with_node(cfgkey("config.toml", "serve.addr", "127.0.0.1:8017"))
3104 // A k8s `Secret`'s data: redacted for where it lives, not what it is
3105 // called, so it is counted but not listed.
3106 .with_node(cfgkey("k8s/secret.yaml", "database-url", "<redacted>"));
3107 store.apply_factset(&facts).expect("apply");
3108 store
3109 }
3110
3111 /// Find an item by dotted name.
3112 fn secret<'a>(report: &'a ConfigSecretReport, name: &str) -> &'a super::ConfigSecretItem {
3113 report
3114 .items
3115 .iter()
3116 .find(|i| i.name == name)
3117 .unwrap_or_else(|| panic!("`{name}` missing from {:?}", report.items))
3118 }
3119
3120 #[test]
3121 fn the_inventory_reports_presence_and_redaction_not_values() {
3122 let report = config_secrets(&configured(), 0).expect("config_secrets");
3123
3124 assert_eq!(report.config_keys, 5, "the population it drew from");
3125 assert_eq!(report.secret_named, 3, "{:?}", report.items);
3126 assert_eq!(report.files, 3);
3127 assert_eq!(report.schema, SCHEMA);
3128
3129 // Paths, key names and state, which is what the lens is for.
3130 assert_eq!(secret(&report, "API_TOKEN").path.as_deref(), Some(".env"));
3131 assert_eq!(
3132 secret(&report, "db.password").key,
3133 "cfgkey:config.toml#db.password"
3134 );
3135 // The state comes from comparing the stored value against the redactor's
3136 // own constant, so asserting it is what keeps reader and writer from
3137 // drifting apart on a spelling.
3138 assert_eq!(
3139 secret(&report, "API_TOKEN").state,
3140 RedactionState::Redacted,
3141 "the placeholder extraction wrote is recognised as a redaction"
3142 );
3143 assert_eq!(report.redacted, 2, "{report:?}");
3144
3145 // No value is carried on any item — there is no field for one. The
3146 // serialised shape is the contract, so assert against that, not the type.
3147 let json = serde_json::to_value(&report).expect("json");
3148 let text = serde_json::to_string(&report).expect("json");
3149 assert!(
3150 json["items"][0].get("value").is_none(),
3151 "an item carries no value field: {text}"
3152 );
3153 assert!(
3154 !text.contains("<redacted>"),
3155 "not even the placeholder is echoed back: {text}"
3156 );
3157
3158 // Ordering is `(path, name, key)` — an inventory, not a ranking.
3159 assert_eq!(
3160 report.items.iter().map(|i| &i.name).collect::<Vec<_>>(),
3161 ["API_TOKEN", "db.password", "serve.api_key"]
3162 );
3163 }
3164
3165 #[test]
3166 fn a_struct_declared_key_is_neither_redacted_nor_a_leak() {
3167 // A `@rto:config` struct field has no literal value in code, so extraction
3168 // omits `meta.value` entirely. Folding that in with a successful redaction
3169 // would claim a redaction that never happened; calling it unredacted would
3170 // report a leak that does not exist.
3171 let report = config_secrets(&configured(), 0).expect("config_secrets");
3172 let declared = secret(&report, "serve.api_key");
3173 assert_eq!(declared.state, RedactionState::Declared);
3174 assert_eq!(declared.source.as_deref(), Some("struct"));
3175
3176 assert_eq!(report.redacted, 2, "the two file-derived keys");
3177 assert_eq!(report.declared, 1);
3178 assert_eq!(
3179 report.unredacted, 0,
3180 "the invariant extraction maintains: {report:?}"
3181 );
3182 }
3183
3184 #[test]
3185 fn an_unredacted_secret_named_value_is_reported_as_a_finding() {
3186 // Extraction redacts every secret-named key, so this state is unreachable
3187 // from extraction — but `apply_import_layer` upserts whatever nodes an
3188 // imported factset carries, so another tool's import can put an unredacted
3189 // value in the store. That is the one path worth reporting, and it is a
3190 // finding about THIS STORE, not about the source repository.
3191 let mut store = configured();
3192 store
3193 .apply_import_layer(
3194 "other-tool",
3195 &FactSet::new().with_node(cfgkey("imported.env", "AWS_SECRET", "AKIAnot-redacted")),
3196 )
3197 .expect("import");
3198
3199 let report = config_secrets(&store, 0).expect("config_secrets");
3200 assert_eq!(report.unredacted, 1, "{report:?}");
3201 assert_eq!(secret(&report, "AWS_SECRET").state, RedactionState::Present);
3202 // And still no value in the report: the lens says *that* something is
3203 // unredacted, and never repeats it.
3204 let text = serde_json::to_string(&report).expect("json");
3205 assert!(
3206 !text.contains("AKIA"),
3207 "the value is not echoed back: {text}"
3208 );
3209 }
3210
3211 #[test]
3212 fn a_redaction_under_an_innocuous_name_is_counted_but_not_listed() {
3213 // A k8s `Secret`'s `data` is redacted because of where it lives, whatever
3214 // the key is called. It is not secret-*named*, so it is not this lens's
3215 // subject — but it is counted, so a reader comparing `redacted` against the
3216 // number of `<redacted>` values in the graph does not find a surplus they
3217 // cannot explain.
3218 let report = config_secrets(&configured(), 0).expect("config_secrets");
3219 assert_eq!(report.redacted_not_secret_named, 1);
3220 assert!(
3221 !report.items.iter().any(|i| i.name == "database-url"),
3222 "not listed: {:?}",
3223 report.items
3224 );
3225 assert_eq!(
3226 report.redacted + report.redacted_not_secret_named,
3227 3,
3228 "and the two figures together account for every redacted value"
3229 );
3230 }
3231
3232 #[test]
3233 fn the_inventory_cannot_see_a_credential_that_is_not_a_config_key() {
3234 // The load-bearing limitation, asserted rather than only documented: a
3235 // credential in a Rust string literal produces no `config_key` node, so it
3236 // is invisible here. No extension of this lens can change that — which is
3237 // why it is named for the inventory it is, not the scanner it is not.
3238 let mut store = configured();
3239 let mut hardcoded = Node::new("sym:rust:src/main.rs#connect", NodeKind::Fn, "connect");
3240 hardcoded.path = Some("src/main.rs".into());
3241 // Split at the prefix for the same reason as `FAKE_TOKEN` in
3242 // `roteiro/tests/config_secrets_cli.rs`: assembled, this is AWS's own
3243 // documentation placeholder, but it matches the canonical access-key-id
3244 // rule exactly and a regex-rule scanner cannot know the difference. The
3245 // assembled value is unchanged; no assertion here matches on its text.
3246 hardcoded.meta = serde_json::json!({
3247 "content": concat!("let token = \"AKIA", "IOSFODNN7EXAMPLE\";"),
3248 });
3249 store
3250 .apply_factset(&FactSet::new().with_node(hardcoded))
3251 .expect("apply");
3252
3253 let report = config_secrets(&store, 0).expect("config_secrets");
3254 assert_eq!(
3255 report.secret_named, 3,
3256 "a hardcoded credential does not appear: {:?}",
3257 report.items
3258 );
3259 assert_eq!(report.config_keys, 5, "and is not a config key at all");
3260 }
3261
3262 #[test]
3263 fn the_inventory_reports_truncation_and_is_deterministic() {
3264 let store = configured();
3265 let capped = config_secrets(&store, 1).expect("config_secrets");
3266 assert_eq!(capped.items.len(), 1);
3267 assert_eq!(capped.limit, 1);
3268 assert_eq!(
3269 capped.secret_named, 3,
3270 "the population is reported, so a capped list cannot read as a clean repository"
3271 );
3272 // The state counts are over the whole population too, not the shown rows —
3273 // otherwise a cap could hide an `unredacted` finding.
3274 assert_eq!((capped.redacted, capped.declared), (2, 1));
3275
3276 let a = serde_json::to_string(&capped).expect("json");
3277 let b = serde_json::to_string(&config_secrets(&store, 1).expect("config_secrets"))
3278 .expect("json");
3279 assert_eq!(a, b, "deterministic serialisation");
3280 }
3281
3282 #[test]
3283 fn an_empty_report_means_no_secret_named_key_not_no_secret() {
3284 // The distinction the lens must never blur: a credential under an
3285 // innocuous key name (`dsn`) is not secret-named, is not redacted, and does
3286 // not appear. So "nothing found" is a statement about naming.
3287 let mut store = Store::open_in_memory().expect("store");
3288 store
3289 .apply_factset(&FactSet::new().with_node(cfgkey(
3290 ".env",
3291 "DSN",
3292 "postgres://u:pw@host/db",
3293 )))
3294 .expect("apply");
3295
3296 let report = config_secrets(&store, 0).expect("config_secrets");
3297 assert_eq!(report.secret_named, 0, "nothing is secret-*named*");
3298 assert_eq!(report.redacted_not_secret_named, 0);
3299 assert_eq!(
3300 report.config_keys, 1,
3301 "while the graph does hold a config key with a credential in it"
3302 );
3303 }
3304
3305 #[test]
3306 fn redaction_state_tokens_match_their_serialisation() {
3307 // The token and the wire form are the same string, so a caller matching on
3308 // the JSON and a caller matching on `as_str` cannot disagree.
3309 for state in [
3310 RedactionState::Redacted,
3311 RedactionState::Declared,
3312 RedactionState::Present,
3313 ] {
3314 let json = serde_json::to_string(&state).expect("json");
3315 assert_eq!(json, format!("\"{}\"", state.as_str()));
3316 }
3317 }
3318}