1use anyhow::{Context, Result};
19use rusqlite::Connection;
20use serde::Deserialize;
21use std::collections::HashMap;
22
23use crate::cache::CacheManager;
24use crate::models::SearchResult;
25
26const ANCHOR_SYMBOLS_PER_MODULE: usize = 5;
28
29const MAX_MODULES_IN_EVIDENCE: usize = 25;
32
33#[derive(Debug, Clone)]
35pub struct Concept {
36 pub name: String,
38 pub definition: String,
40 pub related_modules: Vec<String>,
43 pub category: Option<String>,
45}
46
47#[derive(Debug, Clone, Default)]
49pub struct GlossaryData {
50 pub concepts: Vec<Concept>,
51 pub intro: Option<String>,
53}
54
55#[derive(Debug, Clone)]
57pub struct ModuleEvidence {
58 pub path: String,
60 pub file_count: usize,
62 pub anchor_symbols: Vec<String>,
64}
65
66#[derive(Debug, Clone, Default)]
68pub struct GlossaryEvidence {
69 pub total_files: usize,
70 pub total_lines: usize,
71 pub language_mix: Vec<(String, usize)>,
72 pub dependency_edges: usize,
73 pub hotspot_files: Vec<String>,
74 pub modules: Vec<ModuleEvidence>,
75}
76
77#[derive(Debug, Clone, Deserialize)]
80pub struct ConceptsResponse {
81 #[serde(default)]
82 pub intro: Option<String>,
83 #[serde(default)]
84 pub concepts: Vec<RawConcept>,
85}
86
87#[derive(Debug, Clone, Deserialize)]
88pub struct RawConcept {
89 pub name: String,
90 #[serde(default)]
91 pub definition: String,
92 #[serde(default)]
93 pub category: Option<String>,
94 #[serde(default)]
95 pub related_modules: Vec<String>,
96}
97
98impl From<RawConcept> for Concept {
99 fn from(raw: RawConcept) -> Self {
100 Concept {
101 name: raw.name,
102 definition: raw.definition,
103 category: raw.category,
104 related_modules: raw.related_modules,
105 }
106 }
107}
108
109impl From<ConceptsResponse> for GlossaryData {
110 fn from(resp: ConceptsResponse) -> Self {
111 GlossaryData {
112 concepts: resp.concepts.into_iter().map(Into::into).collect(),
113 intro: resp.intro,
114 }
115 }
116}
117
118fn module_of(file_path: &str) -> String {
124 let parts: Vec<&str> = file_path.split('/').collect();
125 match parts.len() {
126 0 | 1 => String::new(),
127 2 => parts[0].to_string(),
128 _ => format!("{}/{}", parts[0], parts[1]),
129 }
130}
131
132fn module_slug(module_path: &str) -> String {
134 module_path.replace('/', "-")
135}
136
137fn anchor_priority(kind: &str) -> u8 {
142 match kind.to_lowercase().as_str() {
143 "struct" | "class" | "trait" | "interface" | "enum" | "type" | "typedef" => 0,
144 "function" | "method" | "macro" | "module" => 1,
145 "constant" | "property" | "event" | "attribute" | "export" => 2,
146 _ => 3,
149 }
150}
151
152pub fn collect_glossary_evidence(cache: &CacheManager) -> Result<Option<GlossaryEvidence>> {
159 let db_path = cache.path().join("meta.db");
160 let conn = Connection::open(&db_path).context("Failed to open meta.db")?;
161
162 let has_symbols: bool = conn
163 .query_row(
164 "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='symbols'",
165 [],
166 |row| row.get::<_, i64>(0),
167 )
168 .map(|c| c > 0)
169 .unwrap_or(false);
170
171 if !has_symbols {
172 return Ok(None);
173 }
174
175 let total_files: usize = conn
176 .query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0))
177 .unwrap_or(0);
178 let total_lines: usize = conn
179 .query_row("SELECT COALESCE(SUM(line_count), 0) FROM files", [], |r| {
180 r.get(0)
181 })
182 .unwrap_or(0);
183
184 let mut language_mix: Vec<(String, usize)> = Vec::new();
186 if let Ok(mut stmt) = conn.prepare(
187 "SELECT COALESCE(language, 'other'), COUNT(*) FROM files \
188 GROUP BY language ORDER BY COUNT(*) DESC LIMIT 10",
189 ) && let Ok(rows) = stmt.query_map([], |row| {
190 Ok((row.get::<_, String>(0)?, row.get::<_, usize>(1)?))
191 }) {
192 language_mix = rows.flatten().collect();
193 }
194
195 let dependency_edges: usize = conn
197 .query_row::<usize, _, _>(
198 "SELECT COUNT(*) FROM file_dependencies WHERE resolved_file_id IS NOT NULL",
199 [],
200 |row| row.get(0),
201 )
202 .unwrap_or(0);
203
204 let mut hotspot_files: Vec<String> = Vec::new();
206 if dependency_edges > 0
207 && let Ok(mut stmt) = conn.prepare(
208 "SELECT f.path, COUNT(DISTINCT fd.file_id) as dep_count \
209 FROM file_dependencies fd JOIN files f ON fd.resolved_file_id = f.id \
210 GROUP BY fd.resolved_file_id ORDER BY dep_count DESC LIMIT 8",
211 )
212 && let Ok(rows) = stmt.query_map([], |row| row.get::<_, String>(0))
213 {
214 hotspot_files = rows.flatten().collect();
215 }
216
217 let mut stmt = conn.prepare(
221 "SELECT s.symbols_json, f.path, f.line_count \
222 FROM symbols s JOIN files f ON s.file_id = f.id",
223 )?;
224 let rows: Vec<(String, String, usize)> = stmt
225 .query_map([], |row| {
226 Ok((
227 row.get::<_, String>(0)?,
228 row.get::<_, String>(1)?,
229 row.get::<_, usize>(2).unwrap_or(0),
230 ))
231 })?
232 .filter_map(|r| r.ok())
233 .collect();
234
235 #[derive(Default)]
236 struct ModuleBucket {
237 file_count: usize,
238 candidates: Vec<(u8, String)>,
240 }
241
242 let mut by_module: HashMap<String, ModuleBucket> = HashMap::new();
243
244 for (symbols_json, file_path, _line_count) in rows {
245 let module = module_of(&file_path);
246 if module.is_empty() {
247 continue;
248 }
249 let bucket = by_module.entry(module.clone()).or_default();
250 bucket.file_count += 1;
251
252 let symbols: Vec<SearchResult> = match serde_json::from_str(&symbols_json) {
253 Ok(s) => s,
254 Err(_) => continue,
255 };
256
257 for sr in symbols {
258 let Some(name) = sr.symbol else { continue };
259 if name.len() < 3 {
260 continue;
261 }
262 let kind_str = sr.kind.to_string();
263 let kl = kind_str.to_lowercase();
265 if kl == "variable" || kl == "import" || kl == "export" || kl == "unknown" {
266 continue;
267 }
268 let priority = anchor_priority(&kind_str);
269 bucket.candidates.push((priority, name));
270 }
271 }
272
273 let mut modules: Vec<ModuleEvidence> = by_module
276 .into_iter()
277 .map(|(path, mut bucket)| {
278 bucket
279 .candidates
280 .sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1)));
281 let mut anchors: Vec<String> = Vec::new();
282 let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
283 for (_, name) in bucket.candidates {
284 if seen.insert(name.clone()) {
285 anchors.push(name);
286 if anchors.len() >= ANCHOR_SYMBOLS_PER_MODULE {
287 break;
288 }
289 }
290 }
291 ModuleEvidence {
292 path,
293 file_count: bucket.file_count,
294 anchor_symbols: anchors,
295 }
296 })
297 .collect();
298
299 modules.sort_by(|a, b| {
301 b.file_count
302 .cmp(&a.file_count)
303 .then_with(|| a.path.cmp(&b.path))
304 });
305 modules.truncate(MAX_MODULES_IN_EVIDENCE);
306
307 Ok(Some(GlossaryEvidence {
308 total_files,
309 total_lines,
310 language_mix,
311 dependency_edges,
312 hotspot_files,
313 modules,
314 }))
315}
316
317pub fn build_concepts_context(evidence: &GlossaryEvidence, project_name: &str) -> String {
321 let mut ctx = String::new();
322
323 ctx.push_str(&format!("Project: {}\n", project_name));
324 ctx.push_str(&format!(
325 "Scale: {} files, {} lines, {} modules, {} dependency edges\n",
326 evidence.total_files,
327 evidence.total_lines,
328 evidence.modules.len(),
329 evidence.dependency_edges,
330 ));
331
332 if !evidence.language_mix.is_empty() {
333 let langs: Vec<String> = evidence
334 .language_mix
335 .iter()
336 .map(|(lang, count)| format!("{} ({})", lang, count))
337 .collect();
338 ctx.push_str(&format!("Languages: {}\n", langs.join(", ")));
339 }
340 ctx.push('\n');
341
342 ctx.push_str("Top-level modules (with anchor symbol names):\n");
343 for m in &evidence.modules {
344 if m.anchor_symbols.is_empty() {
345 ctx.push_str(&format!("- {} ({} files)\n", m.path, m.file_count));
346 } else {
347 ctx.push_str(&format!(
348 "- {} ({} files) — key symbols: {}\n",
349 m.path,
350 m.file_count,
351 m.anchor_symbols.join(", ")
352 ));
353 }
354 }
355 ctx.push('\n');
356
357 if !evidence.hotspot_files.is_empty() {
358 ctx.push_str("Dependency hotspots (most-imported files):\n");
359 for path in &evidence.hotspot_files {
360 ctx.push_str(&format!("- {}\n", path));
361 }
362 ctx.push('\n');
363 }
364
365 ctx
366}
367
368pub fn parse_concepts_response(raw: &str) -> Result<ConceptsResponse> {
375 let trimmed = raw.trim();
376
377 let cleaned: &str = if let Some(rest) = trimmed.strip_prefix("```json") {
379 rest.trim_start().trim_end_matches("```").trim()
380 } else if let Some(rest) = trimmed.strip_prefix("```") {
381 rest.trim_start().trim_end_matches("```").trim()
382 } else {
383 trimmed
384 };
385
386 let slice = if cleaned.starts_with('{') {
389 cleaned
390 } else if let (Some(start), Some(end)) = (cleaned.find('{'), cleaned.rfind('}')) {
391 &cleaned[start..=end]
392 } else {
393 cleaned
394 };
395
396 serde_json::from_str::<ConceptsResponse>(slice)
397 .context("Failed to parse concepts JSON response from LLM")
398}
399
400pub fn render_glossary_markdown(data: &GlossaryData) -> String {
404 if data.concepts.is_empty() {
405 return "*Concepts are generated by the LLM narration pipeline. \
406 Re-run `rfx pulse generate` with LLM enabled to populate this page.*\n"
407 .to_string();
408 }
409
410 let mut md = String::new();
411
412 if let Some(ref intro) = data.intro {
413 md.push_str(intro.trim());
414 md.push_str("\n\n");
415 }
416
417 let mut order: Vec<String> = Vec::new();
420 let mut grouped: HashMap<String, Vec<&Concept>> = HashMap::new();
421 for concept in &data.concepts {
422 let cat = concept
423 .category
424 .clone()
425 .unwrap_or_else(|| "Concepts".to_string());
426 if !grouped.contains_key(&cat) {
427 order.push(cat.clone());
428 }
429 grouped.entry(cat).or_default().push(concept);
430 }
431
432 md.push_str(&format!(
433 "**{}** core concepts across {} {}.\n\n",
434 data.concepts.len(),
435 order.len(),
436 if order.len() == 1 {
437 "category"
438 } else {
439 "categories"
440 },
441 ));
442
443 for cat in &order {
444 md.push_str(&format!("## {}\n\n", cat));
445 if let Some(items) = grouped.get(cat) {
446 for concept in items {
447 md.push_str(&format!("### {}\n\n", concept.name));
448
449 for line in concept.definition.trim().lines() {
451 md.push_str("> ");
452 md.push_str(line);
453 md.push('\n');
454 }
455 md.push('\n');
456
457 if !concept.related_modules.is_empty() {
458 let links: Vec<String> = concept
459 .related_modules
460 .iter()
461 .map(|m| format!("[`{}`](/wiki/{}/)", m.trim(), module_slug(m.trim())))
462 .collect();
463 md.push_str(&format!("*Implemented in {}*\n\n", links.join(", ")));
464 }
465 }
466 }
467 }
468
469 md
470}
471
472pub fn render_glossary_no_llm(evidence: &GlossaryEvidence) -> String {
476 let mut md = String::new();
477 md.push_str(
478 "*Concepts are generated by the LLM narration pipeline. \
479 Re-run `rfx pulse generate` with LLM enabled to populate this page.*\n\n",
480 );
481
482 if evidence.modules.is_empty() {
483 return md;
484 }
485
486 md.push_str("**Modules in this codebase:**\n\n");
487 for m in &evidence.modules {
488 md.push_str(&format!(
489 "- [`{}`](/wiki/{}/) ({} files)\n",
490 m.path,
491 module_slug(&m.path),
492 m.file_count
493 ));
494 }
495 md.push('\n');
496 md
497}
498
499#[cfg(test)]
500mod tests {
501 use super::*;
502 use crate::cache::CacheManager;
503 use tempfile::TempDir;
504
505 fn empty_cache() -> (TempDir, CacheManager) {
506 let tmp = TempDir::new().unwrap();
507 let cache = CacheManager::new(tmp.path().to_str().unwrap());
508 cache.init().unwrap();
509 (tmp, cache)
510 }
511
512 #[test]
513 fn test_module_of() {
514 assert_eq!(module_of("src/models.rs"), "src");
515 assert_eq!(module_of("src/pulse/wiki.rs"), "src/pulse");
516 assert_eq!(module_of("src/parsers/rust/mod.rs"), "src/parsers");
517 assert_eq!(module_of("README.md"), "");
518 }
519
520 #[test]
521 fn test_module_slug() {
522 assert_eq!(module_slug("src"), "src");
523 assert_eq!(module_slug("src/pulse"), "src-pulse");
524 assert_eq!(module_slug("src/parsers/rust"), "src-parsers-rust");
525 }
526
527 #[test]
528 fn test_anchor_priority_orders_types_first() {
529 assert!(anchor_priority("struct") < anchor_priority("function"));
530 assert!(anchor_priority("trait") < anchor_priority("constant"));
531 assert!(anchor_priority("enum") < anchor_priority("variable"));
532 }
533
534 #[test]
535 fn test_collect_glossary_evidence_empty_cache() {
536 let (_tmp, cache) = empty_cache();
537 let result = collect_glossary_evidence(&cache).unwrap();
538 assert!(result.is_none());
540 }
541
542 #[test]
543 fn test_build_concepts_context_includes_modules() {
544 let evidence = GlossaryEvidence {
545 total_files: 120,
546 total_lines: 18_500,
547 language_mix: vec![("rust".to_string(), 110), ("toml".to_string(), 10)],
548 dependency_edges: 340,
549 hotspot_files: vec!["src/models.rs".to_string()],
550 modules: vec![
551 ModuleEvidence {
552 path: "src".to_string(),
553 file_count: 42,
554 anchor_symbols: vec![
555 "Cli".to_string(),
556 "SearchResult".to_string(),
557 "run".to_string(),
558 ],
559 },
560 ModuleEvidence {
561 path: "src/pulse".to_string(),
562 file_count: 18,
563 anchor_symbols: vec!["generate_site".to_string(), "PulseReport".to_string()],
564 },
565 ModuleEvidence {
566 path: "src/query".to_string(),
567 file_count: 9,
568 anchor_symbols: vec!["QueryEngine".to_string()],
569 },
570 ],
571 };
572 let ctx = build_concepts_context(&evidence, "Reflex");
573
574 assert!(ctx.contains("Project: Reflex"));
575 assert!(ctx.contains("120 files"));
576 assert!(ctx.contains("src (42 files)"));
577 assert!(ctx.contains("src/pulse"));
578 assert!(ctx.contains("src/query"));
579 assert!(ctx.contains("SearchResult"));
580 assert!(ctx.contains("QueryEngine"));
581 assert!(ctx.contains("Languages: rust (110)"));
582 assert!(ctx.contains("Dependency hotspots"));
583 }
584
585 #[test]
586 fn test_parse_concepts_response_valid_json() {
587 let raw = r#"{
588 "intro": "Reflex catalogs search primitives and indexing building blocks.",
589 "concepts": [
590 {
591 "name": "Trigram Index",
592 "category": "Core Capabilities",
593 "definition": "A fast inverted index built from three-character substrings.",
594 "related_modules": ["src/index", "src/query"]
595 },
596 {
597 "name": "Symbol Cache",
598 "category": "Data Model",
599 "definition": "A persistent store of parsed language symbols keyed by content hash.",
600 "related_modules": ["src/cache"]
601 }
602 ]
603 }"#;
604
605 let parsed = parse_concepts_response(raw).expect("should parse");
606 assert_eq!(parsed.concepts.len(), 2);
607 assert_eq!(parsed.concepts[0].name, "Trigram Index");
608 assert_eq!(
609 parsed.concepts[0].related_modules,
610 vec!["src/index", "src/query"]
611 );
612 assert!(parsed.intro.as_ref().unwrap().contains("search primitives"));
613 }
614
615 #[test]
616 fn test_parse_concepts_response_strips_markdown_fence() {
617 let raw = "```json\n{\"intro\":\"x\",\"concepts\":[]}\n```";
618 let parsed = parse_concepts_response(raw).expect("should parse");
619 assert_eq!(parsed.concepts.len(), 0);
620 assert_eq!(parsed.intro.as_deref(), Some("x"));
621 }
622
623 #[test]
624 fn test_parse_concepts_response_extracts_embedded_json() {
625 let raw = "Here is the output you requested:\n\
626 {\"intro\":\"y\",\"concepts\":[{\"name\":\"X\",\"definition\":\"d\"}]}\n\
627 Hope that helps!";
628 let parsed = parse_concepts_response(raw).expect("should parse");
629 assert_eq!(parsed.concepts.len(), 1);
630 assert_eq!(parsed.concepts[0].name, "X");
631 }
632
633 #[test]
634 fn test_parse_concepts_response_rejects_malformed() {
635 let raw = "this is definitely not JSON at all";
636 assert!(parse_concepts_response(raw).is_err());
637 }
638
639 #[test]
640 fn test_render_with_concepts() {
641 let data = GlossaryData {
642 intro: Some(
643 "Reflex catalogs the core pieces of a local code-search engine.".to_string(),
644 ),
645 concepts: vec![
646 Concept {
647 name: "Trigram Index".to_string(),
648 definition: "A fast inverted index built from three-character substrings."
649 .to_string(),
650 category: Some("Core Capabilities".to_string()),
651 related_modules: vec!["src/index".to_string(), "src/query".to_string()],
652 },
653 Concept {
654 name: "Symbol Cache".to_string(),
655 definition: "A persistent store of parsed language symbols.".to_string(),
656 category: Some("Data Model".to_string()),
657 related_modules: vec!["src/cache".to_string()],
658 },
659 ],
660 };
661
662 let md = render_glossary_markdown(&data);
663
664 assert!(md.contains("Reflex catalogs"));
666 assert!(md.contains("## Core Capabilities"));
667 assert!(md.contains("## Data Model"));
668 assert!(md.contains("### Trigram Index"));
669 assert!(md.contains("### Symbol Cache"));
670 assert!(md.contains("> A fast inverted index"));
671 assert!(md.contains("[`src/index`](/wiki/src-index/)"));
672 assert!(md.contains("[`src/query`](/wiki/src-query/)"));
673 assert!(md.contains("Implemented in"));
674
675 assert!(!md.contains("```rust"), "no signature code blocks");
677 assert!(!md.contains(":1"), "no file:line markers (cheap check)");
678 assert!(!md.contains("| Symbol | Kind"), "no flat table");
679 }
680
681 #[test]
682 fn test_render_no_llm_fallback() {
683 let data = GlossaryData::default();
684 let md = render_glossary_markdown(&data);
685 assert!(md.contains("LLM narration pipeline"));
686 assert!(md.contains("rfx pulse generate"));
687 }
688
689 #[test]
690 fn test_render_no_llm_fallback_with_evidence_lists_modules() {
691 let evidence = GlossaryEvidence {
692 total_files: 10,
693 total_lines: 500,
694 language_mix: vec![],
695 dependency_edges: 0,
696 hotspot_files: vec![],
697 modules: vec![
698 ModuleEvidence {
699 path: "src".to_string(),
700 file_count: 5,
701 anchor_symbols: vec![],
702 },
703 ModuleEvidence {
704 path: "src/pulse".to_string(),
705 file_count: 3,
706 anchor_symbols: vec![],
707 },
708 ],
709 };
710 let md = render_glossary_no_llm(&evidence);
711 assert!(md.contains("LLM narration pipeline"));
712 assert!(md.contains("[`src`](/wiki/src/)"));
713 assert!(md.contains("[`src/pulse`](/wiki/src-pulse/)"));
714 assert!(md.contains("(5 files)"));
715 }
716
717 #[test]
718 fn test_concepts_response_into_glossary_data() {
719 let resp = ConceptsResponse {
720 intro: Some("hi".to_string()),
721 concepts: vec![RawConcept {
722 name: "Concept".to_string(),
723 definition: "def".to_string(),
724 category: Some("Cat".to_string()),
725 related_modules: vec!["src".to_string()],
726 }],
727 };
728 let data: GlossaryData = resp.into();
729 assert_eq!(data.concepts.len(), 1);
730 assert_eq!(data.concepts[0].name, "Concept");
731 assert_eq!(data.intro.as_deref(), Some("hi"));
732 }
733}