turbovault_tools/
grounding.rs

1//! Grounding primitives — data an external LLM judge consumes.
2//!
3//! The competitor's evaluation harness scores enrichment with LLM-judge metrics
4//! (hallucination-free, redundancy, contradiction, disambiguation). TurboVault
5//! deliberately does **not** run a judge in-process; instead it surfaces the raw
6//! material a judge needs to score those dimensions, computed deterministically:
7//!
8//! - **claims** — candidate factual statements extracted from the prose body.
9//! - **citations** — the sources declared under `# Citations` (spec §8).
10//! - **structural signals** — presence of `# Schema` / `# Examples` sections.
11//! - **coverage flags** — e.g. a note that makes claims but cites nothing is a
12//!   hallucination-risk candidate worth a judge's attention.
13//!
14//! These are intentionally heuristic (sentence-level extraction, not semantic
15//! parsing). They are inputs to grounding evaluation, not a grounding verdict.
16
17use std::sync::Arc;
18
19use serde::{Deserialize, Serialize};
20use turbovault_core::okf::Citation;
21use turbovault_core::prelude::*;
22use turbovault_parser::{ContentBlock, parse_blocks, parse_citations};
23use turbovault_vault::VaultManager;
24
25/// Maximum claims returned per note (keeps responses bounded).
26const MAX_CLAIMS: usize = 200;
27/// Minimum word count for a sentence to count as a candidate claim.
28const MIN_CLAIM_WORDS: usize = 5;
29
30/// Per-note grounding analysis — the material a judge scores.
31#[derive(Debug, Clone, Serialize, Deserialize)]
32pub struct GroundingAnalysis {
33    /// Vault-relative path.
34    pub path: String,
35    /// Number of candidate factual claims extracted from the body.
36    pub claim_count: usize,
37    /// Number of citations declared under `# Citations`.
38    pub citation_count: usize,
39    /// Whether the note declares any citations.
40    pub has_citations: bool,
41    /// Makes claims but cites nothing — a hallucination-risk candidate.
42    pub uncited: bool,
43    /// Whether a `# Schema` section is present (structural enrichment signal).
44    pub has_schema_section: bool,
45    /// Whether an `# Examples` section is present.
46    pub has_examples_section: bool,
47    /// Whether claims were truncated to the internal maximum claim count.
48    pub claims_truncated: bool,
49    /// Extracted candidate claims (declarative prose sentences).
50    pub claims: Vec<String>,
51    /// Declared citations.
52    pub citations: Vec<Citation>,
53    /// How to turn this data into a grounding verdict with an external judge.
54    pub guidance: Vec<String>,
55}
56
57/// A note that asserts claims without citing any source.
58#[derive(Debug, Clone, Serialize, Deserialize)]
59pub struct UngroundedNote {
60    pub path: String,
61    pub claim_count: usize,
62}
63
64/// Vault-wide ungrounded-note report.
65#[derive(Debug, Clone, Serialize, Deserialize)]
66pub struct UngroundedReport {
67    /// Notes scanned.
68    pub total_notes: usize,
69    /// Notes that make claims but declare no citations.
70    pub ungrounded_count: usize,
71    /// The ungrounded notes, most claims first (capped by the requested limit).
72    pub notes: Vec<UngroundedNote>,
73}
74
75/// Grounding analysis over a vault.
76pub struct GroundingTools {
77    manager: Arc<VaultManager>,
78}
79
80impl GroundingTools {
81    pub fn new(manager: Arc<VaultManager>) -> Self {
82        Self { manager }
83    }
84
85    fn rel(&self, path: &std::path::Path) -> String {
86        path.strip_prefix(self.manager.vault_path())
87            .unwrap_or(path)
88            .to_string_lossy()
89            .replace('\\', "/")
90    }
91
92    /// Analyze a single note's grounding.
93    pub async fn analyze_note(&self, path: &str) -> Result<GroundingAnalysis> {
94        let file_path = std::path::PathBuf::from(path);
95        let vault_file = self.manager.parse_file(&file_path).await?;
96        let body = &vault_file.content;
97
98        let all_claims = extract_claims(body);
99        let claim_count = all_claims.len();
100        let claims_truncated = claim_count > MAX_CLAIMS;
101        let claims: Vec<String> = all_claims.into_iter().take(MAX_CLAIMS).collect();
102
103        let citations = parse_citations(body);
104        let citation_count = citations.len();
105        let has_citations = citation_count > 0;
106
107        // Parse headings once for both section checks.
108        let headings = turbovault_parser::parse_headings(body);
109        let has_section = |name: &str| {
110            headings
111                .iter()
112                .any(|h| h.text.trim().eq_ignore_ascii_case(name))
113        };
114        let has_schema_section = has_section("schema");
115        let has_examples_section = has_section("examples");
116        let uncited = claim_count > 0 && citation_count == 0;
117
118        let mut guidance = vec![
119            "Feed `claims` + `citations` (and the cited sources) to an LLM judge to score hallucination-free grounding: the fraction of claims supported by a cited source.".to_string(),
120            "Compare `claims` across related notes to score contradiction (conflicting join keys, enums, definitions) and redundancy (claims that only restate schema).".to_string(),
121        ];
122        if uncited {
123            guidance.push(
124                "This note makes claims but cites no source — prioritize it for grounding review."
125                    .to_string(),
126            );
127        }
128
129        Ok(GroundingAnalysis {
130            path: self.rel(&file_path),
131            claim_count,
132            citation_count,
133            has_citations,
134            uncited,
135            has_schema_section,
136            has_examples_section,
137            claims_truncated,
138            claims,
139            citations,
140            guidance,
141        })
142    }
143
144    /// Scan the vault for notes that make claims but cite no source.
145    pub async fn find_ungrounded_notes(&self, limit: usize) -> Result<UngroundedReport> {
146        // Cache-first: parsed notes validated against disk mtime, no re-scan.
147        let files = self.manager.vault_files_validated().await;
148        let mut total_notes = 0usize;
149        let mut ungrounded: Vec<UngroundedNote> = Vec::new();
150
151        for vault_file in &files {
152            total_notes += 1;
153            let body = &vault_file.content;
154            if parse_citations(body).is_empty() {
155                let claim_count = extract_claims(body).len();
156                if claim_count > 0 {
157                    ungrounded.push(UngroundedNote {
158                        path: self.rel(&vault_file.path),
159                        claim_count,
160                    });
161                }
162            }
163        }
164
165        ungrounded.sort_by(|a, b| b.claim_count.cmp(&a.claim_count).then(a.path.cmp(&b.path)));
166        let ungrounded_count = ungrounded.len();
167        ungrounded.truncate(limit);
168
169        Ok(UngroundedReport {
170            total_notes,
171            ungrounded_count,
172            notes: ungrounded,
173        })
174    }
175}
176
177/// Extract candidate factual claims (declarative sentences) from prose blocks.
178///
179/// Walks paragraphs, blockquotes, and list items (skipping code, tables, and
180/// headings — they aren't prose claims), splits their plain text into
181/// sentences, and keeps those with at least [`MIN_CLAIM_WORDS`] words.
182fn extract_claims(body: &str) -> Vec<String> {
183    let mut prose = String::new();
184    for block in parse_blocks(body) {
185        collect_prose(&block, &mut prose);
186    }
187
188    let mut claims = Vec::new();
189    for sentence in split_sentences(&prose) {
190        let s = sentence.trim();
191        let words = s.split_whitespace().count();
192        if words >= MIN_CLAIM_WORDS
193            && s.chars().any(|c| c.is_alphabetic())
194            && !s.starts_with("http://")
195            && !s.starts_with("https://")
196        {
197            claims.push(s.to_string());
198        }
199    }
200    claims
201}
202
203/// Append the plain-text prose of claim-bearing blocks to `out`.
204fn collect_prose(block: &ContentBlock, out: &mut String) {
205    match block {
206        ContentBlock::Paragraph { .. }
207        | ContentBlock::Blockquote { .. }
208        | ContentBlock::List { .. } => {
209            let text = block.to_plain_text();
210            if !text.trim().is_empty() {
211                out.push_str(text.trim());
212                out.push('\n');
213            }
214        }
215        // Headings, code, tables, images, rules, details: not prose claims.
216        _ => {}
217    }
218}
219
220/// Split text into sentences on `.`/`?`/`!` boundaries (newlines also separate).
221fn split_sentences(text: &str) -> Vec<String> {
222    let mut sentences = Vec::new();
223    let mut current = String::new();
224    let chars: Vec<char> = text.chars().collect();
225    for (i, &c) in chars.iter().enumerate() {
226        if c == '\n' {
227            if !current.trim().is_empty() {
228                sentences.push(std::mem::take(&mut current));
229            } else {
230                current.clear();
231            }
232            continue;
233        }
234        current.push(c);
235        if matches!(c, '.' | '?' | '!') {
236            // Sentence boundary when followed by whitespace/end (avoids "3.5", "e.g").
237            let next_is_break = chars.get(i + 1).map(|n| n.is_whitespace()).unwrap_or(true);
238            if next_is_break && !current.trim().is_empty() {
239                sentences.push(std::mem::take(&mut current));
240            }
241        }
242    }
243    if !current.trim().is_empty() {
244        sentences.push(current);
245    }
246    sentences
247}
248
249#[cfg(test)]
250mod tests {
251    use super::*;
252    use std::path::Path;
253
254    fn make_manager(vault_dir: &Path) -> Arc<VaultManager> {
255        use turbovault_core::{ServerConfig, VaultConfig};
256        let mut config = ServerConfig::new();
257        config
258            .vaults
259            .push(VaultConfig::builder("test", vault_dir).build().unwrap());
260        Arc::new(VaultManager::new(config).unwrap())
261    }
262
263    #[test]
264    fn extracts_prose_sentences_as_claims() {
265        let body = "# Schema\n\nThe orders table has one row per completed order. It is joined to customers on customer_id.\n\n```sql\nSELECT 1\n```\n";
266        let claims = extract_claims(body);
267        assert_eq!(claims.len(), 2);
268        assert!(claims[0].contains("one row per completed order"));
269        // Code block content is not a claim.
270        assert!(!claims.iter().any(|c| c.contains("SELECT")));
271    }
272
273    #[test]
274    fn short_fragments_are_not_claims() {
275        let body = "Hello world.\n\nThis sentence is long enough to count as a claim here.\n";
276        let claims = extract_claims(body);
277        assert_eq!(claims.len(), 1);
278    }
279
280    #[tokio::test]
281    async fn analyze_note_flags_uncited_claims() {
282        let temp = tempfile::TempDir::new().unwrap();
283        std::fs::write(
284            temp.path().join("uncited.md"),
285            "---\ntype: Table\n---\n# Schema\n\nThe orders table holds one row per completed order in USD.\n",
286        )
287        .unwrap();
288        std::fs::write(
289            temp.path().join("cited.md"),
290            "---\ntype: Table\n---\n# Notes\n\nThe customers table holds one row per registered customer account.\n\n# Citations\n\n[1] [src](https://x.example)\n",
291        )
292        .unwrap();
293
294        let manager = make_manager(temp.path());
295        manager.initialize().await.unwrap();
296        let tools = GroundingTools::new(manager);
297
298        let uncited = tools.analyze_note("uncited.md").await.unwrap();
299        assert!(uncited.claim_count >= 1);
300        assert_eq!(uncited.citation_count, 0);
301        assert!(uncited.uncited);
302        assert!(uncited.has_schema_section);
303
304        let cited = tools.analyze_note("cited.md").await.unwrap();
305        assert_eq!(cited.citation_count, 1);
306        assert!(!cited.uncited);
307    }
308
309    #[tokio::test]
310    async fn find_ungrounded_lists_only_uncited_claim_notes() {
311        let temp = tempfile::TempDir::new().unwrap();
312        std::fs::write(
313            temp.path().join("uncited.md"),
314            "---\ntype: Table\n---\nThe orders table holds one row per completed order today.\n",
315        )
316        .unwrap();
317        std::fs::write(
318            temp.path().join("cited.md"),
319            "The customers table holds one row per registered account here.\n\n# Citations\n\n[1] [s](https://x.example)\n",
320        )
321        .unwrap();
322        std::fs::write(
323            temp.path().join("empty.md"),
324            "---\ntype: Table\n---\n# Just a heading\n",
325        )
326        .unwrap();
327
328        let manager = make_manager(temp.path());
329        manager.initialize().await.unwrap();
330        let tools = GroundingTools::new(manager);
331
332        let report = tools.find_ungrounded_notes(10).await.unwrap();
333        assert_eq!(report.total_notes, 3);
334        assert_eq!(report.ungrounded_count, 1);
335        assert_eq!(report.notes[0].path, "uncited.md");
336    }
337}
turbovault_tools/grounding.rs

turbovault_tools/
grounding.rs
