codetether_rlm/oracle/
mod.rs

1//! Deterministic oracle system for validating RLM REPL trace outputs.
2//!
3//! This module provides oracles that can verify FINAL() answers from RLM traces
4//! without requiring cloud LLM judges. This enables synthetic training data
5//! generation for the BitNet distilled navigation model.
6//!
7//! # Architecture
8//!
9//! - **Grep Oracle**: Pattern-match verification (e.g., "find all async functions")
10//! - **Tree-sitter Oracle**: Structural AST verification (function signatures, etc.)
11//! - **Validator Pipeline**: Routes queries to appropriate oracles and outputs golden traces
12//!
13//! # Usage
14//!
15//! ```ignore
16//! use codetether_agent::rlm::oracle::{TraceValidator, OracleResult};
17//!
18//! let validator = TraceValidator::new();
19//! let result = validator.validate(&analysis_result, &source_file).await;
20//!
21//! match result {
22//!     OracleResult::Golden(trace) => save_to_jsonl(trace),
23//!     OracleResult::Unverified => {} // No oracle available
24//!     OracleResult::Failed(reason) => {} // Oracle disagrees
25//! }
26//! ```
27
28mod ast_validation;
29mod batch;
30mod batch_write;
31mod consensus;
32mod consensus_helpers;
33mod grep_oracle;
34mod grep_validation;
35mod query_type;
36mod record;
37mod schema;
38#[path = "storage/mod.rs"]
39mod storage;
40mod templates;
41mod trace_types;
42mod tree_sitter_oracle;
43mod types;
44#[path = "validator/mod.rs"]
45mod validator;
46
47pub use grep_oracle::{GrepOracle, GrepVerification};
48pub use query_type::QueryType;
49pub use record::OracleTraceRecord;
50pub use schema::{AstPayload, AstResult, FinalPayload, GrepMatch, GrepPayload, SemanticPayload};
51pub use storage::{
52    OracleTracePersistResult, OracleTraceStorage, OracleTraceSyncStats, default_spool_dir,
53};
54pub use templates::{GeneratedQuery, QueryTemplate, TemplateKind};
55pub use trace_types::{OracleResult, ValidatedTrace};
56pub use tree_sitter_oracle::{TreeSitterOracle, TreeSitterVerification};
57pub use types::{TraceStep, VerificationMethod};
58pub use validator::{BatchValidationStats, SplitWriteStats, TraceValidator};
59
60/// Classification of an RLM FINAL() answer format.
61#[derive(Debug, Clone, PartialEq)]
62pub enum FinalAnswerFormat {
63    /// Line-numbered matches (e.g., "42:async fn foo()", "100:pub struct Bar")
64    LineNumberedMatches { matches: Vec<(usize, String)> },
65    /// Count result (e.g., "Found 15 occurrences")
66    CountResult { count: usize },
67    /// Structured data (e.g., function signature JSON)
68    StructuredData { data: serde_json::Value },
69    /// Free-form text (semantic - no deterministic verification)
70    FreeFormText { text: String },
71}
72
73impl FinalAnswerFormat {
74    /// Parse a FINAL() answer string into its classified format.
75    pub fn parse(answer: &str) -> Self {
76        // Try to parse as line-numbered matches
77        let lines: Vec<&str> = answer.lines().collect();
78        let mut numbered_matches = Vec::new();
79        let mut all_valid = true;
80
81        for line in &lines {
82            // Pattern: "42:text" or "42: text" or "L42: text"
83            let trimmed = line.trim();
84            if let Some(colon_pos) = trimmed.find(':') {
85                let num_part = trimmed[..colon_pos].trim().trim_start_matches('L').trim();
86                if let Ok(line_num) = num_part.parse::<usize>() {
87                    let text_part = trimmed[colon_pos + 1..].trim().to_string();
88                    numbered_matches.push((line_num, text_part));
89                } else {
90                    all_valid = false;
91                    break;
92                }
93            } else if !trimmed.is_empty() {
94                // Non-empty line without line number
95                all_valid = false;
96                break;
97            }
98        }
99
100        if all_valid && !numbered_matches.is_empty() {
101            return Self::LineNumberedMatches {
102                matches: numbered_matches,
103            };
104        }
105
106        // Try to parse as count result
107        let lower = answer.to_lowercase();
108        if lower.contains("found") || lower.contains("count:") || lower.contains("occurrences") {
109            // Extract number from text like "Found 15 async functions"
110            if let Some(count) = extract_count_from_text(answer) {
111                return Self::CountResult { count };
112            }
113        }
114
115        // Try to parse as JSON
116        if (answer.trim().starts_with('{') || answer.trim().starts_with('['))
117            && let Ok(data) = serde_json::from_str::<serde_json::Value>(answer)
118        {
119            return Self::StructuredData { data };
120        }
121
122        // Default to free-form text
123        Self::FreeFormText {
124            text: answer.to_string(),
125        }
126    }
127}
128
129/// Extract a count number from natural language text.
130fn extract_count_from_text(text: &str) -> Option<usize> {
131    // Look for patterns like "15 functions", "count: 42", "Found 7"
132    let re = regex::Regex::new(r"(?i)(?:found|count:?\s*)\s*(\d+)|(\d+)\s+(?:functions?|matches?|occurrences?|items?|results?)").ok()?;
133
134    for cap in re.captures_iter(text) {
135        // Try first group (found/count)
136        if let Some(m) = cap.get(1)
137            && let Ok(n) = m.as_str().parse()
138        {
139            return Some(n);
140        }
141        // Try second group (number before word)
142        if let Some(m) = cap.get(2)
143            && let Ok(n) = m.as_str().parse()
144        {
145            return Some(n);
146        }
147    }
148
149    None
150}
151
152#[cfg(test)]
153mod tests {
154    use super::*;
155
156    #[test]
157    fn parse_line_numbered_matches() {
158        let answer = "42:async fn foo()\n100:pub struct Bar\n";
159        let format = FinalAnswerFormat::parse(answer);
160        match format {
161            FinalAnswerFormat::LineNumberedMatches { matches } => {
162                assert_eq!(matches.len(), 2);
163                assert_eq!(matches[0], (42, "async fn foo()".to_string()));
164                assert_eq!(matches[1], (100, "pub struct Bar".to_string()));
165            }
166            _ => panic!("Expected LineNumberedMatches"),
167        }
168    }
169
170    #[test]
171    fn parse_count_result() {
172        let answer = "Found 15 async functions";
173        let format = FinalAnswerFormat::parse(answer);
174        match format {
175            FinalAnswerFormat::CountResult { count } => assert_eq!(count, 15),
176            _ => panic!("Expected CountResult"),
177        }
178    }
179
180    #[test]
181    fn parse_structured_data() {
182        let answer = r#"{"name": "foo", "args": ["x", "y"]}"#;
183        let format = FinalAnswerFormat::parse(answer);
184        match format {
185            FinalAnswerFormat::StructuredData { data } => {
186                assert_eq!(data["name"], "foo");
187            }
188            _ => panic!("Expected StructuredData"),
189        }
190    }
191
192    #[test]
193    fn parse_free_form_text() {
194        let answer = "This function handles error cases by using the ? operator";
195        let format = FinalAnswerFormat::parse(answer);
196        match format {
197            FinalAnswerFormat::FreeFormText { text } => {
198                assert!(text.contains("error cases"));
199            }
200            _ => panic!("Expected FreeFormText"),
201        }
202    }
203}
codetether_rlm/oracle/mod.rs

codetether_rlm/oracle/
mod.rs
