use serde::Deserialize;
use sniff::analyzer::analyze_with_client;
use sniff::config::{LLMConfig, ResolvedConfig, ThresholdsConfig};
use sniff::llm::LLMClient;
use sniff::parser::parse_file;
use sniff::types::{FileRecord, FindingTier};
use std::collections::{BTreeMap, BTreeSet};
use std::fs;
use std::io::{BufRead, BufReader, Read, Write};
use std::net::{Shutdown, TcpListener, TcpStream};
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::thread;
#[derive(Debug, Deserialize)]
struct Manifest {
#[serde(rename = "case")]
cases: Vec<GoldCase>,
}
#[derive(Debug, Deserialize, Clone)]
struct GoldCase {
language: String,
path: String,
method: String,
tier: String,
pattern: String,
evidence: String,
explanation: String,
}
fn read_http_request(stream: TcpStream) -> (TcpStream, String) {
let mut reader = BufReader::new(stream);
let mut headers = String::new();
loop {
let mut line = String::new();
if reader.read_line(&mut line).unwrap_or(0) == 0 {
break;
}
headers.push_str(&line);
if line == "\r\n" || line == "\n" {
break;
}
}
let content_length = headers
.lines()
.find_map(|line| {
let (name, value) = line.split_once(':')?;
if name.eq_ignore_ascii_case("content-length") {
value.trim().parse::<usize>().ok()
} else {
None
}
})
.unwrap_or(0);
let mut body = vec![0u8; content_length];
let _ = reader.read_exact(&mut body);
(
reader.into_inner(),
format!("{}{}", headers, String::from_utf8_lossy(&body)),
)
}
fn prompt_source(request: &str) -> String {
request
.split_once("Method source:\\n---\\n")
.and_then(|(_, rest)| rest.split_once("\\n---"))
.map(|(source, _)| source.replace("\\n", "\n").replace("\\\"", "\""))
.unwrap_or_default()
}
fn prompt_line_range(request: &str) -> (usize, usize) {
request
.split_once("absolute file line numbers from ")
.and_then(|(_, rest)| rest.split_once(" through "))
.and_then(|(start, rest)| {
let start = start.trim().parse::<usize>().ok()?;
let end = rest
.split(|ch: char| !ch.is_ascii_digit())
.next()
.and_then(|value| value.parse::<usize>().ok())?;
Some((start, end))
})
.unwrap_or((1, 1))
}
fn semantic_response(request: &str, expected: &GoldCase) -> String {
let (method_start, _) = prompt_line_range(request);
let source = prompt_source(request);
let evidence = if expected.evidence.is_empty() {
serde_json::json!([])
} else {
let offset = source
.lines()
.position(|line| line.contains(&expected.evidence))
.unwrap_or(0);
serde_json::json!([{
"start_line": method_start + offset,
"end_line": method_start + offset + expected.evidence.lines().count().saturating_sub(1),
"quote": expected.evidence
}])
};
let inner = serde_json::json!({
"smelly": expected.tier != "clean",
"tier": expected.tier,
"pattern": expected.pattern,
"intent": "The method performs the operation described by its name.",
"reason": if expected.tier == "clean" { "" } else { "The implementation adds unnecessary conceptual machinery." },
"necessity_check": expected.explanation,
"evidence": evidence
});
serde_json::json!({
"choices": [{"message": {"content": serde_json::to_string(&inner).unwrap()}}]
})
.to_string()
}
fn spawn_gold_server(cases: Vec<GoldCase>) -> (String, Arc<AtomicUsize>, Arc<AtomicUsize>) {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let method_requests = Arc::new(AtomicUsize::new(0));
let adjudication_requests = Arc::new(AtomicUsize::new(0));
let method_requests_for_server = Arc::clone(&method_requests);
let adjudication_requests_for_server = Arc::clone(&adjudication_requests);
thread::spawn(move || {
for stream in listener.incoming() {
let Ok(stream) = stream else { break };
let (mut stream, request) = read_http_request(stream);
if request.contains("Method source:") {
method_requests_for_server.fetch_add(1, Ordering::Relaxed);
}
if request.contains("final adjudicator") {
adjudication_requests_for_server.fetch_add(1, Ordering::Relaxed);
}
let body = if request.contains("Filename:") {
serde_json::json!({
"choices": [{"message": {"content": "{\"smelly\":false,\"tier\":\"clean\",\"evidence\":\"\",\"cohesive\":true,\"name_accurate\":true,\"reason\":\"clean\"}"}}]
})
.to_string()
} else {
let expected = cases
.iter()
.find(|case| {
request.contains(&case.path)
&& request.contains(&format!("Method: {}", case.method))
})
.unwrap_or_else(|| panic!("gold server received unknown method prompt"));
semantic_response(&request, expected)
};
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
body.len(),
body
);
let _ = stream.write_all(response.as_bytes());
let _ = stream.flush();
let _ = stream.shutdown(Shutdown::Both);
}
});
(
format!("http://{}", address),
method_requests,
adjudication_requests,
)
}
fn fixture_root() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
}
#[test]
fn semantic_gold_corpus_is_complete_and_parser_addressable() {
let root = fixture_root();
let manifest_text = fs::read_to_string(root.join("tests/semantic_gold_manifest.toml"))
.expect("semantic gold manifest should exist");
let manifest: Manifest = toml::from_str(&manifest_text).expect("semantic gold TOML is valid");
let expected_languages = BTreeSet::from([
"python".to_string(),
"javascript".to_string(),
"typescript".to_string(),
"rust".to_string(),
"go".to_string(),
"kotlin".to_string(),
]);
let mut languages = BTreeSet::new();
let mut counts = BTreeMap::<String, usize>::new();
for case in &manifest.cases {
let path = root.join(&case.path);
assert!(
path.is_file(),
"missing semantic gold fixture: {}",
path.display()
);
let path_text = path.to_string_lossy().to_string();
let record = parse_file(&path_text);
let method = record
.methods
.iter()
.find(|method| method.name == case.method)
.unwrap_or_else(|| panic!("missing method {} in {}", case.method, case.path));
assert!(["slop", "kinda_slop", "clean"].contains(&case.tier.as_str()));
assert!(!case.explanation.trim().is_empty());
if case.tier == "clean" {
assert_eq!(case.pattern, "none");
assert!(case.evidence.is_empty());
} else {
assert!(!case.evidence.is_empty());
assert!(
method.source.contains(&case.evidence),
"evidence missing from {}: {}",
case.path,
case.evidence
);
assert!(case.pattern != "none");
}
assert!(method.loc > 0);
languages.insert(case.language.clone());
*counts.entry(case.language.clone()).or_default() += 1;
}
assert_eq!(languages, expected_languages);
assert!(counts.values().all(|count| *count >= 2));
}
#[tokio::test]
async fn semantic_gold_corpus_runs_through_both_method_passes() {
let root = fixture_root();
let manifest_text = fs::read_to_string(root.join("tests/semantic_gold_manifest.toml"))
.expect("semantic gold manifest should exist");
let manifest: Manifest = toml::from_str(&manifest_text).expect("semantic gold TOML is valid");
let mut files = Vec::<FileRecord>::new();
let mut seen_paths = BTreeSet::new();
for case in &manifest.cases {
if !seen_paths.insert(case.path.clone()) {
continue;
}
let path = root.join(&case.path).to_string_lossy().to_string();
files.push(parse_file(&path));
}
let (endpoint, method_requests, adjudication_requests) =
spawn_gold_server(manifest.cases.clone());
let config = ResolvedConfig {
thresholds: ThresholdsConfig::default(),
ignore: vec![],
generic_names: vec![],
generic_file_names: vec![],
model: "gold-test-model".to_string(),
llm: LLMConfig {
system_context: String::new(),
endpoint,
},
};
let client = Arc::new(LLMClient::new(config, Some("gold-test-key".to_string())));
let (verdicts, _, _) = analyze_with_client(&files, &[], client, false, None)
.await
.expect("gold semantic review should complete");
let method_verdicts = verdicts
.iter()
.filter(|verdict| verdict.check_type == "method")
.collect::<Vec<_>>();
assert_eq!(method_verdicts.len(), manifest.cases.len());
let mut slop_true_positive = 0usize;
let mut slop_false_positive = 0usize;
let mut slop_false_negative = 0usize;
let mut kinda_true_positive = 0usize;
let mut kinda_false_positive = 0usize;
let mut unsupported_evidence = 0usize;
for case in &manifest.cases {
let verdict = method_verdicts
.iter()
.find(|verdict| {
verdict.method_name.as_deref() == Some(case.method.as_str())
&& verdict.file_path.replace('\\', "/").ends_with(&case.path)
})
.unwrap_or_else(|| panic!("missing gold verdict for {}", case.method));
let expected_tier = match case.tier.as_str() {
"slop" => FindingTier::Slop,
"kinda_slop" => FindingTier::KindaSlop,
"clean" => FindingTier::Clean,
other => panic!("invalid expected tier {other}"),
};
assert_eq!(
verdict.tier, expected_tier,
"wrong tier for {}",
case.method
);
if !case.evidence.is_empty() {
assert!(
verdict.evidence.contains(&case.evidence),
"missing evidence for {}: {}",
case.path,
verdict.evidence
);
}
match (case.tier.as_str(), verdict.tier) {
("slop", FindingTier::Slop) => slop_true_positive += 1,
("slop", _) => slop_false_negative += 1,
(_, FindingTier::Slop) => slop_false_positive += 1,
_ => {}
}
match (case.tier.as_str(), verdict.tier) {
("kinda_slop", FindingTier::KindaSlop) => kinda_true_positive += 1,
(_, FindingTier::KindaSlop) => kinda_false_positive += 1,
_ => {}
}
if case.tier != "clean" && verdict.evidence.trim().is_empty() {
unsupported_evidence += 1;
}
}
let slop_precision =
100 * slop_true_positive / (slop_true_positive + slop_false_positive).max(1);
let slop_recall = 100 * slop_true_positive / (slop_true_positive + slop_false_negative).max(1);
let kinda_precision =
100 * kinda_true_positive / (kinda_true_positive + kinda_false_positive).max(1);
eprintln!(
"semantic gold metrics: Slop precision={slop_precision}%, recall={slop_recall}%, Kinda Slop precision={kinda_precision}%, unsupported evidence={unsupported_evidence}, method passes={}, adjudications={}",
method_requests.load(Ordering::Relaxed),
adjudication_requests.load(Ordering::Relaxed),
);
assert!(slop_precision >= 90, "Slop precision was {slop_precision}%");
assert!(slop_recall >= 80, "Slop recall was {slop_recall}%");
assert!(
kinda_precision >= 80,
"Kinda Slop precision was {kinda_precision}%"
);
assert_eq!(
unsupported_evidence, 0,
"non-clean gold findings need evidence"
);
assert_eq!(
method_requests.load(Ordering::Relaxed),
manifest.cases.len() * 2,
"each gold method needs both semantic passes"
);
assert_eq!(
adjudication_requests.load(Ordering::Relaxed),
0,
"identical gold passes should not trigger adjudication"
);
}