1use std::fs;
13use std::path::Path;
14use std::sync::OnceLock;
15
16use regex::Regex;
17use rusqlite::params;
18use serde::Deserialize;
19use serde_json::{Map, Value};
20
21#[cfg(feature = "native")]
22#[path = "normalizer_xml.rs"]
23mod normalizer_xml;
24
25use crate::layer0::functional::first_match;
26use crate::layer0::mcard_record::MCardRecord;
27use crate::layer0::vocabulary_env::{ENV_CLM_ACTIVE_A_HASH, ENV_CLM_ACTIVE_C_HASH, PILLAR_EXECUTION_LOG};
28use crate::layer2::record_store::require_verified;
29
30#[derive(Debug, Clone, PartialEq, Eq)]
32pub struct PastExecRecord {
33 pub exec_hash: String,
34 pub g_time: String,
35 pub content: String,
36}
37
38#[derive(Clone, Copy, Debug, PartialEq, Eq)]
39enum SourceKind {
40 Json,
41 Jsonl,
42 JsonOrJsonl,
43 Xml,
44 Yaml,
45}
46
47type KindRule = fn(&str) -> Option<SourceKind>;
48
49const TAG_RULES: &[KindRule] = &[
50 |tag| (tag == "json").then_some(SourceKind::Json),
51 |tag| (tag == "jsonl").then_some(SourceKind::Jsonl),
52 |tag| (tag == "xml").then_some(SourceKind::Xml),
53 |tag| matches!(tag, "yaml" | "yml").then_some(SourceKind::Yaml),
54];
55
56const SNIFF_RULES: &[KindRule] = &[
57 |text| text.starts_with('<').then_some(SourceKind::Xml),
58 |text| (text.starts_with('{') || text.starts_with('[')).then_some(SourceKind::JsonOrJsonl),
59];
60
61pub fn normalize_syntax(target_file: &str, clm_root: &str) -> Result<String, String> {
63 let path = Path::new(target_file);
64 if !path.exists() {
65 return Err(format!("Error: file not found: {}", target_file));
66 }
67
68 let ext = path.extension().and_then(|s| s.to_str()).unwrap_or("");
69 let content = fs::read_to_string(path)
70 .map_err(|e| format!("Failed to read target file: {}", e))?;
71
72 let past_records = load_verified_exec_records(clm_root);
73 let yaml_out = normalize_syntax_str(&content, ext, &past_records)?;
74
75 let tmp_dir = if crate::utils::is_zero_fs() {
76 std::env::temp_dir().join("clm_execution_logs/.tmp")
77 } else {
78 Path::new(clm_root).join("data/execution_logs/.tmp")
79 };
80 let _ = fs::create_dir_all(&tmp_dir);
81
82 let count = crate::TEMP_COUNTER.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
83 let temp_path = tmp_dir.join(format!("normalized_{}_{}.yaml", count, std::process::id()));
84 fs::write(&temp_path, yaml_out)
85 .map_err(|e| format!("Failed to write normalized temp file: {}", e))?;
86
87 Ok(temp_path.to_string_lossy().to_string())
88}
89
90pub fn normalize_syntax_str(
92 content: &str,
93 ext: &str,
94 past_records: &[PastExecRecord],
95) -> Result<String, String> {
96 let ext_lower = ext.to_lowercase();
97 let (parse_slice, kind) = if let Some(kind) = first_match(TAG_RULES, ext_lower.as_str()) {
98 (content, kind)
99 } else if let Some((lang, block)) = extract_code_block(content) {
100 let kind = first_match(TAG_RULES, lang.as_str())
101 .or_else(|| first_match(SNIFF_RULES, block.trim()))
102 .unwrap_or(SourceKind::Yaml);
103 (block, kind)
104 } else {
105 let kind = first_match(SNIFF_RULES, content.trim()).unwrap_or(SourceKind::Yaml);
106 (content, kind)
107 };
108
109 let (mut data_val, is_raw_yaml) = parse_source(parse_slice, kind);
110 if is_raw_yaml {
111 static FALLBACK_RE: OnceLock<Regex> = OnceLock::new();
112 let re = FALLBACK_RE.get_or_init(|| Regex::new(r"\btested_cases\b").unwrap());
113 return Ok(re.replace_all(content, "pending_cases").into_owned());
114 }
115
116 if data_val.is_object() {
117 normalize_tested_cases(&mut data_val);
118 normalize_implementations_order(&mut data_val);
119 inject_past_test_cases(&mut data_val, past_records);
120 }
121
122 serde_yaml::to_string(&data_val)
123 .map_err(|e| format!("Failed to serialize normalized data to YAML: {}", e))
124}
125
126fn parse_source(text: &str, kind: SourceKind) -> (Value, bool) {
127 match kind {
128 SourceKind::Json => (serde_json::from_str(text).unwrap_or(Value::Null), false),
129 SourceKind::Jsonl => (parse_jsonl_string(text).unwrap_or(Value::Null), false),
130 SourceKind::JsonOrJsonl => (
131 serde_json::from_str(text)
132 .ok()
133 .or_else(|| parse_jsonl_string(text).ok())
134 .unwrap_or(Value::Null),
135 false,
136 ),
137 SourceKind::Xml => (parse_xml_string(text).unwrap_or(Value::Null), false),
138 SourceKind::Yaml => match serde_yaml::from_str::<Value>(text) {
139 Ok(v) => (v, false),
140 Err(_) => (Value::Null, true),
141 },
142 }
143}
144
145fn extract_code_block(text: &str) -> Option<(String, &str)> {
146 static BLOCK_RE: OnceLock<Regex> = OnceLock::new();
147 let re = BLOCK_RE.get_or_init(|| Regex::new(r"(?s)```([a-zA-Z0-9_\-]+)?\s*(.*?)\s*```").unwrap());
148 let caps = re.captures(text)?;
149 let lang = caps.get(1).map(|m| m.as_str().trim().to_lowercase()).unwrap_or_default();
150 let content = caps.get(2).map(|m| m.as_str().trim()).unwrap_or_default();
151 Some((lang, content))
152}
153
154fn parse_jsonl_string(jsonl_str: &str) -> Result<Value, String> {
155 let mut obj = Value::Object(Map::new());
156 for item in serde_json::Deserializer::from_str(jsonl_str).into_iter::<Value>() {
157 let line_val = item.map_err(|e| format!("Failed to parse JSONL stream: {}", e))?;
158 deep_merge(&mut obj, line_val);
159 }
160 Ok(obj)
161}
162
163fn deep_merge(dict1: &mut Value, dict2: Value) {
164 if let (Value::Object(map1), Value::Object(map2)) = (dict1, dict2) {
165 for (k, v) in map2 {
166 if map1.get(&k).is_some_and(Value::is_object) && v.is_object() {
167 deep_merge(map1.get_mut(&k).unwrap(), v);
168 } else {
169 map1.insert(k, v);
170 }
171 }
172 }
173}
174
175#[cfg(feature = "native")]
176use normalizer_xml::parse_xml_string;
177
178#[cfg(not(feature = "native"))]
179fn parse_xml_string(_xml_str: &str) -> Result<Value, String> {
180 Err("XML normalization requires the 'native' feature".to_string())
181}
182
183fn balanced_exp_mut(data: &mut Value) -> Option<&mut Map<String, Value>> {
184 data.get_mut("clm")?.get_mut("balanced_exp")?.as_object_mut()
185}
186
187fn normalize_implementations_order(data: &mut Value) {
188 let Some(obj) = data
189 .get_mut("clm")
190 .and_then(|v| v.get_mut("concrete_impl"))
191 .and_then(|v| v.get_mut("process"))
192 .and_then(|v| v.get_mut("implementations"))
193 .and_then(Value::as_object_mut)
194 else {
195 return;
196 };
197 let mut keys: Vec<String> = obj.keys().cloned().collect();
198 keys.sort();
199 let mut sorted = Map::new();
200 for k in keys {
201 if let Some(v) = obj.remove(&k) {
202 sorted.insert(k, v);
203 }
204 }
205 *obj = sorted;
206}
207
208fn normalize_tested_cases(data: &mut Value) {
209 let Some(balanced_exp) = balanced_exp_mut(data) else {
210 return;
211 };
212 let mut pending = balanced_exp
213 .get("pending_cases")
214 .and_then(Value::as_array)
215 .cloned()
216 .unwrap_or_default();
217
218 if let Some(tested) = balanced_exp.get("tested_cases").and_then(Value::as_array) {
219 for case_obj in tested.iter().filter_map(Value::as_object) {
220 let when_val = case_obj.get("when");
221 if !pending.iter().any(|p| p.get("when") == when_val) {
222 let mut clean = case_obj.clone();
223 clean.remove("exec_hash");
224 clean.remove("exec_time");
225 pending.push(Value::Object(clean));
226 }
227 }
228 }
229 balanced_exp.remove("tested_cases");
230 balanced_exp.insert("pending_cases".to_string(), Value::Array(pending));
231}
232
233#[derive(Debug, Deserialize, Default)]
234#[serde(default)]
235struct ExecRecordEnv {
236 os: Option<String>,
237 arch: Option<String>,
238}
239
240#[derive(Debug, Deserialize)]
241#[serde(untagged)]
242enum RuntimeFactsField {
243 Map(ExecRecordEnv),
244 Scalar(serde::de::IgnoredAny),
245}
246
247#[derive(Debug, Deserialize, Default)]
248#[serde(default)]
249struct ExecRecordFields {
250 operation: Option<String>,
251 command: Option<String>,
252 exit_code: Option<i32>,
253 os: Option<String>,
254 arch: Option<String>,
255 runtime_facts: Option<RuntimeFactsField>,
256}
257
258#[derive(Debug, Deserialize, Default)]
259#[serde(default)]
260struct ExecRecordBody {
261 #[serde(flatten)]
262 fields: ExecRecordFields,
263 source: Option<ExecRecordFields>,
264}
265
266#[derive(Debug, Deserialize, Default)]
267#[serde(default)]
268struct ExecRecordDocument {
269 mcard: Option<ExecRecordBody>,
270 #[serde(flatten)]
271 top: ExecRecordBody,
272}
273
274impl RuntimeFactsField {
275 fn as_env(&self) -> Option<&ExecRecordEnv> {
276 match self {
277 Self::Map(env) => Some(env),
278 Self::Scalar(_) => None,
279 }
280 }
281}
282
283fn load_verified_exec_records(clm_root: &str) -> Vec<PastExecRecord> {
284 let env_trim = |k| crate::layer0::environment::var(k).unwrap_or_default().trim().to_string();
285 let (a_hash, c_hash) = (env_trim(ENV_CLM_ACTIVE_A_HASH), env_trim(ENV_CLM_ACTIVE_C_HASH));
286 let db_path = crate::db::get_db_path(PILLAR_EXECUTION_LOG, clm_root);
287 if a_hash.is_empty() || c_hash.is_empty() || !db_path.exists() {
288 return Vec::new();
289 }
290 let Ok(conn) = crate::db::get_connection(PILLAR_EXECUTION_LOG, clm_root) else { return Vec::new(); };
291 let query = "SELECT hash, g_time, content FROM card WHERE CAST(content AS TEXT) LIKE ? AND CAST(content AS TEXT) LIKE ? ORDER BY g_time DESC";
292 let Ok(mut stmt) = conn.prepare(query) else { return Vec::new(); };
293 let (a_like, c_like) = (format!("%{}%", a_hash), format!("%{}%", c_hash));
294 let Ok(rows) = stmt.query_map(params![a_like, c_like], |row| {
295 let bytes = row.get_ref(2)?.as_bytes()?.to_vec();
296 let content = String::from_utf8(bytes).map_err(|e| rusqlite::Error::FromSqlConversionFailure(2, rusqlite::types::Type::Blob, Box::new(e)))?;
297 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?, content))
298 }) else { return Vec::new(); };
299 rows.flatten()
300 .filter(|(h, g_time, content)| !content.is_empty() && require_verified(&MCardRecord { hash: h.clone(), content: content.as_bytes().to_vec(), g_time: g_time.clone() }).is_ok())
301 .map(|(exec_hash, g_time, content)| PastExecRecord { exec_hash, g_time, content })
302 .collect()
303}
304
305fn pick_non_empty<'a>(layers: &[&'a ExecRecordFields], f: impl Fn(&'a ExecRecordFields) -> Option<&'a str>) -> Option<&'a str> {
306 layers.iter().find_map(|layer| f(layer).filter(|s| !s.is_empty()))
307}
308
309fn inject_past_test_cases(data: &mut Value, records: &[PastExecRecord]) {
310 let mut total_executions = 0usize;
311 let mut successful_runs = 0usize;
312 let mut failed_runs = 0usize;
313 let mut last_executed_at = String::new();
314 let mut executions_list = Vec::new();
315
316 for rec in records.iter().filter(|r| !r.content.is_empty()) {
317 let doc: ExecRecordDocument = serde_yaml::from_str(&rec.content).unwrap_or_default();
318 let mut layers: Vec<&ExecRecordFields> = Vec::with_capacity(4);
319 if let Some(m) = &doc.mcard {
320 if let Some(src) = &m.source { layers.push(src); }
321 layers.push(&m.fields);
322 }
323 if let Some(src) = &doc.top.source { layers.push(src); }
324 layers.push(&doc.top.fields);
325
326 let exit_code = layers.iter().find_map(|l| l.exit_code).unwrap_or(0);
327 let command = pick_non_empty(&layers, |l| l.command.as_deref())
328 .map(str::to_string)
329 .unwrap_or_else(|| match pick_non_empty(&layers, |l| l.operation.as_deref()) {
330 Some(op) => format!("{} evaluation", op),
331 None => "evaluation".to_string(),
332 });
333 let os_name = pick_non_empty(&layers, |l| {
334 l.os.as_deref().or_else(|| l.runtime_facts.as_ref().and_then(RuntimeFactsField::as_env)?.os.as_deref())
335 });
336 let arch = pick_non_empty(&layers, |l| {
337 l.arch.as_deref().or_else(|| l.runtime_facts.as_ref().and_then(RuntimeFactsField::as_env)?.arch.as_deref())
338 });
339
340 total_executions += 1;
341 if exit_code == 0 { successful_runs += 1; } else { failed_runs += 1; }
342 if last_executed_at.is_empty() || rec.g_time > last_executed_at {
343 last_executed_at = rec.g_time.clone();
344 }
345
346 let mut entry = serde_json::json!({
347 "exec_hash": rec.exec_hash,
348 "g_time": rec.g_time,
349 "command": command,
350 });
351 if os_name.is_some() || arch.is_some() {
352 let mut env_map = Map::new();
353 if let Some(o) = os_name { env_map.insert("os".to_string(), Value::String(o.to_string())); }
354 if let Some(a) = arch { env_map.insert("arch".to_string(), Value::String(a.to_string())); }
355 entry.as_object_mut().unwrap().insert("environment".to_string(), Value::Object(env_map));
356 }
357 executions_list.push(entry);
358 }
359
360 executions_list.sort_by(|a, b| {
361 let key = |v: &Value| (
362 v.get("command").and_then(Value::as_str).unwrap_or("").to_string(),
363 v.get("exec_hash").and_then(Value::as_str).unwrap_or("").to_string(),
364 );
365 key(a).cmp(&key(b))
366 });
367
368 let stats = if total_executions > 0 {
369 serde_json::json!({
370 "total_executions": total_executions,
371 "successful_runs": successful_runs,
372 "failed_runs": failed_runs,
373 "coverage_ratio": (successful_runs as f64) / (total_executions as f64),
374 "last_executed_at": last_executed_at,
375 })
376 } else {
377 serde_json::json!({
378 "total_executions": 0, "successful_runs": 0, "failed_runs": 0,
379 "coverage_ratio": 0.0, "last_executed_at": "",
380 })
381 };
382
383 if let Some(obj) = balanced_exp_mut(data) {
384 obj.insert("past_test_cases".to_string(), serde_json::json!({
385 "statistics": stats,
386 "executions": executions_list,
387 }));
388 }
389}
390
391#[cfg(test)]
392mod tests {
393 use super::*;
394
395 #[test]
396 fn normalizer_format_parity_json_jsonl_xml_markdown() {
397 let md = "intro\n```xml\n<clm><abstract_spec><success_criteria><item>a</item></success_criteria></abstract_spec>\
398 <concrete_impl><process><implementations><z>1</z><a>2</a></implementations></process></concrete_impl>\
399 <balanced_exp><ok>true</ok><n>42</n></balanced_exp></clm>\n```";
400 let yaml_from_md = normalize_syntax_str(md, "md", &[]).unwrap();
401 let val_md: Value = serde_yaml::from_str(&yaml_from_md).unwrap();
402 assert_eq!(val_md.pointer("/clm/abstract_spec/success_criteria"), Some(&serde_json::json!(["a"])));
403 assert_eq!(val_md.pointer("/clm/balanced_exp/ok"), Some(&serde_json::json!(true)));
404 assert_eq!(val_md.pointer("/clm/balanced_exp/n"), Some(&serde_json::json!(42)));
405 let keys: Vec<&str> = val_md
406 .pointer("/clm/concrete_impl/process/implementations")
407 .and_then(Value::as_object).unwrap().keys().map(String::as_str).collect();
408 assert_eq!(keys, vec!["a", "z"]);
409
410 let jsonl = "{\"clm\":{\"abstract_spec\":{\"goal\":\"g\"}}}\n{\"clm\":{\"balanced_exp\":{\"expectation\":\"e\"}}}";
411 let val_jsonl: Value = serde_yaml::from_str(&normalize_syntax_str(jsonl, "jsonl", &[]).unwrap()).unwrap();
412 assert_eq!(val_jsonl.pointer("/clm/abstract_spec/goal"), Some(&serde_json::json!("g")));
413 assert_eq!(val_jsonl.pointer("/clm/balanced_exp/expectation"), Some(&serde_json::json!("e")));
414 }
415
416 #[test]
417 fn normalizer_tested_cases_promotion_and_raw_yaml_fallback() {
418 let yaml = "clm:\n balanced_exp:\n pending_cases:\n - when: existing\n tested_cases:\n - when: existing\n exec_hash: x\n - when: added\n exec_hash: sha256:1\n exec_time: t\n";
419 let val: Value = serde_yaml::from_str(&normalize_syntax_str(yaml, "yaml", &[]).unwrap()).unwrap();
420 let pending = val.pointer("/clm/balanced_exp/pending_cases").and_then(Value::as_array).unwrap();
421 assert_eq!(pending.len(), 2);
422 assert!(val.pointer("/clm/balanced_exp/tested_cases").is_none());
423 assert!(pending[1].get("exec_hash").is_none());
424
425 let broken_yaml = "clm:\n balanced_exp: [\n tested_cases:\n";
426 let fallback = normalize_syntax_str(broken_yaml, "yml", &[]).unwrap();
427 assert!(fallback.contains("pending_cases:") && !fallback.contains("tested_cases"));
428 }
429
430 #[test]
431 fn normalizer_past_test_cases_serde_yaml_mcard_and_source() {
432 let records = vec![
433 PastExecRecord {
434 exec_hash: "sha256:bbb".to_string(),
435 g_time: "2026-01-02T00:00:00Z".to_string(),
436 content: "mcard:\n operation: op_a\n source:\n command: run b\n exit_code: 0\n".to_string(),
437 },
438 PastExecRecord {
439 exec_hash: "sha256:aaa".to_string(),
440 g_time: "2026-01-03T00:00:00Z".to_string(),
441 content: "mcard:\n operation: op_b\n exit_code: 2\n source:\n runtime_facts:\n os: darwin\n arch: arm64\n".to_string(),
442 },
443 ];
444 let val: Value = serde_yaml::from_str(&normalize_syntax_str("{\"clm\":{\"balanced_exp\":{}}}", "json", &records).unwrap()).unwrap();
445 assert_eq!(val.pointer("/clm/balanced_exp/past_test_cases/statistics/total_executions"), Some(&serde_json::json!(2)));
446 assert_eq!(val.pointer("/clm/balanced_exp/past_test_cases/statistics/successful_runs"), Some(&serde_json::json!(1)));
447 assert_eq!(val.pointer("/clm/balanced_exp/past_test_cases/statistics/last_executed_at"), Some(&serde_json::json!("2026-01-03T00:00:00Z")));
448 assert_eq!(val.pointer("/clm/balanced_exp/past_test_cases/executions/0/environment/os"), Some(&serde_json::json!("darwin")));
449 assert_eq!(val.pointer("/clm/balanced_exp/past_test_cases/executions/1/command"), Some(&serde_json::json!("run b")));
450 }
451}