use std::fs;
use std::path::Path;
use std::sync::OnceLock;
use regex::Regex;
use rusqlite::params;
use serde::Deserialize;
use serde_json::{Map, Value};
#[cfg(feature = "native")]
#[path = "normalizer_xml.rs"]
mod normalizer_xml;
use crate::layer0::functional::first_match;
use crate::layer0::mcard_record::MCardRecord;
use crate::layer0::vocabulary_env::{ENV_CLM_ACTIVE_A_HASH, ENV_CLM_ACTIVE_C_HASH, PILLAR_EXECUTION_LOG};
use crate::layer2::record_store::require_verified;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PastExecRecord {
pub exec_hash: String,
pub g_time: String,
pub content: String,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum SourceKind {
Json,
Jsonl,
JsonOrJsonl,
Xml,
Yaml,
}
type KindRule = fn(&str) -> Option<SourceKind>;
const TAG_RULES: &[KindRule] = &[
|tag| (tag == "json").then_some(SourceKind::Json),
|tag| (tag == "jsonl").then_some(SourceKind::Jsonl),
|tag| (tag == "xml").then_some(SourceKind::Xml),
|tag| matches!(tag, "yaml" | "yml").then_some(SourceKind::Yaml),
];
const SNIFF_RULES: &[KindRule] = &[
|text| text.starts_with('<').then_some(SourceKind::Xml),
|text| (text.starts_with('{') || text.starts_with('[')).then_some(SourceKind::JsonOrJsonl),
];
pub fn normalize_syntax(target_file: &str, clm_root: &str) -> Result<String, String> {
let path = Path::new(target_file);
if !path.exists() {
return Err(format!("Error: file not found: {}", target_file));
}
let ext = path.extension().and_then(|s| s.to_str()).unwrap_or("");
let content = fs::read_to_string(path)
.map_err(|e| format!("Failed to read target file: {}", e))?;
let past_records = load_verified_exec_records(clm_root);
let yaml_out = normalize_syntax_str(&content, ext, &past_records)?;
let tmp_dir = if crate::utils::is_zero_fs() {
std::env::temp_dir().join("clm_execution_logs/.tmp")
} else {
Path::new(clm_root).join("data/execution_logs/.tmp")
};
let _ = fs::create_dir_all(&tmp_dir);
let count = crate::TEMP_COUNTER.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
let temp_path = tmp_dir.join(format!("normalized_{}_{}.yaml", count, std::process::id()));
fs::write(&temp_path, yaml_out)
.map_err(|e| format!("Failed to write normalized temp file: {}", e))?;
Ok(temp_path.to_string_lossy().to_string())
}
pub fn normalize_syntax_str(
content: &str,
ext: &str,
past_records: &[PastExecRecord],
) -> Result<String, String> {
let ext_lower = ext.to_lowercase();
let (parse_slice, kind) = if let Some(kind) = first_match(TAG_RULES, ext_lower.as_str()) {
(content, kind)
} else if let Some((lang, block)) = extract_code_block(content) {
let kind = first_match(TAG_RULES, lang.as_str())
.or_else(|| first_match(SNIFF_RULES, block.trim()))
.unwrap_or(SourceKind::Yaml);
(block, kind)
} else {
let kind = first_match(SNIFF_RULES, content.trim()).unwrap_or(SourceKind::Yaml);
(content, kind)
};
let (mut data_val, is_raw_yaml) = parse_source(parse_slice, kind);
if is_raw_yaml {
static FALLBACK_RE: OnceLock<Regex> = OnceLock::new();
let re = FALLBACK_RE.get_or_init(|| Regex::new(r"\btested_cases\b").unwrap());
return Ok(re.replace_all(content, "pending_cases").into_owned());
}
if data_val.is_object() {
normalize_tested_cases(&mut data_val);
normalize_implementations_order(&mut data_val);
inject_past_test_cases(&mut data_val, past_records);
}
serde_yaml::to_string(&data_val)
.map_err(|e| format!("Failed to serialize normalized data to YAML: {}", e))
}
fn parse_source(text: &str, kind: SourceKind) -> (Value, bool) {
match kind {
SourceKind::Json => (serde_json::from_str(text).unwrap_or(Value::Null), false),
SourceKind::Jsonl => (parse_jsonl_string(text).unwrap_or(Value::Null), false),
SourceKind::JsonOrJsonl => (
serde_json::from_str(text)
.ok()
.or_else(|| parse_jsonl_string(text).ok())
.unwrap_or(Value::Null),
false,
),
SourceKind::Xml => (parse_xml_string(text).unwrap_or(Value::Null), false),
SourceKind::Yaml => match serde_yaml::from_str::<Value>(text) {
Ok(v) => (v, false),
Err(_) => (Value::Null, true),
},
}
}
fn extract_code_block(text: &str) -> Option<(String, &str)> {
static BLOCK_RE: OnceLock<Regex> = OnceLock::new();
let re = BLOCK_RE.get_or_init(|| Regex::new(r"(?s)```([a-zA-Z0-9_\-]+)?\s*(.*?)\s*```").unwrap());
let caps = re.captures(text)?;
let lang = caps.get(1).map(|m| m.as_str().trim().to_lowercase()).unwrap_or_default();
let content = caps.get(2).map(|m| m.as_str().trim()).unwrap_or_default();
Some((lang, content))
}
fn parse_jsonl_string(jsonl_str: &str) -> Result<Value, String> {
let mut obj = Value::Object(Map::new());
for item in serde_json::Deserializer::from_str(jsonl_str).into_iter::<Value>() {
let line_val = item.map_err(|e| format!("Failed to parse JSONL stream: {}", e))?;
deep_merge(&mut obj, line_val);
}
Ok(obj)
}
fn deep_merge(dict1: &mut Value, dict2: Value) {
if let (Value::Object(map1), Value::Object(map2)) = (dict1, dict2) {
for (k, v) in map2 {
if map1.get(&k).is_some_and(Value::is_object) && v.is_object() {
deep_merge(map1.get_mut(&k).unwrap(), v);
} else {
map1.insert(k, v);
}
}
}
}
#[cfg(feature = "native")]
use normalizer_xml::parse_xml_string;
#[cfg(not(feature = "native"))]
fn parse_xml_string(_xml_str: &str) -> Result<Value, String> {
Err("XML normalization requires the 'native' feature".to_string())
}
fn balanced_exp_mut(data: &mut Value) -> Option<&mut Map<String, Value>> {
data.get_mut("clm")?.get_mut("balanced_exp")?.as_object_mut()
}
fn normalize_implementations_order(data: &mut Value) {
let Some(obj) = data
.get_mut("clm")
.and_then(|v| v.get_mut("concrete_impl"))
.and_then(|v| v.get_mut("process"))
.and_then(|v| v.get_mut("implementations"))
.and_then(Value::as_object_mut)
else {
return;
};
let mut keys: Vec<String> = obj.keys().cloned().collect();
keys.sort();
let mut sorted = Map::new();
for k in keys {
if let Some(v) = obj.remove(&k) {
sorted.insert(k, v);
}
}
*obj = sorted;
}
fn normalize_tested_cases(data: &mut Value) {
let Some(balanced_exp) = balanced_exp_mut(data) else {
return;
};
let mut pending = balanced_exp
.get("pending_cases")
.and_then(Value::as_array)
.cloned()
.unwrap_or_default();
if let Some(tested) = balanced_exp.get("tested_cases").and_then(Value::as_array) {
for case_obj in tested.iter().filter_map(Value::as_object) {
let when_val = case_obj.get("when");
if !pending.iter().any(|p| p.get("when") == when_val) {
let mut clean = case_obj.clone();
clean.remove("exec_hash");
clean.remove("exec_time");
pending.push(Value::Object(clean));
}
}
}
balanced_exp.remove("tested_cases");
balanced_exp.insert("pending_cases".to_string(), Value::Array(pending));
}
#[derive(Debug, Deserialize, Default)]
#[serde(default)]
struct ExecRecordEnv {
os: Option<String>,
arch: Option<String>,
}
#[derive(Debug, Deserialize)]
#[serde(untagged)]
enum RuntimeFactsField {
Map(ExecRecordEnv),
Scalar(serde::de::IgnoredAny),
}
#[derive(Debug, Deserialize, Default)]
#[serde(default)]
struct ExecRecordFields {
operation: Option<String>,
command: Option<String>,
exit_code: Option<i32>,
os: Option<String>,
arch: Option<String>,
runtime_facts: Option<RuntimeFactsField>,
}
#[derive(Debug, Deserialize, Default)]
#[serde(default)]
struct ExecRecordBody {
#[serde(flatten)]
fields: ExecRecordFields,
source: Option<ExecRecordFields>,
}
#[derive(Debug, Deserialize, Default)]
#[serde(default)]
struct ExecRecordDocument {
mcard: Option<ExecRecordBody>,
#[serde(flatten)]
top: ExecRecordBody,
}
impl RuntimeFactsField {
fn as_env(&self) -> Option<&ExecRecordEnv> {
match self {
Self::Map(env) => Some(env),
Self::Scalar(_) => None,
}
}
}
fn load_verified_exec_records(clm_root: &str) -> Vec<PastExecRecord> {
let env_trim = |k| crate::layer0::environment::var(k).unwrap_or_default().trim().to_string();
let (a_hash, c_hash) = (env_trim(ENV_CLM_ACTIVE_A_HASH), env_trim(ENV_CLM_ACTIVE_C_HASH));
let db_path = crate::db::get_db_path(PILLAR_EXECUTION_LOG, clm_root);
if a_hash.is_empty() || c_hash.is_empty() || !db_path.exists() {
return Vec::new();
}
let Ok(conn) = crate::db::get_connection(PILLAR_EXECUTION_LOG, clm_root) else { return Vec::new(); };
let query = "SELECT hash, g_time, content FROM card WHERE CAST(content AS TEXT) LIKE ? AND CAST(content AS TEXT) LIKE ? ORDER BY g_time DESC";
let Ok(mut stmt) = conn.prepare(query) else { return Vec::new(); };
let (a_like, c_like) = (format!("%{}%", a_hash), format!("%{}%", c_hash));
let Ok(rows) = stmt.query_map(params![a_like, c_like], |row| {
let bytes = row.get_ref(2)?.as_bytes()?.to_vec();
let content = String::from_utf8(bytes).map_err(|e| rusqlite::Error::FromSqlConversionFailure(2, rusqlite::types::Type::Blob, Box::new(e)))?;
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?, content))
}) else { return Vec::new(); };
rows.flatten()
.filter(|(h, g_time, content)| !content.is_empty() && require_verified(&MCardRecord { hash: h.clone(), content: content.as_bytes().to_vec(), g_time: g_time.clone() }).is_ok())
.map(|(exec_hash, g_time, content)| PastExecRecord { exec_hash, g_time, content })
.collect()
}
fn pick_non_empty<'a>(layers: &[&'a ExecRecordFields], f: impl Fn(&'a ExecRecordFields) -> Option<&'a str>) -> Option<&'a str> {
layers.iter().find_map(|layer| f(layer).filter(|s| !s.is_empty()))
}
fn inject_past_test_cases(data: &mut Value, records: &[PastExecRecord]) {
let mut total_executions = 0usize;
let mut successful_runs = 0usize;
let mut failed_runs = 0usize;
let mut last_executed_at = String::new();
let mut executions_list = Vec::new();
for rec in records.iter().filter(|r| !r.content.is_empty()) {
let doc: ExecRecordDocument = serde_yaml::from_str(&rec.content).unwrap_or_default();
let mut layers: Vec<&ExecRecordFields> = Vec::with_capacity(4);
if let Some(m) = &doc.mcard {
if let Some(src) = &m.source { layers.push(src); }
layers.push(&m.fields);
}
if let Some(src) = &doc.top.source { layers.push(src); }
layers.push(&doc.top.fields);
let exit_code = layers.iter().find_map(|l| l.exit_code).unwrap_or(0);
let command = pick_non_empty(&layers, |l| l.command.as_deref())
.map(str::to_string)
.unwrap_or_else(|| match pick_non_empty(&layers, |l| l.operation.as_deref()) {
Some(op) => format!("{} evaluation", op),
None => "evaluation".to_string(),
});
let os_name = pick_non_empty(&layers, |l| {
l.os.as_deref().or_else(|| l.runtime_facts.as_ref().and_then(RuntimeFactsField::as_env)?.os.as_deref())
});
let arch = pick_non_empty(&layers, |l| {
l.arch.as_deref().or_else(|| l.runtime_facts.as_ref().and_then(RuntimeFactsField::as_env)?.arch.as_deref())
});
total_executions += 1;
if exit_code == 0 { successful_runs += 1; } else { failed_runs += 1; }
if last_executed_at.is_empty() || rec.g_time > last_executed_at {
last_executed_at = rec.g_time.clone();
}
let mut entry = serde_json::json!({
"exec_hash": rec.exec_hash,
"g_time": rec.g_time,
"command": command,
});
if os_name.is_some() || arch.is_some() {
let mut env_map = Map::new();
if let Some(o) = os_name { env_map.insert("os".to_string(), Value::String(o.to_string())); }
if let Some(a) = arch { env_map.insert("arch".to_string(), Value::String(a.to_string())); }
entry.as_object_mut().unwrap().insert("environment".to_string(), Value::Object(env_map));
}
executions_list.push(entry);
}
executions_list.sort_by(|a, b| {
let key = |v: &Value| (
v.get("command").and_then(Value::as_str).unwrap_or("").to_string(),
v.get("exec_hash").and_then(Value::as_str).unwrap_or("").to_string(),
);
key(a).cmp(&key(b))
});
let stats = if total_executions > 0 {
serde_json::json!({
"total_executions": total_executions,
"successful_runs": successful_runs,
"failed_runs": failed_runs,
"coverage_ratio": (successful_runs as f64) / (total_executions as f64),
"last_executed_at": last_executed_at,
})
} else {
serde_json::json!({
"total_executions": 0, "successful_runs": 0, "failed_runs": 0,
"coverage_ratio": 0.0, "last_executed_at": "",
})
};
if let Some(obj) = balanced_exp_mut(data) {
obj.insert("past_test_cases".to_string(), serde_json::json!({
"statistics": stats,
"executions": executions_list,
}));
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn normalizer_format_parity_json_jsonl_xml_markdown() {
let md = "intro\n```xml\n<clm><abstract_spec><success_criteria><item>a</item></success_criteria></abstract_spec>\
<concrete_impl><process><implementations><z>1</z><a>2</a></implementations></process></concrete_impl>\
<balanced_exp><ok>true</ok><n>42</n></balanced_exp></clm>\n```";
let yaml_from_md = normalize_syntax_str(md, "md", &[]).unwrap();
let val_md: Value = serde_yaml::from_str(&yaml_from_md).unwrap();
assert_eq!(val_md.pointer("/clm/abstract_spec/success_criteria"), Some(&serde_json::json!(["a"])));
assert_eq!(val_md.pointer("/clm/balanced_exp/ok"), Some(&serde_json::json!(true)));
assert_eq!(val_md.pointer("/clm/balanced_exp/n"), Some(&serde_json::json!(42)));
let keys: Vec<&str> = val_md
.pointer("/clm/concrete_impl/process/implementations")
.and_then(Value::as_object).unwrap().keys().map(String::as_str).collect();
assert_eq!(keys, vec!["a", "z"]);
let jsonl = "{\"clm\":{\"abstract_spec\":{\"goal\":\"g\"}}}\n{\"clm\":{\"balanced_exp\":{\"expectation\":\"e\"}}}";
let val_jsonl: Value = serde_yaml::from_str(&normalize_syntax_str(jsonl, "jsonl", &[]).unwrap()).unwrap();
assert_eq!(val_jsonl.pointer("/clm/abstract_spec/goal"), Some(&serde_json::json!("g")));
assert_eq!(val_jsonl.pointer("/clm/balanced_exp/expectation"), Some(&serde_json::json!("e")));
}
#[test]
fn normalizer_tested_cases_promotion_and_raw_yaml_fallback() {
let yaml = "clm:\n balanced_exp:\n pending_cases:\n - when: existing\n tested_cases:\n - when: existing\n exec_hash: x\n - when: added\n exec_hash: sha256:1\n exec_time: t\n";
let val: Value = serde_yaml::from_str(&normalize_syntax_str(yaml, "yaml", &[]).unwrap()).unwrap();
let pending = val.pointer("/clm/balanced_exp/pending_cases").and_then(Value::as_array).unwrap();
assert_eq!(pending.len(), 2);
assert!(val.pointer("/clm/balanced_exp/tested_cases").is_none());
assert!(pending[1].get("exec_hash").is_none());
let broken_yaml = "clm:\n balanced_exp: [\n tested_cases:\n";
let fallback = normalize_syntax_str(broken_yaml, "yml", &[]).unwrap();
assert!(fallback.contains("pending_cases:") && !fallback.contains("tested_cases"));
}
#[test]
fn normalizer_past_test_cases_serde_yaml_mcard_and_source() {
let records = vec![
PastExecRecord {
exec_hash: "sha256:bbb".to_string(),
g_time: "2026-01-02T00:00:00Z".to_string(),
content: "mcard:\n operation: op_a\n source:\n command: run b\n exit_code: 0\n".to_string(),
},
PastExecRecord {
exec_hash: "sha256:aaa".to_string(),
g_time: "2026-01-03T00:00:00Z".to_string(),
content: "mcard:\n operation: op_b\n exit_code: 2\n source:\n runtime_facts:\n os: darwin\n arch: arm64\n".to_string(),
},
];
let val: Value = serde_yaml::from_str(&normalize_syntax_str("{\"clm\":{\"balanced_exp\":{}}}", "json", &records).unwrap()).unwrap();
assert_eq!(val.pointer("/clm/balanced_exp/past_test_cases/statistics/total_executions"), Some(&serde_json::json!(2)));
assert_eq!(val.pointer("/clm/balanced_exp/past_test_cases/statistics/successful_runs"), Some(&serde_json::json!(1)));
assert_eq!(val.pointer("/clm/balanced_exp/past_test_cases/statistics/last_executed_at"), Some(&serde_json::json!("2026-01-03T00:00:00Z")));
assert_eq!(val.pointer("/clm/balanced_exp/past_test_cases/executions/0/environment/os"), Some(&serde_json::json!("darwin")));
assert_eq!(val.pointer("/clm/balanced_exp/past_test_cases/executions/1/command"), Some(&serde_json::json!("run b")));
}
}