use serde_json::{Value, Map};
use std::fs;
use std::path::Path;
use regex::Regex;
use quick_xml::Reader;
use quick_xml::events::Event;
use rusqlite::params;
pub fn normalize_syntax(target_file: &str, clm_root: &str) -> Result<String, String> {
let path = Path::new(target_file);
if !path.exists() {
return Err(format!("Error: file not found: {}", target_file));
}
let ext = path.extension().and_then(|s| s.to_str()).unwrap_or("").to_lowercase();
let content = fs::read_to_string(path)
.map_err(|e| format!("Failed to read target file: {}", e))?;
#[allow(unused_assignments)]
let mut normalized_data: Option<Value> = None;
let mut is_raw_yaml = false;
if ext == "json" {
normalized_data = serde_json::from_str(&content).ok();
} else if ext == "jsonl" {
normalized_data = parse_jsonl_string(&content).ok();
} else if ext == "xml" {
normalized_data = parse_xml_string(&content).ok();
} else if ext == "yaml" || ext == "yml" {
normalized_data = serde_yaml::from_str(&content).ok();
if normalized_data.is_none() {
is_raw_yaml = true;
}
} else {
if let Some((lang, block_content)) = extract_code_block(&content) {
if lang == "json" {
normalized_data = serde_json::from_str(&block_content).ok();
} else if lang == "jsonl" {
normalized_data = parse_jsonl_string(&block_content).ok();
} else if lang == "xml" {
normalized_data = parse_xml_string(&block_content).ok();
} else if lang == "yaml" || lang == "yml" {
normalized_data = serde_yaml::from_str(&block_content).ok();
if normalized_data.is_none() {
is_raw_yaml = true;
}
} else {
let stripped = block_content.trim();
if stripped.starts_with('<') {
normalized_data = parse_xml_string(&block_content).ok();
} else if stripped.starts_with('{') || stripped.starts_with('[') {
normalized_data = serde_json::from_str(&block_content)
.ok()
.or_else(|| parse_jsonl_string(&block_content).ok());
} else {
normalized_data = serde_yaml::from_str(&block_content).ok();
if normalized_data.is_none() {
is_raw_yaml = true;
}
}
}
} else {
let stripped = content.trim();
if stripped.starts_with('<') {
normalized_data = parse_xml_string(&content).ok();
} else if stripped.starts_with('{') || stripped.starts_with('[') {
normalized_data = serde_json::from_str(&content)
.ok()
.or_else(|| parse_jsonl_string(&content).ok());
} else {
normalized_data = serde_yaml::from_str(&content).ok();
if normalized_data.is_none() {
is_raw_yaml = true;
}
}
}
}
let mut data_val = normalized_data.unwrap_or(Value::Null);
if data_val.is_object() {
normalize_tested_cases(&mut data_val);
normalize_implementations_order(&mut data_val);
inject_past_test_cases(&mut data_val, clm_root);
}
let zero_fs = crate::utils::is_zero_fs();
let tmp_dir = if zero_fs {
std::env::temp_dir().join("clm_execution_logs/.tmp")
} else {
Path::new(clm_root).join("data/execution_logs/.tmp")
};
let _ = fs::create_dir_all(&tmp_dir);
let count = crate::TEMP_COUNTER.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
let temp_path = tmp_dir.join(format!("normalized_{}_{}.yaml", count, std::process::id()));
if is_raw_yaml {
let fallback_re = Regex::new(r"\btested_cases\b").unwrap();
let fallback_content = fallback_re.replace_all(&content, "pending_cases").to_string();
fs::write(&temp_path, fallback_content)
.map_err(|e| format!("Failed to write normalized temp file: {}", e))?;
} else {
let yaml_str = serde_yaml::to_string(&data_val)
.map_err(|e| format!("Failed to serialize normalized data to YAML: {}", e))?;
fs::write(&temp_path, yaml_str)
.map_err(|e| format!("Failed to write normalized temp file: {}", e))?;
}
Ok(temp_path.to_string_lossy().to_string())
}
fn extract_code_block(text: &str) -> Option<(String, String)> {
let re = Regex::new(r"(?s)```([a-zA-Z0-9_\-]+)?\s*(.*?)\s*```").ok()?;
if let Some(caps) = re.captures(text) {
let lang = caps.get(1).map(|m| m.as_str().trim().to_lowercase()).unwrap_or_default();
let content = caps.get(2).map(|m| m.as_str().trim().to_string()).unwrap_or_default();
return Some((lang, content));
}
None
}
fn parse_jsonl_string(jsonl_str: &str) -> Result<Value, String> {
let mut obj = Value::Object(Map::new());
for line in jsonl_str.lines() {
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
let line_val: Value = serde_json::from_str(trimmed)
.map_err(|e| format!("Failed to parse JSONL line: {}", e))?;
deep_merge(&mut obj, line_val);
}
Ok(obj)
}
fn deep_merge(dict1: &mut Value, dict2: Value) {
if let (Value::Object(map1), Value::Object(map2)) = (dict1, dict2) {
for (k, v) in map2 {
if map1.contains_key(&k) && map1.get(&k).unwrap().is_object() && v.is_object() {
let existing = map1.get_mut(&k).unwrap();
deep_merge(existing, v);
} else {
map1.insert(k, v);
}
}
}
}
fn parse_xml_string(xml_str: &str) -> Result<Value, String> {
let mut reader = Reader::from_str(xml_str);
reader.trim_text(true);
let mut buf = Vec::new();
loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(ref e)) => {
let tag_name = String::from_utf8_lossy(e.name().as_ref()).into_owned();
let val = parse_xml_element(&mut reader, &tag_name)?;
let mut map = Map::new();
map.insert(tag_name, val);
return Ok(Value::Object(map));
}
Ok(Event::Eof) => return Err("Empty XML document".to_string()),
Err(e) => return Err(format!("XML error: {}", e)),
_ => {}
}
buf.clear();
}
}
fn parse_xml_element(reader: &mut Reader<&[u8]>, current_tag: &str) -> Result<Value, String> {
let mut buf = Vec::new();
let mut children_tags: Vec<String> = Vec::new();
let mut children_values: Vec<Value> = Vec::new();
let mut text_content = String::new();
loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(ref e)) => {
let tag_name = String::from_utf8_lossy(e.name().as_ref()).into_owned();
let child_val = parse_xml_element(reader, &tag_name)?;
children_tags.push(tag_name);
children_values.push(child_val);
}
Ok(Event::Text(ref e)) => {
text_content.push_str(&e.unescape().map_err(|err| err.to_string())?);
}
Ok(Event::End(ref e)) => {
let tag_name = String::from_utf8_lossy(e.name().as_ref()).into_owned();
if tag_name == current_tag {
break;
}
}
Ok(Event::Eof) => return Err(format!("Unexpected EOF inside tag: {}", current_tag)),
Err(e) => return Err(format!("XML error: {}", e)),
_ => {}
}
buf.clear();
}
if children_tags.is_empty() {
let text = text_content.trim();
if text.eq_ignore_ascii_case("true") {
return Ok(Value::Bool(true));
}
if text.eq_ignore_ascii_case("false") {
return Ok(Value::Bool(false));
}
if let Ok(i) = text.parse::<i64>() {
return Ok(Value::Number(serde_json::Number::from(i)));
}
if let Ok(f) = text.parse::<f64>()
&& let Some(num) = serde_json::Number::from_f64(f) {
return Ok(Value::Number(num));
}
return Ok(Value::String(text.to_string()));
}
let all_same = children_tags.iter().all(|t| t == &children_tags[0]);
if children_tags.len() > 1 && all_same {
return Ok(Value::Array(children_values));
}
if children_tags.len() == 1 && (children_tags[0] == "item" || children_tags[0] == "step") {
return Ok(Value::Array(vec![children_values[0].clone()]));
}
let mut map = Map::new();
for (tag, val) in children_tags.into_iter().zip(children_values) {
if map.contains_key(&tag) {
let existing = map.get_mut(&tag).unwrap();
if let Value::Array(arr) = existing {
arr.push(val);
} else {
let prev = existing.clone();
*existing = Value::Array(vec![prev, val]);
}
} else {
map.insert(tag, val);
}
}
Ok(Value::Object(map))
}
fn normalize_implementations_order(data: &mut Value) {
if let Some(clm) = data.get_mut("clm")
&& let Some(concrete_impl) = clm.get_mut("concrete_impl")
&& let Some(process) = concrete_impl.get_mut("process")
&& let Some(implementations) = process.get_mut("implementations")
&& let Some(obj) = implementations.as_object_mut() {
let mut sorted_map = Map::new();
let mut keys: Vec<String> = obj.keys().cloned().collect();
keys.sort();
for k in keys {
if let Some(v) = obj.remove(&k) {
sorted_map.insert(k, v);
}
}
*obj = sorted_map;
}
}
fn normalize_tested_cases(data: &mut Value) {
if let Some(clm) = data.get_mut("clm")
&& let Some(balanced_exp) = clm.get_mut("balanced_exp") {
let mut pending_cases = balanced_exp.get("pending_cases")
.and_then(|v| v.as_array())
.cloned()
.unwrap_or_default();
if let Some(tested_cases) = balanced_exp.get("tested_cases").and_then(|v| v.as_array()) {
for case in tested_cases {
if let Some(case_obj) = case.as_object() {
let when_val = case_obj.get("when");
let exists = pending_cases.iter().any(|p_case| {
p_case.get("when") == when_val
});
if !exists {
let mut clean_case = case_obj.clone();
clean_case.remove("exec_hash");
clean_case.remove("exec_time");
pending_cases.push(Value::Object(clean_case));
}
}
}
}
if let Some(obj) = balanced_exp.as_object_mut() {
obj.remove("tested_cases");
obj.insert("pending_cases".to_string(), Value::Array(pending_cases));
}
}
}
fn inject_past_test_cases(data: &mut Value, clm_root: &str) {
let a_hash = crate::layer0::environment::var(crate::layer0::vocabulary_env::ENV_CLM_ACTIVE_A_HASH).unwrap_or_default().trim().to_string();
let c_hash = crate::layer0::environment::var(crate::layer0::vocabulary_env::ENV_CLM_ACTIVE_C_HASH).unwrap_or_default().trim().to_string();
let db_path = crate::db::get_db_path(crate::layer0::vocabulary_env::PILLAR_EXECUTION_LOG, clm_root);
let mut statistics = serde_json::json!({
"total_executions": 0,
"successful_runs": 0,
"failed_runs": 0,
"coverage_ratio": 0.0,
"last_executed_at": ""
});
let mut executions = Value::Array(Vec::new());
if !a_hash.is_empty() && !c_hash.is_empty() && db_path.exists()
&& let Ok(conn) = crate::db::get_connection(crate::layer0::vocabulary_env::PILLAR_EXECUTION_LOG, clm_root) {
let query = "SELECT hash, g_time, content FROM card \
WHERE CAST(content AS TEXT) LIKE ? AND CAST(content AS TEXT) LIKE ? \
ORDER BY g_time DESC";
let a_like = format!("%{}%", a_hash);
let c_like = format!("%{}%", c_hash);
if let Ok(mut stmt) = conn.prepare(query) {
let rows_res = stmt.query_map(params![a_like, c_like], |row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, String>(1)?,
String::from_utf8(row.get_ref(2)?.as_bytes()?.to_vec()).map_err(|e| rusqlite::Error::FromSqlConversionFailure(2, rusqlite::types::Type::Blob, Box::new(e)))?,
))
});
if let Ok(rows) = rows_res {
let mut total_executions = 0;
let mut successful_runs = 0;
let mut failed_runs = 0;
let mut last_executed_at = String::new();
let mut executions_list = Vec::new();
let exit_code_re = Regex::new(r"exit_code:\s*(\d+)").unwrap();
let arch_re = Regex::new(r#"arch:\s*"([^"]*)""#).unwrap();
let os_re = Regex::new(r#"os:\s*"([^"]*)""#).unwrap();
let command_re = Regex::new(r#"command:\s*"([^"]*)""#).unwrap();
let op_re = Regex::new(r#"operation:\s*"([^"]*)""#).unwrap();
for row_val in rows.flatten() {
let (h, g_time, content) = row_val;
let record = crate::layer0::mcard_record::MCardRecord { hash: h.clone(), content: content.as_bytes().to_vec(), g_time: g_time.clone() };
if crate::layer2::record_store::require_verified(&record).is_err() { continue; }
if content.is_empty() {
continue;
}
let exit_code = exit_code_re.captures(&content)
.and_then(|c| c.get(1))
.and_then(|m| m.as_str().parse::<i32>().ok())
.unwrap_or(0);
let arch = arch_re.captures(&content)
.and_then(|c| c.get(1))
.map(|m| m.as_str().to_string())
.unwrap_or_default();
let os_name = os_re.captures(&content)
.and_then(|c| c.get(1))
.map(|m| m.as_str().to_string())
.unwrap_or_default();
let mut command = command_re.captures(&content)
.and_then(|c| c.get(1))
.map(|m| m.as_str().to_string())
.unwrap_or_default();
if command.is_empty() {
if let Some(op) = op_re.captures(&content).and_then(|c| c.get(1)) {
command = format!("{} evaluation", op.as_str());
} else {
command = "evaluation".to_string();
}
}
total_executions += 1;
if exit_code == 0 {
successful_runs += 1;
} else {
failed_runs += 1;
}
if last_executed_at.is_empty() || g_time > last_executed_at {
last_executed_at = g_time.clone();
}
let mut exec_entry = serde_json::json!({
"exec_hash": h,
"g_time": g_time,
"command": command
});
if !os_name.is_empty() || !arch.is_empty() {
let mut env_map = Map::new();
if !os_name.is_empty() {
env_map.insert("os".to_string(), Value::String(os_name));
}
if !arch.is_empty() {
env_map.insert("arch".to_string(), Value::String(arch));
}
exec_entry.as_object_mut().unwrap().insert("environment".to_string(), Value::Object(env_map));
}
executions_list.push(exec_entry);
}
executions_list.sort_by(|a, b| {
let a_cmd = a.get("command").and_then(|v| v.as_str()).unwrap_or("");
let b_cmd = b.get("command").and_then(|v| v.as_str()).unwrap_or("");
let a_hash = a.get("exec_hash").and_then(|v| v.as_str()).unwrap_or("");
let b_hash = b.get("exec_hash").and_then(|v| v.as_str()).unwrap_or("");
let cmd_cmp = a_cmd.cmp(b_cmd);
if cmd_cmp == std::cmp::Ordering::Equal {
a_hash.cmp(b_hash)
} else {
cmd_cmp
}
});
if total_executions > 0 {
statistics = serde_json::json!({
"total_executions": total_executions,
"successful_runs": successful_runs,
"failed_runs": failed_runs,
"coverage_ratio": (successful_runs as f64) / (total_executions as f64),
"last_executed_at": last_executed_at
});
executions = Value::Array(executions_list);
}
}
}
}
if let Some(clm) = data.get_mut("clm")
&& let Some(balanced_exp) = clm.get_mut("balanced_exp")
&& let Some(obj) = balanced_exp.as_object_mut() {
let past_test = serde_json::json!({
"statistics": statistics,
"executions": executions
});
obj.insert("past_test_cases".to_string(), past_test);
}
}