Skip to main content

clm_rust_core/
normalizer.rs

1use serde_json::{Value, Map};
2use std::fs;
3use std::path::Path;
4use regex::Regex;
5use quick_xml::Reader;
6use quick_xml::events::Event;
7use rusqlite::params;
8
9pub fn normalize_syntax(target_file: &str, clm_root: &str) -> Result<String, String> {
10    let path = Path::new(target_file);
11    if !path.exists() {
12        return Err(format!("Error: file not found: {}", target_file));
13    }
14
15    let ext = path.extension().and_then(|s| s.to_str()).unwrap_or("").to_lowercase();
16    let content = fs::read_to_string(path)
17        .map_err(|e| format!("Failed to read target file: {}", e))?;
18
19    #[allow(unused_assignments)]
20    let mut normalized_data: Option<Value> = None;
21    let mut is_raw_yaml = false;
22
23    // Parse according to extension or content sniffing
24    if ext == "json" {
25        normalized_data = serde_json::from_str(&content).ok();
26    } else if ext == "jsonl" {
27        normalized_data = parse_jsonl_string(&content).ok();
28    } else if ext == "xml" {
29        normalized_data = parse_xml_string(&content).ok();
30    } else if ext == "yaml" || ext == "yml" {
31        normalized_data = serde_yaml::from_str(&content).ok();
32        if normalized_data.is_none() {
33            is_raw_yaml = true;
34        }
35    } else {
36        // Markdown / Text imprecise format parsing
37        if let Some((lang, block_content)) = extract_code_block(&content) {
38            if lang == "json" {
39                normalized_data = serde_json::from_str(&block_content).ok();
40            } else if lang == "jsonl" {
41                normalized_data = parse_jsonl_string(&block_content).ok();
42            } else if lang == "xml" {
43                normalized_data = parse_xml_string(&block_content).ok();
44            } else if lang == "yaml" || lang == "yml" {
45                normalized_data = serde_yaml::from_str(&block_content).ok();
46                if normalized_data.is_none() {
47                    is_raw_yaml = true;
48                }
49            } else {
50                // Sniff block content
51                let stripped = block_content.trim();
52                if stripped.starts_with('<') {
53                    normalized_data = parse_xml_string(&block_content).ok();
54                } else if stripped.starts_with('{') || stripped.starts_with('[') {
55                    normalized_data = serde_json::from_str(&block_content)
56                        .ok()
57                        .or_else(|| parse_jsonl_string(&block_content).ok());
58                } else {
59                    normalized_data = serde_yaml::from_str(&block_content).ok();
60                    if normalized_data.is_none() {
61                        is_raw_yaml = true;
62                    }
63                }
64            }
65        } else {
66            // Sniff raw content
67            let stripped = content.trim();
68            if stripped.starts_with('<') {
69                normalized_data = parse_xml_string(&content).ok();
70            } else if stripped.starts_with('{') || stripped.starts_with('[') {
71                normalized_data = serde_json::from_str(&content)
72                    .ok()
73                    .or_else(|| parse_jsonl_string(&content).ok());
74            } else {
75                normalized_data = serde_yaml::from_str(&content).ok();
76                if normalized_data.is_none() {
77                    is_raw_yaml = true;
78                }
79            }
80        }
81    }
82
83    let mut data_val = normalized_data.unwrap_or(Value::Null);
84
85    // Normalize grammar if parsed successfully
86    if data_val.is_object() {
87        normalize_tested_cases(&mut data_val);
88        normalize_implementations_order(&mut data_val);
89        inject_past_test_cases(&mut data_val, clm_root);
90    }
91
92    // Resolve output directory
93    let zero_fs = crate::utils::is_zero_fs();
94    let tmp_dir = if zero_fs {
95        std::env::temp_dir().join("clm_execution_logs/.tmp")
96    } else {
97        Path::new(clm_root).join("data/execution_logs/.tmp")
98    };
99    let _ = fs::create_dir_all(&tmp_dir);
100
101    // Generate temp file name
102    let count = crate::TEMP_COUNTER.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
103    let temp_path = tmp_dir.join(format!("normalized_{}_{}.yaml", count, std::process::id()));
104
105    if is_raw_yaml {
106        let fallback_re = Regex::new(r"\btested_cases\b").unwrap();
107        let fallback_content = fallback_re.replace_all(&content, "pending_cases").to_string();
108        fs::write(&temp_path, fallback_content)
109            .map_err(|e| format!("Failed to write normalized temp file: {}", e))?;
110    } else {
111        let yaml_str = serde_yaml::to_string(&data_val)
112            .map_err(|e| format!("Failed to serialize normalized data to YAML: {}", e))?;
113        fs::write(&temp_path, yaml_str)
114            .map_err(|e| format!("Failed to write normalized temp file: {}", e))?;
115    }
116
117    Ok(temp_path.to_string_lossy().to_string())
118}
119
120fn extract_code_block(text: &str) -> Option<(String, String)> {
121    let re = Regex::new(r"(?s)```([a-zA-Z0-9_\-]+)?\s*(.*?)\s*```").ok()?;
122    if let Some(caps) = re.captures(text) {
123        let lang = caps.get(1).map(|m| m.as_str().trim().to_lowercase()).unwrap_or_default();
124        let content = caps.get(2).map(|m| m.as_str().trim().to_string()).unwrap_or_default();
125        return Some((lang, content));
126    }
127    None
128}
129
130fn parse_jsonl_string(jsonl_str: &str) -> Result<Value, String> {
131    let mut obj = Value::Object(Map::new());
132    for line in jsonl_str.lines() {
133        let trimmed = line.trim();
134        if trimmed.is_empty() {
135            continue;
136        }
137        let line_val: Value = serde_json::from_str(trimmed)
138            .map_err(|e| format!("Failed to parse JSONL line: {}", e))?;
139        deep_merge(&mut obj, line_val);
140    }
141    Ok(obj)
142}
143
144fn deep_merge(dict1: &mut Value, dict2: Value) {
145    if let (Value::Object(map1), Value::Object(map2)) = (dict1, dict2) {
146        for (k, v) in map2 {
147            if map1.contains_key(&k) && map1.get(&k).unwrap().is_object() && v.is_object() {
148                let existing = map1.get_mut(&k).unwrap();
149                deep_merge(existing, v);
150            } else {
151                map1.insert(k, v);
152            }
153        }
154    }
155}
156
157fn parse_xml_string(xml_str: &str) -> Result<Value, String> {
158    let mut reader = Reader::from_str(xml_str);
159    reader.trim_text(true);
160    let mut buf = Vec::new();
161    loop {
162        match reader.read_event_into(&mut buf) {
163            Ok(Event::Start(ref e)) => {
164                let tag_name = String::from_utf8_lossy(e.name().as_ref()).into_owned();
165                let val = parse_xml_element(&mut reader, &tag_name)?;
166                let mut map = Map::new();
167                map.insert(tag_name, val);
168                return Ok(Value::Object(map));
169            }
170            Ok(Event::Eof) => return Err("Empty XML document".to_string()),
171            Err(e) => return Err(format!("XML error: {}", e)),
172            _ => {}
173        }
174        buf.clear();
175    }
176}
177
178fn parse_xml_element(reader: &mut Reader<&[u8]>, current_tag: &str) -> Result<Value, String> {
179    let mut buf = Vec::new();
180    let mut children_tags: Vec<String> = Vec::new();
181    let mut children_values: Vec<Value> = Vec::new();
182    let mut text_content = String::new();
183    
184    loop {
185        match reader.read_event_into(&mut buf) {
186            Ok(Event::Start(ref e)) => {
187                let tag_name = String::from_utf8_lossy(e.name().as_ref()).into_owned();
188                let child_val = parse_xml_element(reader, &tag_name)?;
189                children_tags.push(tag_name);
190                children_values.push(child_val);
191            }
192            Ok(Event::Text(ref e)) => {
193                text_content.push_str(&e.unescape().map_err(|err| err.to_string())?);
194            }
195            Ok(Event::End(ref e)) => {
196                let tag_name = String::from_utf8_lossy(e.name().as_ref()).into_owned();
197                if tag_name == current_tag {
198                    break;
199                }
200            }
201            Ok(Event::Eof) => return Err(format!("Unexpected EOF inside tag: {}", current_tag)),
202            Err(e) => return Err(format!("XML error: {}", e)),
203            _ => {}
204        }
205        buf.clear();
206    }
207    
208    if children_tags.is_empty() {
209        let text = text_content.trim();
210        if text.eq_ignore_ascii_case("true") {
211            return Ok(Value::Bool(true));
212        }
213        if text.eq_ignore_ascii_case("false") {
214            return Ok(Value::Bool(false));
215        }
216        if let Ok(i) = text.parse::<i64>() {
217            return Ok(Value::Number(serde_json::Number::from(i)));
218        }
219        if let Ok(f) = text.parse::<f64>() {
220            if let Some(num) = serde_json::Number::from_f64(f) {
221                return Ok(Value::Number(num));
222            }
223        }
224        return Ok(Value::String(text.to_string()));
225    }
226    
227    let all_same = children_tags.iter().all(|t| t == &children_tags[0]);
228    if children_tags.len() > 1 && all_same {
229        return Ok(Value::Array(children_values));
230    }
231    if children_tags.len() == 1 && (children_tags[0] == "item" || children_tags[0] == "step") {
232        return Ok(Value::Array(vec![children_values[0].clone()]));
233    }
234    
235    let mut map = Map::new();
236    for (tag, val) in children_tags.into_iter().zip(children_values.into_iter()) {
237        if map.contains_key(&tag) {
238            let existing = map.get_mut(&tag).unwrap();
239            if let Value::Array(arr) = existing {
240                arr.push(val);
241            } else {
242                let prev = existing.clone();
243                *existing = Value::Array(vec![prev, val]);
244            }
245        } else {
246            map.insert(tag, val);
247        }
248    }
249    Ok(Value::Object(map))
250}
251
252fn normalize_implementations_order(data: &mut Value) {
253    if let Some(clm) = data.get_mut("clm") {
254        if let Some(concrete_impl) = clm.get_mut("concrete_impl") {
255            if let Some(process) = concrete_impl.get_mut("process") {
256                if let Some(implementations) = process.get_mut("implementations") {
257                    if let Some(obj) = implementations.as_object_mut() {
258                        let mut sorted_map = Map::new();
259                        let mut keys: Vec<String> = obj.keys().cloned().collect();
260                        keys.sort();
261                        for k in keys {
262                            if let Some(v) = obj.remove(&k) {
263                                sorted_map.insert(k, v);
264                            }
265                        }
266                        *obj = sorted_map;
267                    }
268                }
269            }
270        }
271    }
272}
273
274fn normalize_tested_cases(data: &mut Value) {
275    if let Some(clm) = data.get_mut("clm") {
276        if let Some(balanced_exp) = clm.get_mut("balanced_exp") {
277            let mut pending_cases = balanced_exp.get("pending_cases")
278                .and_then(|v| v.as_array())
279                .cloned()
280                .unwrap_or_default();
281            
282            if let Some(tested_cases) = balanced_exp.get("tested_cases").and_then(|v| v.as_array()) {
283                for case in tested_cases {
284                    if let Some(case_obj) = case.as_object() {
285                        let when_val = case_obj.get("when");
286                        let exists = pending_cases.iter().any(|p_case| {
287                            p_case.get("when") == when_val
288                        });
289                        if !exists {
290                            let mut clean_case = case_obj.clone();
291                            clean_case.remove("exec_hash");
292                            clean_case.remove("exec_time");
293                            pending_cases.push(Value::Object(clean_case));
294                        }
295                    }
296                }
297            }
298            
299            if let Some(obj) = balanced_exp.as_object_mut() {
300                obj.remove("tested_cases");
301                obj.insert("pending_cases".to_string(), Value::Array(pending_cases));
302            }
303        }
304    }
305}
306
307fn inject_past_test_cases(data: &mut Value, clm_root: &str) {
308    let a_hash = std::env::var("CLM_ACTIVE_A_HASH").unwrap_or_default().trim().to_string();
309    let c_hash = std::env::var("CLM_ACTIVE_C_HASH").unwrap_or_default().trim().to_string();
310    let db_path = crate::db::get_db_path("execution_log", clm_root);
311
312    let mut statistics = serde_json::json!({
313        "total_executions": 0,
314        "successful_runs": 0,
315        "failed_runs": 0,
316        "coverage_ratio": 0.0,
317        "last_executed_at": ""
318    });
319    let mut executions = Value::Array(Vec::new());
320
321    if !a_hash.is_empty() && !c_hash.is_empty() && db_path.exists() {
322        if let Ok(conn) = crate::db::get_connection("execution_log", clm_root) {
323            let query = "SELECT hash, g_time, content FROM card \
324                         WHERE content LIKE ? AND content LIKE ? \
325                         ORDER BY g_time DESC";
326            let a_like = format!("%{}%", a_hash);
327            let c_like = format!("%{}%", c_hash);
328            
329            if let Ok(mut stmt) = conn.prepare(query) {
330                let rows_res = stmt.query_map(params![a_like, c_like], |row| {
331                    Ok((
332                        row.get::<_, String>(0)?,
333                        row.get::<_, String>(1)?,
334                        row.get::<_, String>(2)?,
335                    ))
336                });
337                
338                if let Ok(rows) = rows_res {
339                    let mut total_executions = 0;
340                    let mut successful_runs = 0;
341                    let mut failed_runs = 0;
342                    let mut last_executed_at = String::new();
343                    let mut executions_list = Vec::new();
344
345                    let exit_code_re = Regex::new(r"exit_code:\s*(\d+)").unwrap();
346                    let arch_re = Regex::new(r#"arch:\s*"([^"]*)""#).unwrap();
347                    let os_re = Regex::new(r#"os:\s*"([^"]*)""#).unwrap();
348                    let command_re = Regex::new(r#"command:\s*"([^"]*)""#).unwrap();
349                    let op_re = Regex::new(r#"operation:\s*"([^"]*)""#).unwrap();
350
351                    for row_val in rows.flatten() {
352                        let (h, g_time, content) = row_val;
353                        if content.is_empty() {
354                            continue;
355                        }
356                        
357                        let exit_code = exit_code_re.captures(&content)
358                            .and_then(|c| c.get(1))
359                            .and_then(|m| m.as_str().parse::<i32>().ok())
360                            .unwrap_or(0);
361                        
362                        let arch = arch_re.captures(&content)
363                            .and_then(|c| c.get(1))
364                            .map(|m| m.as_str().to_string())
365                            .unwrap_or_default();
366                            
367                        let os_name = os_re.captures(&content)
368                            .and_then(|c| c.get(1))
369                            .map(|m| m.as_str().to_string())
370                            .unwrap_or_default();
371                            
372                        let mut command = command_re.captures(&content)
373                            .and_then(|c| c.get(1))
374                            .map(|m| m.as_str().to_string())
375                            .unwrap_or_default();
376                            
377                        if command.is_empty() {
378                            if let Some(op) = op_re.captures(&content).and_then(|c| c.get(1)) {
379                                command = format!("{} evaluation", op.as_str());
380                            } else {
381                                command = "evaluation".to_string();
382                            }
383                        }
384
385                        total_executions += 1;
386                        if exit_code == 0 {
387                            successful_runs += 1;
388                        } else {
389                            failed_runs += 1;
390                        }
391
392                        if last_executed_at.is_empty() || g_time > last_executed_at {
393                            last_executed_at = g_time.clone();
394                        }
395
396                        let mut exec_entry = serde_json::json!({
397                            "exec_hash": h,
398                            "g_time": g_time,
399                            "command": command
400                        });
401                        
402                        if !os_name.is_empty() || !arch.is_empty() {
403                            let mut env_map = Map::new();
404                            if !os_name.is_empty() {
405                                env_map.insert("os".to_string(), Value::String(os_name));
406                            }
407                            if !arch.is_empty() {
408                                env_map.insert("arch".to_string(), Value::String(arch));
409                            }
410                            exec_entry.as_object_mut().unwrap().insert("environment".to_string(), Value::Object(env_map));
411                        }
412
413                        executions_list.push(exec_entry);
414                    }
415
416                    // Sort executions list deterministically: command asc, then exec_hash asc
417                    executions_list.sort_by(|a, b| {
418                        let a_cmd = a.get("command").and_then(|v| v.as_str()).unwrap_or("");
419                        let b_cmd = b.get("command").and_then(|v| v.as_str()).unwrap_or("");
420                        let a_hash = a.get("exec_hash").and_then(|v| v.as_str()).unwrap_or("");
421                        let b_hash = b.get("exec_hash").and_then(|v| v.as_str()).unwrap_or("");
422                        
423                        let cmd_cmp = a_cmd.cmp(b_cmd);
424                        if cmd_cmp == std::cmp::Ordering::Equal {
425                            a_hash.cmp(b_hash)
426                        } else {
427                            cmd_cmp
428                        }
429                    });
430
431                    if total_executions > 0 {
432                        statistics = serde_json::json!({
433                            "total_executions": total_executions,
434                            "successful_runs": successful_runs,
435                            "failed_runs": failed_runs,
436                            "coverage_ratio": (successful_runs as f64) / (total_executions as f64),
437                            "last_executed_at": last_executed_at
438                        });
439                        executions = Value::Array(executions_list);
440                    }
441                }
442            }
443        }
444    }
445
446    if let Some(clm) = data.get_mut("clm") {
447        if let Some(balanced_exp) = clm.get_mut("balanced_exp") {
448            if let Some(obj) = balanced_exp.as_object_mut() {
449                let past_test = serde_json::json!({
450                    "statistics": statistics,
451                    "executions": executions
452                });
453                obj.insert("past_test_cases".to_string(), past_test);
454            }
455        }
456    }
457}