Skip to main content

clm_rust_core/layer2/
normalizer.rs

1use serde_json::{Value, Map};
2use std::fs;
3use std::path::Path;
4use regex::Regex;
5use quick_xml::Reader;
6use quick_xml::events::Event;
7use rusqlite::params;
8
9pub fn normalize_syntax(target_file: &str, clm_root: &str) -> Result<String, String> {
10    let path = Path::new(target_file);
11    if !path.exists() {
12        return Err(format!("Error: file not found: {}", target_file));
13    }
14
15    let ext = path.extension().and_then(|s| s.to_str()).unwrap_or("").to_lowercase();
16    let content = fs::read_to_string(path)
17        .map_err(|e| format!("Failed to read target file: {}", e))?;
18
19    #[allow(unused_assignments)]
20    let mut normalized_data: Option<Value> = None;
21    let mut is_raw_yaml = false;
22
23    // Parse according to extension or content sniffing
24    if ext == "json" {
25        normalized_data = serde_json::from_str(&content).ok();
26    } else if ext == "jsonl" {
27        normalized_data = parse_jsonl_string(&content).ok();
28    } else if ext == "xml" {
29        normalized_data = parse_xml_string(&content).ok();
30    } else if ext == "yaml" || ext == "yml" {
31        normalized_data = serde_yaml::from_str(&content).ok();
32        if normalized_data.is_none() {
33            is_raw_yaml = true;
34        }
35    } else {
36        // Markdown / Text imprecise format parsing
37        if let Some((lang, block_content)) = extract_code_block(&content) {
38            if lang == "json" {
39                normalized_data = serde_json::from_str(&block_content).ok();
40            } else if lang == "jsonl" {
41                normalized_data = parse_jsonl_string(&block_content).ok();
42            } else if lang == "xml" {
43                normalized_data = parse_xml_string(&block_content).ok();
44            } else if lang == "yaml" || lang == "yml" {
45                normalized_data = serde_yaml::from_str(&block_content).ok();
46                if normalized_data.is_none() {
47                    is_raw_yaml = true;
48                }
49            } else {
50                // Sniff block content
51                let stripped = block_content.trim();
52                if stripped.starts_with('<') {
53                    normalized_data = parse_xml_string(&block_content).ok();
54                } else if stripped.starts_with('{') || stripped.starts_with('[') {
55                    normalized_data = serde_json::from_str(&block_content)
56                        .ok()
57                        .or_else(|| parse_jsonl_string(&block_content).ok());
58                } else {
59                    normalized_data = serde_yaml::from_str(&block_content).ok();
60                    if normalized_data.is_none() {
61                        is_raw_yaml = true;
62                    }
63                }
64            }
65        } else {
66            // Sniff raw content
67            let stripped = content.trim();
68            if stripped.starts_with('<') {
69                normalized_data = parse_xml_string(&content).ok();
70            } else if stripped.starts_with('{') || stripped.starts_with('[') {
71                normalized_data = serde_json::from_str(&content)
72                    .ok()
73                    .or_else(|| parse_jsonl_string(&content).ok());
74            } else {
75                normalized_data = serde_yaml::from_str(&content).ok();
76                if normalized_data.is_none() {
77                    is_raw_yaml = true;
78                }
79            }
80        }
81    }
82
83    let mut data_val = normalized_data.unwrap_or(Value::Null);
84
85    // Normalize grammar if parsed successfully
86    if data_val.is_object() {
87        normalize_tested_cases(&mut data_val);
88        normalize_implementations_order(&mut data_val);
89        inject_past_test_cases(&mut data_val, clm_root);
90    }
91
92    // Resolve output directory
93    let zero_fs = crate::utils::is_zero_fs();
94    let tmp_dir = if zero_fs {
95        std::env::temp_dir().join("clm_execution_logs/.tmp")
96    } else {
97        Path::new(clm_root).join("data/execution_logs/.tmp")
98    };
99    let _ = fs::create_dir_all(&tmp_dir);
100
101    // Generate temp file name
102    let count = crate::TEMP_COUNTER.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
103    let temp_path = tmp_dir.join(format!("normalized_{}_{}.yaml", count, std::process::id()));
104
105    if is_raw_yaml {
106        let fallback_re = Regex::new(r"\btested_cases\b").unwrap();
107        let fallback_content = fallback_re.replace_all(&content, "pending_cases").to_string();
108        fs::write(&temp_path, fallback_content)
109            .map_err(|e| format!("Failed to write normalized temp file: {}", e))?;
110    } else {
111        let yaml_str = serde_yaml::to_string(&data_val)
112            .map_err(|e| format!("Failed to serialize normalized data to YAML: {}", e))?;
113        fs::write(&temp_path, yaml_str)
114            .map_err(|e| format!("Failed to write normalized temp file: {}", e))?;
115    }
116
117    Ok(temp_path.to_string_lossy().to_string())
118}
119
120fn extract_code_block(text: &str) -> Option<(String, String)> {
121    let re = Regex::new(r"(?s)```([a-zA-Z0-9_\-]+)?\s*(.*?)\s*```").ok()?;
122    if let Some(caps) = re.captures(text) {
123        let lang = caps.get(1).map(|m| m.as_str().trim().to_lowercase()).unwrap_or_default();
124        let content = caps.get(2).map(|m| m.as_str().trim().to_string()).unwrap_or_default();
125        return Some((lang, content));
126    }
127    None
128}
129
130fn parse_jsonl_string(jsonl_str: &str) -> Result<Value, String> {
131    let mut obj = Value::Object(Map::new());
132    for line in jsonl_str.lines() {
133        let trimmed = line.trim();
134        if trimmed.is_empty() {
135            continue;
136        }
137        let line_val: Value = serde_json::from_str(trimmed)
138            .map_err(|e| format!("Failed to parse JSONL line: {}", e))?;
139        deep_merge(&mut obj, line_val);
140    }
141    Ok(obj)
142}
143
144fn deep_merge(dict1: &mut Value, dict2: Value) {
145    if let (Value::Object(map1), Value::Object(map2)) = (dict1, dict2) {
146        for (k, v) in map2 {
147            if map1.contains_key(&k) && map1.get(&k).unwrap().is_object() && v.is_object() {
148                let existing = map1.get_mut(&k).unwrap();
149                deep_merge(existing, v);
150            } else {
151                map1.insert(k, v);
152            }
153        }
154    }
155}
156
157fn parse_xml_string(xml_str: &str) -> Result<Value, String> {
158    let mut reader = Reader::from_str(xml_str);
159    reader.trim_text(true);
160    let mut buf = Vec::new();
161    loop {
162        match reader.read_event_into(&mut buf) {
163            Ok(Event::Start(ref e)) => {
164                let tag_name = String::from_utf8_lossy(e.name().as_ref()).into_owned();
165                let val = parse_xml_element(&mut reader, &tag_name)?;
166                let mut map = Map::new();
167                map.insert(tag_name, val);
168                return Ok(Value::Object(map));
169            }
170            Ok(Event::Eof) => return Err("Empty XML document".to_string()),
171            Err(e) => return Err(format!("XML error: {}", e)),
172            _ => {}
173        }
174        buf.clear();
175    }
176}
177
178fn parse_xml_element(reader: &mut Reader<&[u8]>, current_tag: &str) -> Result<Value, String> {
179    let mut buf = Vec::new();
180    let mut children_tags: Vec<String> = Vec::new();
181    let mut children_values: Vec<Value> = Vec::new();
182    let mut text_content = String::new();
183    
184    loop {
185        match reader.read_event_into(&mut buf) {
186            Ok(Event::Start(ref e)) => {
187                let tag_name = String::from_utf8_lossy(e.name().as_ref()).into_owned();
188                let child_val = parse_xml_element(reader, &tag_name)?;
189                children_tags.push(tag_name);
190                children_values.push(child_val);
191            }
192            Ok(Event::Text(ref e)) => {
193                text_content.push_str(&e.unescape().map_err(|err| err.to_string())?);
194            }
195            Ok(Event::End(ref e)) => {
196                let tag_name = String::from_utf8_lossy(e.name().as_ref()).into_owned();
197                if tag_name == current_tag {
198                    break;
199                }
200            }
201            Ok(Event::Eof) => return Err(format!("Unexpected EOF inside tag: {}", current_tag)),
202            Err(e) => return Err(format!("XML error: {}", e)),
203            _ => {}
204        }
205        buf.clear();
206    }
207    
208    if children_tags.is_empty() {
209        let text = text_content.trim();
210        if text.eq_ignore_ascii_case("true") {
211            return Ok(Value::Bool(true));
212        }
213        if text.eq_ignore_ascii_case("false") {
214            return Ok(Value::Bool(false));
215        }
216        if let Ok(i) = text.parse::<i64>() {
217            return Ok(Value::Number(serde_json::Number::from(i)));
218        }
219        if let Ok(f) = text.parse::<f64>()
220            && let Some(num) = serde_json::Number::from_f64(f) {
221                return Ok(Value::Number(num));
222            }
223        return Ok(Value::String(text.to_string()));
224    }
225    
226    let all_same = children_tags.iter().all(|t| t == &children_tags[0]);
227    if children_tags.len() > 1 && all_same {
228        return Ok(Value::Array(children_values));
229    }
230    if children_tags.len() == 1 && (children_tags[0] == "item" || children_tags[0] == "step") {
231        return Ok(Value::Array(vec![children_values[0].clone()]));
232    }
233    
234    let mut map = Map::new();
235    for (tag, val) in children_tags.into_iter().zip(children_values) {
236        if map.contains_key(&tag) {
237            let existing = map.get_mut(&tag).unwrap();
238            if let Value::Array(arr) = existing {
239                arr.push(val);
240            } else {
241                let prev = existing.clone();
242                *existing = Value::Array(vec![prev, val]);
243            }
244        } else {
245            map.insert(tag, val);
246        }
247    }
248    Ok(Value::Object(map))
249}
250
251fn normalize_implementations_order(data: &mut Value) {
252    if let Some(clm) = data.get_mut("clm")
253        && let Some(concrete_impl) = clm.get_mut("concrete_impl")
254            && let Some(process) = concrete_impl.get_mut("process")
255                && let Some(implementations) = process.get_mut("implementations")
256                    && let Some(obj) = implementations.as_object_mut() {
257                        let mut sorted_map = Map::new();
258                        let mut keys: Vec<String> = obj.keys().cloned().collect();
259                        keys.sort();
260                        for k in keys {
261                            if let Some(v) = obj.remove(&k) {
262                                sorted_map.insert(k, v);
263                            }
264                        }
265                        *obj = sorted_map;
266                    }
267}
268
269fn normalize_tested_cases(data: &mut Value) {
270    if let Some(clm) = data.get_mut("clm")
271        && let Some(balanced_exp) = clm.get_mut("balanced_exp") {
272            let mut pending_cases = balanced_exp.get("pending_cases")
273                .and_then(|v| v.as_array())
274                .cloned()
275                .unwrap_or_default();
276            
277            if let Some(tested_cases) = balanced_exp.get("tested_cases").and_then(|v| v.as_array()) {
278                for case in tested_cases {
279                    if let Some(case_obj) = case.as_object() {
280                        let when_val = case_obj.get("when");
281                        let exists = pending_cases.iter().any(|p_case| {
282                            p_case.get("when") == when_val
283                        });
284                        if !exists {
285                            let mut clean_case = case_obj.clone();
286                            clean_case.remove("exec_hash");
287                            clean_case.remove("exec_time");
288                            pending_cases.push(Value::Object(clean_case));
289                        }
290                    }
291                }
292            }
293            
294            if let Some(obj) = balanced_exp.as_object_mut() {
295                obj.remove("tested_cases");
296                obj.insert("pending_cases".to_string(), Value::Array(pending_cases));
297            }
298        }
299}
300
301fn inject_past_test_cases(data: &mut Value, clm_root: &str) {
302    let a_hash = crate::layer0::environment::var(crate::layer0::vocabulary_env::ENV_CLM_ACTIVE_A_HASH).unwrap_or_default().trim().to_string();
303    let c_hash = crate::layer0::environment::var(crate::layer0::vocabulary_env::ENV_CLM_ACTIVE_C_HASH).unwrap_or_default().trim().to_string();
304    let db_path = crate::db::get_db_path(crate::layer0::vocabulary_env::PILLAR_EXECUTION_LOG, clm_root);
305
306    let mut statistics = serde_json::json!({
307        "total_executions": 0,
308        "successful_runs": 0,
309        "failed_runs": 0,
310        "coverage_ratio": 0.0,
311        "last_executed_at": ""
312    });
313    let mut executions = Value::Array(Vec::new());
314
315    if !a_hash.is_empty() && !c_hash.is_empty() && db_path.exists()
316        && let Ok(conn) = crate::db::get_connection(crate::layer0::vocabulary_env::PILLAR_EXECUTION_LOG, clm_root) {
317            let query = "SELECT hash, g_time, content FROM card \
318                         WHERE content LIKE ? AND content LIKE ? \
319                         ORDER BY g_time DESC";
320            let a_like = format!("%{}%", a_hash);
321            let c_like = format!("%{}%", c_hash);
322            
323            if let Ok(mut stmt) = conn.prepare(query) {
324                let rows_res = stmt.query_map(params![a_like, c_like], |row| {
325                    Ok((
326                        row.get::<_, String>(0)?,
327                        row.get::<_, String>(1)?,
328                        row.get::<_, String>(2)?,
329                    ))
330                });
331                
332                if let Ok(rows) = rows_res {
333                    let mut total_executions = 0;
334                    let mut successful_runs = 0;
335                    let mut failed_runs = 0;
336                    let mut last_executed_at = String::new();
337                    let mut executions_list = Vec::new();
338
339                    let exit_code_re = Regex::new(r"exit_code:\s*(\d+)").unwrap();
340                    let arch_re = Regex::new(r#"arch:\s*"([^"]*)""#).unwrap();
341                    let os_re = Regex::new(r#"os:\s*"([^"]*)""#).unwrap();
342                    let command_re = Regex::new(r#"command:\s*"([^"]*)""#).unwrap();
343                    let op_re = Regex::new(r#"operation:\s*"([^"]*)""#).unwrap();
344
345                    for row_val in rows.flatten() {
346                        let (h, g_time, content) = row_val;
347                        if content.is_empty() {
348                            continue;
349                        }
350                        
351                        let exit_code = exit_code_re.captures(&content)
352                            .and_then(|c| c.get(1))
353                            .and_then(|m| m.as_str().parse::<i32>().ok())
354                            .unwrap_or(0);
355                        
356                        let arch = arch_re.captures(&content)
357                            .and_then(|c| c.get(1))
358                            .map(|m| m.as_str().to_string())
359                            .unwrap_or_default();
360                            
361                        let os_name = os_re.captures(&content)
362                            .and_then(|c| c.get(1))
363                            .map(|m| m.as_str().to_string())
364                            .unwrap_or_default();
365                            
366                        let mut command = command_re.captures(&content)
367                            .and_then(|c| c.get(1))
368                            .map(|m| m.as_str().to_string())
369                            .unwrap_or_default();
370                            
371                        if command.is_empty() {
372                            if let Some(op) = op_re.captures(&content).and_then(|c| c.get(1)) {
373                                command = format!("{} evaluation", op.as_str());
374                            } else {
375                                command = "evaluation".to_string();
376                            }
377                        }
378
379                        total_executions += 1;
380                        if exit_code == 0 {
381                            successful_runs += 1;
382                        } else {
383                            failed_runs += 1;
384                        }
385
386                        if last_executed_at.is_empty() || g_time > last_executed_at {
387                            last_executed_at = g_time.clone();
388                        }
389
390                        let mut exec_entry = serde_json::json!({
391                            "exec_hash": h,
392                            "g_time": g_time,
393                            "command": command
394                        });
395                        
396                        if !os_name.is_empty() || !arch.is_empty() {
397                            let mut env_map = Map::new();
398                            if !os_name.is_empty() {
399                                env_map.insert("os".to_string(), Value::String(os_name));
400                            }
401                            if !arch.is_empty() {
402                                env_map.insert("arch".to_string(), Value::String(arch));
403                            }
404                            exec_entry.as_object_mut().unwrap().insert("environment".to_string(), Value::Object(env_map));
405                        }
406
407                        executions_list.push(exec_entry);
408                    }
409
410                    // Sort executions list deterministically: command asc, then exec_hash asc
411                    executions_list.sort_by(|a, b| {
412                        let a_cmd = a.get("command").and_then(|v| v.as_str()).unwrap_or("");
413                        let b_cmd = b.get("command").and_then(|v| v.as_str()).unwrap_or("");
414                        let a_hash = a.get("exec_hash").and_then(|v| v.as_str()).unwrap_or("");
415                        let b_hash = b.get("exec_hash").and_then(|v| v.as_str()).unwrap_or("");
416                        
417                        let cmd_cmp = a_cmd.cmp(b_cmd);
418                        if cmd_cmp == std::cmp::Ordering::Equal {
419                            a_hash.cmp(b_hash)
420                        } else {
421                            cmd_cmp
422                        }
423                    });
424
425                    if total_executions > 0 {
426                        statistics = serde_json::json!({
427                            "total_executions": total_executions,
428                            "successful_runs": successful_runs,
429                            "failed_runs": failed_runs,
430                            "coverage_ratio": (successful_runs as f64) / (total_executions as f64),
431                            "last_executed_at": last_executed_at
432                        });
433                        executions = Value::Array(executions_list);
434                    }
435                }
436            }
437        }
438
439    if let Some(clm) = data.get_mut("clm")
440        && let Some(balanced_exp) = clm.get_mut("balanced_exp")
441            && let Some(obj) = balanced_exp.as_object_mut() {
442                let past_test = serde_json::json!({
443                    "statistics": statistics,
444                    "executions": executions
445                });
446                obj.insert("past_test_cases".to_string(), past_test);
447            }
448}