clm_rust_core/layer2/
normalizer.rs1use serde_json::{Value, Map};
2use std::fs;
3use std::path::Path;
4use regex::Regex;
5use quick_xml::Reader;
6use quick_xml::events::Event;
7use rusqlite::params;
8
9pub fn normalize_syntax(target_file: &str, clm_root: &str) -> Result<String, String> {
10 let path = Path::new(target_file);
11 if !path.exists() {
12 return Err(format!("Error: file not found: {}", target_file));
13 }
14
15 let ext = path.extension().and_then(|s| s.to_str()).unwrap_or("").to_lowercase();
16 let content = fs::read_to_string(path)
17 .map_err(|e| format!("Failed to read target file: {}", e))?;
18
19 #[allow(unused_assignments)]
20 let mut normalized_data: Option<Value> = None;
21 let mut is_raw_yaml = false;
22
23 if ext == "json" {
25 normalized_data = serde_json::from_str(&content).ok();
26 } else if ext == "jsonl" {
27 normalized_data = parse_jsonl_string(&content).ok();
28 } else if ext == "xml" {
29 normalized_data = parse_xml_string(&content).ok();
30 } else if ext == "yaml" || ext == "yml" {
31 normalized_data = serde_yaml::from_str(&content).ok();
32 if normalized_data.is_none() {
33 is_raw_yaml = true;
34 }
35 } else {
36 if let Some((lang, block_content)) = extract_code_block(&content) {
38 if lang == "json" {
39 normalized_data = serde_json::from_str(&block_content).ok();
40 } else if lang == "jsonl" {
41 normalized_data = parse_jsonl_string(&block_content).ok();
42 } else if lang == "xml" {
43 normalized_data = parse_xml_string(&block_content).ok();
44 } else if lang == "yaml" || lang == "yml" {
45 normalized_data = serde_yaml::from_str(&block_content).ok();
46 if normalized_data.is_none() {
47 is_raw_yaml = true;
48 }
49 } else {
50 let stripped = block_content.trim();
52 if stripped.starts_with('<') {
53 normalized_data = parse_xml_string(&block_content).ok();
54 } else if stripped.starts_with('{') || stripped.starts_with('[') {
55 normalized_data = serde_json::from_str(&block_content)
56 .ok()
57 .or_else(|| parse_jsonl_string(&block_content).ok());
58 } else {
59 normalized_data = serde_yaml::from_str(&block_content).ok();
60 if normalized_data.is_none() {
61 is_raw_yaml = true;
62 }
63 }
64 }
65 } else {
66 let stripped = content.trim();
68 if stripped.starts_with('<') {
69 normalized_data = parse_xml_string(&content).ok();
70 } else if stripped.starts_with('{') || stripped.starts_with('[') {
71 normalized_data = serde_json::from_str(&content)
72 .ok()
73 .or_else(|| parse_jsonl_string(&content).ok());
74 } else {
75 normalized_data = serde_yaml::from_str(&content).ok();
76 if normalized_data.is_none() {
77 is_raw_yaml = true;
78 }
79 }
80 }
81 }
82
83 let mut data_val = normalized_data.unwrap_or(Value::Null);
84
85 if data_val.is_object() {
87 normalize_tested_cases(&mut data_val);
88 normalize_implementations_order(&mut data_val);
89 inject_past_test_cases(&mut data_val, clm_root);
90 }
91
92 let zero_fs = crate::utils::is_zero_fs();
94 let tmp_dir = if zero_fs {
95 std::env::temp_dir().join("clm_execution_logs/.tmp")
96 } else {
97 Path::new(clm_root).join("data/execution_logs/.tmp")
98 };
99 let _ = fs::create_dir_all(&tmp_dir);
100
101 let count = crate::TEMP_COUNTER.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
103 let temp_path = tmp_dir.join(format!("normalized_{}_{}.yaml", count, std::process::id()));
104
105 if is_raw_yaml {
106 let fallback_re = Regex::new(r"\btested_cases\b").unwrap();
107 let fallback_content = fallback_re.replace_all(&content, "pending_cases").to_string();
108 fs::write(&temp_path, fallback_content)
109 .map_err(|e| format!("Failed to write normalized temp file: {}", e))?;
110 } else {
111 let yaml_str = serde_yaml::to_string(&data_val)
112 .map_err(|e| format!("Failed to serialize normalized data to YAML: {}", e))?;
113 fs::write(&temp_path, yaml_str)
114 .map_err(|e| format!("Failed to write normalized temp file: {}", e))?;
115 }
116
117 Ok(temp_path.to_string_lossy().to_string())
118}
119
120fn extract_code_block(text: &str) -> Option<(String, String)> {
121 let re = Regex::new(r"(?s)```([a-zA-Z0-9_\-]+)?\s*(.*?)\s*```").ok()?;
122 if let Some(caps) = re.captures(text) {
123 let lang = caps.get(1).map(|m| m.as_str().trim().to_lowercase()).unwrap_or_default();
124 let content = caps.get(2).map(|m| m.as_str().trim().to_string()).unwrap_or_default();
125 return Some((lang, content));
126 }
127 None
128}
129
130fn parse_jsonl_string(jsonl_str: &str) -> Result<Value, String> {
131 let mut obj = Value::Object(Map::new());
132 for line in jsonl_str.lines() {
133 let trimmed = line.trim();
134 if trimmed.is_empty() {
135 continue;
136 }
137 let line_val: Value = serde_json::from_str(trimmed)
138 .map_err(|e| format!("Failed to parse JSONL line: {}", e))?;
139 deep_merge(&mut obj, line_val);
140 }
141 Ok(obj)
142}
143
144fn deep_merge(dict1: &mut Value, dict2: Value) {
145 if let (Value::Object(map1), Value::Object(map2)) = (dict1, dict2) {
146 for (k, v) in map2 {
147 if map1.contains_key(&k) && map1.get(&k).unwrap().is_object() && v.is_object() {
148 let existing = map1.get_mut(&k).unwrap();
149 deep_merge(existing, v);
150 } else {
151 map1.insert(k, v);
152 }
153 }
154 }
155}
156
157fn parse_xml_string(xml_str: &str) -> Result<Value, String> {
158 let mut reader = Reader::from_str(xml_str);
159 reader.trim_text(true);
160 let mut buf = Vec::new();
161 loop {
162 match reader.read_event_into(&mut buf) {
163 Ok(Event::Start(ref e)) => {
164 let tag_name = String::from_utf8_lossy(e.name().as_ref()).into_owned();
165 let val = parse_xml_element(&mut reader, &tag_name)?;
166 let mut map = Map::new();
167 map.insert(tag_name, val);
168 return Ok(Value::Object(map));
169 }
170 Ok(Event::Eof) => return Err("Empty XML document".to_string()),
171 Err(e) => return Err(format!("XML error: {}", e)),
172 _ => {}
173 }
174 buf.clear();
175 }
176}
177
178fn parse_xml_element(reader: &mut Reader<&[u8]>, current_tag: &str) -> Result<Value, String> {
179 let mut buf = Vec::new();
180 let mut children_tags: Vec<String> = Vec::new();
181 let mut children_values: Vec<Value> = Vec::new();
182 let mut text_content = String::new();
183
184 loop {
185 match reader.read_event_into(&mut buf) {
186 Ok(Event::Start(ref e)) => {
187 let tag_name = String::from_utf8_lossy(e.name().as_ref()).into_owned();
188 let child_val = parse_xml_element(reader, &tag_name)?;
189 children_tags.push(tag_name);
190 children_values.push(child_val);
191 }
192 Ok(Event::Text(ref e)) => {
193 text_content.push_str(&e.unescape().map_err(|err| err.to_string())?);
194 }
195 Ok(Event::End(ref e)) => {
196 let tag_name = String::from_utf8_lossy(e.name().as_ref()).into_owned();
197 if tag_name == current_tag {
198 break;
199 }
200 }
201 Ok(Event::Eof) => return Err(format!("Unexpected EOF inside tag: {}", current_tag)),
202 Err(e) => return Err(format!("XML error: {}", e)),
203 _ => {}
204 }
205 buf.clear();
206 }
207
208 if children_tags.is_empty() {
209 let text = text_content.trim();
210 if text.eq_ignore_ascii_case("true") {
211 return Ok(Value::Bool(true));
212 }
213 if text.eq_ignore_ascii_case("false") {
214 return Ok(Value::Bool(false));
215 }
216 if let Ok(i) = text.parse::<i64>() {
217 return Ok(Value::Number(serde_json::Number::from(i)));
218 }
219 if let Ok(f) = text.parse::<f64>()
220 && let Some(num) = serde_json::Number::from_f64(f) {
221 return Ok(Value::Number(num));
222 }
223 return Ok(Value::String(text.to_string()));
224 }
225
226 let all_same = children_tags.iter().all(|t| t == &children_tags[0]);
227 if children_tags.len() > 1 && all_same {
228 return Ok(Value::Array(children_values));
229 }
230 if children_tags.len() == 1 && (children_tags[0] == "item" || children_tags[0] == "step") {
231 return Ok(Value::Array(vec![children_values[0].clone()]));
232 }
233
234 let mut map = Map::new();
235 for (tag, val) in children_tags.into_iter().zip(children_values) {
236 if map.contains_key(&tag) {
237 let existing = map.get_mut(&tag).unwrap();
238 if let Value::Array(arr) = existing {
239 arr.push(val);
240 } else {
241 let prev = existing.clone();
242 *existing = Value::Array(vec![prev, val]);
243 }
244 } else {
245 map.insert(tag, val);
246 }
247 }
248 Ok(Value::Object(map))
249}
250
251fn normalize_implementations_order(data: &mut Value) {
252 if let Some(clm) = data.get_mut("clm")
253 && let Some(concrete_impl) = clm.get_mut("concrete_impl")
254 && let Some(process) = concrete_impl.get_mut("process")
255 && let Some(implementations) = process.get_mut("implementations")
256 && let Some(obj) = implementations.as_object_mut() {
257 let mut sorted_map = Map::new();
258 let mut keys: Vec<String> = obj.keys().cloned().collect();
259 keys.sort();
260 for k in keys {
261 if let Some(v) = obj.remove(&k) {
262 sorted_map.insert(k, v);
263 }
264 }
265 *obj = sorted_map;
266 }
267}
268
269fn normalize_tested_cases(data: &mut Value) {
270 if let Some(clm) = data.get_mut("clm")
271 && let Some(balanced_exp) = clm.get_mut("balanced_exp") {
272 let mut pending_cases = balanced_exp.get("pending_cases")
273 .and_then(|v| v.as_array())
274 .cloned()
275 .unwrap_or_default();
276
277 if let Some(tested_cases) = balanced_exp.get("tested_cases").and_then(|v| v.as_array()) {
278 for case in tested_cases {
279 if let Some(case_obj) = case.as_object() {
280 let when_val = case_obj.get("when");
281 let exists = pending_cases.iter().any(|p_case| {
282 p_case.get("when") == when_val
283 });
284 if !exists {
285 let mut clean_case = case_obj.clone();
286 clean_case.remove("exec_hash");
287 clean_case.remove("exec_time");
288 pending_cases.push(Value::Object(clean_case));
289 }
290 }
291 }
292 }
293
294 if let Some(obj) = balanced_exp.as_object_mut() {
295 obj.remove("tested_cases");
296 obj.insert("pending_cases".to_string(), Value::Array(pending_cases));
297 }
298 }
299}
300
301fn inject_past_test_cases(data: &mut Value, clm_root: &str) {
302 let a_hash = crate::layer0::environment::var(crate::layer0::vocabulary_env::ENV_CLM_ACTIVE_A_HASH).unwrap_or_default().trim().to_string();
303 let c_hash = crate::layer0::environment::var(crate::layer0::vocabulary_env::ENV_CLM_ACTIVE_C_HASH).unwrap_or_default().trim().to_string();
304 let db_path = crate::db::get_db_path(crate::layer0::vocabulary_env::PILLAR_EXECUTION_LOG, clm_root);
305
306 let mut statistics = serde_json::json!({
307 "total_executions": 0,
308 "successful_runs": 0,
309 "failed_runs": 0,
310 "coverage_ratio": 0.0,
311 "last_executed_at": ""
312 });
313 let mut executions = Value::Array(Vec::new());
314
315 if !a_hash.is_empty() && !c_hash.is_empty() && db_path.exists()
316 && let Ok(conn) = crate::db::get_connection(crate::layer0::vocabulary_env::PILLAR_EXECUTION_LOG, clm_root) {
317 let query = "SELECT hash, g_time, content FROM card \
318 WHERE content LIKE ? AND content LIKE ? \
319 ORDER BY g_time DESC";
320 let a_like = format!("%{}%", a_hash);
321 let c_like = format!("%{}%", c_hash);
322
323 if let Ok(mut stmt) = conn.prepare(query) {
324 let rows_res = stmt.query_map(params![a_like, c_like], |row| {
325 Ok((
326 row.get::<_, String>(0)?,
327 row.get::<_, String>(1)?,
328 row.get::<_, String>(2)?,
329 ))
330 });
331
332 if let Ok(rows) = rows_res {
333 let mut total_executions = 0;
334 let mut successful_runs = 0;
335 let mut failed_runs = 0;
336 let mut last_executed_at = String::new();
337 let mut executions_list = Vec::new();
338
339 let exit_code_re = Regex::new(r"exit_code:\s*(\d+)").unwrap();
340 let arch_re = Regex::new(r#"arch:\s*"([^"]*)""#).unwrap();
341 let os_re = Regex::new(r#"os:\s*"([^"]*)""#).unwrap();
342 let command_re = Regex::new(r#"command:\s*"([^"]*)""#).unwrap();
343 let op_re = Regex::new(r#"operation:\s*"([^"]*)""#).unwrap();
344
345 for row_val in rows.flatten() {
346 let (h, g_time, content) = row_val;
347 if content.is_empty() {
348 continue;
349 }
350
351 let exit_code = exit_code_re.captures(&content)
352 .and_then(|c| c.get(1))
353 .and_then(|m| m.as_str().parse::<i32>().ok())
354 .unwrap_or(0);
355
356 let arch = arch_re.captures(&content)
357 .and_then(|c| c.get(1))
358 .map(|m| m.as_str().to_string())
359 .unwrap_or_default();
360
361 let os_name = os_re.captures(&content)
362 .and_then(|c| c.get(1))
363 .map(|m| m.as_str().to_string())
364 .unwrap_or_default();
365
366 let mut command = command_re.captures(&content)
367 .and_then(|c| c.get(1))
368 .map(|m| m.as_str().to_string())
369 .unwrap_or_default();
370
371 if command.is_empty() {
372 if let Some(op) = op_re.captures(&content).and_then(|c| c.get(1)) {
373 command = format!("{} evaluation", op.as_str());
374 } else {
375 command = "evaluation".to_string();
376 }
377 }
378
379 total_executions += 1;
380 if exit_code == 0 {
381 successful_runs += 1;
382 } else {
383 failed_runs += 1;
384 }
385
386 if last_executed_at.is_empty() || g_time > last_executed_at {
387 last_executed_at = g_time.clone();
388 }
389
390 let mut exec_entry = serde_json::json!({
391 "exec_hash": h,
392 "g_time": g_time,
393 "command": command
394 });
395
396 if !os_name.is_empty() || !arch.is_empty() {
397 let mut env_map = Map::new();
398 if !os_name.is_empty() {
399 env_map.insert("os".to_string(), Value::String(os_name));
400 }
401 if !arch.is_empty() {
402 env_map.insert("arch".to_string(), Value::String(arch));
403 }
404 exec_entry.as_object_mut().unwrap().insert("environment".to_string(), Value::Object(env_map));
405 }
406
407 executions_list.push(exec_entry);
408 }
409
410 executions_list.sort_by(|a, b| {
412 let a_cmd = a.get("command").and_then(|v| v.as_str()).unwrap_or("");
413 let b_cmd = b.get("command").and_then(|v| v.as_str()).unwrap_or("");
414 let a_hash = a.get("exec_hash").and_then(|v| v.as_str()).unwrap_or("");
415 let b_hash = b.get("exec_hash").and_then(|v| v.as_str()).unwrap_or("");
416
417 let cmd_cmp = a_cmd.cmp(b_cmd);
418 if cmd_cmp == std::cmp::Ordering::Equal {
419 a_hash.cmp(b_hash)
420 } else {
421 cmd_cmp
422 }
423 });
424
425 if total_executions > 0 {
426 statistics = serde_json::json!({
427 "total_executions": total_executions,
428 "successful_runs": successful_runs,
429 "failed_runs": failed_runs,
430 "coverage_ratio": (successful_runs as f64) / (total_executions as f64),
431 "last_executed_at": last_executed_at
432 });
433 executions = Value::Array(executions_list);
434 }
435 }
436 }
437 }
438
439 if let Some(clm) = data.get_mut("clm")
440 && let Some(balanced_exp) = clm.get_mut("balanced_exp")
441 && let Some(obj) = balanced_exp.as_object_mut() {
442 let past_test = serde_json::json!({
443 "statistics": statistics,
444 "executions": executions
445 });
446 obj.insert("past_test_cases".to_string(), past_test);
447 }
448}