clm_rust_core/
normalizer.rs1use serde_json::{Value, Map};
2use std::fs;
3use std::path::Path;
4use regex::Regex;
5use quick_xml::Reader;
6use quick_xml::events::Event;
7use rusqlite::params;
8
9pub fn normalize_syntax(target_file: &str, clm_root: &str) -> Result<String, String> {
10 let path = Path::new(target_file);
11 if !path.exists() {
12 return Err(format!("Error: file not found: {}", target_file));
13 }
14
15 let ext = path.extension().and_then(|s| s.to_str()).unwrap_or("").to_lowercase();
16 let content = fs::read_to_string(path)
17 .map_err(|e| format!("Failed to read target file: {}", e))?;
18
19 #[allow(unused_assignments)]
20 let mut normalized_data: Option<Value> = None;
21 let mut is_raw_yaml = false;
22
23 if ext == "json" {
25 normalized_data = serde_json::from_str(&content).ok();
26 } else if ext == "jsonl" {
27 normalized_data = parse_jsonl_string(&content).ok();
28 } else if ext == "xml" {
29 normalized_data = parse_xml_string(&content).ok();
30 } else if ext == "yaml" || ext == "yml" {
31 normalized_data = serde_yaml::from_str(&content).ok();
32 if normalized_data.is_none() {
33 is_raw_yaml = true;
34 }
35 } else {
36 if let Some((lang, block_content)) = extract_code_block(&content) {
38 if lang == "json" {
39 normalized_data = serde_json::from_str(&block_content).ok();
40 } else if lang == "jsonl" {
41 normalized_data = parse_jsonl_string(&block_content).ok();
42 } else if lang == "xml" {
43 normalized_data = parse_xml_string(&block_content).ok();
44 } else if lang == "yaml" || lang == "yml" {
45 normalized_data = serde_yaml::from_str(&block_content).ok();
46 if normalized_data.is_none() {
47 is_raw_yaml = true;
48 }
49 } else {
50 let stripped = block_content.trim();
52 if stripped.starts_with('<') {
53 normalized_data = parse_xml_string(&block_content).ok();
54 } else if stripped.starts_with('{') || stripped.starts_with('[') {
55 normalized_data = serde_json::from_str(&block_content)
56 .ok()
57 .or_else(|| parse_jsonl_string(&block_content).ok());
58 } else {
59 normalized_data = serde_yaml::from_str(&block_content).ok();
60 if normalized_data.is_none() {
61 is_raw_yaml = true;
62 }
63 }
64 }
65 } else {
66 let stripped = content.trim();
68 if stripped.starts_with('<') {
69 normalized_data = parse_xml_string(&content).ok();
70 } else if stripped.starts_with('{') || stripped.starts_with('[') {
71 normalized_data = serde_json::from_str(&content)
72 .ok()
73 .or_else(|| parse_jsonl_string(&content).ok());
74 } else {
75 normalized_data = serde_yaml::from_str(&content).ok();
76 if normalized_data.is_none() {
77 is_raw_yaml = true;
78 }
79 }
80 }
81 }
82
83 let mut data_val = normalized_data.unwrap_or(Value::Null);
84
85 if data_val.is_object() {
87 normalize_tested_cases(&mut data_val);
88 normalize_implementations_order(&mut data_val);
89 inject_past_test_cases(&mut data_val, clm_root);
90 }
91
92 let zero_fs = crate::utils::is_zero_fs();
94 let tmp_dir = if zero_fs {
95 std::env::temp_dir().join("clm_execution_logs/.tmp")
96 } else {
97 Path::new(clm_root).join("data/execution_logs/.tmp")
98 };
99 let _ = fs::create_dir_all(&tmp_dir);
100
101 let count = crate::TEMP_COUNTER.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
103 let temp_path = tmp_dir.join(format!("normalized_{}_{}.yaml", count, std::process::id()));
104
105 if is_raw_yaml {
106 let fallback_re = Regex::new(r"\btested_cases\b").unwrap();
107 let fallback_content = fallback_re.replace_all(&content, "pending_cases").to_string();
108 fs::write(&temp_path, fallback_content)
109 .map_err(|e| format!("Failed to write normalized temp file: {}", e))?;
110 } else {
111 let yaml_str = serde_yaml::to_string(&data_val)
112 .map_err(|e| format!("Failed to serialize normalized data to YAML: {}", e))?;
113 fs::write(&temp_path, yaml_str)
114 .map_err(|e| format!("Failed to write normalized temp file: {}", e))?;
115 }
116
117 Ok(temp_path.to_string_lossy().to_string())
118}
119
120fn extract_code_block(text: &str) -> Option<(String, String)> {
121 let re = Regex::new(r"(?s)```([a-zA-Z0-9_\-]+)?\s*(.*?)\s*```").ok()?;
122 if let Some(caps) = re.captures(text) {
123 let lang = caps.get(1).map(|m| m.as_str().trim().to_lowercase()).unwrap_or_default();
124 let content = caps.get(2).map(|m| m.as_str().trim().to_string()).unwrap_or_default();
125 return Some((lang, content));
126 }
127 None
128}
129
130fn parse_jsonl_string(jsonl_str: &str) -> Result<Value, String> {
131 let mut obj = Value::Object(Map::new());
132 for line in jsonl_str.lines() {
133 let trimmed = line.trim();
134 if trimmed.is_empty() {
135 continue;
136 }
137 let line_val: Value = serde_json::from_str(trimmed)
138 .map_err(|e| format!("Failed to parse JSONL line: {}", e))?;
139 deep_merge(&mut obj, line_val);
140 }
141 Ok(obj)
142}
143
144fn deep_merge(dict1: &mut Value, dict2: Value) {
145 if let (Value::Object(map1), Value::Object(map2)) = (dict1, dict2) {
146 for (k, v) in map2 {
147 if map1.contains_key(&k) && map1.get(&k).unwrap().is_object() && v.is_object() {
148 let existing = map1.get_mut(&k).unwrap();
149 deep_merge(existing, v);
150 } else {
151 map1.insert(k, v);
152 }
153 }
154 }
155}
156
157fn parse_xml_string(xml_str: &str) -> Result<Value, String> {
158 let mut reader = Reader::from_str(xml_str);
159 reader.trim_text(true);
160 let mut buf = Vec::new();
161 loop {
162 match reader.read_event_into(&mut buf) {
163 Ok(Event::Start(ref e)) => {
164 let tag_name = String::from_utf8_lossy(e.name().as_ref()).into_owned();
165 let val = parse_xml_element(&mut reader, &tag_name)?;
166 let mut map = Map::new();
167 map.insert(tag_name, val);
168 return Ok(Value::Object(map));
169 }
170 Ok(Event::Eof) => return Err("Empty XML document".to_string()),
171 Err(e) => return Err(format!("XML error: {}", e)),
172 _ => {}
173 }
174 buf.clear();
175 }
176}
177
178fn parse_xml_element(reader: &mut Reader<&[u8]>, current_tag: &str) -> Result<Value, String> {
179 let mut buf = Vec::new();
180 let mut children_tags: Vec<String> = Vec::new();
181 let mut children_values: Vec<Value> = Vec::new();
182 let mut text_content = String::new();
183
184 loop {
185 match reader.read_event_into(&mut buf) {
186 Ok(Event::Start(ref e)) => {
187 let tag_name = String::from_utf8_lossy(e.name().as_ref()).into_owned();
188 let child_val = parse_xml_element(reader, &tag_name)?;
189 children_tags.push(tag_name);
190 children_values.push(child_val);
191 }
192 Ok(Event::Text(ref e)) => {
193 text_content.push_str(&e.unescape().map_err(|err| err.to_string())?);
194 }
195 Ok(Event::End(ref e)) => {
196 let tag_name = String::from_utf8_lossy(e.name().as_ref()).into_owned();
197 if tag_name == current_tag {
198 break;
199 }
200 }
201 Ok(Event::Eof) => return Err(format!("Unexpected EOF inside tag: {}", current_tag)),
202 Err(e) => return Err(format!("XML error: {}", e)),
203 _ => {}
204 }
205 buf.clear();
206 }
207
208 if children_tags.is_empty() {
209 let text = text_content.trim();
210 if text.eq_ignore_ascii_case("true") {
211 return Ok(Value::Bool(true));
212 }
213 if text.eq_ignore_ascii_case("false") {
214 return Ok(Value::Bool(false));
215 }
216 if let Ok(i) = text.parse::<i64>() {
217 return Ok(Value::Number(serde_json::Number::from(i)));
218 }
219 if let Ok(f) = text.parse::<f64>() {
220 if let Some(num) = serde_json::Number::from_f64(f) {
221 return Ok(Value::Number(num));
222 }
223 }
224 return Ok(Value::String(text.to_string()));
225 }
226
227 let all_same = children_tags.iter().all(|t| t == &children_tags[0]);
228 if children_tags.len() > 1 && all_same {
229 return Ok(Value::Array(children_values));
230 }
231 if children_tags.len() == 1 && (children_tags[0] == "item" || children_tags[0] == "step") {
232 return Ok(Value::Array(vec![children_values[0].clone()]));
233 }
234
235 let mut map = Map::new();
236 for (tag, val) in children_tags.into_iter().zip(children_values.into_iter()) {
237 if map.contains_key(&tag) {
238 let existing = map.get_mut(&tag).unwrap();
239 if let Value::Array(arr) = existing {
240 arr.push(val);
241 } else {
242 let prev = existing.clone();
243 *existing = Value::Array(vec![prev, val]);
244 }
245 } else {
246 map.insert(tag, val);
247 }
248 }
249 Ok(Value::Object(map))
250}
251
252fn normalize_implementations_order(data: &mut Value) {
253 if let Some(clm) = data.get_mut("clm") {
254 if let Some(concrete_impl) = clm.get_mut("concrete_impl") {
255 if let Some(process) = concrete_impl.get_mut("process") {
256 if let Some(implementations) = process.get_mut("implementations") {
257 if let Some(obj) = implementations.as_object_mut() {
258 let mut sorted_map = Map::new();
259 let mut keys: Vec<String> = obj.keys().cloned().collect();
260 keys.sort();
261 for k in keys {
262 if let Some(v) = obj.remove(&k) {
263 sorted_map.insert(k, v);
264 }
265 }
266 *obj = sorted_map;
267 }
268 }
269 }
270 }
271 }
272}
273
274fn normalize_tested_cases(data: &mut Value) {
275 if let Some(clm) = data.get_mut("clm") {
276 if let Some(balanced_exp) = clm.get_mut("balanced_exp") {
277 let mut pending_cases = balanced_exp.get("pending_cases")
278 .and_then(|v| v.as_array())
279 .cloned()
280 .unwrap_or_default();
281
282 if let Some(tested_cases) = balanced_exp.get("tested_cases").and_then(|v| v.as_array()) {
283 for case in tested_cases {
284 if let Some(case_obj) = case.as_object() {
285 let when_val = case_obj.get("when");
286 let exists = pending_cases.iter().any(|p_case| {
287 p_case.get("when") == when_val
288 });
289 if !exists {
290 let mut clean_case = case_obj.clone();
291 clean_case.remove("exec_hash");
292 clean_case.remove("exec_time");
293 pending_cases.push(Value::Object(clean_case));
294 }
295 }
296 }
297 }
298
299 if let Some(obj) = balanced_exp.as_object_mut() {
300 obj.remove("tested_cases");
301 obj.insert("pending_cases".to_string(), Value::Array(pending_cases));
302 }
303 }
304 }
305}
306
307fn inject_past_test_cases(data: &mut Value, clm_root: &str) {
308 let a_hash = std::env::var("CLM_ACTIVE_A_HASH").unwrap_or_default().trim().to_string();
309 let c_hash = std::env::var("CLM_ACTIVE_C_HASH").unwrap_or_default().trim().to_string();
310 let db_path = crate::db::get_db_path("execution_log", clm_root);
311
312 let mut statistics = serde_json::json!({
313 "total_executions": 0,
314 "successful_runs": 0,
315 "failed_runs": 0,
316 "coverage_ratio": 0.0,
317 "last_executed_at": ""
318 });
319 let mut executions = Value::Array(Vec::new());
320
321 if !a_hash.is_empty() && !c_hash.is_empty() && db_path.exists() {
322 if let Ok(conn) = crate::db::get_connection("execution_log", clm_root) {
323 let query = "SELECT hash, g_time, content FROM card \
324 WHERE content LIKE ? AND content LIKE ? \
325 ORDER BY g_time DESC";
326 let a_like = format!("%{}%", a_hash);
327 let c_like = format!("%{}%", c_hash);
328
329 if let Ok(mut stmt) = conn.prepare(query) {
330 let rows_res = stmt.query_map(params![a_like, c_like], |row| {
331 Ok((
332 row.get::<_, String>(0)?,
333 row.get::<_, String>(1)?,
334 row.get::<_, String>(2)?,
335 ))
336 });
337
338 if let Ok(rows) = rows_res {
339 let mut total_executions = 0;
340 let mut successful_runs = 0;
341 let mut failed_runs = 0;
342 let mut last_executed_at = String::new();
343 let mut executions_list = Vec::new();
344
345 let exit_code_re = Regex::new(r"exit_code:\s*(\d+)").unwrap();
346 let arch_re = Regex::new(r#"arch:\s*"([^"]*)""#).unwrap();
347 let os_re = Regex::new(r#"os:\s*"([^"]*)""#).unwrap();
348 let command_re = Regex::new(r#"command:\s*"([^"]*)""#).unwrap();
349 let op_re = Regex::new(r#"operation:\s*"([^"]*)""#).unwrap();
350
351 for row_val in rows.flatten() {
352 let (h, g_time, content) = row_val;
353 if content.is_empty() {
354 continue;
355 }
356
357 let exit_code = exit_code_re.captures(&content)
358 .and_then(|c| c.get(1))
359 .and_then(|m| m.as_str().parse::<i32>().ok())
360 .unwrap_or(0);
361
362 let arch = arch_re.captures(&content)
363 .and_then(|c| c.get(1))
364 .map(|m| m.as_str().to_string())
365 .unwrap_or_default();
366
367 let os_name = os_re.captures(&content)
368 .and_then(|c| c.get(1))
369 .map(|m| m.as_str().to_string())
370 .unwrap_or_default();
371
372 let mut command = command_re.captures(&content)
373 .and_then(|c| c.get(1))
374 .map(|m| m.as_str().to_string())
375 .unwrap_or_default();
376
377 if command.is_empty() {
378 if let Some(op) = op_re.captures(&content).and_then(|c| c.get(1)) {
379 command = format!("{} evaluation", op.as_str());
380 } else {
381 command = "evaluation".to_string();
382 }
383 }
384
385 total_executions += 1;
386 if exit_code == 0 {
387 successful_runs += 1;
388 } else {
389 failed_runs += 1;
390 }
391
392 if last_executed_at.is_empty() || g_time > last_executed_at {
393 last_executed_at = g_time.clone();
394 }
395
396 let mut exec_entry = serde_json::json!({
397 "exec_hash": h,
398 "g_time": g_time,
399 "command": command
400 });
401
402 if !os_name.is_empty() || !arch.is_empty() {
403 let mut env_map = Map::new();
404 if !os_name.is_empty() {
405 env_map.insert("os".to_string(), Value::String(os_name));
406 }
407 if !arch.is_empty() {
408 env_map.insert("arch".to_string(), Value::String(arch));
409 }
410 exec_entry.as_object_mut().unwrap().insert("environment".to_string(), Value::Object(env_map));
411 }
412
413 executions_list.push(exec_entry);
414 }
415
416 executions_list.sort_by(|a, b| {
418 let a_cmd = a.get("command").and_then(|v| v.as_str()).unwrap_or("");
419 let b_cmd = b.get("command").and_then(|v| v.as_str()).unwrap_or("");
420 let a_hash = a.get("exec_hash").and_then(|v| v.as_str()).unwrap_or("");
421 let b_hash = b.get("exec_hash").and_then(|v| v.as_str()).unwrap_or("");
422
423 let cmd_cmp = a_cmd.cmp(b_cmd);
424 if cmd_cmp == std::cmp::Ordering::Equal {
425 a_hash.cmp(b_hash)
426 } else {
427 cmd_cmp
428 }
429 });
430
431 if total_executions > 0 {
432 statistics = serde_json::json!({
433 "total_executions": total_executions,
434 "successful_runs": successful_runs,
435 "failed_runs": failed_runs,
436 "coverage_ratio": (successful_runs as f64) / (total_executions as f64),
437 "last_executed_at": last_executed_at
438 });
439 executions = Value::Array(executions_list);
440 }
441 }
442 }
443 }
444 }
445
446 if let Some(clm) = data.get_mut("clm") {
447 if let Some(balanced_exp) = clm.get_mut("balanced_exp") {
448 if let Some(obj) = balanced_exp.as_object_mut() {
449 let past_test = serde_json::json!({
450 "statistics": statistics,
451 "executions": executions
452 });
453 obj.insert("past_test_cases".to_string(), past_test);
454 }
455 }
456 }
457}