Skip to main content

clm_rust_core/layer2/
normalizer.rs

1//! Syntax normalization for Layer 2 CLM specifications (`normalizer.rs`).
2//!
3//! Translates JSON, streaming JSONL, XML, YAML, and Markdown-fenced code blocks into a canonical
4//! temporary YAML file with `tested_cases` promoted to `pending_cases`, sorted implementation
5//! keys, and verified `past_test_cases` history attached.
6//!
7//! Markdown fence extraction (`extract_code_block`) intentionally remains here: CLM allows
8//! imprecise LLM/Markdown documents (`test_syntax_imprecise.txt`) whose embedded `` ```<lang> ``
9//! body delegates immediately to `serde_json`, `serde_yaml`, or `quick_xml` without pulling a
10//! full Markdown parser crate into the kernel.
11
12use std::fs;
13use std::path::Path;
14use std::sync::OnceLock;
15
16use regex::Regex;
17use rusqlite::params;
18use serde::Deserialize;
19use serde_json::{Map, Value};
20
21#[cfg(feature = "native")]
22#[path = "normalizer_xml.rs"]
23mod normalizer_xml;
24
25use crate::layer0::functional::first_match;
26use crate::layer0::mcard_record::MCardRecord;
27use crate::layer0::vocabulary_env::{ENV_CLM_ACTIVE_A_HASH, ENV_CLM_ACTIVE_C_HASH, PILLAR_EXECUTION_LOG};
28use crate::layer2::record_store::require_verified;
29
30/// Verified row from the `execution_log` ledger passed into pure normalization.
31#[derive(Debug, Clone, PartialEq, Eq)]
32pub struct PastExecRecord {
33    pub exec_hash: String,
34    pub g_time: String,
35    pub content: String,
36}
37
38#[derive(Clone, Copy, Debug, PartialEq, Eq)]
39enum SourceKind {
40    Json,
41    Jsonl,
42    JsonOrJsonl,
43    Xml,
44    Yaml,
45}
46
47type KindRule = fn(&str) -> Option<SourceKind>;
48
49const TAG_RULES: &[KindRule] = &[
50    |tag| (tag == "json").then_some(SourceKind::Json),
51    |tag| (tag == "jsonl").then_some(SourceKind::Jsonl),
52    |tag| (tag == "xml").then_some(SourceKind::Xml),
53    |tag| matches!(tag, "yaml" | "yml").then_some(SourceKind::Yaml),
54];
55
56const SNIFF_RULES: &[KindRule] = &[
57    |text| text.starts_with('<').then_some(SourceKind::Xml),
58    |text| (text.starts_with('{') || text.starts_with('[')).then_some(SourceKind::JsonOrJsonl),
59];
60
61/// Normalizes a CLM specification file on disk and writes the canonical YAML artifact to `.tmp`.
62pub fn normalize_syntax(target_file: &str, clm_root: &str) -> Result<String, String> {
63    let path = Path::new(target_file);
64    if !path.exists() {
65        return Err(format!("Error: file not found: {}", target_file));
66    }
67
68    let ext = path.extension().and_then(|s| s.to_str()).unwrap_or("");
69    let content = fs::read_to_string(path)
70        .map_err(|e| format!("Failed to read target file: {}", e))?;
71
72    let past_records = load_verified_exec_records(clm_root);
73    let yaml_out = normalize_syntax_str(&content, ext, &past_records)?;
74
75    let tmp_dir = if crate::utils::is_zero_fs() {
76        std::env::temp_dir().join("clm_execution_logs/.tmp")
77    } else {
78        Path::new(clm_root).join("data/execution_logs/.tmp")
79    };
80    let _ = fs::create_dir_all(&tmp_dir);
81
82    let count = crate::TEMP_COUNTER.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
83    let temp_path = tmp_dir.join(format!("normalized_{}_{}.yaml", count, std::process::id()));
84    fs::write(&temp_path, yaml_out)
85        .map_err(|e| format!("Failed to write normalized temp file: {}", e))?;
86
87    Ok(temp_path.to_string_lossy().to_string())
88}
89
90/// Pure syntax normalization from raw source text, file extension hint, and verified past records.
91pub fn normalize_syntax_str(
92    content: &str,
93    ext: &str,
94    past_records: &[PastExecRecord],
95) -> Result<String, String> {
96    let ext_lower = ext.to_lowercase();
97    let (parse_slice, kind) = if let Some(kind) = first_match(TAG_RULES, ext_lower.as_str()) {
98        (content, kind)
99    } else if let Some((lang, block)) = extract_code_block(content) {
100        let kind = first_match(TAG_RULES, lang.as_str())
101            .or_else(|| first_match(SNIFF_RULES, block.trim()))
102            .unwrap_or(SourceKind::Yaml);
103        (block, kind)
104    } else {
105        let kind = first_match(SNIFF_RULES, content.trim()).unwrap_or(SourceKind::Yaml);
106        (content, kind)
107    };
108
109    let (mut data_val, is_raw_yaml) = parse_source(parse_slice, kind);
110    if is_raw_yaml {
111        static FALLBACK_RE: OnceLock<Regex> = OnceLock::new();
112        let re = FALLBACK_RE.get_or_init(|| Regex::new(r"\btested_cases\b").unwrap());
113        return Ok(re.replace_all(content, "pending_cases").into_owned());
114    }
115
116    if data_val.is_object() {
117        normalize_tested_cases(&mut data_val);
118        normalize_implementations_order(&mut data_val);
119        inject_past_test_cases(&mut data_val, past_records);
120    }
121
122    serde_yaml::to_string(&data_val)
123        .map_err(|e| format!("Failed to serialize normalized data to YAML: {}", e))
124}
125
126fn parse_source(text: &str, kind: SourceKind) -> (Value, bool) {
127    match kind {
128        SourceKind::Json => (serde_json::from_str(text).unwrap_or(Value::Null), false),
129        SourceKind::Jsonl => (parse_jsonl_string(text).unwrap_or(Value::Null), false),
130        SourceKind::JsonOrJsonl => (
131            serde_json::from_str(text)
132                .ok()
133                .or_else(|| parse_jsonl_string(text).ok())
134                .unwrap_or(Value::Null),
135            false,
136        ),
137        SourceKind::Xml => (parse_xml_string(text).unwrap_or(Value::Null), false),
138        SourceKind::Yaml => match serde_yaml::from_str::<Value>(text) {
139            Ok(v) => (v, false),
140            Err(_) => (Value::Null, true),
141        },
142    }
143}
144
145fn extract_code_block(text: &str) -> Option<(String, &str)> {
146    static BLOCK_RE: OnceLock<Regex> = OnceLock::new();
147    let re = BLOCK_RE.get_or_init(|| Regex::new(r"(?s)```([a-zA-Z0-9_\-]+)?\s*(.*?)\s*```").unwrap());
148    let caps = re.captures(text)?;
149    let lang = caps.get(1).map(|m| m.as_str().trim().to_lowercase()).unwrap_or_default();
150    let content = caps.get(2).map(|m| m.as_str().trim()).unwrap_or_default();
151    Some((lang, content))
152}
153
154fn parse_jsonl_string(jsonl_str: &str) -> Result<Value, String> {
155    let mut obj = Value::Object(Map::new());
156    for item in serde_json::Deserializer::from_str(jsonl_str).into_iter::<Value>() {
157        let line_val = item.map_err(|e| format!("Failed to parse JSONL stream: {}", e))?;
158        deep_merge(&mut obj, line_val);
159    }
160    Ok(obj)
161}
162
163fn deep_merge(dict1: &mut Value, dict2: Value) {
164    if let (Value::Object(map1), Value::Object(map2)) = (dict1, dict2) {
165        for (k, v) in map2 {
166            if map1.get(&k).is_some_and(Value::is_object) && v.is_object() {
167                deep_merge(map1.get_mut(&k).unwrap(), v);
168            } else {
169                map1.insert(k, v);
170            }
171        }
172    }
173}
174
175#[cfg(feature = "native")]
176use normalizer_xml::parse_xml_string;
177
178#[cfg(not(feature = "native"))]
179fn parse_xml_string(_xml_str: &str) -> Result<Value, String> {
180    Err("XML normalization requires the 'native' feature".to_string())
181}
182
183fn balanced_exp_mut(data: &mut Value) -> Option<&mut Map<String, Value>> {
184    data.get_mut("clm")?.get_mut("balanced_exp")?.as_object_mut()
185}
186
187fn normalize_implementations_order(data: &mut Value) {
188    let Some(obj) = data
189        .get_mut("clm")
190        .and_then(|v| v.get_mut("concrete_impl"))
191        .and_then(|v| v.get_mut("process"))
192        .and_then(|v| v.get_mut("implementations"))
193        .and_then(Value::as_object_mut)
194    else {
195        return;
196    };
197    let mut keys: Vec<String> = obj.keys().cloned().collect();
198    keys.sort();
199    let mut sorted = Map::new();
200    for k in keys {
201        if let Some(v) = obj.remove(&k) {
202            sorted.insert(k, v);
203        }
204    }
205    *obj = sorted;
206}
207
208fn normalize_tested_cases(data: &mut Value) {
209    let Some(balanced_exp) = balanced_exp_mut(data) else {
210        return;
211    };
212    let mut pending = balanced_exp
213        .get("pending_cases")
214        .and_then(Value::as_array)
215        .cloned()
216        .unwrap_or_default();
217
218    if let Some(tested) = balanced_exp.get("tested_cases").and_then(Value::as_array) {
219        for case_obj in tested.iter().filter_map(Value::as_object) {
220            let when_val = case_obj.get("when");
221            if !pending.iter().any(|p| p.get("when") == when_val) {
222                let mut clean = case_obj.clone();
223                clean.remove("exec_hash");
224                clean.remove("exec_time");
225                pending.push(Value::Object(clean));
226            }
227        }
228    }
229    balanced_exp.remove("tested_cases");
230    balanced_exp.insert("pending_cases".to_string(), Value::Array(pending));
231}
232
233#[derive(Debug, Deserialize, Default)]
234#[serde(default)]
235struct ExecRecordEnv {
236    os: Option<String>,
237    arch: Option<String>,
238}
239
240#[derive(Debug, Deserialize)]
241#[serde(untagged)]
242enum RuntimeFactsField {
243    Map(ExecRecordEnv),
244    Scalar(serde::de::IgnoredAny),
245}
246
247#[derive(Debug, Deserialize, Default)]
248#[serde(default)]
249struct ExecRecordFields {
250    operation: Option<String>,
251    command: Option<String>,
252    exit_code: Option<i32>,
253    os: Option<String>,
254    arch: Option<String>,
255    runtime_facts: Option<RuntimeFactsField>,
256}
257
258#[derive(Debug, Deserialize, Default)]
259#[serde(default)]
260struct ExecRecordBody {
261    #[serde(flatten)]
262    fields: ExecRecordFields,
263    source: Option<ExecRecordFields>,
264}
265
266#[derive(Debug, Deserialize, Default)]
267#[serde(default)]
268struct ExecRecordDocument {
269    mcard: Option<ExecRecordBody>,
270    #[serde(flatten)]
271    top: ExecRecordBody,
272}
273
274impl RuntimeFactsField {
275    fn as_env(&self) -> Option<&ExecRecordEnv> {
276        match self {
277            Self::Map(env) => Some(env),
278            Self::Scalar(_) => None,
279        }
280    }
281}
282
283fn load_verified_exec_records(clm_root: &str) -> Vec<PastExecRecord> {
284    let env_trim = |k| crate::layer0::environment::var(k).unwrap_or_default().trim().to_string();
285    let (a_hash, c_hash) = (env_trim(ENV_CLM_ACTIVE_A_HASH), env_trim(ENV_CLM_ACTIVE_C_HASH));
286    let db_path = crate::db::get_db_path(PILLAR_EXECUTION_LOG, clm_root);
287    if a_hash.is_empty() || c_hash.is_empty() || !db_path.exists() {
288        return Vec::new();
289    }
290    let Ok(conn) = crate::db::get_connection(PILLAR_EXECUTION_LOG, clm_root) else { return Vec::new(); };
291    let query = "SELECT hash, g_time, content FROM card WHERE CAST(content AS TEXT) LIKE ? AND CAST(content AS TEXT) LIKE ? ORDER BY g_time DESC";
292    let Ok(mut stmt) = conn.prepare(query) else { return Vec::new(); };
293    let (a_like, c_like) = (format!("%{}%", a_hash), format!("%{}%", c_hash));
294    let Ok(rows) = stmt.query_map(params![a_like, c_like], |row| {
295        let bytes = row.get_ref(2)?.as_bytes()?.to_vec();
296        let content = String::from_utf8(bytes).map_err(|e| rusqlite::Error::FromSqlConversionFailure(2, rusqlite::types::Type::Blob, Box::new(e)))?;
297        Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?, content))
298    }) else { return Vec::new(); };
299    rows.flatten()
300        .filter(|(h, g_time, content)| !content.is_empty() && require_verified(&MCardRecord { hash: h.clone(), content: content.as_bytes().to_vec(), g_time: g_time.clone() }).is_ok())
301        .map(|(exec_hash, g_time, content)| PastExecRecord { exec_hash, g_time, content })
302        .collect()
303}
304
305fn pick_non_empty<'a>(layers: &[&'a ExecRecordFields], f: impl Fn(&'a ExecRecordFields) -> Option<&'a str>) -> Option<&'a str> {
306    layers.iter().find_map(|layer| f(layer).filter(|s| !s.is_empty()))
307}
308
309fn inject_past_test_cases(data: &mut Value, records: &[PastExecRecord]) {
310    let mut total_executions = 0usize;
311    let mut successful_runs = 0usize;
312    let mut failed_runs = 0usize;
313    let mut last_executed_at = String::new();
314    let mut executions_list = Vec::new();
315
316    for rec in records.iter().filter(|r| !r.content.is_empty()) {
317        let doc: ExecRecordDocument = serde_yaml::from_str(&rec.content).unwrap_or_default();
318        let mut layers: Vec<&ExecRecordFields> = Vec::with_capacity(4);
319        if let Some(m) = &doc.mcard {
320            if let Some(src) = &m.source { layers.push(src); }
321            layers.push(&m.fields);
322        }
323        if let Some(src) = &doc.top.source { layers.push(src); }
324        layers.push(&doc.top.fields);
325
326        let exit_code = layers.iter().find_map(|l| l.exit_code).unwrap_or(0);
327        let command = pick_non_empty(&layers, |l| l.command.as_deref())
328            .map(str::to_string)
329            .unwrap_or_else(|| match pick_non_empty(&layers, |l| l.operation.as_deref()) {
330                Some(op) => format!("{} evaluation", op),
331                None => "evaluation".to_string(),
332            });
333        let os_name = pick_non_empty(&layers, |l| {
334            l.os.as_deref().or_else(|| l.runtime_facts.as_ref().and_then(RuntimeFactsField::as_env)?.os.as_deref())
335        });
336        let arch = pick_non_empty(&layers, |l| {
337            l.arch.as_deref().or_else(|| l.runtime_facts.as_ref().and_then(RuntimeFactsField::as_env)?.arch.as_deref())
338        });
339
340        total_executions += 1;
341        if exit_code == 0 { successful_runs += 1; } else { failed_runs += 1; }
342        if last_executed_at.is_empty() || rec.g_time > last_executed_at {
343            last_executed_at = rec.g_time.clone();
344        }
345
346        let mut entry = serde_json::json!({
347            "exec_hash": rec.exec_hash,
348            "g_time": rec.g_time,
349            "command": command,
350        });
351        if os_name.is_some() || arch.is_some() {
352            let mut env_map = Map::new();
353            if let Some(o) = os_name { env_map.insert("os".to_string(), Value::String(o.to_string())); }
354            if let Some(a) = arch { env_map.insert("arch".to_string(), Value::String(a.to_string())); }
355            entry.as_object_mut().unwrap().insert("environment".to_string(), Value::Object(env_map));
356        }
357        executions_list.push(entry);
358    }
359
360    executions_list.sort_by(|a, b| {
361        let key = |v: &Value| (
362            v.get("command").and_then(Value::as_str).unwrap_or("").to_string(),
363            v.get("exec_hash").and_then(Value::as_str).unwrap_or("").to_string(),
364        );
365        key(a).cmp(&key(b))
366    });
367
368    let stats = if total_executions > 0 {
369        serde_json::json!({
370            "total_executions": total_executions,
371            "successful_runs": successful_runs,
372            "failed_runs": failed_runs,
373            "coverage_ratio": (successful_runs as f64) / (total_executions as f64),
374            "last_executed_at": last_executed_at,
375        })
376    } else {
377        serde_json::json!({
378            "total_executions": 0, "successful_runs": 0, "failed_runs": 0,
379            "coverage_ratio": 0.0, "last_executed_at": "",
380        })
381    };
382
383    if let Some(obj) = balanced_exp_mut(data) {
384        obj.insert("past_test_cases".to_string(), serde_json::json!({
385            "statistics": stats,
386            "executions": executions_list,
387        }));
388    }
389}
390
391#[cfg(test)]
392mod tests {
393    use super::*;
394
395    #[test]
396    fn normalizer_format_parity_json_jsonl_xml_markdown() {
397        let md = "intro\n```xml\n<clm><abstract_spec><success_criteria><item>a</item></success_criteria></abstract_spec>\
398                  <concrete_impl><process><implementations><z>1</z><a>2</a></implementations></process></concrete_impl>\
399                  <balanced_exp><ok>true</ok><n>42</n></balanced_exp></clm>\n```";
400        let yaml_from_md = normalize_syntax_str(md, "md", &[]).unwrap();
401        let val_md: Value = serde_yaml::from_str(&yaml_from_md).unwrap();
402        assert_eq!(val_md.pointer("/clm/abstract_spec/success_criteria"), Some(&serde_json::json!(["a"])));
403        assert_eq!(val_md.pointer("/clm/balanced_exp/ok"), Some(&serde_json::json!(true)));
404        assert_eq!(val_md.pointer("/clm/balanced_exp/n"), Some(&serde_json::json!(42)));
405        let keys: Vec<&str> = val_md
406            .pointer("/clm/concrete_impl/process/implementations")
407            .and_then(Value::as_object).unwrap().keys().map(String::as_str).collect();
408        assert_eq!(keys, vec!["a", "z"]);
409
410        let jsonl = "{\"clm\":{\"abstract_spec\":{\"goal\":\"g\"}}}\n{\"clm\":{\"balanced_exp\":{\"expectation\":\"e\"}}}";
411        let val_jsonl: Value = serde_yaml::from_str(&normalize_syntax_str(jsonl, "jsonl", &[]).unwrap()).unwrap();
412        assert_eq!(val_jsonl.pointer("/clm/abstract_spec/goal"), Some(&serde_json::json!("g")));
413        assert_eq!(val_jsonl.pointer("/clm/balanced_exp/expectation"), Some(&serde_json::json!("e")));
414    }
415
416    #[test]
417    fn normalizer_tested_cases_promotion_and_raw_yaml_fallback() {
418        let yaml = "clm:\n  balanced_exp:\n    pending_cases:\n      - when: existing\n    tested_cases:\n      - when: existing\n        exec_hash: x\n      - when: added\n        exec_hash: sha256:1\n        exec_time: t\n";
419        let val: Value = serde_yaml::from_str(&normalize_syntax_str(yaml, "yaml", &[]).unwrap()).unwrap();
420        let pending = val.pointer("/clm/balanced_exp/pending_cases").and_then(Value::as_array).unwrap();
421        assert_eq!(pending.len(), 2);
422        assert!(val.pointer("/clm/balanced_exp/tested_cases").is_none());
423        assert!(pending[1].get("exec_hash").is_none());
424
425        let broken_yaml = "clm:\n  balanced_exp: [\n    tested_cases:\n";
426        let fallback = normalize_syntax_str(broken_yaml, "yml", &[]).unwrap();
427        assert!(fallback.contains("pending_cases:") && !fallback.contains("tested_cases"));
428    }
429
430    #[test]
431    fn normalizer_past_test_cases_serde_yaml_mcard_and_source() {
432        let records = vec![
433            PastExecRecord {
434                exec_hash: "sha256:bbb".to_string(),
435                g_time: "2026-01-02T00:00:00Z".to_string(),
436                content: "mcard:\n  operation: op_a\n  source:\n    command: run b\n    exit_code: 0\n".to_string(),
437            },
438            PastExecRecord {
439                exec_hash: "sha256:aaa".to_string(),
440                g_time: "2026-01-03T00:00:00Z".to_string(),
441                content: "mcard:\n  operation: op_b\n  exit_code: 2\n  source:\n    runtime_facts:\n      os: darwin\n      arch: arm64\n".to_string(),
442            },
443        ];
444        let val: Value = serde_yaml::from_str(&normalize_syntax_str("{\"clm\":{\"balanced_exp\":{}}}", "json", &records).unwrap()).unwrap();
445        assert_eq!(val.pointer("/clm/balanced_exp/past_test_cases/statistics/total_executions"), Some(&serde_json::json!(2)));
446        assert_eq!(val.pointer("/clm/balanced_exp/past_test_cases/statistics/successful_runs"), Some(&serde_json::json!(1)));
447        assert_eq!(val.pointer("/clm/balanced_exp/past_test_cases/statistics/last_executed_at"), Some(&serde_json::json!("2026-01-03T00:00:00Z")));
448        assert_eq!(val.pointer("/clm/balanced_exp/past_test_cases/executions/0/environment/os"), Some(&serde_json::json!("darwin")));
449        assert_eq!(val.pointer("/clm/balanced_exp/past_test_cases/executions/1/command"), Some(&serde_json::json!("run b")));
450    }
451}