Skip to main content

mdql_core/
validator.rs

1//! Validate parsed markdown files against a schema.
2
3use std::collections::{HashMap, HashSet};
4
5use crate::database::DatabaseConfig;
6use crate::errors::ValidationError;
7use crate::model::{Row, Value};
8use crate::parser::ParsedFile;
9use crate::schema::Schema;
10use crate::stamp::TIMESTAMP_FIELDS;
11
12pub fn validate_file(parsed: &ParsedFile, schema: &Schema) -> Vec<ValidationError> {
13    let mut errors = Vec::new();
14    let fp = &parsed.path;
15
16    // Parse-level errors
17    for msg in &parsed.parse_errors {
18        errors.push(ValidationError {
19            file_path: fp.clone(),
20            error_type: "parse_error".to_string(),
21            field: None,
22            message: msg.clone(),
23            line_number: None,
24        });
25    }
26
27    if errors.iter().any(|e| e.error_type == "parse_error") {
28        return errors;
29    }
30
31    let fm = &parsed.raw_frontmatter;
32    let fm_map = match fm.as_mapping() {
33        Some(m) => m,
34        None => return errors,
35    };
36
37    // --- Frontmatter field checks ---
38    for (name, field_def) in &schema.frontmatter {
39        let key = serde_yaml::Value::String(name.clone());
40        match fm_map.get(&key) {
41            None => {
42                if field_def.required {
43                    errors.push(ValidationError {
44                        file_path: fp.clone(),
45                        error_type: "missing_field".to_string(),
46                        field: Some(name.clone()),
47                        message: format!("Missing required frontmatter field '{}'", name),
48                        line_number: None,
49                    });
50                }
51            }
52            Some(value) => {
53                if let Some(type_err) = check_type(value, &field_def.field_type, name) {
54                    errors.push(ValidationError {
55                        file_path: fp.clone(),
56                        error_type: "type_mismatch".to_string(),
57                        field: Some(name.clone()),
58                        message: type_err,
59                        line_number: None,
60                    });
61                }
62
63                if let Some(ref enum_vals) = field_def.enum_values {
64                    if !value.is_null() {
65                        let str_val = yaml_value_to_string(value);
66                        if !enum_vals.contains(&str_val) {
67                            errors.push(ValidationError {
68                                file_path: fp.clone(),
69                                error_type: "enum_violation".to_string(),
70                                field: Some(name.clone()),
71                                message: format!(
72                                    "Field '{}' value '{}' not in allowed values: {:?}",
73                                    name, str_val, enum_vals
74                                ),
75                                line_number: None,
76                            });
77                        }
78                    }
79                }
80            }
81        }
82    }
83
84    // Validate timestamp fields as datetime (ISO 8601)
85    for ts_field in TIMESTAMP_FIELDS {
86        let key = serde_yaml::Value::String(ts_field.to_string());
87        if let Some(value) = fm_map.get(&key) {
88            if let Some(type_err) = check_type(
89                value,
90                &crate::schema::FieldType::DateTime,
91                ts_field,
92            ) {
93                errors.push(ValidationError {
94                    file_path: fp.clone(),
95                    error_type: "type_mismatch".to_string(),
96                    field: Some(ts_field.to_string()),
97                    message: type_err,
98                    line_number: None,
99                });
100            }
101        }
102    }
103
104    // Unknown frontmatter
105    if schema.rules.reject_unknown_frontmatter {
106        for (key_val, _) in fm_map {
107            if let Some(key) = key_val.as_str() {
108                if !schema.frontmatter.contains_key(key)
109                    && !TIMESTAMP_FIELDS.contains(&key)
110                {
111                    errors.push(ValidationError {
112                        file_path: fp.clone(),
113                        error_type: "unknown_field".to_string(),
114                        field: Some(key.to_string()),
115                        message: format!(
116                            "Unknown frontmatter field '{}' (not in schema)",
117                            key
118                        ),
119                        line_number: None,
120                    });
121                }
122            }
123        }
124    }
125
126    // --- H1 checks ---
127    if schema.h1_required && parsed.h1.is_none() {
128        errors.push(ValidationError {
129            file_path: fp.clone(),
130            error_type: "missing_h1".to_string(),
131            field: None,
132            message: "Missing required H1 heading".to_string(),
133            line_number: None,
134        });
135    }
136
137    if let Some(ref h1_field) = schema.h1_must_equal_frontmatter {
138        if let Some(ref h1) = parsed.h1 {
139            let key = serde_yaml::Value::String(h1_field.clone());
140            if let Some(expected_val) = fm_map.get(&key) {
141                let expected = yaml_value_to_string(expected_val);
142                if h1 != &expected {
143                    errors.push(ValidationError {
144                        file_path: fp.clone(),
145                        error_type: "h1_mismatch".to_string(),
146                        field: None,
147                        message: format!(
148                            "H1 '{}' does not match frontmatter '{}' (expected '{}')",
149                            h1, h1_field, expected
150                        ),
151                        line_number: parsed.h1_line_number,
152                    });
153                }
154            }
155        }
156    }
157
158    // --- Section checks ---
159    let section_names: Vec<&str> = parsed
160        .sections
161        .iter()
162        .map(|s| s.normalized_heading.as_str())
163        .collect();
164
165    // Count occurrences
166    let mut section_counter: HashMap<&str, usize> = HashMap::new();
167    for name in &section_names {
168        *section_counter.entry(name).or_insert(0) += 1;
169    }
170
171    // Duplicate sections
172    if schema.rules.reject_duplicate_sections {
173        for (name, count) in &section_counter {
174            if *count > 1 {
175                errors.push(ValidationError {
176                    file_path: fp.clone(),
177                    error_type: "duplicate_section".to_string(),
178                    field: Some(name.to_string()),
179                    message: format!(
180                        "Duplicate section '{}' (appears {} times)",
181                        name, count
182                    ),
183                    line_number: None,
184                });
185            }
186        }
187    }
188
189    // Required sections
190    for (name, section_def) in &schema.sections {
191        if section_def.required && !section_names.contains(&name.as_str()) {
192            errors.push(ValidationError {
193                file_path: fp.clone(),
194                error_type: "missing_section".to_string(),
195                field: Some(name.clone()),
196                message: format!("Missing required section '{}'", name),
197                line_number: None,
198            });
199        }
200    }
201
202    // Unknown sections
203    if schema.rules.reject_unknown_sections {
204        for section in &parsed.sections {
205            if !schema.sections.contains_key(&section.normalized_heading) {
206                errors.push(ValidationError {
207                    file_path: fp.clone(),
208                    error_type: "unknown_section".to_string(),
209                    field: Some(section.normalized_heading.clone()),
210                    message: format!(
211                        "Unknown section '{}' (not in schema)",
212                        section.normalized_heading
213                    ),
214                    line_number: Some(section.line_number),
215                });
216            }
217        }
218    }
219
220    errors
221}
222
223fn check_type(
224    value: &serde_yaml::Value,
225    expected: &crate::schema::FieldType,
226    field_name: &str,
227) -> Option<String> {
228    use crate::schema::FieldType;
229
230    if value.is_null() {
231        return None;
232    }
233
234    match expected {
235        FieldType::String => {
236            if !value.is_string() {
237                return Some(format!(
238                    "Field '{}' expected string, got {}",
239                    field_name,
240                    yaml_type_name(value)
241                ));
242            }
243        }
244        FieldType::Int => {
245            if value.is_bool() {
246                return Some(format!(
247                    "Field '{}' expected int, got bool",
248                    field_name
249                ));
250            }
251            // serde_yaml may parse integers as i64 or u64
252            if !value.is_i64() && !value.is_u64() {
253                return Some(format!(
254                    "Field '{}' expected int, got {}",
255                    field_name,
256                    yaml_type_name(value)
257                ));
258            }
259        }
260        FieldType::Float => {
261            if value.is_bool() {
262                return Some(format!(
263                    "Field '{}' expected float, got bool",
264                    field_name
265                ));
266            }
267            if !value.is_f64() && !value.is_i64() && !value.is_u64() {
268                return Some(format!(
269                    "Field '{}' expected float, got {}",
270                    field_name,
271                    yaml_type_name(value)
272                ));
273            }
274        }
275        FieldType::Bool => {
276            if !value.is_bool() {
277                return Some(format!(
278                    "Field '{}' expected bool, got {}",
279                    field_name,
280                    yaml_type_name(value)
281                ));
282            }
283        }
284        FieldType::Date => {
285            if let Some(s) = value.as_str() {
286                if chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d").is_err() {
287                    return Some(format!(
288                        "Field '{}' expected date (YYYY-MM-DD), got string '{}'",
289                        field_name, s
290                    ));
291                }
292                return None;
293            }
294            if !value.is_string() {
295                return Some(format!(
296                    "Field '{}' expected date, got {}",
297                    field_name,
298                    yaml_type_name(value)
299                ));
300            }
301        }
302        FieldType::DateTime => {
303            if let Some(s) = value.as_str() {
304                let ok = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S").is_ok()
305                    || chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S%.f").is_ok();
306                if !ok {
307                    return Some(format!(
308                        "Field '{}' expected datetime (ISO 8601), got string '{}'",
309                        field_name, s
310                    ));
311                }
312                return None;
313            }
314            if !value.is_string() {
315                return Some(format!(
316                    "Field '{}' expected datetime, got {}",
317                    field_name,
318                    yaml_type_name(value)
319                ));
320            }
321        }
322        FieldType::StringArray => {
323            match value.as_sequence() {
324                None => {
325                    return Some(format!(
326                        "Field '{}' expected string[], got {}",
327                        field_name,
328                        yaml_type_name(value)
329                    ));
330                }
331                Some(seq) => {
332                    for (i, item) in seq.iter().enumerate() {
333                        if !item.is_string() {
334                            return Some(format!(
335                                "Field '{}[{}]' expected string, got {}",
336                                field_name,
337                                i,
338                                yaml_type_name(item)
339                            ));
340                        }
341                    }
342                }
343            }
344        }
345        FieldType::Dict => {
346            if !value.is_mapping() {
347                return Some(format!(
348                    "Field '{}' expected dict (mapping), got {}",
349                    field_name,
350                    yaml_type_name(value)
351                ));
352            }
353            if let Some(mapping) = value.as_mapping() {
354                for (k, v) in mapping {
355                    if v.is_mapping() || v.is_sequence() {
356                        return Some(format!(
357                            "Field '{}' dict value for key '{}' must be a scalar, got {}",
358                            field_name,
359                            k.as_str().unwrap_or("?"),
360                            yaml_type_name(v)
361                        ));
362                    }
363                }
364            }
365        }
366    }
367
368    None
369}
370
371fn yaml_type_name(value: &serde_yaml::Value) -> &'static str {
372    match value {
373        serde_yaml::Value::Null => "null",
374        serde_yaml::Value::Bool(_) => "bool",
375        serde_yaml::Value::Number(_) => {
376            if value.is_f64() && !value.is_i64() && !value.is_u64() {
377                "float"
378            } else {
379                "int"
380            }
381        }
382        serde_yaml::Value::String(_) => "str",
383        serde_yaml::Value::Sequence(_) => "list",
384        serde_yaml::Value::Mapping(_) => "mapping",
385        _ => "unknown",
386    }
387}
388
389fn yaml_value_to_string(value: &serde_yaml::Value) -> String {
390    match value {
391        serde_yaml::Value::String(s) => s.clone(),
392        serde_yaml::Value::Number(n) => n.to_string(),
393        serde_yaml::Value::Bool(b) => b.to_string(),
394        serde_yaml::Value::Null => "null".to_string(),
395        _ => format!("{:?}", value),
396    }
397}
398
399/// Validate all foreign key constraints across a loaded database.
400pub fn validate_foreign_keys(
401    db_config: &DatabaseConfig,
402    tables: &HashMap<String, (Schema, Vec<Row>)>,
403) -> Vec<ValidationError> {
404    let mut errors = Vec::new();
405
406    for fk in &db_config.foreign_keys {
407        let to_table = match tables.get(&fk.to_table) {
408            Some(t) => t,
409            None => {
410                errors.push(ValidationError {
411                    file_path: format!("_mdql.md"),
412                    error_type: "fk_missing_table".to_string(),
413                    field: None,
414                    message: format!(
415                        "Foreign key references unknown table '{}'",
416                        fk.to_table
417                    ),
418                    line_number: None,
419                });
420                continue;
421            }
422        };
423
424        let from_table = match tables.get(&fk.from_table) {
425            Some(t) => t,
426            None => {
427                errors.push(ValidationError {
428                    file_path: format!("_mdql.md"),
429                    error_type: "fk_missing_table".to_string(),
430                    field: None,
431                    message: format!(
432                        "Foreign key references unknown table '{}'",
433                        fk.from_table
434                    ),
435                    line_number: None,
436                });
437                continue;
438            }
439        };
440
441        // Build set of valid target values
442        let valid_values: HashSet<String> = to_table
443            .1
444            .iter()
445            .filter_map(|row| {
446                row.get(&fk.to_column).and_then(|v| match v {
447                    Value::Null => None,
448                    _ => Some(v.to_display_string()),
449                })
450            })
451            .collect();
452
453        // Check each row in the referencing table
454        for row in &from_table.1 {
455            let value = match row.get(&fk.from_column) {
456                Some(Value::Null) | None => continue,
457                Some(v) => v,
458            };
459
460            let file_path = row
461                .get("path")
462                .map(|v| format!("{}/{}", fk.from_table, v.to_display_string()))
463                .unwrap_or_else(|| fk.from_table.clone());
464
465            let value_str = value.to_display_string();
466            if !valid_values.contains(&value_str) {
467                errors.push(ValidationError {
468                    file_path,
469                    error_type: "fk_violation".to_string(),
470                    field: Some(fk.from_column.clone()),
471                    message: format!(
472                        "{} = '{}' not found in {}.{}",
473                        fk.from_column, value_str, fk.to_table, fk.to_column
474                    ),
475                    line_number: None,
476                });
477            }
478        }
479    }
480
481    errors
482}
483
484#[cfg(test)]
485mod tests {
486    use super::*;
487    use crate::parser::parse_text;
488    use crate::schema::*;
489    use indexmap::IndexMap;
490
491    fn make_schema() -> Schema {
492        let mut frontmatter = IndexMap::new();
493        frontmatter.insert("title".to_string(), FieldDef {
494            field_type: FieldType::String,
495            required: true,
496            enum_values: None,
497        });
498        frontmatter.insert("count".to_string(), FieldDef {
499            field_type: FieldType::Int,
500            required: true,
501            enum_values: None,
502        });
503        frontmatter.insert("status".to_string(), FieldDef {
504            field_type: FieldType::String,
505            required: false,
506            enum_values: Some(vec!["ACTIVE".into(), "ARCHIVED".into()]),
507        });
508
509        let mut sections = IndexMap::new();
510        sections.insert("Summary".to_string(), SectionDef {
511            content_type: "markdown".to_string(),
512            required: true,
513        });
514
515        Schema {
516            table: "test".to_string(),
517            primary_key: "path".to_string(),
518            frontmatter,
519            h1_required: false,
520            h1_must_equal_frontmatter: None,
521            sections,
522            rules: Rules {
523                reject_unknown_frontmatter: true,
524                reject_unknown_sections: false,
525                reject_duplicate_sections: true,
526                normalize_numbered_headings: false,
527            },
528        }
529    }
530
531    #[test]
532    fn test_valid_file() {
533        let text = "---\ntitle: \"Hello\"\ncount: 5\n---\n\n## Summary\n\nA summary.\n";
534        let parsed = parse_text(text, "test.md", false);
535        let errors = validate_file(&parsed, &make_schema());
536        assert!(errors.is_empty(), "Expected no errors, got: {:?}", errors);
537    }
538
539    #[test]
540    fn test_missing_required_field() {
541        let text = "---\ntitle: \"Hello\"\n---\n\n## Summary\n\nText.\n";
542        let parsed = parse_text(text, "test.md", false);
543        let errors = validate_file(&parsed, &make_schema());
544        assert!(errors.iter().any(|e| e.error_type == "missing_field" && e.field.as_deref() == Some("count")));
545    }
546
547    #[test]
548    fn test_type_mismatch() {
549        let text = "---\ntitle: \"Hello\"\ncount: \"not a number\"\n---\n\n## Summary\n\nText.\n";
550        let parsed = parse_text(text, "test.md", false);
551        let errors = validate_file(&parsed, &make_schema());
552        assert!(errors.iter().any(|e| e.error_type == "type_mismatch" && e.field.as_deref() == Some("count")));
553    }
554
555    #[test]
556    fn test_enum_violation() {
557        let text = "---\ntitle: \"Hello\"\ncount: 5\nstatus: INVALID\n---\n\n## Summary\n\nText.\n";
558        let parsed = parse_text(text, "test.md", false);
559        let errors = validate_file(&parsed, &make_schema());
560        assert!(errors.iter().any(|e| e.error_type == "enum_violation"));
561    }
562
563    #[test]
564    fn test_unknown_frontmatter() {
565        let text = "---\ntitle: \"Hello\"\ncount: 5\nextra: bad\n---\n\n## Summary\n\nText.\n";
566        let parsed = parse_text(text, "test.md", false);
567        let errors = validate_file(&parsed, &make_schema());
568        assert!(errors.iter().any(|e| e.error_type == "unknown_field" && e.field.as_deref() == Some("extra")));
569    }
570
571    #[test]
572    fn test_missing_required_section() {
573        let text = "---\ntitle: \"Hello\"\ncount: 5\n---\n\n## Other\n\nText.\n";
574        let parsed = parse_text(text, "test.md", false);
575        let errors = validate_file(&parsed, &make_schema());
576        assert!(errors.iter().any(|e| e.error_type == "missing_section"));
577    }
578
579    #[test]
580    fn test_duplicate_section() {
581        let text = "---\ntitle: \"Hello\"\ncount: 5\n---\n\n## Summary\n\nFirst.\n\n## Summary\n\nSecond.\n";
582        let parsed = parse_text(text, "test.md", false);
583        let errors = validate_file(&parsed, &make_schema());
584        assert!(errors.iter().any(|e| e.error_type == "duplicate_section"));
585    }
586
587    // --- Foreign key validation tests ---
588
589    use crate::database::{DatabaseConfig, ForeignKey};
590
591    fn make_fk_tables() -> HashMap<String, (Schema, Vec<Row>)> {
592        let strategy_schema = Schema {
593            table: "strategies".to_string(),
594            primary_key: "path".to_string(),
595            frontmatter: IndexMap::new(),
596            h1_required: false,
597            h1_must_equal_frontmatter: None,
598            sections: IndexMap::new(),
599            rules: Rules {
600                reject_unknown_frontmatter: false,
601                reject_unknown_sections: false,
602                reject_duplicate_sections: false,
603                normalize_numbered_headings: false,
604            },
605        };
606
607        let backtest_schema = Schema {
608            table: "backtests".to_string(),
609            primary_key: "path".to_string(),
610            frontmatter: IndexMap::new(),
611            h1_required: false,
612            h1_must_equal_frontmatter: None,
613            sections: IndexMap::new(),
614            rules: Rules {
615                reject_unknown_frontmatter: false,
616                reject_unknown_sections: false,
617                reject_duplicate_sections: false,
618                normalize_numbered_headings: false,
619            },
620        };
621
622        let mut s1 = Row::new();
623        s1.insert("path".into(), Value::String("alpha.md".into()));
624        let mut s2 = Row::new();
625        s2.insert("path".into(), Value::String("beta.md".into()));
626
627        let mut b1 = Row::new();
628        b1.insert("path".into(), Value::String("bt-alpha.md".into()));
629        b1.insert("strategy".into(), Value::String("alpha.md".into()));
630        let mut b2 = Row::new();
631        b2.insert("path".into(), Value::String("bt-beta.md".into()));
632        b2.insert("strategy".into(), Value::String("beta.md".into()));
633
634        let mut tables = HashMap::new();
635        tables.insert("strategies".into(), (strategy_schema, vec![s1, s2]));
636        tables.insert("backtests".into(), (backtest_schema, vec![b1, b2]));
637        tables
638    }
639
640    fn make_fk_config() -> DatabaseConfig {
641        DatabaseConfig {
642            name: "test".into(),
643            foreign_keys: vec![ForeignKey {
644                from_table: "backtests".into(),
645                from_column: "strategy".into(),
646                to_table: "strategies".into(),
647                to_column: "path".into(),
648            }],
649        }
650    }
651
652    #[test]
653    fn test_fk_valid() {
654        let tables = make_fk_tables();
655        let config = make_fk_config();
656        let errors = validate_foreign_keys(&config, &tables);
657        assert!(errors.is_empty(), "Expected no FK errors, got: {:?}", errors);
658    }
659
660    #[test]
661    fn test_fk_violation() {
662        let mut tables = make_fk_tables();
663        // Add a backtest referencing a nonexistent strategy
664        let mut broken = Row::new();
665        broken.insert("path".into(), Value::String("bt-broken.md".into()));
666        broken.insert("strategy".into(), Value::String("nonexistent.md".into()));
667        tables.get_mut("backtests").unwrap().1.push(broken);
668
669        let config = make_fk_config();
670        let errors = validate_foreign_keys(&config, &tables);
671        assert_eq!(errors.len(), 1);
672        assert_eq!(errors[0].error_type, "fk_violation");
673        assert!(errors[0].message.contains("nonexistent.md"));
674    }
675
676    #[test]
677    fn test_fk_null_not_violation() {
678        let mut tables = make_fk_tables();
679        // Add a backtest with null strategy — should not be a violation
680        let mut nullref = Row::new();
681        nullref.insert("path".into(), Value::String("bt-null.md".into()));
682        nullref.insert("strategy".into(), Value::Null);
683        tables.get_mut("backtests").unwrap().1.push(nullref);
684
685        let config = make_fk_config();
686        let errors = validate_foreign_keys(&config, &tables);
687        assert!(errors.is_empty());
688    }
689
690    #[test]
691    fn test_fk_missing_table() {
692        let tables = make_fk_tables();
693        let config = DatabaseConfig {
694            name: "test".into(),
695            foreign_keys: vec![ForeignKey {
696                from_table: "backtests".into(),
697                from_column: "strategy".into(),
698                to_table: "nonexistent_table".into(),
699                to_column: "path".into(),
700            }],
701        };
702        let errors = validate_foreign_keys(&config, &tables);
703        assert_eq!(errors.len(), 1);
704        assert_eq!(errors[0].error_type, "fk_missing_table");
705    }
706}