Skip to main content

mdql_core/
validator.rs

1//! Validate parsed markdown files against a schema.
2
3use std::collections::{HashMap, HashSet};
4
5use crate::database::DatabaseConfig;
6use crate::errors::ValidationError;
7use crate::model::{Row, Value};
8use crate::parser::ParsedFile;
9use crate::schema::Schema;
10use crate::stamp::TIMESTAMP_FIELDS;
11
12pub fn validate_file(parsed: &ParsedFile, schema: &Schema) -> Vec<ValidationError> {
13    let mut errors = Vec::new();
14    let fp = &parsed.path;
15
16    // Parse-level errors
17    for msg in &parsed.parse_errors {
18        errors.push(ValidationError {
19            file_path: fp.clone(),
20            error_type: "parse_error".to_string(),
21            field: None,
22            message: msg.clone(),
23            line_number: None,
24        });
25    }
26
27    if errors.iter().any(|e| e.error_type == "parse_error") {
28        return errors;
29    }
30
31    let fm = &parsed.raw_frontmatter;
32    let fm_map = match fm.as_mapping() {
33        Some(m) => m,
34        None => return errors,
35    };
36
37    // --- Frontmatter field checks ---
38    for (name, field_def) in &schema.frontmatter {
39        let key = serde_yaml::Value::String(name.clone());
40        match fm_map.get(&key) {
41            None => {
42                if field_def.required {
43                    errors.push(ValidationError {
44                        file_path: fp.clone(),
45                        error_type: "missing_field".to_string(),
46                        field: Some(name.clone()),
47                        message: format!("Missing required frontmatter field '{}'", name),
48                        line_number: None,
49                    });
50                }
51            }
52            Some(value) => {
53                if let Some(type_err) = check_type(value, &field_def.field_type, name) {
54                    errors.push(ValidationError {
55                        file_path: fp.clone(),
56                        error_type: "type_mismatch".to_string(),
57                        field: Some(name.clone()),
58                        message: type_err,
59                        line_number: None,
60                    });
61                }
62
63                if let Some(ref enum_vals) = field_def.enum_values {
64                    if !value.is_null() {
65                        let str_val = yaml_value_to_string(value);
66                        if !enum_vals.contains(&str_val) {
67                            errors.push(ValidationError {
68                                file_path: fp.clone(),
69                                error_type: "enum_violation".to_string(),
70                                field: Some(name.clone()),
71                                message: format!(
72                                    "Field '{}' value '{}' not in allowed values: {:?}",
73                                    name, str_val, enum_vals
74                                ),
75                                line_number: None,
76                            });
77                        }
78                    }
79                }
80            }
81        }
82    }
83
84    // Validate timestamp fields as datetime (ISO 8601)
85    for ts_field in TIMESTAMP_FIELDS {
86        let key = serde_yaml::Value::String(ts_field.to_string());
87        if let Some(value) = fm_map.get(&key) {
88            if let Some(type_err) = check_type(
89                value,
90                &crate::schema::FieldType::DateTime,
91                ts_field,
92            ) {
93                errors.push(ValidationError {
94                    file_path: fp.clone(),
95                    error_type: "type_mismatch".to_string(),
96                    field: Some(ts_field.to_string()),
97                    message: type_err,
98                    line_number: None,
99                });
100            }
101        }
102    }
103
104    // Unknown frontmatter
105    if schema.rules.reject_unknown_frontmatter {
106        for (key_val, _) in fm_map {
107            if let Some(key) = key_val.as_str() {
108                if !schema.frontmatter.contains_key(key)
109                    && !TIMESTAMP_FIELDS.contains(&key)
110                {
111                    errors.push(ValidationError {
112                        file_path: fp.clone(),
113                        error_type: "unknown_field".to_string(),
114                        field: Some(key.to_string()),
115                        message: format!(
116                            "Unknown frontmatter field '{}' (not in schema)",
117                            key
118                        ),
119                        line_number: None,
120                    });
121                }
122            }
123        }
124    }
125
126    // --- H1 checks ---
127    if schema.h1_required && parsed.h1.is_none() {
128        errors.push(ValidationError {
129            file_path: fp.clone(),
130            error_type: "missing_h1".to_string(),
131            field: None,
132            message: "Missing required H1 heading".to_string(),
133            line_number: None,
134        });
135    }
136
137    if let Some(ref h1_field) = schema.h1_must_equal_frontmatter {
138        if let Some(ref h1) = parsed.h1 {
139            let key = serde_yaml::Value::String(h1_field.clone());
140            if let Some(expected_val) = fm_map.get(&key) {
141                let expected = yaml_value_to_string(expected_val);
142                if h1 != &expected {
143                    errors.push(ValidationError {
144                        file_path: fp.clone(),
145                        error_type: "h1_mismatch".to_string(),
146                        field: None,
147                        message: format!(
148                            "H1 '{}' does not match frontmatter '{}' (expected '{}')",
149                            h1, h1_field, expected
150                        ),
151                        line_number: parsed.h1_line_number,
152                    });
153                }
154            }
155        }
156    }
157
158    // --- Section checks ---
159    let section_names: Vec<&str> = parsed
160        .sections
161        .iter()
162        .map(|s| s.normalized_heading.as_str())
163        .collect();
164
165    // Count occurrences
166    let mut section_counter: HashMap<&str, usize> = HashMap::new();
167    for name in &section_names {
168        *section_counter.entry(name).or_insert(0) += 1;
169    }
170
171    // Duplicate sections
172    if schema.rules.reject_duplicate_sections {
173        for (name, count) in &section_counter {
174            if *count > 1 {
175                errors.push(ValidationError {
176                    file_path: fp.clone(),
177                    error_type: "duplicate_section".to_string(),
178                    field: Some(name.to_string()),
179                    message: format!(
180                        "Duplicate section '{}' (appears {} times)",
181                        name, count
182                    ),
183                    line_number: None,
184                });
185            }
186        }
187    }
188
189    // Required sections
190    for (name, section_def) in &schema.sections {
191        if section_def.required && !section_names.contains(&name.as_str()) {
192            errors.push(ValidationError {
193                file_path: fp.clone(),
194                error_type: "missing_section".to_string(),
195                field: Some(name.clone()),
196                message: format!("Missing required section '{}'", name),
197                line_number: None,
198            });
199        }
200    }
201
202    // Unknown sections
203    if schema.rules.reject_unknown_sections {
204        for section in &parsed.sections {
205            if !schema.sections.contains_key(&section.normalized_heading) {
206                errors.push(ValidationError {
207                    file_path: fp.clone(),
208                    error_type: "unknown_section".to_string(),
209                    field: Some(section.normalized_heading.clone()),
210                    message: format!(
211                        "Unknown section '{}' (not in schema)",
212                        section.normalized_heading
213                    ),
214                    line_number: Some(section.line_number),
215                });
216            }
217        }
218    }
219
220    errors
221}
222
223fn check_type(
224    value: &serde_yaml::Value,
225    expected: &crate::schema::FieldType,
226    field_name: &str,
227) -> Option<String> {
228    use crate::schema::FieldType;
229
230    if value.is_null() {
231        return None;
232    }
233
234    match expected {
235        FieldType::String => {
236            if !value.is_string() {
237                return Some(format!(
238                    "Field '{}' expected string, got {}",
239                    field_name,
240                    yaml_type_name(value)
241                ));
242            }
243        }
244        FieldType::Int => {
245            if value.is_bool() {
246                return Some(format!(
247                    "Field '{}' expected int, got bool",
248                    field_name
249                ));
250            }
251            // serde_yaml may parse integers as i64 or u64
252            if !value.is_i64() && !value.is_u64() {
253                return Some(format!(
254                    "Field '{}' expected int, got {}",
255                    field_name,
256                    yaml_type_name(value)
257                ));
258            }
259        }
260        FieldType::Float => {
261            if value.is_bool() {
262                return Some(format!(
263                    "Field '{}' expected float, got bool",
264                    field_name
265                ));
266            }
267            if !value.is_f64() && !value.is_i64() && !value.is_u64() {
268                return Some(format!(
269                    "Field '{}' expected float, got {}",
270                    field_name,
271                    yaml_type_name(value)
272                ));
273            }
274        }
275        FieldType::Bool => {
276            if !value.is_bool() {
277                return Some(format!(
278                    "Field '{}' expected bool, got {}",
279                    field_name,
280                    yaml_type_name(value)
281                ));
282            }
283        }
284        FieldType::Date => {
285            if let Some(s) = value.as_str() {
286                if chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d").is_err() {
287                    return Some(format!(
288                        "Field '{}' expected date (YYYY-MM-DD), got string '{}'",
289                        field_name, s
290                    ));
291                }
292                return None;
293            }
294            if !value.is_string() {
295                return Some(format!(
296                    "Field '{}' expected date, got {}",
297                    field_name,
298                    yaml_type_name(value)
299                ));
300            }
301        }
302        FieldType::DateTime => {
303            if let Some(s) = value.as_str() {
304                let ok = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S").is_ok()
305                    || chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S%.f").is_ok();
306                if !ok {
307                    return Some(format!(
308                        "Field '{}' expected datetime (ISO 8601), got string '{}'",
309                        field_name, s
310                    ));
311                }
312                return None;
313            }
314            if !value.is_string() {
315                return Some(format!(
316                    "Field '{}' expected datetime, got {}",
317                    field_name,
318                    yaml_type_name(value)
319                ));
320            }
321        }
322        FieldType::StringArray => {
323            match value.as_sequence() {
324                None => {
325                    return Some(format!(
326                        "Field '{}' expected string[], got {}",
327                        field_name,
328                        yaml_type_name(value)
329                    ));
330                }
331                Some(seq) => {
332                    for (i, item) in seq.iter().enumerate() {
333                        if !item.is_string() {
334                            return Some(format!(
335                                "Field '{}[{}]' expected string, got {}",
336                                field_name,
337                                i,
338                                yaml_type_name(item)
339                            ));
340                        }
341                    }
342                }
343            }
344        }
345    }
346
347    None
348}
349
350fn yaml_type_name(value: &serde_yaml::Value) -> &'static str {
351    match value {
352        serde_yaml::Value::Null => "null",
353        serde_yaml::Value::Bool(_) => "bool",
354        serde_yaml::Value::Number(_) => {
355            if value.is_f64() && !value.is_i64() && !value.is_u64() {
356                "float"
357            } else {
358                "int"
359            }
360        }
361        serde_yaml::Value::String(_) => "str",
362        serde_yaml::Value::Sequence(_) => "list",
363        serde_yaml::Value::Mapping(_) => "mapping",
364        _ => "unknown",
365    }
366}
367
368fn yaml_value_to_string(value: &serde_yaml::Value) -> String {
369    match value {
370        serde_yaml::Value::String(s) => s.clone(),
371        serde_yaml::Value::Number(n) => n.to_string(),
372        serde_yaml::Value::Bool(b) => b.to_string(),
373        serde_yaml::Value::Null => "null".to_string(),
374        _ => format!("{:?}", value),
375    }
376}
377
378/// Validate all foreign key constraints across a loaded database.
379pub fn validate_foreign_keys(
380    db_config: &DatabaseConfig,
381    tables: &HashMap<String, (Schema, Vec<Row>)>,
382) -> Vec<ValidationError> {
383    let mut errors = Vec::new();
384
385    for fk in &db_config.foreign_keys {
386        let to_table = match tables.get(&fk.to_table) {
387            Some(t) => t,
388            None => {
389                errors.push(ValidationError {
390                    file_path: format!("_mdql.md"),
391                    error_type: "fk_missing_table".to_string(),
392                    field: None,
393                    message: format!(
394                        "Foreign key references unknown table '{}'",
395                        fk.to_table
396                    ),
397                    line_number: None,
398                });
399                continue;
400            }
401        };
402
403        let from_table = match tables.get(&fk.from_table) {
404            Some(t) => t,
405            None => {
406                errors.push(ValidationError {
407                    file_path: format!("_mdql.md"),
408                    error_type: "fk_missing_table".to_string(),
409                    field: None,
410                    message: format!(
411                        "Foreign key references unknown table '{}'",
412                        fk.from_table
413                    ),
414                    line_number: None,
415                });
416                continue;
417            }
418        };
419
420        // Build set of valid target values
421        let valid_values: HashSet<String> = to_table
422            .1
423            .iter()
424            .filter_map(|row| {
425                row.get(&fk.to_column).and_then(|v| match v {
426                    Value::Null => None,
427                    _ => Some(v.to_display_string()),
428                })
429            })
430            .collect();
431
432        // Check each row in the referencing table
433        for row in &from_table.1 {
434            let value = match row.get(&fk.from_column) {
435                Some(Value::Null) | None => continue,
436                Some(v) => v,
437            };
438
439            let file_path = row
440                .get("path")
441                .map(|v| format!("{}/{}", fk.from_table, v.to_display_string()))
442                .unwrap_or_else(|| fk.from_table.clone());
443
444            let value_str = value.to_display_string();
445            if !valid_values.contains(&value_str) {
446                errors.push(ValidationError {
447                    file_path,
448                    error_type: "fk_violation".to_string(),
449                    field: Some(fk.from_column.clone()),
450                    message: format!(
451                        "{} = '{}' not found in {}.{}",
452                        fk.from_column, value_str, fk.to_table, fk.to_column
453                    ),
454                    line_number: None,
455                });
456            }
457        }
458    }
459
460    errors
461}
462
463#[cfg(test)]
464mod tests {
465    use super::*;
466    use crate::parser::parse_text;
467    use crate::schema::*;
468    use indexmap::IndexMap;
469
470    fn make_schema() -> Schema {
471        let mut frontmatter = IndexMap::new();
472        frontmatter.insert("title".to_string(), FieldDef {
473            field_type: FieldType::String,
474            required: true,
475            enum_values: None,
476        });
477        frontmatter.insert("count".to_string(), FieldDef {
478            field_type: FieldType::Int,
479            required: true,
480            enum_values: None,
481        });
482        frontmatter.insert("status".to_string(), FieldDef {
483            field_type: FieldType::String,
484            required: false,
485            enum_values: Some(vec!["ACTIVE".into(), "ARCHIVED".into()]),
486        });
487
488        let mut sections = IndexMap::new();
489        sections.insert("Summary".to_string(), SectionDef {
490            content_type: "markdown".to_string(),
491            required: true,
492        });
493
494        Schema {
495            table: "test".to_string(),
496            primary_key: "path".to_string(),
497            frontmatter,
498            h1_required: false,
499            h1_must_equal_frontmatter: None,
500            sections,
501            rules: Rules {
502                reject_unknown_frontmatter: true,
503                reject_unknown_sections: false,
504                reject_duplicate_sections: true,
505                normalize_numbered_headings: false,
506            },
507        }
508    }
509
510    #[test]
511    fn test_valid_file() {
512        let text = "---\ntitle: \"Hello\"\ncount: 5\n---\n\n## Summary\n\nA summary.\n";
513        let parsed = parse_text(text, "test.md", false);
514        let errors = validate_file(&parsed, &make_schema());
515        assert!(errors.is_empty(), "Expected no errors, got: {:?}", errors);
516    }
517
518    #[test]
519    fn test_missing_required_field() {
520        let text = "---\ntitle: \"Hello\"\n---\n\n## Summary\n\nText.\n";
521        let parsed = parse_text(text, "test.md", false);
522        let errors = validate_file(&parsed, &make_schema());
523        assert!(errors.iter().any(|e| e.error_type == "missing_field" && e.field.as_deref() == Some("count")));
524    }
525
526    #[test]
527    fn test_type_mismatch() {
528        let text = "---\ntitle: \"Hello\"\ncount: \"not a number\"\n---\n\n## Summary\n\nText.\n";
529        let parsed = parse_text(text, "test.md", false);
530        let errors = validate_file(&parsed, &make_schema());
531        assert!(errors.iter().any(|e| e.error_type == "type_mismatch" && e.field.as_deref() == Some("count")));
532    }
533
534    #[test]
535    fn test_enum_violation() {
536        let text = "---\ntitle: \"Hello\"\ncount: 5\nstatus: INVALID\n---\n\n## Summary\n\nText.\n";
537        let parsed = parse_text(text, "test.md", false);
538        let errors = validate_file(&parsed, &make_schema());
539        assert!(errors.iter().any(|e| e.error_type == "enum_violation"));
540    }
541
542    #[test]
543    fn test_unknown_frontmatter() {
544        let text = "---\ntitle: \"Hello\"\ncount: 5\nextra: bad\n---\n\n## Summary\n\nText.\n";
545        let parsed = parse_text(text, "test.md", false);
546        let errors = validate_file(&parsed, &make_schema());
547        assert!(errors.iter().any(|e| e.error_type == "unknown_field" && e.field.as_deref() == Some("extra")));
548    }
549
550    #[test]
551    fn test_missing_required_section() {
552        let text = "---\ntitle: \"Hello\"\ncount: 5\n---\n\n## Other\n\nText.\n";
553        let parsed = parse_text(text, "test.md", false);
554        let errors = validate_file(&parsed, &make_schema());
555        assert!(errors.iter().any(|e| e.error_type == "missing_section"));
556    }
557
558    #[test]
559    fn test_duplicate_section() {
560        let text = "---\ntitle: \"Hello\"\ncount: 5\n---\n\n## Summary\n\nFirst.\n\n## Summary\n\nSecond.\n";
561        let parsed = parse_text(text, "test.md", false);
562        let errors = validate_file(&parsed, &make_schema());
563        assert!(errors.iter().any(|e| e.error_type == "duplicate_section"));
564    }
565
566    // --- Foreign key validation tests ---
567
568    use crate::database::{DatabaseConfig, ForeignKey};
569
570    fn make_fk_tables() -> HashMap<String, (Schema, Vec<Row>)> {
571        let strategy_schema = Schema {
572            table: "strategies".to_string(),
573            primary_key: "path".to_string(),
574            frontmatter: IndexMap::new(),
575            h1_required: false,
576            h1_must_equal_frontmatter: None,
577            sections: IndexMap::new(),
578            rules: Rules {
579                reject_unknown_frontmatter: false,
580                reject_unknown_sections: false,
581                reject_duplicate_sections: false,
582                normalize_numbered_headings: false,
583            },
584        };
585
586        let backtest_schema = Schema {
587            table: "backtests".to_string(),
588            primary_key: "path".to_string(),
589            frontmatter: IndexMap::new(),
590            h1_required: false,
591            h1_must_equal_frontmatter: None,
592            sections: IndexMap::new(),
593            rules: Rules {
594                reject_unknown_frontmatter: false,
595                reject_unknown_sections: false,
596                reject_duplicate_sections: false,
597                normalize_numbered_headings: false,
598            },
599        };
600
601        let mut s1 = Row::new();
602        s1.insert("path".into(), Value::String("alpha.md".into()));
603        let mut s2 = Row::new();
604        s2.insert("path".into(), Value::String("beta.md".into()));
605
606        let mut b1 = Row::new();
607        b1.insert("path".into(), Value::String("bt-alpha.md".into()));
608        b1.insert("strategy".into(), Value::String("alpha.md".into()));
609        let mut b2 = Row::new();
610        b2.insert("path".into(), Value::String("bt-beta.md".into()));
611        b2.insert("strategy".into(), Value::String("beta.md".into()));
612
613        let mut tables = HashMap::new();
614        tables.insert("strategies".into(), (strategy_schema, vec![s1, s2]));
615        tables.insert("backtests".into(), (backtest_schema, vec![b1, b2]));
616        tables
617    }
618
619    fn make_fk_config() -> DatabaseConfig {
620        DatabaseConfig {
621            name: "test".into(),
622            foreign_keys: vec![ForeignKey {
623                from_table: "backtests".into(),
624                from_column: "strategy".into(),
625                to_table: "strategies".into(),
626                to_column: "path".into(),
627            }],
628        }
629    }
630
631    #[test]
632    fn test_fk_valid() {
633        let tables = make_fk_tables();
634        let config = make_fk_config();
635        let errors = validate_foreign_keys(&config, &tables);
636        assert!(errors.is_empty(), "Expected no FK errors, got: {:?}", errors);
637    }
638
639    #[test]
640    fn test_fk_violation() {
641        let mut tables = make_fk_tables();
642        // Add a backtest referencing a nonexistent strategy
643        let mut broken = Row::new();
644        broken.insert("path".into(), Value::String("bt-broken.md".into()));
645        broken.insert("strategy".into(), Value::String("nonexistent.md".into()));
646        tables.get_mut("backtests").unwrap().1.push(broken);
647
648        let config = make_fk_config();
649        let errors = validate_foreign_keys(&config, &tables);
650        assert_eq!(errors.len(), 1);
651        assert_eq!(errors[0].error_type, "fk_violation");
652        assert!(errors[0].message.contains("nonexistent.md"));
653    }
654
655    #[test]
656    fn test_fk_null_not_violation() {
657        let mut tables = make_fk_tables();
658        // Add a backtest with null strategy — should not be a violation
659        let mut nullref = Row::new();
660        nullref.insert("path".into(), Value::String("bt-null.md".into()));
661        nullref.insert("strategy".into(), Value::Null);
662        tables.get_mut("backtests").unwrap().1.push(nullref);
663
664        let config = make_fk_config();
665        let errors = validate_foreign_keys(&config, &tables);
666        assert!(errors.is_empty());
667    }
668
669    #[test]
670    fn test_fk_missing_table() {
671        let tables = make_fk_tables();
672        let config = DatabaseConfig {
673            name: "test".into(),
674            foreign_keys: vec![ForeignKey {
675                from_table: "backtests".into(),
676                from_column: "strategy".into(),
677                to_table: "nonexistent_table".into(),
678                to_column: "path".into(),
679            }],
680        };
681        let errors = validate_foreign_keys(&config, &tables);
682        assert_eq!(errors.len(), 1);
683        assert_eq!(errors[0].error_type, "fk_missing_table");
684    }
685}