vespertide_loader/
models.rs

1use std::fs;
2use std::path::{Path, PathBuf};
3
4use anyhow::{Context, Result};
5use vespertide_config::VespertideConfig;
6use vespertide_core::TableDef;
7use vespertide_planner::validate_schema;
8
9use crate::parallel_config::map_paths_with_threshold;
10
11/// Load all model definitions from the models directory (recursively).
12pub fn load_models(config: &VespertideConfig) -> Result<Vec<TableDef>> {
13    let models_dir = config.models_dir();
14    if !models_dir.exists() {
15        return Ok(Vec::new());
16    }
17
18    let mut tables = Vec::new();
19    load_models_recursive(models_dir, &mut tables)?;
20
21    // Validate schema integrity using normalized version
22    // But return the original tables to preserve inline constraints
23    if !tables.is_empty() {
24        let normalized_tables: Vec<TableDef> = tables
25            .iter()
26            .map(|t| {
27                t.normalize()
28                    .map_err(|e| anyhow::anyhow!("Failed to normalize table '{}': {}", t.name, e))
29            })
30            .collect::<Result<Vec<_>, _>>()?;
31
32        validate_schema(&normalized_tables)
33            .map_err(|e| anyhow::anyhow!("schema validation failed: {e}"))?;
34    }
35
36    Ok(tables)
37}
38
39/// Recursively walk directory and load model files.
40fn load_models_recursive(dir: &Path, tables: &mut Vec<TableDef>) -> Result<()> {
41    let paths = collect_model_paths(dir)?;
42    let results = map_paths_with_threshold(&paths, load_model_file);
43
44    for result in results {
45        tables.push(result?);
46    }
47
48    Ok(())
49}
50
51fn collect_model_paths(dir: &Path) -> Result<Vec<PathBuf>> {
52    let entries =
53        fs::read_dir(dir).with_context(|| format!("read models directory: {}", dir.display()))?;
54    let mut paths = Vec::new();
55
56    for entry in entries {
57        let entry = entry.context("read directory entry")?;
58        let path = entry.path();
59
60        // `DirEntry::file_type()` answers dir/file from the single `readdir`
61        // result on most platforms (no extra stat). It does NOT follow
62        // symlinks, whereas `Path::is_dir`/`is_file` DO — so for symlinked
63        // entries we fall back to the path checks to keep behavior identical.
64        let ft = entry.file_type().ok();
65        let is_symlink = ft.is_some_and(|t| t.is_symlink());
66
67        if ft.is_some_and(|t| t.is_dir()) || (is_symlink && path.is_dir()) {
68            paths.extend(collect_model_paths(&path)?);
69        } else if (ft.is_some_and(|t| t.is_file()) || (is_symlink && path.is_file()))
70            && crate::has_supported_extension(&path)
71        {
72            paths.push(path);
73        }
74    }
75
76    Ok(paths)
77}
78
79fn load_model_file(path: &Path) -> Result<TableDef> {
80    let ext = path.extension().and_then(|s| s.to_str());
81    let content =
82        fs::read_to_string(path).with_context(|| format!("read model file: {}", path.display()))?;
83
84    let table: TableDef = if ext == Some("json") {
85        serde_json::from_str(&content)
86            .with_context(|| format!("parse JSON model: {}", path.display()))?
87    } else {
88        serde_yaml::from_str(&content)
89            .with_context(|| format!("parse YAML model: {}", path.display()))?
90    };
91
92    table
93        .validate_unique_column_names()
94        .with_context(|| format!("validate model: {}", path.display()))?;
95
96    Ok(table)
97}
98
99/// Load models from a specific directory (for compile-time use in macros).
100pub fn load_models_from_dir(
101    project_root: Option<PathBuf>,
102) -> Result<Vec<TableDef>, Box<dyn std::error::Error>> {
103    // Locate project root from CARGO_MANIFEST_DIR or use provided path
104    let project_root = if let Some(root) = project_root {
105        root
106    } else {
107        PathBuf::from(
108            std::env::var("CARGO_MANIFEST_DIR")
109                .context("CARGO_MANIFEST_DIR environment variable not set")?,
110        )
111    };
112
113    // Read vespertide.json or use defaults
114    let config = crate::config::load_config_or_default(Some(project_root.clone()))
115        .map_err(|e| format!("Failed to load config: {e}"))?;
116
117    // Read models directory
118    let models_dir = project_root.join(config.models_dir());
119    if !models_dir.exists() {
120        return Ok(Vec::new());
121    }
122
123    let mut tables = Vec::new();
124    load_models_recursive_internal(&models_dir, &mut tables)
125        .map_err(|e| format!("Failed to load models: {e}"))?;
126
127    Ok(tables)
128}
129
130/// Internal recursive function for loading models (used by both runtime and compile-time).
131fn load_models_recursive_internal(
132    dir: &Path,
133    tables: &mut Vec<TableDef>,
134) -> Result<(), Box<dyn std::error::Error>> {
135    let paths = collect_model_paths(dir).map_err(|e| e.to_string())?;
136    let results = map_paths_with_threshold(&paths, load_normalized_model_file_internal);
137
138    for result in results {
139        tables.push(result.map_err(|e| -> Box<dyn std::error::Error> { e.into() })?);
140    }
141
142    Ok(())
143}
144
145/// Compile-time variant of [`load_model_file`]: same read → parse → validate
146/// pipeline (delegated so extension dispatch and validation live in exactly
147/// one place), followed by normalization — the macro path needs inline
148/// constraints resolved to table level.
149fn load_normalized_model_file_internal(path: &Path) -> Result<TableDef, String> {
150    // anyhow's alternate format renders the context chain as
151    // "parse JSON model: <path>: <source>"; prefixing "Failed to " keeps the
152    // messages this pipeline has always produced.
153    let table = load_model_file(path).map_err(|e| format!("Failed to {e:#}"))?;
154    table
155        .normalize()
156        .map_err(|e| format!("Failed to normalize table '{}': {}", table.name, e))
157}
158
159/// Load models at compile time (for macro use).
160pub fn load_models_at_compile_time() -> Result<Vec<TableDef>, Box<dyn std::error::Error>> {
161    load_models_from_dir(None)
162}
163
164#[cfg(test)]
165mod tests {
166    use super::*;
167    use crate::test_support::{CwdGuard, write_default_config};
168    use serial_test::serial;
169    use std::fs;
170    use tempfile::tempdir;
171    use vespertide_core::{
172        ColumnDef, ColumnType, SimpleColumnType, TableConstraint,
173        schema::foreign_key::ForeignKeySyntax,
174    };
175
176    #[test]
177    #[serial]
178    fn load_models_returns_empty_when_no_models_dir() {
179        let tmp = tempdir().unwrap();
180        let _guard = CwdGuard::new(tmp.path());
181        write_default_config("vespertide.json");
182
183        // Don't create models directory
184        let models = load_models(&VespertideConfig::default()).unwrap();
185        assert_eq!(models.len(), 0);
186    }
187
188    #[test]
189    #[serial]
190    fn load_models_reads_yaml_and_validates() {
191        let tmp = tempdir().unwrap();
192        let _guard = CwdGuard::new(tmp.path());
193        write_default_config("vespertide.json");
194
195        fs::create_dir_all("models").unwrap();
196        let table = TableDef {
197            name: "users".into(),
198            description: None,
199            columns: vec![ColumnDef::new(
200                "id",
201                ColumnType::Simple(SimpleColumnType::Integer),
202                false,
203            )],
204            constraints: vec![TableConstraint::PrimaryKey {
205                auto_increment: false,
206                columns: vec!["id".into()],
207                strategy: vespertide_core::PrimaryKeyAdditionStrategy::default(),
208            }],
209        };
210        fs::write("models/users.yaml", serde_yaml::to_string(&table).unwrap()).unwrap();
211
212        let models = load_models(&VespertideConfig::default()).unwrap();
213        assert_eq!(models.len(), 1);
214        assert_eq!(models[0].name, "users");
215    }
216
217    // `DirEntry::file_type()` describes the LINK, never its target, so symlinked
218    // entries fall back to `Path::is_dir` / `Path::is_file`. Three shapes must be
219    // told apart inside `models/`:
220    //   * `order.json`   -> a real file OUTSIDE models/  => loaded
221    //   * `nowhere.json` -> a missing target (dangling)  => skipped
222    //   * `user.json`    -> a plain file                 => loaded
223    // Misreading a file symlink as a directory would `read_dir` a file, and
224    // misreading a dangling symlink as a file would `read_to_string` a path that
225    // does not exist. Both surface as `Err`, so the successful two-table load
226    // pins each `is_symlink && ...` conjunction.
227    #[cfg(unix)]
228    #[test]
229    #[serial]
230    fn load_models_follows_file_symlinks_and_skips_dangling_ones() {
231        use std::os::unix::fs::symlink;
232
233        let tmp = tempdir().unwrap();
234        let _guard = CwdGuard::new(tmp.path());
235        write_default_config("vespertide.json");
236
237        fs::create_dir_all("models").unwrap();
238        fs::create_dir_all("external").unwrap();
239
240        let write_model = |path: &str, name: &str| {
241            let table = TableDef {
242                name: name.into(),
243                description: None,
244                columns: vec![ColumnDef::new(
245                    "id",
246                    ColumnType::Simple(SimpleColumnType::Integer),
247                    false,
248                )],
249                constraints: vec![TableConstraint::PrimaryKey {
250                    auto_increment: false,
251                    columns: vec!["id".into()],
252                    strategy: vespertide_core::PrimaryKeyAdditionStrategy::default(),
253                }],
254            };
255            fs::write(path, serde_json::to_string(&table).unwrap()).unwrap();
256        };
257
258        write_model("models/user.json", "users");
259        write_model("external/order.json", "orders");
260
261        symlink(tmp.path().join("external/order.json"), "models/order.json").unwrap();
262        symlink(
263            tmp.path().join("external/nowhere.json"),
264            "models/nowhere.json",
265        )
266        .unwrap();
267
268        let mut names: Vec<String> = load_models(&VespertideConfig::default())
269            .unwrap()
270            .into_iter()
271            .map(|t| t.name.into_inner())
272            .collect();
273        names.sort();
274
275        assert_eq!(names, ["orders", "users"]);
276    }
277
278    #[test]
279    #[serial]
280    fn load_models_recursive_processes_subdirectories() {
281        let tmp = tempdir().unwrap();
282        let _guard = CwdGuard::new(tmp.path());
283        write_default_config("vespertide.json");
284
285        fs::create_dir_all("models/subdir").unwrap();
286
287        // Create model in subdirectory
288        let table = TableDef {
289            name: "subtable".into(),
290            description: None,
291            columns: vec![ColumnDef::new(
292                "id",
293                ColumnType::Simple(SimpleColumnType::Integer),
294                false,
295            )],
296            constraints: vec![TableConstraint::PrimaryKey {
297                auto_increment: false,
298                columns: vec!["id".into()],
299                strategy: vespertide_core::PrimaryKeyAdditionStrategy::default(),
300            }],
301        };
302        let content = serde_json::to_string_pretty(&table).unwrap();
303        fs::write("models/subdir/subtable.json", content).unwrap();
304
305        let models = load_models(&VespertideConfig::default()).unwrap();
306        assert_eq!(models.len(), 1);
307        assert_eq!(models[0].name, "subtable");
308    }
309
310    #[test]
311    #[serial]
312    fn load_models_fails_on_invalid_fk_format() {
313        let tmp = tempdir().unwrap();
314        let _guard = CwdGuard::new(tmp.path());
315        write_default_config("vespertide.json");
316
317        fs::create_dir_all("models").unwrap();
318
319        // Create a model with invalid FK string format (missing dot separator)
320        let table = TableDef {
321            name: "orders".into(),
322            description: None,
323            columns: vec![
324                // Invalid FK format: should be "table.column" but missing the dot
325                ColumnDef::new(
326                    "user_id",
327                    ColumnType::Simple(SimpleColumnType::Integer),
328                    false,
329                )
330                .foreign_key(ForeignKeySyntax::String("invalid_format".into())),
331            ],
332            constraints: vec![],
333        };
334        fs::write(
335            "models/orders.json",
336            serde_json::to_string_pretty(&table).unwrap(),
337        )
338        .unwrap();
339
340        let result = load_models(&VespertideConfig::default());
341        assert!(result.is_err());
342        let err_msg = result.unwrap_err().to_string();
343        assert!(err_msg.contains("Failed to normalize table 'orders'"));
344    }
345
346    // A non-model-extension file (e.g. `.txt`) must be ignored by the
347    // collector. Pins `path.is_file() && has_model_extension(&path)`: a
348    // `&& -> ||` mutant would pick up the `.txt`, then fail to parse it as a
349    // model and surface an error instead of an empty load.
350    #[test]
351    #[serial]
352    fn load_models_ignores_non_model_extension_files() {
353        let tmp = tempdir().unwrap();
354        let _guard = CwdGuard::new(tmp.path());
355        write_default_config("vespertide.json");
356        fs::create_dir_all("models").unwrap();
357        fs::write("models/README.txt", "not a model: {{{ invalid").unwrap();
358
359        let models = load_models(&VespertideConfig::default()).unwrap();
360        assert_eq!(models.len(), 0, "the .txt file must be skipped");
361    }
362
363    #[test]
364    #[serial]
365    fn load_models_from_dir_ignores_non_model_extension_files() {
366        let temp_dir = tempdir().unwrap();
367        let models_dir = temp_dir.path().join("models");
368        fs::create_dir_all(&models_dir).unwrap();
369        fs::write(models_dir.join("notes.txt"), "not a model: {{{ invalid").unwrap();
370
371        let result = load_models_from_dir(Some(temp_dir.path().to_path_buf()));
372        assert!(
373            result.is_ok(),
374            "the .txt file must be skipped, not parsed: {result:?}"
375        );
376        assert_eq!(result.unwrap().len(), 0);
377    }
378
379    #[test]
380    #[serial]
381    fn test_load_models_from_dir_with_root() {
382        let temp_dir = tempdir().unwrap();
383        let models_dir = temp_dir.path().join("models");
384        fs::create_dir_all(&models_dir).unwrap();
385
386        let table = TableDef {
387            name: "users".into(),
388            description: None,
389            columns: vec![ColumnDef::new(
390                "id",
391                ColumnType::Simple(SimpleColumnType::Integer),
392                false,
393            )],
394            constraints: vec![],
395        };
396        fs::write(
397            models_dir.join("users.json"),
398            serde_json::to_string_pretty(&table).unwrap(),
399        )
400        .unwrap();
401
402        let result = load_models_from_dir(Some(temp_dir.path().to_path_buf()));
403        assert!(result.is_ok());
404        let models = result.unwrap();
405        assert_eq!(models.len(), 1);
406        assert_eq!(models[0].name, "users");
407    }
408
409    #[test]
410    #[serial]
411    fn test_load_models_from_dir_without_root() {
412        use std::env;
413
414        // Save the original value
415        let original = env::var("CARGO_MANIFEST_DIR").ok();
416
417        // Remove CARGO_MANIFEST_DIR to test the error path
418        unsafe {
419            env::remove_var("CARGO_MANIFEST_DIR");
420        }
421
422        let result = load_models_from_dir(None);
423        assert!(result.is_err());
424        let err_msg = result.unwrap_err().to_string();
425        assert!(err_msg.contains("CARGO_MANIFEST_DIR environment variable not set"));
426
427        // Restore the original value if it existed
428        if let Some(val) = original {
429            unsafe {
430                env::set_var("CARGO_MANIFEST_DIR", val);
431            }
432        }
433    }
434
435    #[test]
436    #[serial]
437    fn test_load_models_from_dir_no_models_dir() {
438        let temp_dir = tempdir().unwrap();
439        // Don't create models directory
440
441        let result = load_models_from_dir(Some(temp_dir.path().to_path_buf()));
442        assert!(result.is_ok());
443        let models = result.unwrap();
444        assert_eq!(models.len(), 0);
445    }
446
447    #[test]
448    #[serial]
449    fn test_load_models_from_dir_with_yaml() {
450        let temp_dir = tempdir().unwrap();
451        let models_dir = temp_dir.path().join("models");
452        fs::create_dir_all(&models_dir).unwrap();
453
454        let table = TableDef {
455            name: "users".into(),
456            description: None,
457            columns: vec![ColumnDef::new(
458                "id",
459                ColumnType::Simple(SimpleColumnType::Integer),
460                false,
461            )],
462            constraints: vec![],
463        };
464        fs::write(
465            models_dir.join("users.yaml"),
466            serde_yaml::to_string(&table).unwrap(),
467        )
468        .unwrap();
469
470        let result = load_models_from_dir(Some(temp_dir.path().to_path_buf()));
471        assert!(result.is_ok());
472        let models = result.unwrap();
473        assert_eq!(models.len(), 1);
474        assert_eq!(models[0].name, "users");
475    }
476
477    #[test]
478    #[serial]
479    fn test_load_models_from_dir_with_yml() {
480        let temp_dir = tempdir().unwrap();
481        let models_dir = temp_dir.path().join("models");
482        fs::create_dir_all(&models_dir).unwrap();
483
484        let table = TableDef {
485            name: "users".into(),
486            description: None,
487            columns: vec![ColumnDef::new(
488                "id",
489                ColumnType::Simple(SimpleColumnType::Integer),
490                false,
491            )],
492            constraints: vec![],
493        };
494        fs::write(
495            models_dir.join("users.yml"),
496            serde_yaml::to_string(&table).unwrap(),
497        )
498        .unwrap();
499
500        let result = load_models_from_dir(Some(temp_dir.path().to_path_buf()));
501        assert!(result.is_ok());
502        let models = result.unwrap();
503        assert_eq!(models.len(), 1);
504        assert_eq!(models[0].name, "users");
505    }
506
507    #[test]
508    #[serial]
509    fn test_load_models_from_dir_recursive() {
510        let temp_dir = tempdir().unwrap();
511        let models_dir = temp_dir.path().join("models");
512        let subdir = models_dir.join("subdir");
513        fs::create_dir_all(&subdir).unwrap();
514
515        let table = TableDef {
516            name: "subtable".into(),
517            description: None,
518            columns: vec![ColumnDef::new(
519                "id",
520                ColumnType::Simple(SimpleColumnType::Integer),
521                false,
522            )],
523            constraints: vec![],
524        };
525        fs::write(
526            subdir.join("subtable.json"),
527            serde_json::to_string_pretty(&table).unwrap(),
528        )
529        .unwrap();
530
531        let result = load_models_from_dir(Some(temp_dir.path().to_path_buf()));
532        assert!(result.is_ok());
533        let models = result.unwrap();
534        assert_eq!(models.len(), 1);
535        assert_eq!(models[0].name, "subtable");
536    }
537
538    #[test]
539    #[serial]
540    fn test_load_models_from_dir_with_invalid_json() {
541        let temp_dir = tempdir().unwrap();
542        let models_dir = temp_dir.path().join("models");
543        fs::create_dir_all(&models_dir).unwrap();
544
545        fs::write(models_dir.join("invalid.json"), r#"{"invalid": json}"#).unwrap();
546
547        let result = load_models_from_dir(Some(temp_dir.path().to_path_buf()));
548        assert!(result.is_err());
549        let err_msg = result.unwrap_err().to_string();
550        assert!(err_msg.contains("Failed to parse JSON model"));
551    }
552
553    #[test]
554    #[serial]
555    fn test_load_models_from_dir_with_invalid_yaml() {
556        let temp_dir = tempdir().unwrap();
557        let models_dir = temp_dir.path().join("models");
558        fs::create_dir_all(&models_dir).unwrap();
559
560        fs::write(models_dir.join("invalid.yaml"), r"invalid: [yaml").unwrap();
561
562        let result = load_models_from_dir(Some(temp_dir.path().to_path_buf()));
563        assert!(result.is_err());
564        let err_msg = result.unwrap_err().to_string();
565        assert!(err_msg.contains("Failed to parse YAML model"));
566    }
567
568    #[test]
569    #[serial]
570    fn test_load_models_from_dir_normalization_error() {
571        let temp_dir = tempdir().unwrap();
572        let models_dir = temp_dir.path().join("models");
573        fs::create_dir_all(&models_dir).unwrap();
574
575        // Create a model with invalid FK format
576        let table = TableDef {
577            name: "orders".into(),
578            description: None,
579            columns: vec![
580                ColumnDef::new(
581                    "user_id",
582                    ColumnType::Simple(SimpleColumnType::Integer),
583                    false,
584                )
585                .foreign_key(ForeignKeySyntax::String("invalid_format".into())),
586            ],
587            constraints: vec![],
588        };
589        fs::write(
590            models_dir.join("orders.json"),
591            serde_json::to_string_pretty(&table).unwrap(),
592        )
593        .unwrap();
594
595        let result = load_models_from_dir(Some(temp_dir.path().to_path_buf()));
596        assert!(result.is_err());
597        let err_msg = result.unwrap_err().to_string();
598        assert!(err_msg.contains("Failed to normalize table 'orders'"));
599    }
600
601    #[test]
602    #[serial]
603    fn test_load_models_from_dir_with_cargo_manifest_dir() {
604        // Test the path where CARGO_MANIFEST_DIR is set (line 87)
605        // In cargo test environment, CARGO_MANIFEST_DIR is usually set
606        let result = load_models_from_dir(None);
607        // This might succeed if CARGO_MANIFEST_DIR is set (like in cargo test)
608        // or fail if it's not set
609        // Either way, we're testing the code path including line 87
610        let _ = result;
611    }
612
613    #[test]
614    #[serial]
615    fn test_load_models_at_compile_time() {
616        // This function just calls load_models_from_dir(None)
617        // We can't easily test it without CARGO_MANIFEST_DIR, but we can verify
618        // it doesn't panic
619        let result = load_models_at_compile_time();
620        // This might succeed if CARGO_MANIFEST_DIR is set (like in cargo test)
621        // or fail if it's not set
622        // Either way, we're testing the code path
623        let _ = result;
624    }
625}
vespertide_loader/models.rs

vespertide_loader/
models.rs
