Skip to main content

rok_db_codegen/
generate.rs

1//! Turn the `.sql` files into the generated crate's files.
2
3use std::collections::{BTreeMap, BTreeSet};
4use std::fs;
5use std::path::{Path, PathBuf};
6
7use crate::ddl::{self, Rename};
8use crate::describe::{self, Cache};
9use crate::emit;
10use crate::ir::Schema;
11use crate::parse::{self, SourceFile};
12use crate::queries;
13use crate::types::Feature;
14use crate::{Error, naming};
15
16/// Where rok-db comes from in the generated `Cargo.toml`.
17#[derive(Debug, Clone, PartialEq, Eq)]
18pub enum Dependency {
19    /// `rok-db = { version = "..." }`.
20    Version(String),
21    /// `rok-db = { path = "..." }`, relative to the project root.
22    Path(PathBuf),
23}
24
25/// What to generate, and how.
26#[derive(Debug, Clone)]
27pub struct Options {
28    /// Directory holding the `.sql` files.
29    pub input: PathBuf,
30    /// Directory of the generated crate.
31    pub output: PathBuf,
32    /// Package name of the generated crate.
33    pub crate_name: String,
34    /// The rok-db dependency of the generated crate.
35    pub rok_db: Dependency,
36    /// Treat warnings (queries that skip tenant or soft-delete filters) as errors.
37    pub strict: bool,
38    /// PostgreSQL server for checking queries that aren't in `queries.json`.
39    pub database_url: Option<String>,
40    /// Name for a new migration, when the schema changed.
41    pub migration: Option<String>,
42    /// Renames to apply instead of drop-and-add.
43    pub renames: Vec<Rename>,
44    /// Allow steps that can lose data or fail on existing rows.
45    pub allow_destructive: bool,
46}
47
48impl Options {
49    /// Defaults: `db/` to `db-gen/`, rok-db from crates.io.
50    pub fn new() -> Self {
51        Self {
52            input: PathBuf::from("db"),
53            output: PathBuf::from("db-gen"),
54            crate_name: "db-gen".to_owned(),
55            rok_db: Dependency::Version(default_version()),
56            strict: false,
57            database_url: None,
58            migration: None,
59            renames: Vec::new(),
60            allow_destructive: false,
61        }
62    }
63}
64
65/// `0.4` for rok-db-codegen 0.4.x: the matching rok-db release.
66fn default_version() -> String {
67    env!("CARGO_PKG_VERSION")
68        .splitn(3, '.')
69        .take(2)
70        .collect::<Vec<_>>()
71        .join(".")
72}
73
74impl Default for Options {
75    fn default() -> Self {
76        Self::new()
77    }
78}
79
80/// The generated files, relative to the output directory.
81#[derive(Debug, Default)]
82pub struct Generated {
83    /// Files to write.
84    pub files: BTreeMap<PathBuf, String>,
85    /// Previously generated files that are no longer produced.
86    pub stale: Vec<PathBuf>,
87    /// The migration this run adds, if any.
88    pub new_migration: Option<String>,
89    /// Warnings to show.
90    pub warnings: Vec<String>,
91}
92
93const HEADER: &str = "// @generated by rok-db-gen. Do not edit: change the .sql files and run `rok-db-gen generate`.";
94
95/// Generate everything in memory. Nothing is written.
96pub fn generate(options: &Options) -> Result<Generated, Error> {
97    let sources = read_sources(&options.input)?;
98    let parsed = parse::parse(&sources)?;
99    let output = &options.output;
100    let mut generated = Generated::default();
101
102    // Migrations: compare with the snapshot of the latest one.
103    let snapshot_path = output.join("schema.json");
104    let snapshot: Schema = match fs::read_to_string(&snapshot_path) {
105        Ok(text) => serde_json::from_str(&text)
106            .map_err(|e| Error::new(format!("{}: {e}", snapshot_path.display())))?,
107        Err(_) => Schema::default(),
108    };
109    let mut migrations = existing_migrations(&output.join("migrations"))?;
110    if migrations.is_empty() && snapshot != Schema::default() {
111        return Err(Error::new(format!(
112            "{} exists but {} has no migrations; restore them or delete the snapshot",
113            snapshot_path.display(),
114            output.join("migrations").display()
115        )));
116    }
117    let ops = ddl::diff(&snapshot, &parsed.schema, &options.renames).map_err(Error::new)?;
118    let name = match (&options.migration, ops.is_empty()) {
119        (Some(_), true) if options.renames.is_empty() => {
120            return Err(Error::new(
121                "the schema hasn't changed since the last migration; nothing to migrate",
122            ));
123        }
124        (Some(name), _) => Some(naming::snake_ident(name)),
125        (None, false) if migrations.is_empty() => Some("init".to_owned()),
126        (None, false) => {
127            let preview: Vec<String> = ops
128                .iter()
129                .take(5)
130                .map(|op| format!("  {}", first_line(&op.up)))
131                .collect();
132            return Err(Error::new(format!(
133                "the .sql files changed since the last migration:\n{}{}\nrun `rok-db-gen generate --migration <name>` to add a migration",
134                preview.join("\n"),
135                if ops.len() > 5 { "\n  ..." } else { "" }
136            )));
137        }
138        (None, true) => None,
139    };
140    if let Some(name) = name {
141        let reasons: Vec<&str> = ops
142            .iter()
143            .filter_map(|op| op.destructive.as_deref())
144            .collect();
145        if !reasons.is_empty() && !options.allow_destructive {
146            return Err(Error::new(format!(
147                "migration `{name}` would:\n  - {}\nre-run with --allow-destructive if that is intended, or use --rename for renames",
148                reasons.join("\n  - ")
149            )));
150        }
151        let version = migrations.last().map_or(1, |m| m.version + 1);
152        let (up, down) = ddl::scripts(&ops);
153        let stem = format!("{version:04}_{name}");
154        generated.files.insert(
155            PathBuf::from(format!("migrations/{stem}.up.sql")),
156            up.clone(),
157        );
158        if !down.is_empty() {
159            generated.files.insert(
160                PathBuf::from(format!("migrations/{stem}.down.sql")),
161                down.clone(),
162            );
163        } else {
164            generated.warnings.push(format!(
165                "migration `{stem}` can't be reverted automatically (it adds enum values or removes statements without down SQL)"
166            ));
167        }
168        migrations.push(MigrationFile {
169            version,
170            name: name.clone(),
171            stem,
172            up,
173            reversible: !down.is_empty(),
174        });
175        generated.new_migration = Some(name);
176    }
177    generated
178        .files
179        .insert(PathBuf::from("schema.json"), json(&parsed.schema)?);
180
181    // Queries: types from PostgreSQL (or the cache).
182    let cache_path = output.join("queries.json");
183    let cache: Cache = match fs::read_to_string(&cache_path) {
184        Ok(text) => serde_json::from_str(&text).unwrap_or_default(),
185        Err(_) => Cache::default(),
186    };
187    let up_scripts: Vec<String> = migrations.iter().map(|m| m.up.clone()).collect();
188    let (descriptions, cache) = describe::describe_all(
189        &parsed.queries,
190        &up_scripts,
191        &cache,
192        options.database_url.as_deref(),
193    )?;
194    generated
195        .files
196        .insert(PathBuf::from("queries.json"), json(&cache)?);
197
198    // Rust.
199    let mut features = BTreeSet::new();
200    let mut modules = emit::models(&parsed.schema, &mut features)?;
201    let mut warnings = Vec::new();
202    let all_modules: BTreeSet<String> = parsed
203        .modules
204        .iter()
205        .chain(modules.keys())
206        .cloned()
207        .collect();
208    for module in &all_modules {
209        let mine: Vec<_> = parsed
210            .queries
211            .iter()
212            .zip(&descriptions)
213            .filter(|(q, _)| &q.module == module)
214            .collect();
215        let code = queries::functions(&mine, &parsed.schema, &mut features, &mut warnings)?;
216        modules.entry(module.clone()).or_default().push_str(&code);
217    }
218    if options.strict && !warnings.is_empty() {
219        return Err(Error::new(format!("--strict: {}", warnings.join("\n"))));
220    }
221    generated.warnings.extend(warnings);
222
223    let input_display = options.input.file_name().map_or_else(
224        || options.input.display().to_string(),
225        |n| n.to_string_lossy().into_owned(),
226    );
227    for (module, code) in &modules {
228        let source = parsed_source(&sources, module);
229        let doc = format!("//! Generated from `{source}`.\n\n{code}");
230        generated.files.insert(
231            PathBuf::from(format!("src/{module}.rs")),
232            emit::format(&doc, HEADER)?,
233        );
234    }
235    generated.files.insert(
236        PathBuf::from("src/lib.rs"),
237        lib_rs(&modules, &input_display)?,
238    );
239    generated.files.insert(
240        PathBuf::from("src/migrations.rs"),
241        migrations_rs(&migrations)?,
242    );
243    generated
244        .files
245        .insert(PathBuf::from("src/up.rs"), emit::format(UP_RS, HEADER)?);
246    generated
247        .files
248        .insert(PathBuf::from("src/down.rs"), emit::format(DOWN_RS, HEADER)?);
249    generated
250        .files
251        .insert(PathBuf::from("Cargo.toml"), cargo_toml(options, &features));
252
253    // Generated sources that are no longer produced (a removed .sql file).
254    if let Ok(entries) = fs::read_dir(output.join("src")) {
255        for entry in entries.flatten() {
256            let rel = PathBuf::from("src").join(entry.file_name());
257            let is_generated = fs::read_to_string(entry.path())
258                .is_ok_and(|t| t.starts_with("// @generated by rok-db-gen"));
259            if is_generated && !generated.files.contains_key(&rel) {
260                generated.stale.push(rel);
261            }
262        }
263    }
264    Ok(generated)
265}
266
267/// Write the files that changed and remove stale ones. Returns the paths
268/// written.
269pub fn write(options: &Options, generated: &Generated) -> Result<Vec<PathBuf>, Error> {
270    let io = |path: &Path, e: std::io::Error| Error::new(format!("{}: {e}", path.display()));
271    let mut written = Vec::new();
272    for (rel, content) in &generated.files {
273        let path = options.output.join(rel);
274        if fs::read_to_string(&path).is_ok_and(|old| &old == content) {
275            continue;
276        }
277        if let Some(dir) = path.parent() {
278            fs::create_dir_all(dir).map_err(|e| io(dir, e))?;
279        }
280        fs::write(&path, content).map_err(|e| io(&path, e))?;
281        written.push(rel.clone());
282    }
283    for rel in &generated.stale {
284        let path = options.output.join(rel);
285        fs::remove_file(&path).map_err(|e| io(&path, e))?;
286        written.push(rel.clone());
287    }
288    Ok(written)
289}
290
291/// Files whose content on disk differs from `generated` (CI check).
292pub fn differences(options: &Options, generated: &Generated) -> Vec<PathBuf> {
293    let mut out: Vec<PathBuf> = generated
294        .files
295        .iter()
296        .filter(|(rel, content)| {
297            fs::read_to_string(options.output.join(rel)).map_or(true, |old| &old != *content)
298        })
299        .map(|(rel, _)| rel.clone())
300        .collect();
301    out.extend(generated.stale.iter().cloned());
302    out
303}
304
305fn read_sources(dir: &Path) -> Result<Vec<SourceFile>, Error> {
306    let entries = fs::read_dir(dir).map_err(|e| Error::new(format!("{}: {e}", dir.display())))?;
307    let mut paths: Vec<PathBuf> = entries
308        .flatten()
309        .map(|e| e.path())
310        .filter(|p| p.extension().is_some_and(|x| x == "sql"))
311        .collect();
312    paths.sort();
313    if paths.is_empty() {
314        return Err(Error::new(format!("no .sql files in {}", dir.display())));
315    }
316    paths
317        .into_iter()
318        .map(|path| {
319            let text = fs::read_to_string(&path)
320                .map_err(|e| Error::new(format!("{}: {e}", path.display())))?;
321            let stem = path
322                .file_stem()
323                .map(|s| s.to_string_lossy().into_owned())
324                .unwrap_or_default();
325            let module = if stem.starts_with('_') {
326                format!("_{}", naming::snake_ident(&stem))
327            } else {
328                naming::snake_ident(&stem)
329            };
330            // Relative to the input directory's parent, so the generated
331            // files don't depend on where the command runs.
332            let shown = match (dir.file_name(), path.file_name()) {
333                (Some(d), Some(f)) => format!("{}/{}", d.to_string_lossy(), f.to_string_lossy()),
334                _ => path.display().to_string(),
335            };
336            Ok(SourceFile {
337                path: shown,
338                module,
339                text,
340            })
341        })
342        .collect()
343}
344
345fn parsed_source(sources: &[SourceFile], module: &str) -> String {
346    let paths: Vec<&str> = sources
347        .iter()
348        .filter(|s| s.module == module || (module == "types" && s.module.starts_with('_')))
349        .map(|s| s.path.as_str())
350        .collect();
351    paths.join("`, `")
352}
353
354/// A migration on disk: `migrations/<version>_<name>.up.sql`.
355struct MigrationFile {
356    version: i64,
357    name: String,
358    stem: String,
359    up: String,
360    reversible: bool,
361}
362
363fn existing_migrations(dir: &Path) -> Result<Vec<MigrationFile>, Error> {
364    let Ok(entries) = fs::read_dir(dir) else {
365        return Ok(Vec::new());
366    };
367    let mut out = Vec::new();
368    for entry in entries.flatten() {
369        let file = entry.file_name().to_string_lossy().into_owned();
370        let Some(stem) = file.strip_suffix(".up.sql") else {
371            continue;
372        };
373        let Some((version, name)) = stem.split_once('_') else {
374            continue;
375        };
376        let Ok(version) = version.parse::<i64>() else {
377            continue;
378        };
379        let up =
380            fs::read_to_string(entry.path()).map_err(|e| Error::new(format!("{file}: {e}")))?;
381        out.push(MigrationFile {
382            version,
383            name: name.to_owned(),
384            stem: stem.to_owned(),
385            up,
386            reversible: dir.join(format!("{stem}.down.sql")).exists(),
387        });
388    }
389    out.sort_by_key(|m| m.version);
390    Ok(out)
391}
392
393/// Resolve `.` and `..` without touching the file system.
394fn normalize(path: &Path) -> PathBuf {
395    let mut out = PathBuf::new();
396    for component in path.components() {
397        match component {
398            std::path::Component::CurDir => {}
399            std::path::Component::ParentDir => {
400                if matches!(
401                    out.components().next_back(),
402                    Some(std::path::Component::Normal(_))
403                ) {
404                    out.pop();
405                } else {
406                    out.push("..");
407                }
408            }
409            other => out.push(other),
410        }
411    }
412    out
413}
414
415/// `to` relative to the directory `from` (both relative to the same place,
416/// or both absolute).
417fn relative_path(from: &Path, to: &Path) -> PathBuf {
418    let from = normalize(from);
419    let to = normalize(to);
420    let from: Vec<_> = from.components().collect();
421    let to: Vec<_> = to.components().collect();
422    let common = from.iter().zip(&to).take_while(|(a, b)| a == b).count();
423    let mut out = PathBuf::new();
424    for _ in common..from.len() {
425        out.push("..");
426    }
427    for c in &to[common..] {
428        out.push(c);
429    }
430    out
431}
432
433fn first_line(sql: &str) -> &str {
434    sql.lines().next().unwrap_or_default()
435}
436
437fn json<T: serde::Serialize>(value: &T) -> Result<String, Error> {
438    serde_json::to_string_pretty(value)
439        .map(|s| s + "\n")
440        .map_err(|e| Error::new(e.to_string()))
441}
442
443fn lib_rs(modules: &BTreeMap<String, String>, input: &str) -> Result<String, Error> {
444    // `#[rustfmt::skip]` keeps `cargo fmt` from reformatting generated
445    // modules (which would make `rok-db-gen check` fail); declarations are
446    // sorted the way rustfmt sorts them.
447    let mut mods: Vec<(String, &str)> = modules.keys().map(|m| (m.clone(), "pub ")).collect();
448    mods.push(("migrations".to_owned(), "pub "));
449    mods.push(("down".to_owned(), ""));
450    mods.push(("up".to_owned(), ""));
451    mods.sort();
452    let mods: String = mods
453        .iter()
454        .map(|(m, vis)| format!("#[rustfmt::skip]\n{vis}mod {m};\n"))
455        .collect();
456    let source = format!(
457        "//! Database code generated by rok-db-gen from `{input}/*.sql`: one module\n\
458         //! per file with its models, enums and queries, and the migrations.\n\
459         //!\n\
460         //! Apply pending migrations with [`up`] and revert them with [`down`].\n\
461         #![allow(clippy::all, clippy::pedantic, clippy::nursery)]\n\n\
462         {mods}\n\
463         pub use down::down;\n\
464         pub use up::up;\n"
465    );
466    emit::format(&source, HEADER)
467}
468
469fn migrations_rs(migrations: &[MigrationFile]) -> Result<String, Error> {
470    let mut items = String::new();
471    for m in migrations {
472        let down = if m.reversible {
473            format!("include_str!(\"../migrations/{}.down.sql\")", m.stem)
474        } else {
475            "\"\"".to_owned()
476        };
477        items.push_str(&format!(
478            "rok_db::migration::Migration {{ version: {}, name: {:?}, up: include_str!(\"../migrations/{}.up.sql\"), down: {down} }},\n",
479            m.version, m.name, m.stem
480        ));
481    }
482    let source = format!(
483        "//! Every migration, oldest first. Each one is a pair of files in\n\
484         //! `migrations/`; never edit a migration that was applied somewhere.\n\n\
485         /// Every migration, oldest first.\n\
486         pub const MIGRATIONS: &[rok_db::migration::Migration] = &[\n{items}];\n"
487    );
488    emit::format(&source, HEADER)
489}
490
491const UP_RS: &str = "/// Apply every pending migration, oldest first, each in its own transaction.\n\
492/// Returns the versions applied.\n\
493pub async fn up(db: &rok_db::Db) -> rok_db::Result<Vec<i64>> {\n\
494    rok_db::migration::apply(db, crate::migrations::MIGRATIONS).await\n\
495}\n";
496
497const DOWN_RS: &str = "/// Revert the latest `steps` applied migrations, newest first. Returns the\n\
498/// versions reverted.\n\
499pub async fn down(db: &rok_db::Db, steps: usize) -> rok_db::Result<Vec<i64>> {\n\
500    rok_db::migration::revert(db, crate::migrations::MIGRATIONS, steps).await\n\
501}\n";
502
503fn cargo_toml(options: &Options, features: &BTreeSet<Feature>) -> String {
504    let features: Vec<String> = features.iter().map(|f| format!("{:?}", f.name())).collect();
505    let features = if features.is_empty() {
506        String::new()
507    } else {
508        format!(", features = [{}]", features.join(", "))
509    };
510    let source = match &options.rok_db {
511        Dependency::Version(v) => format!("version = {v:?}"),
512        Dependency::Path(p) => {
513            let relative = relative_path(&options.output, p);
514            format!(
515                "path = {:?}, version = {:?}",
516                relative.display().to_string().replace('\\', "/"),
517                default_version()
518            )
519        }
520    };
521    format!(
522        "# @generated by rok-db-gen. Do not edit: settings live in rok-db.toml.\n\
523         [package]\n\
524         name = {:?}\n\
525         version = \"0.1.0\"\n\
526         edition = \"2024\"\n\
527         publish = false\n\
528         description = \"Database models, queries and migrations generated by rok-db-gen\"\n\n\
529         [dependencies]\n\
530         rok-db = {{ {source}{features} }}\n",
531        options.crate_name
532    )
533}