use std::collections::{BTreeMap, BTreeSet};
use std::fs;
use std::path::{Path, PathBuf};
use crate::ddl::{self, Rename};
use crate::describe::{self, Cache};
use crate::emit;
use crate::ir::Schema;
use crate::parse::{self, SourceFile};
use crate::queries;
use crate::types::Feature;
use crate::{Error, naming};
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Dependency {
Version(String),
Path(PathBuf),
}
#[derive(Debug, Clone)]
pub struct Options {
pub input: PathBuf,
pub output: PathBuf,
pub crate_name: String,
pub rok_db: Dependency,
pub strict: bool,
pub database_url: Option<String>,
pub migration: Option<String>,
pub renames: Vec<Rename>,
pub allow_destructive: bool,
}
impl Options {
pub fn new() -> Self {
Self {
input: PathBuf::from("db"),
output: PathBuf::from("db-gen"),
crate_name: "db-gen".to_owned(),
rok_db: Dependency::Version(default_version()),
strict: false,
database_url: None,
migration: None,
renames: Vec::new(),
allow_destructive: false,
}
}
}
fn default_version() -> String {
env!("CARGO_PKG_VERSION")
.splitn(3, '.')
.take(2)
.collect::<Vec<_>>()
.join(".")
}
impl Default for Options {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Default)]
pub struct Generated {
pub files: BTreeMap<PathBuf, String>,
pub stale: Vec<PathBuf>,
pub new_migration: Option<String>,
pub warnings: Vec<String>,
}
const HEADER: &str = "// @generated by rok-db-gen. Do not edit: change the .sql files and run `rok-db-gen generate`.";
pub fn generate(options: &Options) -> Result<Generated, Error> {
let sources = read_sources(&options.input)?;
let parsed = parse::parse(&sources)?;
let output = &options.output;
let mut generated = Generated::default();
let snapshot_path = output.join("schema.json");
let snapshot: Schema = match fs::read_to_string(&snapshot_path) {
Ok(text) => serde_json::from_str(&text)
.map_err(|e| Error::new(format!("{}: {e}", snapshot_path.display())))?,
Err(_) => Schema::default(),
};
let mut migrations = existing_migrations(&output.join("migrations"))?;
if migrations.is_empty() && snapshot != Schema::default() {
return Err(Error::new(format!(
"{} exists but {} has no migrations; restore them or delete the snapshot",
snapshot_path.display(),
output.join("migrations").display()
)));
}
let ops = ddl::diff(&snapshot, &parsed.schema, &options.renames).map_err(Error::new)?;
let name = match (&options.migration, ops.is_empty()) {
(Some(_), true) if options.renames.is_empty() => {
return Err(Error::new(
"the schema hasn't changed since the last migration; nothing to migrate",
));
}
(Some(name), _) => Some(naming::snake_ident(name)),
(None, false) if migrations.is_empty() => Some("init".to_owned()),
(None, false) => {
let preview: Vec<String> = ops
.iter()
.take(5)
.map(|op| format!(" {}", first_line(&op.up)))
.collect();
return Err(Error::new(format!(
"the .sql files changed since the last migration:\n{}{}\nrun `rok-db-gen generate --migration <name>` to add a migration",
preview.join("\n"),
if ops.len() > 5 { "\n ..." } else { "" }
)));
}
(None, true) => None,
};
if let Some(name) = name {
let reasons: Vec<&str> = ops
.iter()
.filter_map(|op| op.destructive.as_deref())
.collect();
if !reasons.is_empty() && !options.allow_destructive {
return Err(Error::new(format!(
"migration `{name}` would:\n - {}\nre-run with --allow-destructive if that is intended, or use --rename for renames",
reasons.join("\n - ")
)));
}
let version = migrations.last().map_or(1, |m| m.version + 1);
let (up, down) = ddl::scripts(&ops);
let stem = format!("{version:04}_{name}");
generated.files.insert(
PathBuf::from(format!("migrations/{stem}.up.sql")),
up.clone(),
);
if !down.is_empty() {
generated.files.insert(
PathBuf::from(format!("migrations/{stem}.down.sql")),
down.clone(),
);
} else {
generated.warnings.push(format!(
"migration `{stem}` can't be reverted automatically (it adds enum values or removes statements without down SQL)"
));
}
migrations.push(MigrationFile {
version,
name: name.clone(),
stem,
up,
reversible: !down.is_empty(),
});
generated.new_migration = Some(name);
}
generated
.files
.insert(PathBuf::from("schema.json"), json(&parsed.schema)?);
let cache_path = output.join("queries.json");
let cache: Cache = match fs::read_to_string(&cache_path) {
Ok(text) => serde_json::from_str(&text).unwrap_or_default(),
Err(_) => Cache::default(),
};
let up_scripts: Vec<String> = migrations.iter().map(|m| m.up.clone()).collect();
let (descriptions, cache) = describe::describe_all(
&parsed.queries,
&up_scripts,
&cache,
options.database_url.as_deref(),
)?;
generated
.files
.insert(PathBuf::from("queries.json"), json(&cache)?);
let mut features = BTreeSet::new();
let mut modules = emit::models(&parsed.schema, &mut features)?;
let mut warnings = Vec::new();
let all_modules: BTreeSet<String> = parsed
.modules
.iter()
.chain(modules.keys())
.cloned()
.collect();
for module in &all_modules {
let mine: Vec<_> = parsed
.queries
.iter()
.zip(&descriptions)
.filter(|(q, _)| &q.module == module)
.collect();
let code = queries::functions(&mine, &parsed.schema, &mut features, &mut warnings)?;
modules.entry(module.clone()).or_default().push_str(&code);
}
if options.strict && !warnings.is_empty() {
return Err(Error::new(format!("--strict: {}", warnings.join("\n"))));
}
generated.warnings.extend(warnings);
let input_display = options.input.file_name().map_or_else(
|| options.input.display().to_string(),
|n| n.to_string_lossy().into_owned(),
);
for (module, code) in &modules {
let source = parsed_source(&sources, module);
let doc = format!("//! Generated from `{source}`.\n\n{code}");
generated.files.insert(
PathBuf::from(format!("src/{module}.rs")),
emit::format(&doc, HEADER)?,
);
}
generated.files.insert(
PathBuf::from("src/lib.rs"),
lib_rs(&modules, &input_display)?,
);
generated.files.insert(
PathBuf::from("src/migrations.rs"),
migrations_rs(&migrations)?,
);
generated
.files
.insert(PathBuf::from("src/up.rs"), emit::format(UP_RS, HEADER)?);
generated
.files
.insert(PathBuf::from("src/down.rs"), emit::format(DOWN_RS, HEADER)?);
generated
.files
.insert(PathBuf::from("Cargo.toml"), cargo_toml(options, &features));
if let Ok(entries) = fs::read_dir(output.join("src")) {
for entry in entries.flatten() {
let rel = PathBuf::from("src").join(entry.file_name());
let is_generated = fs::read_to_string(entry.path())
.is_ok_and(|t| t.starts_with("// @generated by rok-db-gen"));
if is_generated && !generated.files.contains_key(&rel) {
generated.stale.push(rel);
}
}
}
Ok(generated)
}
pub fn write(options: &Options, generated: &Generated) -> Result<Vec<PathBuf>, Error> {
let io = |path: &Path, e: std::io::Error| Error::new(format!("{}: {e}", path.display()));
let mut written = Vec::new();
for (rel, content) in &generated.files {
let path = options.output.join(rel);
if fs::read_to_string(&path).is_ok_and(|old| &old == content) {
continue;
}
if let Some(dir) = path.parent() {
fs::create_dir_all(dir).map_err(|e| io(dir, e))?;
}
fs::write(&path, content).map_err(|e| io(&path, e))?;
written.push(rel.clone());
}
for rel in &generated.stale {
let path = options.output.join(rel);
fs::remove_file(&path).map_err(|e| io(&path, e))?;
written.push(rel.clone());
}
Ok(written)
}
pub fn differences(options: &Options, generated: &Generated) -> Vec<PathBuf> {
let mut out: Vec<PathBuf> = generated
.files
.iter()
.filter(|(rel, content)| {
fs::read_to_string(options.output.join(rel)).map_or(true, |old| &old != *content)
})
.map(|(rel, _)| rel.clone())
.collect();
out.extend(generated.stale.iter().cloned());
out
}
fn read_sources(dir: &Path) -> Result<Vec<SourceFile>, Error> {
let entries = fs::read_dir(dir).map_err(|e| Error::new(format!("{}: {e}", dir.display())))?;
let mut paths: Vec<PathBuf> = entries
.flatten()
.map(|e| e.path())
.filter(|p| p.extension().is_some_and(|x| x == "sql"))
.collect();
paths.sort();
if paths.is_empty() {
return Err(Error::new(format!("no .sql files in {}", dir.display())));
}
paths
.into_iter()
.map(|path| {
let text = fs::read_to_string(&path)
.map_err(|e| Error::new(format!("{}: {e}", path.display())))?;
let stem = path
.file_stem()
.map(|s| s.to_string_lossy().into_owned())
.unwrap_or_default();
let module = if stem.starts_with('_') {
format!("_{}", naming::snake_ident(&stem))
} else {
naming::snake_ident(&stem)
};
let shown = match (dir.file_name(), path.file_name()) {
(Some(d), Some(f)) => format!("{}/{}", d.to_string_lossy(), f.to_string_lossy()),
_ => path.display().to_string(),
};
Ok(SourceFile {
path: shown,
module,
text,
})
})
.collect()
}
fn parsed_source(sources: &[SourceFile], module: &str) -> String {
let paths: Vec<&str> = sources
.iter()
.filter(|s| s.module == module || (module == "types" && s.module.starts_with('_')))
.map(|s| s.path.as_str())
.collect();
paths.join("`, `")
}
struct MigrationFile {
version: i64,
name: String,
stem: String,
up: String,
reversible: bool,
}
fn existing_migrations(dir: &Path) -> Result<Vec<MigrationFile>, Error> {
let Ok(entries) = fs::read_dir(dir) else {
return Ok(Vec::new());
};
let mut out = Vec::new();
for entry in entries.flatten() {
let file = entry.file_name().to_string_lossy().into_owned();
let Some(stem) = file.strip_suffix(".up.sql") else {
continue;
};
let Some((version, name)) = stem.split_once('_') else {
continue;
};
let Ok(version) = version.parse::<i64>() else {
continue;
};
let up =
fs::read_to_string(entry.path()).map_err(|e| Error::new(format!("{file}: {e}")))?;
out.push(MigrationFile {
version,
name: name.to_owned(),
stem: stem.to_owned(),
up,
reversible: dir.join(format!("{stem}.down.sql")).exists(),
});
}
out.sort_by_key(|m| m.version);
Ok(out)
}
fn normalize(path: &Path) -> PathBuf {
let mut out = PathBuf::new();
for component in path.components() {
match component {
std::path::Component::CurDir => {}
std::path::Component::ParentDir => {
if matches!(
out.components().next_back(),
Some(std::path::Component::Normal(_))
) {
out.pop();
} else {
out.push("..");
}
}
other => out.push(other),
}
}
out
}
fn relative_path(from: &Path, to: &Path) -> PathBuf {
let from = normalize(from);
let to = normalize(to);
let from: Vec<_> = from.components().collect();
let to: Vec<_> = to.components().collect();
let common = from.iter().zip(&to).take_while(|(a, b)| a == b).count();
let mut out = PathBuf::new();
for _ in common..from.len() {
out.push("..");
}
for c in &to[common..] {
out.push(c);
}
out
}
fn first_line(sql: &str) -> &str {
sql.lines().next().unwrap_or_default()
}
fn json<T: serde::Serialize>(value: &T) -> Result<String, Error> {
serde_json::to_string_pretty(value)
.map(|s| s + "\n")
.map_err(|e| Error::new(e.to_string()))
}
fn lib_rs(modules: &BTreeMap<String, String>, input: &str) -> Result<String, Error> {
let mut mods: Vec<(String, &str)> = modules.keys().map(|m| (m.clone(), "pub ")).collect();
mods.push(("migrations".to_owned(), "pub "));
mods.push(("down".to_owned(), ""));
mods.push(("up".to_owned(), ""));
mods.sort();
let mods: String = mods
.iter()
.map(|(m, vis)| format!("#[rustfmt::skip]\n{vis}mod {m};\n"))
.collect();
let source = format!(
"//! Database code generated by rok-db-gen from `{input}/*.sql`: one module\n\
//! per file with its models, enums and queries, and the migrations.\n\
//!\n\
//! Apply pending migrations with [`up`] and revert them with [`down`].\n\
#![allow(clippy::all, clippy::pedantic, clippy::nursery)]\n\n\
{mods}\n\
pub use down::down;\n\
pub use up::up;\n"
);
emit::format(&source, HEADER)
}
fn migrations_rs(migrations: &[MigrationFile]) -> Result<String, Error> {
let mut items = String::new();
for m in migrations {
let down = if m.reversible {
format!("include_str!(\"../migrations/{}.down.sql\")", m.stem)
} else {
"\"\"".to_owned()
};
items.push_str(&format!(
"rok_db::migration::Migration {{ version: {}, name: {:?}, up: include_str!(\"../migrations/{}.up.sql\"), down: {down} }},\n",
m.version, m.name, m.stem
));
}
let source = format!(
"//! Every migration, oldest first. Each one is a pair of files in\n\
//! `migrations/`; never edit a migration that was applied somewhere.\n\n\
/// Every migration, oldest first.\n\
pub const MIGRATIONS: &[rok_db::migration::Migration] = &[\n{items}];\n"
);
emit::format(&source, HEADER)
}
const UP_RS: &str = "/// Apply every pending migration, oldest first, each in its own transaction.\n\
/// Returns the versions applied.\n\
pub async fn up(db: &rok_db::Db) -> rok_db::Result<Vec<i64>> {\n\
rok_db::migration::apply(db, crate::migrations::MIGRATIONS).await\n\
}\n";
const DOWN_RS: &str = "/// Revert the latest `steps` applied migrations, newest first. Returns the\n\
/// versions reverted.\n\
pub async fn down(db: &rok_db::Db, steps: usize) -> rok_db::Result<Vec<i64>> {\n\
rok_db::migration::revert(db, crate::migrations::MIGRATIONS, steps).await\n\
}\n";
fn cargo_toml(options: &Options, features: &BTreeSet<Feature>) -> String {
let features: Vec<String> = features.iter().map(|f| format!("{:?}", f.name())).collect();
let features = if features.is_empty() {
String::new()
} else {
format!(", features = [{}]", features.join(", "))
};
let source = match &options.rok_db {
Dependency::Version(v) => format!("version = {v:?}"),
Dependency::Path(p) => {
let relative = relative_path(&options.output, p);
format!(
"path = {:?}, version = {:?}",
relative.display().to_string().replace('\\', "/"),
default_version()
)
}
};
format!(
"# @generated by rok-db-gen. Do not edit: settings live in rok-db.toml.\n\
[package]\n\
name = {:?}\n\
version = \"0.1.0\"\n\
edition = \"2024\"\n\
publish = false\n\
description = \"Database models, queries and migrations generated by rok-db-gen\"\n\n\
[dependencies]\n\
rok-db = {{ {source}{features} }}\n",
options.crate_name
)
}