use std::io::Write as IoWrite;
use std::path::Path;
use std::time::Instant;
use clap::{Arg, ArgMatches, Command};
use crate::cmd::resolve_db_path;
pub fn command() -> Command {
Command::new("validate")
.about("Validate records in the local commonmeta database against the v1.0 schema")
.long_about(
"Reads records from the commonmeta SQLite database and validates each one \
against the commonmeta v1.0 JSON schema. Invalid records are reported with \
their DOI and a description of each schema violation.\n\n\
Pass a DOI or ORCID as the positional argument to validate a single record:\n\
commonmeta validate 10.25490/a97f-egyk\n\
commonmeta validate 0000-0002-0068-716X\n\
Without an identifier, all records in the database are validated.\n\n\
Use --table to select which table to validate (default: works):\n\
commonmeta validate --table people\n\
commonmeta validate --table organizations\n\n\
Each record has a `valid` boolean column (default false). Records that pass \
validation are marked valid = true. Use `--recheck` to skip already-valid \
records and only process those still marked false.\n\n\
Use `--fill` to enrich affiliation and organization identifiers. For each \
record, every contributor affiliation and organization-type contributor is \
inspected:\n\
- Crossref Funder ID or ISNI: replaced with the matching ROR URL and display name.\n\
- Ringgold ID: looked up in organizations.external_ids (populated by \
`import --from ror` via the Wikidata SPARQL bulk import; cached for 30 days).\n\
- ROR URL with an empty name: name filled from the organizations table.\n\n\
Examples:\n\n\
commonmeta validate\n\
commonmeta validate 10.25490/a97f-egyk\n\
commonmeta validate 0000-0002-0068-716X --fill\n\
commonmeta validate --from datacite\n\
commonmeta validate --from datacite --type Dataset\n\
commonmeta validate --number 1000\n\
commonmeta validate --recheck --fix\n\
commonmeta validate --report errors.jsonl\n\
commonmeta validate /path/to/other.sqlite3\n\
commonmeta validate --fill --from crossref --number 10000\n\
commonmeta validate --table people --recheck\n\
commonmeta validate --table organizations --number 1000\n\
commonmeta validate --sample\n\
commonmeta validate --sample --number 500 --from crossref\n\
commonmeta validate --table people --sample --number 200",
)
.arg(
Arg::new("input")
.help("DOI, ORCID, ROR URL, or SQLite database path (default: platform commonmeta.sqlite3)")
.required(false)
.index(1),
)
.arg(
Arg::new("table")
.long("table")
.help("Which table to validate: works (default), people, or organizations")
.value_parser(["works", "people", "organizations"])
.default_value("works"),
)
.arg(
Arg::new("from")
.long("from")
.short('f')
.help("Filter by DOI registration agency (lowercase): crossref, datacite, medra, jalc, cnki, kisti, op, airiti"),
)
.arg(
Arg::new("type")
.long("type")
.help("Filter by work type, e.g. Dataset, JournalArticle"),
)
.arg(
Arg::new("number")
.long("number")
.short('n')
.help("Maximum number of records to validate (0 = all)")
.value_parser(clap::value_parser!(usize))
.default_value("0"),
)
.arg(
Arg::new("report")
.long("report")
.help("Write validation errors as JSONL to this file instead of stderr"),
)
.arg(
Arg::new("fix")
.long("fix")
.help("Repair invalid records in-place by re-applying schema normalization")
.action(clap::ArgAction::SetTrue),
)
.arg(
Arg::new("recheck")
.long("recheck")
.help("Only validate records not yet marked valid (valid = false); skips records already confirmed valid")
.action(clap::ArgAction::SetTrue),
)
.arg(
Arg::new("has-ror-id")
.long("has-ror-id")
.help("Only process records that have at least one ROR affiliation ID")
.action(clap::ArgAction::SetTrue),
)
.arg(
Arg::new("fill")
.long("fill")
.help("Enrich affiliations: replace Crossref Funder IDs, ISNIs, and Ringgold IDs with ROR URLs; fill missing names for bare ROR IDs")
.action(clap::ArgAction::SetTrue),
)
.arg(
Arg::new("sample")
.long("sample")
.help("Validate a random subset of records. Use --number to set the size (default 1000).")
.action(clap::ArgAction::SetTrue),
)
}
pub fn execute(matches: &ArgMatches) -> Result<(), String> {
let table = matches.get_one::<String>("table").map(String::as_str).unwrap_or("works");
let (db_path, record_id) = match matches.get_one::<String>("input").map(String::as_str) {
Some(raw) => {
let doi = crate::doi_utils::normalize_doi(raw);
if !doi.is_empty() {
(resolve_db_path(None), Some(doi))
} else if crate::utils::validate_id(raw).1 == "ORCID" {
let norm = crate::utils::normalize_orcid(raw);
let id = if norm.is_empty() { raw.to_string() } else { norm };
(resolve_db_path(None), Some(id))
} else if raw.starts_with("https://ror.org/") || crate::utils::validate_id(raw).1 == "ROR" {
(resolve_db_path(None), Some(raw.to_string()))
} else {
(raw.to_string(), None)
}
}
None => (resolve_db_path(None), None),
};
let from_raw = matches.get_one::<String>("from").map(String::as_str);
let work_type = matches.get_one::<String>("type").map(String::as_str);
let number = *matches.get_one::<usize>("number").unwrap_or(&0);
let error_report = matches.get_one::<String>("report");
let fix = matches.get_flag("fix");
let recheck = matches.get_flag("recheck");
let has_ror_id = matches.get_flag("has-ror-id");
let fill = matches.get_flag("fill");
let sample = matches.get_flag("sample");
let provider = from_raw.map(|s| match s.to_lowercase().as_str() {
"crossref" => "Crossref",
"datacite" => "DataCite",
"medra" => "mEDRA",
"jalc" => "JaLC",
"cnki" => "CNKI",
"kisti" => "KISTI",
"op" => "OP",
"airiti" => "Airiti",
_ => s,
});
if !Path::new(&db_path).exists() {
return Err(format!("database not found: {}", db_path));
}
const DEFAULT_SAMPLE: usize = 1_000;
let effective_number = if sample && number == 0 { DEFAULT_SAMPLE } else { number };
if fill {
eprintln!("validate: --fill from {}", db_path);
let start = Instant::now();
let report = crate::fill_sqlite(
Path::new(&db_path),
provider,
work_type,
record_id.as_deref(),
has_ror_id,
effective_number,
)
.map_err(|e| e.to_string())?;
eprintln!(
"validate: fill — {} records checked, {} changed, {} affiliations filled in {:.2?}",
report.total, report.changed, report.affiliations_filled, start.elapsed()
);
if record_id.is_some() && report.total == 0 {
eprintln!(
"validate: fill found no record for '{}' — \
ensure `commonmeta import --from ror` has been run to populate \
the organizations table and Wikidata Ringgold→ROR mappings",
record_id.as_deref().unwrap_or("")
);
}
}
eprintln!("validate: reading from {}", db_path);
if recheck {
eprintln!("validate: --recheck — only records not yet marked valid");
}
let start = Instant::now();
if table == "people" {
if fix {
eprintln!("validate: --fix enabled, repairing invalid people records in-place");
}
let report = crate::validate_people_sqlite(
Path::new(&db_path), record_id.as_deref(), effective_number, fix, recheck, sample,
).map_err(|e| e.to_string())?;
return finish_report(report, record_id.as_deref(), error_report, start);
}
if table == "organizations" {
let report = crate::validate_organizations_sqlite(
Path::new(&db_path), record_id.as_deref(), effective_number, recheck, sample,
).map_err(|e| e.to_string())?;
return finish_report(report, record_id.as_deref(), error_report, start);
}
if let Some(p) = provider {
eprintln!("validate: provider filter = {}", p);
}
if let Some(t) = work_type {
eprintln!("validate: type filter = {}", t);
}
if let Some(f) = error_report {
eprintln!("validate: errors → {}", f);
}
if fix {
eprintln!("validate: --fix enabled, repairing invalid records in-place");
}
let report = crate::validate_sqlite(
Path::new(&db_path), provider, work_type, record_id.as_deref(), has_ror_id, effective_number, fix, recheck, sample,
).map_err(|e| e.to_string())?;
if record_id.is_some() && report.total == 0 {
let id = record_id.as_deref().unwrap_or("");
return Err(format!(
"record not found: {}\n hint: run `commonmeta import --from ror` to populate \
the organizations table and Wikidata Ringgold→ROR mappings",
id
));
}
eprintln!(
"validate: checked {} records in {:.2?}",
report.total,
start.elapsed()
);
if report.invalid > 0 {
match error_report {
Some(path) => {
let mut f = std::fs::File::create(path)
.map_err(|e| format!("cannot create '{}': {}", path, e))?;
for ve in &report.errors {
let line = serde_json::json!({
"id": ve.id,
"errors": ve.errors,
});
writeln!(f, "{}", line)
.map_err(|e| format!("write error: {}", e))?;
}
}
None => {
for ve in &report.errors {
for err in &ve.errors {
eprintln!(" INVALID {}: {}", ve.id, err);
}
}
}
}
}
if report.fixed > 0 {
eprintln!("validate: {} record(s) repaired in-place", report.fixed);
}
println!(
"validate: {} records — {} valid, {} invalid{}",
report.total,
report.valid,
report.invalid,
if report.fixed > 0 { format!(" ({} fixed)", report.fixed) } else { String::new() },
);
if report.invalid > 0 {
Err(format!("{} record(s) failed schema validation", report.invalid))
} else {
Ok(())
}
}
fn finish_report(
report: crate::ValidationReport,
record_id: Option<&str>,
error_report: Option<&String>,
start: Instant,
) -> Result<(), String> {
if record_id.is_some() && report.total == 0 {
return Err(format!("record not found: {}", record_id.unwrap_or("")));
}
eprintln!("validate: checked {} records in {:.2?}", report.total, start.elapsed());
if report.invalid > 0 {
match error_report {
Some(path) => {
let mut f = std::fs::File::create(path)
.map_err(|e| format!("cannot create '{}': {}", path, e))?;
for ve in &report.errors {
let line = serde_json::json!({"id": ve.id, "errors": ve.errors});
writeln!(f, "{}", line).map_err(|e| format!("write error: {}", e))?;
}
}
None => {
for ve in &report.errors {
for err in &ve.errors {
eprintln!(" INVALID {}: {}", ve.id, err);
}
}
}
}
}
if report.fixed > 0 {
eprintln!("validate: {} record(s) repaired in-place", report.fixed);
}
println!(
"validate: {} records — {} valid, {} invalid{}",
report.total,
report.valid,
report.invalid,
if report.fixed > 0 { format!(" ({} fixed)", report.fixed) } else { String::new() },
);
if report.invalid > 0 {
Err(format!("{} record(s) failed schema validation", report.invalid))
} else {
Ok(())
}
}