pub mod author_utils;
pub mod constants;
pub mod crockford;
pub mod date_utils;
pub mod data;
pub mod doi_utils;
pub mod error;
pub mod io_utils;
mod formats;
pub mod progress;
pub mod schema_utils;
pub mod spdx;
pub mod utils;
pub mod vocabularies;
pub use data::Data;
pub use error::{Error, Result};
pub use schema_utils::SCHEMA_JSON;
pub use formats::crossref;
pub use formats::pubmed;
pub use formats::inveniordm::PushResult;
pub use formats::ror::AffiliationMatch;
pub use formats::ror::RorRelease;
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
pub fn read(from: &str, input: &str) -> Result<Data> {
formats::read(from, input)
}
pub fn convert(from: &str, to: &str, input: &str) -> Result<Vec<u8>> {
let data = formats::read(from, input)?;
formats::write(to, &data)
}
pub fn write(to: &str, data: &Data) -> Result<Vec<u8>> {
formats::write(to, data)
}
pub fn write_with_style(
to: &str,
data: &Data,
style: Option<&str>,
locale: Option<&str>,
) -> Result<Vec<u8>> {
formats::write_citation(to, data, style, locale)
}
pub fn write_ror_json(data: &Data) -> Result<Vec<u8>> {
formats::ror::write_json(data)
}
pub fn fetch_ror(id: &str) -> Result<Data> {
formats::ror::fetch(id)
}
pub fn fetch_latest_ror_release() -> Result<RorRelease> {
formats::ror::fetch_latest_ror_release()
}
pub fn download_ror_release(release: &RorRelease) -> Result<(Vec<formats::ror::Ror>, bool)> {
formats::ror::download_release(release)
}
pub fn download_ror_all() -> Result<(RorRelease, Vec<formats::ror::Ror>, bool)> {
formats::ror::download_all()
}
pub fn fetch_ror_sqlite(
id: &str,
db_path: &std::path::Path,
) -> Result<Data> {
formats::ror::fetch_sqlite(id, db_path)
}
pub fn write_ror_sqlite(
list: &[formats::ror::Ror],
path: &std::path::Path,
version: Option<&str>,
date: Option<&str>,
) -> Result<()> {
formats::ror::write_sqlite(list, path, version, date)
}
pub fn fetch_installed_ror_version(db_path: &std::path::Path) -> Result<Option<String>> {
formats::ror::fetch_installed_ror_version(db_path)
}
pub fn fetch_installed_vraix_date(db_path: &std::path::Path) -> Result<Option<String>> {
formats::vraix::fetch_installed_vraix_date(db_path)
}
pub fn match_ror_affiliation(affiliation: &str) -> Result<Vec<AffiliationMatch>> {
formats::ror::match_affiliation(affiliation)
}
pub fn match_ror_affiliation_sqlite(
affiliation: &str,
db_path: &std::path::Path,
) -> Result<Vec<AffiliationMatch>> {
formats::ror::match_affiliation_sqlite(affiliation, db_path)
}
pub fn convert_citation(
from: &str,
input: &str,
style: Option<&str>,
locale: Option<&str>,
) -> Result<Vec<u8>> {
let data = formats::read(from, input)?;
formats::write_citation("citation", &data, style, locale)
}
pub fn write_parquet(list: &[Data]) -> Result<Vec<u8>> {
formats::commonmeta::write_parquet_all(list)
}
pub fn read_parquet(bytes: &[u8]) -> Result<Vec<Data>> {
formats::commonmeta::read_parquet_all(bytes)
}
pub fn write_sqlite(list: &[Data], path: &std::path::Path) -> Result<()> {
formats::commonmeta::write_sqlite(list, path)
}
pub fn upsert_sqlite(list: &[Data], path: &std::path::Path) -> Result<()> {
formats::commonmeta::upsert_sqlite(list, path)
}
pub fn count_sqlite_works(path: &std::path::Path) -> Result<usize> {
formats::commonmeta::count_sqlite_works(path)
}
pub fn run_migrations(path: &std::path::Path) -> Result<(usize, u32)> {
formats::commonmeta::run_migrations(path)
}
pub fn read_sqlite_commonmeta(
path: &std::path::Path,
limit: Option<usize>,
offset: usize,
) -> Result<Vec<Data>> {
formats::commonmeta::read_sqlite_commonmeta(path, limit, offset)
}
pub fn read_sqlite_by_id(id: &str, path: &std::path::Path) -> Result<Option<Data>> {
formats::commonmeta::read_sqlite_by_id(id, path)
}
pub fn read_sqlite_by_pmid(pmid: &str, path: &std::path::Path) -> Result<Option<Data>> {
formats::commonmeta::read_sqlite_by_pmid(pmid, path)
}
pub fn read_sqlite_by_pmcid(pmcid: &str, path: &std::path::Path) -> Result<Option<Data>> {
formats::commonmeta::read_sqlite_by_pmcid(pmcid, path)
}
pub fn read_sqlite_by_openalex(openalex: &str, path: &std::path::Path) -> Result<Option<Data>> {
formats::commonmeta::read_sqlite_by_openalex(openalex, path)
}
pub fn read_sqlite_by_arxiv(arxiv: &str, path: &std::path::Path) -> Result<Option<Data>> {
formats::commonmeta::read_sqlite_by_arxiv(arxiv, path)
}
pub use formats::commonmeta::{ValidationError, ValidationReport};
pub fn validate_sqlite(
path: &std::path::Path,
provider: Option<&str>,
work_type: Option<&str>,
limit: usize,
fix: bool,
recheck: bool,
) -> Result<ValidationReport> {
formats::commonmeta::validate_sqlite(path, provider, work_type, limit, fix, recheck)
}
#[allow(clippy::too_many_arguments)]
pub fn crossref_fetch_page_with_cursor(
cursor: &str,
number: usize,
member: &str,
type_: &str,
year: &str,
orcid: &str,
ror: &str,
has_orcid: bool,
has_ror: bool,
has_references: bool,
has_relation: bool,
has_abstract: bool,
has_award: bool,
has_license: bool,
has_archive: bool,
match_ror: bool,
) -> Result<(Vec<Data>, Option<String>)> {
formats::crossref::fetch_page_with_cursor(
cursor, number, member, type_, year, orcid, ror,
has_orcid, has_ror, has_references, has_relation, has_abstract, has_award, has_license, has_archive,
match_ror,
)
}
pub fn stream_vraix_to_sqlite(
input_path: &std::path::Path,
from: &str,
output_path: &std::path::Path,
limit: usize,
update: bool,
) -> Result<usize> {
formats::vraix::stream_dump_to_sqlite(input_path, from, output_path, limit, !update)
}
pub fn stream_pidbox_to_sqlite(
input_path: &std::path::Path,
output_path: &std::path::Path,
limit: usize,
update: bool,
) -> Result<usize> {
formats::vraix::stream_pidbox_to_sqlite(input_path, output_path, limit, !update)
}
pub fn flush_dragoman_cache(path: &std::path::Path) -> Result<usize> {
formats::vraix::flush_transport_table(path)
}
pub fn stream_zst_pidbox_to_sqlite(
zst_path: &std::path::Path,
output_path: &std::path::Path,
limit: usize,
) -> Result<usize> {
formats::sqlite_stream::stream_zst_pidbox_to_sqlite(zst_path, output_path, limit, true)
}
pub fn stream_pmc_ids_to_sqlite(
gz_path: &std::path::Path,
output_path: &std::path::Path,
limit: usize,
) -> Result<usize> {
formats::pubmed::stream_pmc_ids_to_sqlite(gz_path, output_path, limit)
}
pub fn write_list(list: &[Data], to: &str) -> Result<Vec<u8>> {
write_list_citation(list, to, None, None)
}
pub fn write_list_citation(
list: &[Data],
to: &str,
style: Option<&str>,
locale: Option<&str>,
) -> Result<Vec<u8>> {
let bar = progress::count_bar("rendering", list.len() as u64);
if matches!(
to,
"commonmeta"
| "csl"
| "datacite"
| "inveniordm"
| "schemaorg"
| "ror"
| "citation"
| "crossref_xml"
| "datacite_xml"
) {
let bytes = formats::write_all_citation(to, list, style, locale)?;
bar.finish_and_clear();
return Ok(bytes);
}
let mut output = String::new();
for (idx, item) in list.iter().enumerate() {
let rendered = formats::write_citation(to, item, style, locale)?;
if idx > 0 {
output.push('\n');
}
output.push_str(&String::from_utf8_lossy(&rendered));
bar.inc(1);
}
bar.finish_and_clear();
Ok(output.into_bytes())
}
pub fn write_archive(
list: &[Data],
to: &str,
base_name: &str,
batch_size: usize,
) -> Result<Vec<(String, Vec<u8>)>> {
write_archive_citation(list, to, base_name, batch_size, None, None)
}
pub fn write_archive_citation(
list: &[Data],
to: &str,
base_name: &str,
batch_size: usize,
style: Option<&str>,
locale: Option<&str>,
) -> Result<Vec<(String, Vec<u8>)>> {
if list.is_empty() {
return Err(Error::Serialize("no records to write".to_string()));
}
let chunks: Vec<&[Data]> = list.chunks(batch_size.max(1)).collect();
let multi = chunks.len() > 1;
let mut entries = Vec::with_capacity(chunks.len());
for (idx, chunk) in chunks.into_iter().enumerate() {
let bytes = write_list_citation(chunk, to, style, locale)?;
let name = batch_entry_name(base_name, if multi { Some(idx) } else { None });
entries.push((name, bytes));
}
Ok(entries)
}
fn batch_entry_name(base_name: &str, idx: Option<usize>) -> String {
match idx {
None => base_name.to_string(),
Some(i) => {
let path = std::path::Path::new(base_name);
let stem = path
.file_stem()
.unwrap_or_default()
.to_string_lossy()
.to_string();
let ext = path
.extension()
.map(|e| e.to_string_lossy().to_string())
.unwrap_or_default();
if ext.is_empty() {
format!("{}-{:05}", stem, i)
} else {
format!("{}-{:05}.{}", stem, i, ext)
}
}
}
}
pub fn read_vraix_sqlite(
sqlite_path: &str,
from: &str,
limit: Option<usize>,
offset: usize,
) -> Result<Vec<Data>> {
formats::vraix::read_dump(sqlite_path, from, limit, offset)
}
pub fn write_vraix_table_parquet(sqlite_path: &str, batch_size: usize) -> Result<Vec<u8>> {
formats::vraix::write_table_parquet(sqlite_path, batch_size)
}
pub fn fetch_vraix_dump(
from: &str,
date: &str,
input_path: Option<&str>,
limit: Option<usize>,
offset: usize,
cache_ttl: std::time::Duration,
) -> Result<Vec<Data>> {
if let Some(path) = input_path {
return read_vraix_sqlite(path, from, limit, offset);
}
let url = format!("https://metadata.vraix.org/{}-{}.sqlite3.zst", from, date);
let cache_key = format!("{}-{}.sqlite3.zst", from, date);
let (zst_path, _from_cache) =
io_utils::ensure_cached_path(&url, "vraix", &cache_key, cache_ttl)
.map_err(|e| Error::Http(format!("failed to download '{}': {}", url, e)))?;
let tmp_path = std::env::temp_dir().join(format!(
"commonmeta-vraix-{}-{}-{}.sqlite3",
from,
date,
std::process::id()
));
io_utils::decompress_zst_file(&zst_path, &tmp_path)
.map_err(|e| Error::Parse(format!("failed to decompress '{}': {}", url, e)))?;
let result = read_vraix_sqlite(tmp_path.to_str().unwrap(), from, limit, offset);
std::fs::remove_file(&tmp_path).ok();
result
}
pub fn push_inveniordm(list: &[Data], host: &str, token: &str) -> Vec<PushResult> {
formats::inveniordm::upsert_all(list, host, token)
}
pub fn put_inveniordm(data: &Data, host: &str, token: &str) -> PushResult {
formats::inveniordm::upsert(data, host, token)
}
#[cfg(test)]
mod tests {
use super::*;
fn sample_data(id: &str) -> Data {
Data {
id: id.to_string(),
type_: "JournalArticle".to_string(),
..Data::default()
}
}
#[test]
fn test_write_list_json_array_formats() {
let list = vec![
sample_data("https://doi.org/10.1/a"),
sample_data("https://doi.org/10.1/b"),
];
let bytes = write_list(&list, "commonmeta").unwrap();
let value: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
assert_eq!(value.as_array().unwrap().len(), 2);
}
#[test]
fn test_write_list_newline_joined_formats() {
let list = vec![
sample_data("https://doi.org/10.1/a"),
sample_data("https://doi.org/10.1/b"),
];
let bytes = write_list(&list, "ris").unwrap();
let text = String::from_utf8(bytes).unwrap();
assert_eq!(text.lines().filter(|l| l.starts_with("TY -")).count(), 2);
}
#[test]
fn test_write_list_crossref_xml_batches_into_one_doi_batch() {
let list = vec![
sample_data("https://doi.org/10.1/a"),
sample_data("https://doi.org/10.1/b"),
];
let bytes = write_list(&list, "crossref_xml").unwrap();
let text = String::from_utf8(bytes).unwrap();
assert_eq!(text.matches("<doi_batch xmlns=").count(), 1);
assert_eq!(text.matches("<journal_article").count(), 2);
}
#[test]
fn test_write_list_ror_uses_json_array_batch_writer() {
let mut a = sample_data("https://ror.org/0342dzm54");
a.title = "Org A".to_string();
let mut b = sample_data("https://ror.org/0521rfr06");
b.title = "Org B".to_string();
let bytes = write_list(&[a, b], "ror").unwrap();
let value: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
assert_eq!(value.as_array().unwrap().len(), 2);
}
#[test]
fn test_write_list_citation_renders_each_record() {
let mut a = sample_data("https://doi.org/10.1/a");
a.title = "Title A".to_string();
a.date_published = "2020".to_string();
let mut b = sample_data("https://doi.org/10.1/b");
b.title = "Title B".to_string();
b.date_published = "2021".to_string();
let bytes = write_list(&[a, b], "citation").unwrap();
let text = String::from_utf8(bytes).unwrap();
let lines: Vec<&str> = text.lines().collect();
assert_eq!(lines.len(), 2);
assert!(lines[0].contains("Title A"));
assert!(lines[1].contains("Title B"));
}
#[test]
fn test_write_list_citation_respects_style() {
let mut a = sample_data("https://doi.org/10.1/a");
a.title = "Title A".to_string();
a.date_published = "2020".to_string();
let apa = write_list_citation(&[a.clone()], "citation", None, None).unwrap();
let chicago =
write_list_citation(&[a], "citation", Some("chicago-author-date"), None).unwrap();
assert_ne!(apa, chicago);
}
#[test]
fn test_write_archive_single_batch_uses_base_name() {
let list = vec![sample_data("https://doi.org/10.1/a")];
let entries = write_archive(&list, "commonmeta", "out.json", 100_000).unwrap();
assert_eq!(entries.len(), 1);
assert_eq!(entries[0].0, "out.json");
}
#[test]
fn test_write_archive_numbered_batches() {
let list = vec![
sample_data("https://doi.org/10.1/a"),
sample_data("https://doi.org/10.1/b"),
sample_data("https://doi.org/10.1/c"),
];
let entries = write_archive(&list, "commonmeta", "out.json", 1).unwrap();
assert_eq!(entries.len(), 3);
assert_eq!(entries[0].0, "out-00000.json");
assert_eq!(entries[1].0, "out-00001.json");
assert_eq!(entries[2].0, "out-00002.json");
}
#[test]
fn test_write_archive_no_extension_base_name() {
let list = vec![
sample_data("https://doi.org/10.1/a"),
sample_data("https://doi.org/10.1/b"),
];
let entries = write_archive(&list, "commonmeta", "out", 1).unwrap();
assert_eq!(entries[0].0, "out-00000");
assert_eq!(entries[1].0, "out-00001");
}
#[test]
fn test_write_archive_empty_list_errors() {
assert!(write_archive(&[], "commonmeta", "out.json", 100_000).is_err());
}
#[test]
fn test_fetch_vraix_dump_uses_local_input_path_without_network() {
let dir = std::env::temp_dir().join("commonmeta_lib_fetch_vraix_dump");
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("datacite.sqlite3");
std::fs::remove_file(&path).ok();
{
let conn = rusqlite::Connection::open(&path).unwrap();
conn.execute_batch("CREATE TABLE works (pid TEXT, source_id INTEGER, raw_metadata TEXT);")
.unwrap();
conn.execute(
"INSERT INTO works (pid, source_id, raw_metadata) VALUES (?1, ?2, ?3)",
rusqlite::params![
"pid-0",
1i64,
r#"{"data":{"id":"10.5678/b","attributes":{"doi":"10.5678/b"}}}"#
],
)
.unwrap();
}
let data = fetch_vraix_dump(
"datacite",
"2026-06-14",
Some(path.to_str().unwrap()),
None,
0,
std::time::Duration::from_secs(30 * 24 * 60 * 60),
)
.unwrap();
assert_eq!(data.len(), 1);
assert_eq!(data[0].id, "https://doi.org/10.5678/b");
std::fs::remove_dir_all(&dir).ok();
}
}