#![allow(clippy::doc_markdown, reason = "documentation quotes upstream table and column names throughout")]
use std::collections::HashMap;
use std::fs::File;
use std::io::{BufRead, BufReader, Read};
use std::path::PathBuf;
use anyhow::{Context, Result, bail};
use bzip2::read::MultiBzDecoder;
use clap::Args as ClapArgs;
use sqlx::PgPool;
use super::copy_text::{Reader, Row};
const BATCH: usize = 8192;
#[derive(ClapArgs)]
pub struct Args {
#[arg(long, value_name = "FILE")]
pub dump: PathBuf,
#[arg(long = "dump-version", value_name = "VERSION")]
pub version: Option<String>,
}
#[derive(Default)]
struct Tables {
artist_types: HashMap<i32, String>,
release_group_types: HashMap<i32, String>,
areas: HashMap<i32, String>,
area_codes: HashMap<i32, String>,
release_groups_of_releases: HashMap<i32, i32>,
release_years: HashMap<i32, i16>,
links: HashMap<i32, i32>,
url_link_types: HashMap<i32, String>,
urls: HashMap<i32, String>,
artists: Vec<Artist>,
release_groups: Vec<ReleaseGroup>,
artist_url_links: Vec<(i32, i32, i32)>,
credit_artists: HashMap<i32, i32>,
timestamp: Option<String>,
}
struct Artist {
id: i32,
mbid: uuid::Uuid,
name: String,
sort_name: String,
type_id: Option<i32>,
area_id: Option<i32>,
begin_year: Option<i16>,
end_year: Option<i16>,
ended: bool,
comment: Option<String>,
}
struct ReleaseGroup {
id: i32,
mbid: uuid::Uuid,
name: String,
type_id: Option<i32>,
credit_id: Option<i32>,
}
pub async fn run(pool: &PgPool, args: &Args) -> Result<()> {
let file = File::open(&args.dump).with_context(|| format!("cannot open {}", args.dump.display()))?;
let size = file.metadata().map_or(0, |m| m.len());
tracing::info!(dump = %args.dump.display(), size_mb = size / 1_048_576, "reading the MusicBrainz export");
let tables = read_archive(file)?;
let version = args
.version
.clone()
.or_else(|| tables.timestamp.clone())
.context("the dump carries no TIMESTAMP file; pass --version with the fullexport timestamp")?;
tracing::info!(
version = %version,
artists = tables.artists.len(),
release_groups = tables.release_groups.len(),
artist_urls = tables.artist_url_links.len(),
"archive read; writing to PostgreSQL"
);
write(pool, &tables, &version).await
}
fn read_archive(file: File) -> Result<Tables> {
let decoder = MultiBzDecoder::new(BufReader::with_capacity(1 << 20, file));
let mut archive = tar::Archive::new(decoder);
let mut tables = Tables::default();
for entry in archive.entries().context("the dump is not a readable tar archive")? {
let entry = entry.context("failed to read an archive entry")?;
let path = entry.path().context("archive entry has an unreadable path")?.into_owned();
let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
continue;
};
if name == "TIMESTAMP" {
let mut text = String::new();
let mut entry = entry;
entry.read_to_string(&mut text).ok();
tables.timestamp = Some(text.trim().to_string());
continue;
}
if path.parent().and_then(|p| p.file_name()).and_then(|n| n.to_str()) != Some("mbdump") {
continue;
}
let reader = BufReader::with_capacity(1 << 20, entry);
match name {
"artist" => read_artists(reader, &mut tables)?,
"artist_type" => read_names(reader, &mut tables.artist_types)?,
"release_group" => read_release_groups(reader, &mut tables)?,
"release_group_primary_type" => read_names(reader, &mut tables.release_group_types)?,
"area" => read_areas(reader, &mut tables)?,
"iso_3166_1" => read_area_codes(reader, &mut tables)?,
"release" => read_releases(reader, &mut tables)?,
"release_country" | "release_unknown_country" => read_release_dates(reader, name, &mut tables)?,
"artist_credit_name" => read_credit_names(reader, &mut tables)?,
"url" => read_urls(reader, &mut tables)?,
"link" => read_links(reader, &mut tables)?,
"link_type" => read_link_types(reader, &mut tables)?,
"l_artist_url" => read_artist_url_links(reader, &mut tables)?,
_ => {}
}
}
if tables.artists.is_empty() {
bail!("no artists found in the archive: is this mbdump.tar.bz2 from a full export?");
}
Ok(tables)
}
fn each_row<R: BufRead>(reader: R, mut handle: impl FnMut(&Row)) -> Result<()> {
let mut rows = Reader::new(reader);
while let Some(row) = rows.next_row().context("failed to read a dump row")? {
handle(&row);
}
Ok(())
}
fn read_names<R: BufRead>(reader: R, out: &mut HashMap<i32, String>) -> Result<()> {
each_row(reader, |row| {
if let (Some(id), Some(name)) = (row.parse::<i32>(0), row.get(1)) {
out.insert(id, name.to_string());
}
})
}
fn read_artists<R: BufRead>(reader: R, tables: &mut Tables) -> Result<()> {
each_row(reader, |row| {
let (Some(id), Some(mbid), Some(name), Some(sort_name)) = (row.parse::<i32>(0), row.parse::<uuid::Uuid>(1), row.get(2), row.get(3)) else {
return;
};
tables.artists.push(Artist {
id,
mbid,
name: name.to_string(),
sort_name: sort_name.to_string(),
begin_year: row.parse(4),
end_year: row.parse(7),
type_id: row.parse(10),
area_id: row.parse(11),
comment: row.get(13).filter(|c| !c.is_empty()).map(str::to_string),
ended: row.get(16) == Some("t"),
});
})
}
fn read_areas<R: BufRead>(reader: R, tables: &mut Tables) -> Result<()> {
each_row(reader, |row| {
if let (Some(id), Some(name)) = (row.parse::<i32>(0), row.get(2)) {
tables.areas.insert(id, name.to_string());
}
})
}
fn read_area_codes<R: BufRead>(reader: R, tables: &mut Tables) -> Result<()> {
each_row(reader, |row| {
if let (Some(area), Some(code)) = (row.parse::<i32>(0), row.get(1)) {
tables.area_codes.insert(area, code.to_string());
}
})
}
fn read_releases<R: BufRead>(reader: R, tables: &mut Tables) -> Result<()> {
each_row(reader, |row| {
if let (Some(id), Some(group)) = (row.parse::<i32>(0), row.parse::<i32>(4)) {
tables.release_groups_of_releases.insert(id, group);
}
})
}
fn read_release_dates<R: BufRead>(reader: R, table: &str, tables: &mut Tables) -> Result<()> {
let year_column = if table == "release_country" { 2 } else { 1 };
each_row(reader, |row| {
let (Some(release), Some(year)) = (row.parse::<i32>(0), row.parse::<i16>(year_column)) else {
return;
};
tables
.release_years
.entry(release)
.and_modify(|earliest| *earliest = (*earliest).min(year))
.or_insert(year);
})
}
fn read_release_groups<R: BufRead>(reader: R, tables: &mut Tables) -> Result<()> {
each_row(reader, |row| {
let (Some(id), Some(mbid), Some(name)) = (row.parse::<i32>(0), row.parse::<uuid::Uuid>(1), row.get(2)) else {
return;
};
tables.release_groups.push(ReleaseGroup {
id,
mbid,
name: name.to_string(),
credit_id: row.parse(3),
type_id: row.parse(4),
});
})
}
fn read_credit_names<R: BufRead>(reader: R, tables: &mut Tables) -> Result<()> {
each_row(reader, |row| {
if row.parse::<i32>(1) != Some(0) {
return;
}
if let (Some(credit), Some(artist)) = (row.parse::<i32>(0), row.parse::<i32>(2)) {
tables.credit_artists.insert(credit, artist);
}
})
}
fn read_urls<R: BufRead>(reader: R, tables: &mut Tables) -> Result<()> {
each_row(reader, |row| {
if let (Some(id), Some(url)) = (row.parse::<i32>(0), row.get(2)) {
tables.urls.insert(id, url.to_string());
}
})
}
fn read_links<R: BufRead>(reader: R, tables: &mut Tables) -> Result<()> {
each_row(reader, |row| {
if let (Some(id), Some(link_type)) = (row.parse::<i32>(0), row.parse::<i32>(1)) {
tables.links.insert(id, link_type);
}
})
}
fn read_link_types<R: BufRead>(reader: R, tables: &mut Tables) -> Result<()> {
each_row(reader, |row| {
if row.get(4) != Some("artist") || row.get(5) != Some("url") {
return;
}
if let (Some(id), Some(name)) = (row.parse::<i32>(0), row.get(6)) {
tables.url_link_types.insert(id, name.to_string());
}
})
}
fn read_artist_url_links<R: BufRead>(reader: R, tables: &mut Tables) -> Result<()> {
each_row(reader, |row| {
if let (Some(link), Some(artist), Some(url)) = (row.parse::<i32>(1), row.parse::<i32>(2), row.parse::<i32>(3)) {
tables.artist_url_links.push((artist, link, url));
}
})
}
async fn write(pool: &PgPool, tables: &Tables, version: &str) -> Result<()> {
let mut tx = pool.begin().await.context("failed to open the import transaction")?;
let import_id: i32 = sqlx::query_scalar(
"INSERT INTO dump_import (source, version) VALUES ('musicbrainz', $1)
ON CONFLICT (source, version) DO UPDATE SET started_at = now(), finished_at = NULL, rows_imported = NULL
RETURNING id",
)
.bind(version)
.fetch_one(&mut *tx)
.await
.context("failed to record the import")?;
sqlx::query("TRUNCATE artist, release_group, artist_url, artist_credit RESTART IDENTITY CASCADE")
.execute(&mut *tx)
.await
.context("failed to clear the previous canon")?;
let mut written: i64 = 0;
written += write_artists(&mut tx, tables).await?;
written += write_artist_credits(&mut tx, tables).await?;
written += write_release_groups(&mut tx, tables).await?;
written += write_artist_urls(&mut tx, tables).await?;
sqlx::query("UPDATE dump_import SET finished_at = now(), rows_imported = $2 WHERE id = $1")
.bind(import_id)
.bind(written)
.execute(&mut *tx)
.await
.context("failed to close the import record")?;
tx.commit().await.context("failed to commit the import")?;
tracing::info!(version, rows = written, "import complete");
Ok(())
}
async fn write_artist_credits(tx: &mut sqlx::PgTransaction<'_>, tables: &Tables) -> Result<i64> {
let known: std::collections::HashSet<i32> = tables.artists.iter().map(|a| a.id).collect();
let resolved: Vec<(i32, i32)> = tables
.credit_artists
.iter()
.filter(|(_, artist)| known.contains(artist))
.map(|(&credit, &artist)| (credit, artist))
.collect();
let mut written = 0i64;
for chunk in resolved.chunks(BATCH) {
let credits: Vec<i32> = chunk.iter().map(|(c, _)| *c).collect();
let artists: Vec<i32> = chunk.iter().map(|(_, a)| *a).collect();
sqlx::query(
"INSERT INTO artist_credit (id, artist_id)
SELECT * FROM UNNEST($1::int[], $2::int[])",
)
.bind(&credits)
.bind(&artists)
.execute(&mut **tx)
.await
.context("failed to write artist credits")?;
written += i64::try_from(chunk.len()).unwrap_or(i64::MAX);
}
tracing::info!(rows = written, "artist credits written");
Ok(written)
}
async fn write_artists(tx: &mut sqlx::PgTransaction<'_>, tables: &Tables) -> Result<i64> {
let mut written = 0i64;
for chunk in tables.artists.chunks(BATCH) {
let ids: Vec<i32> = chunk.iter().map(|a| a.id).collect();
let mbids: Vec<uuid::Uuid> = chunk.iter().map(|a| a.mbid).collect();
let names: Vec<&str> = chunk.iter().map(|a| a.name.as_str()).collect();
let sort_names: Vec<&str> = chunk.iter().map(|a| a.sort_name.as_str()).collect();
let kinds: Vec<Option<&str>> = chunk
.iter()
.map(|a| a.type_id.and_then(|t| tables.artist_types.get(&t)).map(String::as_str))
.collect();
let areas: Vec<Option<&str>> = chunk.iter().map(|a| a.area_id.and_then(|t| tables.areas.get(&t)).map(String::as_str)).collect();
let area_codes: Vec<Option<&str>> = chunk
.iter()
.map(|a| a.area_id.and_then(|t| tables.area_codes.get(&t)).map(String::as_str))
.collect();
let begins: Vec<Option<i16>> = chunk.iter().map(|a| a.begin_year).collect();
let ends: Vec<Option<i16>> = chunk.iter().map(|a| a.end_year).collect();
let ended: Vec<bool> = chunk.iter().map(|a| a.ended).collect();
let comments: Vec<Option<&str>> = chunk.iter().map(|a| a.comment.as_deref()).collect();
sqlx::query(
"INSERT INTO artist (id, mbid, name, sort_name, kind, area, area_code, begin_year, end_year, ended, comment)
SELECT * FROM UNNEST($1::int[], $2::uuid[], $3::text[], $4::text[], $5::text[], $6::text[], $7::text[], $8::smallint[], $9::smallint[], $10::bool[], $11::text[])",
)
.bind(&ids)
.bind(&mbids)
.bind(&names)
.bind(&sort_names)
.bind(&kinds)
.bind(&areas)
.bind(&area_codes)
.bind(&begins)
.bind(&ends)
.bind(&ended)
.bind(&comments)
.execute(&mut **tx)
.await
.context("failed to write artists")?;
written += i64::try_from(chunk.len()).unwrap_or(i64::MAX);
}
tracing::info!(rows = written, "artists written");
Ok(written)
}
fn release_group_years(tables: &Tables) -> HashMap<i32, i16> {
let mut years: HashMap<i32, i16> = HashMap::with_capacity(tables.release_groups.len());
for (release, year) in &tables.release_years {
let Some(group) = tables.release_groups_of_releases.get(release) else {
continue;
};
years.entry(*group).and_modify(|earliest| *earliest = (*earliest).min(*year)).or_insert(*year);
}
years
}
async fn write_release_groups(tx: &mut sqlx::PgTransaction<'_>, tables: &Tables) -> Result<i64> {
let known: std::collections::HashSet<i32> = tables.artists.iter().map(|a| a.id).collect();
let resolved: Vec<(&ReleaseGroup, i32)> = tables
.release_groups
.iter()
.filter_map(|rg| {
let artist = rg.credit_id.and_then(|c| tables.credit_artists.get(&c)).copied()?;
known.contains(&artist).then_some((rg, artist))
})
.collect();
let years = release_group_years(tables);
let mut written = 0i64;
for chunk in resolved.chunks(BATCH) {
let ids: Vec<i32> = chunk.iter().map(|(rg, _)| rg.id).collect();
let mbids: Vec<uuid::Uuid> = chunk.iter().map(|(rg, _)| rg.mbid).collect();
let names: Vec<&str> = chunk.iter().map(|(rg, _)| rg.name.as_str()).collect();
let types: Vec<Option<&str>> = chunk
.iter()
.map(|(rg, _)| rg.type_id.and_then(|t| tables.release_group_types.get(&t)).map(String::as_str))
.collect();
let artists: Vec<i32> = chunk.iter().map(|(_, artist)| *artist).collect();
let group_years: Vec<Option<i16>> = chunk.iter().map(|(rg, _)| years.get(&rg.id).copied()).collect();
sqlx::query(
"INSERT INTO release_group (id, mbid, name, primary_type, artist_id, year)
SELECT * FROM UNNEST($1::int[], $2::uuid[], $3::text[], $4::text[], $5::int[], $6::smallint[])",
)
.bind(&ids)
.bind(&mbids)
.bind(&names)
.bind(&types)
.bind(&artists)
.bind(&group_years)
.execute(&mut **tx)
.await
.context("failed to write release groups")?;
written += i64::try_from(chunk.len()).unwrap_or(i64::MAX);
}
tracing::info!(rows = written, dropped = tables.release_groups.len() - resolved.len(), "release groups written");
Ok(written)
}
async fn write_artist_urls(tx: &mut sqlx::PgTransaction<'_>, tables: &Tables) -> Result<i64> {
let known: std::collections::HashSet<i32> = tables.artists.iter().map(|a| a.id).collect();
let resolved: Vec<(i32, &str, &str)> = tables
.artist_url_links
.iter()
.filter_map(|&(artist, link, url)| {
let link_type = tables.links.get(&link)?;
let kind = tables.url_link_types.get(link_type)?;
let address = tables.urls.get(&url)?;
known.contains(&artist).then_some((artist, kind.as_str(), address.as_str()))
})
.collect();
let mut written = 0i64;
for chunk in resolved.chunks(BATCH) {
let artists: Vec<i32> = chunk.iter().map(|(a, _, _)| *a).collect();
let kinds: Vec<&str> = chunk.iter().map(|(_, k, _)| *k).collect();
let urls: Vec<&str> = chunk.iter().map(|(_, _, u)| *u).collect();
sqlx::query(
"INSERT INTO artist_url (artist_id, kind, url)
SELECT * FROM UNNEST($1::int[], $2::text[], $3::text[])
ON CONFLICT DO NOTHING",
)
.bind(&artists)
.bind(&kinds)
.bind(&urls)
.execute(&mut **tx)
.await
.context("failed to write artist URLs")?;
written += i64::try_from(chunk.len()).unwrap_or(i64::MAX);
}
tracing::info!(rows = written, "artist URLs written");
Ok(written)
}
#[cfg(test)]
mod tests {
use super::*;
fn read_table(table: &str, rows: &str) -> Tables {
let mut tables = Tables::default();
match table {
"iso_3166_1" => read_area_codes(rows.as_bytes(), &mut tables).unwrap(),
"release" => read_releases(rows.as_bytes(), &mut tables).unwrap(),
other => read_release_dates(rows.as_bytes(), other, &mut tables).unwrap(),
}
tables
}
#[test]
fn reads_country_codes_from_iso_3166_1() {
let tables = read_table("iso_3166_1", "222\tUS\n81\tDE\n");
assert_eq!(tables.area_codes.get(&222).map(String::as_str), Some("US"));
assert_eq!(tables.area_codes.get(&81).map(String::as_str), Some("DE"));
}
#[test]
fn takes_the_year_from_the_right_column_in_each_date_table() {
let with_country = read_table("release_country", "10\t222\t1991\t9\t24\n");
assert_eq!(with_country.release_years.get(&10), Some(&1991));
let without = read_table("release_unknown_country", "11\t1994\t4\t5\n");
assert_eq!(without.release_years.get(&11), Some(&1994));
}
#[test]
fn keeps_the_earliest_year_of_a_release_issued_in_several_countries() {
let tables = read_table("release_country", "10\t222\t1992\t1\t1\n10\t81\t1991\t9\t24\n");
assert_eq!(tables.release_years.get(&10), Some(&1991));
}
#[test]
fn dates_a_release_group_by_its_earliest_release() {
let mut tables = Tables::default();
tables.release_groups_of_releases.insert(10, 500);
tables.release_groups_of_releases.insert(11, 500);
tables.release_years.insert(10, 2011);
tables.release_years.insert(11, 1991);
let years = release_group_years(&tables);
assert_eq!(years.get(&500), Some(&1991));
}
#[test]
fn leaves_a_release_group_undated_when_no_release_carries_a_year() {
let mut tables = Tables::default();
tables.release_groups_of_releases.insert(10, 500);
assert!(!release_group_years(&tables).contains_key(&500));
}
#[test]
fn ignores_a_date_on_a_release_belonging_to_no_group() {
let mut tables = Tables::default();
tables.release_years.insert(10, 1991);
assert!(release_group_years(&tables).is_empty());
}
}