#![allow(clippy::doc_markdown, reason = "documentation quotes upstream element and file names throughout")]
use std::collections::{HashMap, HashSet};
use std::fs::File;
use std::io::{BufRead, BufReader};
use std::path::{Path, PathBuf};
use anyhow::{Context, Result, bail};
use clap::Args as ClapArgs;
use flate2::read::MultiGzDecoder;
use sqlx::PgPool;
use super::discogs_xml::{Attributes, Record, Records};
const BATCH: usize = 8192;
#[derive(ClapArgs)]
pub struct Args {
#[arg(long, value_name = "FILE")]
pub masters: PathBuf,
#[arg(long, value_name = "FILE")]
pub labels: Option<PathBuf>,
#[arg(long, value_name = "FILE")]
pub artists: Option<PathBuf>,
#[arg(long = "dump-version", value_name = "VERSION")]
pub version: Option<String>,
}
#[derive(Default)]
struct Master {
artists: Vec<i32>,
genres: Vec<String>,
styles: Vec<String>,
}
impl Record for Master {
const ELEMENT: &'static str = "master";
fn open(_: Attributes<'_>) -> Self {
Self::default()
}
fn field(&mut self, path: &[&str], text: &str, _: Attributes<'_>) {
match path {
["artists", "artist", "id"] => {
if let Ok(id) = text.parse() {
self.artists.push(id);
}
}
["genres", "genre"] if !text.is_empty() => self.genres.push(text.to_string()),
["styles", "style"] if !text.is_empty() => self.styles.push(text.to_string()),
_ => {}
}
}
}
#[derive(Default)]
struct Label {
id: Option<i32>,
name: Option<String>,
profile: Option<String>,
parent_id: Option<i32>,
}
impl Record for Label {
const ELEMENT: &'static str = "label";
fn open(_: Attributes<'_>) -> Self {
Self::default()
}
fn field(&mut self, path: &[&str], text: &str, attributes: Attributes<'_>) {
match path {
["id"] => self.id = text.parse().ok(),
["name"] => self.name = Some(text.to_string()),
["profile"] if !text.is_empty() => self.profile = Some(text.to_string()),
["parentLabel"] => self.parent_id = attributes.parse("id"),
_ => {}
}
}
}
struct DiscogsArtist {
id: Option<i32>,
}
impl Record for DiscogsArtist {
const ELEMENT: &'static str = "artist";
fn open(_: Attributes<'_>) -> Self {
Self { id: None }
}
fn field(&mut self, path: &[&str], text: &str, _: Attributes<'_>) {
if path == ["id"] {
self.id = text.parse().ok();
}
}
}
type GenreCounts = HashMap<i32, HashMap<(String, bool), i32>>;
pub async fn run(pool: &PgPool, args: &Args) -> Result<()> {
let mapping = load_discogs_ids(pool).await?;
if mapping.is_empty() {
bail!(
"no artist has a Discogs link in the canon: run `lyrid import musicbrainz` first \
(the link comes from MusicBrainz's `discogs` artist-URL relationship)"
);
}
tracing::info!(linked = mapping.len(), "resolving Discogs data against the canon");
let version = args
.version
.clone()
.or_else(|| version_from_filename(&args.masters))
.context("cannot tell the dump version from the filename; pass --dump-version")?;
let wanted: HashSet<i32> = mapping.keys().copied().collect();
let known_artists = match &args.artists {
Some(path) => Some(read_artist_ids(path, &wanted)?),
None => None,
};
let counts = read_masters(&args.masters, &wanted, known_artists.as_ref())?;
let labels = match &args.labels {
Some(path) => read_labels(path)?,
None => Vec::new(),
};
write(pool, &mapping, &counts, &labels, &version).await
}
async fn load_discogs_ids(pool: &PgPool) -> Result<HashMap<i32, Vec<i32>>> {
let rows: Vec<(i32, String)> = sqlx::query_as(
"SELECT artist_id, url FROM artist_url
WHERE kind = 'discogs' AND url LIKE '%discogs.com/artist/%'",
)
.fetch_all(pool)
.await
.context("failed to read Discogs links from the canon")?;
let mut mapping: HashMap<i32, Vec<i32>> = HashMap::new();
for (artist_id, url) in rows {
if let Some(discogs_id) = discogs_id_from_url(&url) {
mapping.entry(discogs_id).or_default().push(artist_id);
}
}
Ok(mapping)
}
fn discogs_id_from_url(url: &str) -> Option<i32> {
let after = url.split("/artist/").nth(1)?;
let digits: String = after.chars().take_while(char::is_ascii_digit).collect();
digits.parse().ok()
}
fn version_from_filename(path: &Path) -> Option<String> {
let name = path.file_name()?.to_str()?;
let after = name.strip_prefix("discogs_")?;
let digits: String = after.chars().take_while(char::is_ascii_digit).collect();
(digits.len() == 8).then_some(digits)
}
fn open_dump(path: &Path) -> Result<impl BufRead> {
let file = File::open(path).with_context(|| format!("cannot open {}", path.display()))?;
let size = file.metadata().map_or(0, |m| m.len());
tracing::info!(dump = %path.display(), size_mb = size / 1_048_576, "reading a Discogs dump");
let decoder = MultiGzDecoder::new(BufReader::with_capacity(1 << 20, file));
Ok(BufReader::with_capacity(1 << 20, decoder))
}
fn read_artist_ids(path: &Path, wanted: &HashSet<i32>) -> Result<HashSet<i32>> {
let mut records = Records::<_, DiscogsArtist>::new(open_dump(path)?);
let mut found = HashSet::with_capacity(wanted.len());
let mut total: u64 = 0;
while let Some(record) = records.next_record()? {
total += 1;
if let Some(id) = record.id
&& wanted.contains(&id)
{
found.insert(id);
}
}
tracing::info!(records = total, linked_found = found.len(), "artists file read");
Ok(found)
}
fn tally(master: &Master, wanted: &HashSet<i32>, known: Option<&HashSet<i32>>, counts: &mut GenreCounts) -> u64 {
if master.genres.is_empty() && master.styles.is_empty() {
return 0;
}
let mut credited = 0;
for artist in &master.artists {
if !wanted.contains(artist) {
continue;
}
if known.is_some_and(|known| !known.contains(artist)) {
continue;
}
credited += 1;
let per_artist = counts.entry(*artist).or_default();
for genre in &master.genres {
*per_artist.entry((genre.clone(), false)).or_insert(0) += 1;
}
for style in &master.styles {
*per_artist.entry((style.clone(), true)).or_insert(0) += 1;
}
}
credited
}
fn read_masters(path: &Path, wanted: &HashSet<i32>, known: Option<&HashSet<i32>>) -> Result<GenreCounts> {
let mut records = Records::<_, Master>::new(open_dump(path)?);
let mut counts: GenreCounts = HashMap::new();
let mut total: u64 = 0;
let mut credited: u64 = 0;
while let Some(master) = records.next_record()? {
total += 1;
credited += tally(&master, wanted, known, &mut counts);
}
tracing::info!(
records = total,
artists_with_genres = counts.len(),
credits_counted = credited,
"masters file read"
);
Ok(counts)
}
fn read_labels(path: &Path) -> Result<Vec<Label>> {
let mut records = Records::<_, Label>::new(open_dump(path)?);
let mut labels = Vec::new();
let mut total: u64 = 0;
while let Some(label) = records.next_record()? {
total += 1;
if label.id.is_some() && label.name.is_some() {
labels.push(label);
}
}
tracing::info!(records = total, kept = labels.len(), "labels file read");
Ok(labels)
}
async fn write(pool: &PgPool, mapping: &HashMap<i32, Vec<i32>>, counts: &GenreCounts, labels: &[Label], version: &str) -> Result<()> {
let mut tx = pool.begin().await.context("failed to open the import transaction")?;
let import_id: i32 = sqlx::query_scalar(
"INSERT INTO dump_import (source, version) VALUES ('discogs', $1)
ON CONFLICT (source, version) DO UPDATE SET started_at = now(), finished_at = NULL, rows_imported = NULL
RETURNING id",
)
.bind(version)
.fetch_one(&mut *tx)
.await
.context("failed to record the import")?;
sqlx::query("TRUNCATE artist_genre, artist_discogs, genre RESTART IDENTITY CASCADE")
.execute(&mut *tx)
.await
.context("failed to clear the previous genres")?;
let mut written: i64 = 0;
written += write_artist_discogs(&mut tx, mapping).await?;
written += write_genres(&mut tx, mapping, counts).await?;
if !labels.is_empty() {
written += write_labels(&mut tx, labels).await?;
}
sqlx::query("UPDATE dump_import SET finished_at = now(), rows_imported = $2 WHERE id = $1")
.bind(import_id)
.bind(written)
.execute(&mut *tx)
.await
.context("failed to close the import record")?;
tx.commit().await.context("failed to commit the import")?;
tracing::info!(version, rows = written, "Discogs import complete");
Ok(())
}
async fn write_artist_discogs(tx: &mut sqlx::PgTransaction<'_>, mapping: &HashMap<i32, Vec<i32>>) -> Result<i64> {
let pairs: Vec<(i32, i32)> = mapping
.iter()
.flat_map(|(discogs_id, artists)| artists.iter().map(move |artist| (*artist, *discogs_id)))
.collect();
let mut written = 0i64;
for chunk in pairs.chunks(BATCH) {
let artists: Vec<i32> = chunk.iter().map(|(a, _)| *a).collect();
let discogs: Vec<i32> = chunk.iter().map(|(_, d)| *d).collect();
sqlx::query(
"INSERT INTO artist_discogs (artist_id, discogs_id)
SELECT * FROM UNNEST($1::int[], $2::int[])
ON CONFLICT (artist_id) DO NOTHING",
)
.bind(&artists)
.bind(&discogs)
.execute(&mut **tx)
.await
.context("failed to write Discogs links")?;
written += i64::try_from(chunk.len()).unwrap_or(i64::MAX);
}
tracing::info!(rows = written, "Discogs links written");
Ok(written)
}
async fn write_genres(tx: &mut sqlx::PgTransaction<'_>, mapping: &HashMap<i32, Vec<i32>>, counts: &GenreCounts) -> Result<i64> {
let vocabulary: HashSet<(&str, bool)> = counts
.values()
.flat_map(|per_artist| per_artist.keys().map(|(name, is_style)| (name.as_str(), *is_style)))
.collect();
let names: Vec<&str> = vocabulary.iter().map(|(name, _)| *name).collect();
let styles: Vec<bool> = vocabulary.iter().map(|(_, is_style)| *is_style).collect();
let ids: Vec<(i32, String, bool)> = sqlx::query_as(
"INSERT INTO genre (name, is_style)
SELECT * FROM UNNEST($1::text[], $2::bool[])
RETURNING id, name, is_style",
)
.bind(&names)
.bind(&styles)
.fetch_all(&mut **tx)
.await
.context("failed to write the genre vocabulary")?;
tracing::info!(rows = ids.len(), "genre vocabulary written");
let genre_ids: HashMap<(String, bool), i32> = ids.into_iter().map(|(id, name, is_style)| ((name, is_style), id)).collect();
let mut rows: Vec<(i32, i32, i32)> = Vec::new();
for (discogs_id, per_artist) in counts {
let Some(artists) = mapping.get(discogs_id) else {
continue;
};
for artist in artists {
for (key, releases) in per_artist {
if let Some(genre_id) = genre_ids.get(key) {
rows.push((*artist, *genre_id, *releases));
}
}
}
}
let mut written = 0i64;
for chunk in rows.chunks(BATCH) {
let artists: Vec<i32> = chunk.iter().map(|(a, _, _)| *a).collect();
let genres: Vec<i32> = chunk.iter().map(|(_, g, _)| *g).collect();
let releases: Vec<i32> = chunk.iter().map(|(_, _, r)| *r).collect();
sqlx::query(
"INSERT INTO artist_genre (artist_id, genre_id, releases)
SELECT * FROM UNNEST($1::int[], $2::int[], $3::int[])
ON CONFLICT (artist_id, genre_id) DO UPDATE SET releases = artist_genre.releases + EXCLUDED.releases",
)
.bind(&artists)
.bind(&genres)
.bind(&releases)
.execute(&mut **tx)
.await
.context("failed to write artist genres")?;
written += i64::try_from(chunk.len()).unwrap_or(i64::MAX);
}
tracing::info!(rows = written, "artist genres written");
Ok(written)
}
async fn write_labels(tx: &mut sqlx::PgTransaction<'_>, labels: &[Label]) -> Result<i64> {
sqlx::query("TRUNCATE label CASCADE")
.execute(&mut **tx)
.await
.context("failed to clear the previous labels")?;
let mut written = 0i64;
for chunk in labels.chunks(BATCH) {
let ids: Vec<i32> = chunk.iter().filter_map(|l| l.id).collect();
let names: Vec<&str> = chunk.iter().filter_map(|l| l.name.as_deref()).collect();
let profiles: Vec<Option<&str>> = chunk.iter().map(|l| l.profile.as_deref()).collect();
sqlx::query(
"INSERT INTO label (id, name, profile)
SELECT * FROM UNNEST($1::int[], $2::text[], $3::text[])
ON CONFLICT (id) DO NOTHING",
)
.bind(&ids)
.bind(&names)
.bind(&profiles)
.execute(&mut **tx)
.await
.context("failed to write labels")?;
written += i64::try_from(chunk.len()).unwrap_or(i64::MAX);
}
let children: Vec<i32> = labels.iter().filter(|l| l.parent_id.is_some()).filter_map(|l| l.id).collect();
let parents: Vec<i32> = labels.iter().filter(|l| l.id.is_some()).filter_map(|l| l.parent_id).collect();
let updated = sqlx::query(
"UPDATE label SET parent_label_id = pairs.parent
FROM UNNEST($1::int[], $2::int[]) AS pairs(child, parent)
WHERE label.id = pairs.child
AND pairs.parent <> pairs.child
AND EXISTS (SELECT 1 FROM label AS p WHERE p.id = pairs.parent)",
)
.bind(&children)
.bind(&parents)
.execute(&mut **tx)
.await
.context("failed to link labels to their parents")?;
tracing::info!(rows = written, parents = updated.rows_affected(), "labels written");
Ok(written)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn reads_the_discogs_id_out_of_every_url_form() {
for url in [
"https://www.discogs.com/artist/11136",
"http://discogs.com/artist/11136",
"https://www.discogs.com/artist/11136-Peter-Gabriel",
"https://www.discogs.com/artist/11136/",
] {
assert_eq!(discogs_id_from_url(url), Some(11136), "failed on {url}");
}
}
#[test]
fn ignores_urls_that_are_not_artist_pages() {
assert_eq!(discogs_id_from_url("https://www.discogs.com/label/1-Planet-E"), None);
assert_eq!(discogs_id_from_url("https://www.discogs.com/release/116925"), None);
assert_eq!(discogs_id_from_url("https://www.discogs.com/artist/none"), None);
}
#[test]
fn takes_the_version_from_the_filename() {
assert_eq!(
version_from_filename(Path::new("/dumps/discogs_20260801_masters.xml.gz")).as_deref(),
Some("20260801")
);
assert_eq!(version_from_filename(Path::new("/dumps/masters.xml.gz")), None);
assert_eq!(version_from_filename(Path::new("/dumps/discogs_2026_masters.xml.gz")), None);
}
fn count(xml: &str, wanted: &[i32]) -> GenreCounts {
let wanted: HashSet<i32> = wanted.iter().copied().collect();
let mut records = Records::<_, Master>::new(xml.as_bytes());
let mut counts: GenreCounts = HashMap::new();
while let Some(master) = records.next_record().unwrap() {
tally(&master, &wanted, None, &mut counts);
}
counts
}
#[test]
fn counts_a_genre_once_per_release() {
let xml = concat!(
"<masters>",
"<master id=\"1\"><artists><artist><id>7</id><name>A</name></artist></artists>",
"<genres><genre>Electronic</genre></genres><styles><style>Techno</style></styles></master>",
"<master id=\"2\"><artists><artist><id>7</id><name>A</name></artist></artists>",
"<genres><genre>Electronic</genre></genres><styles><style>Techno</style></styles></master>",
"</masters>"
);
let counts = count(xml, &[7]);
assert_eq!(counts[&7][&("Techno".to_string(), true)], 2);
assert_eq!(counts[&7][&("Electronic".to_string(), false)], 2);
}
#[test]
fn keeps_genres_and_styles_apart() {
let xml = concat!(
"<masters><master id=\"1\"><artists><artist><id>7</id></artist></artists>",
"<genres><genre>Rock</genre></genres><styles><style>Rock & Roll</style></styles>",
"</master></masters>"
);
let counts = count(xml, &[7]);
assert!(counts[&7].contains_key(&("Rock".to_string(), false)));
assert!(counts[&7].contains_key(&("Rock & Roll".to_string(), true)));
}
#[test]
fn credits_every_artist_on_a_collaboration() {
let xml = concat!(
"<masters><master id=\"1\">",
"<artists><artist><id>7</id><join>&</join></artist><artist><id>8</id></artist></artists>",
"<styles><style>Techno</style></styles></master></masters>"
);
let counts = count(xml, &[7, 8]);
assert_eq!(counts[&7][&("Techno".to_string(), true)], 1);
assert_eq!(counts[&8][&("Techno".to_string(), true)], 1);
}
#[test]
fn ignores_artists_outside_the_canon() {
let xml = concat!(
"<masters><master id=\"1\"><artists><artist><id>999</id></artist></artists>",
"<styles><style>Techno</style></styles></master></masters>"
);
assert!(count(xml, &[7]).is_empty());
}
#[test]
fn does_not_take_the_release_name_as_an_artist_id() {
let xml = concat!(
"<masters><master id=\"1\">",
"<artists><artist><id>7</id><name>Samuel L Session</name><anv>Samuel L</anv></artist></artists>",
"<styles><style>Techno</style></styles></master></masters>"
);
let counts = count(xml, &[7]);
assert_eq!(counts.len(), 1);
assert_eq!(counts[&7].len(), 1);
}
#[test]
fn skips_a_master_with_no_genres_at_all() {
let xml = "<masters><master id=\"1\"><artists><artist><id>7</id></artist></artists><title>Untitled</title></master></masters>";
assert!(count(xml, &[7]).is_empty());
}
fn labels_of(xml: &str) -> Vec<Label> {
let mut records = Records::<_, Label>::new(xml.as_bytes());
let mut out = Vec::new();
while let Some(label) = records.next_record().unwrap() {
out.push(label);
}
out
}
#[test]
fn reads_a_label_with_its_parent() {
let xml = concat!(
"<labels><label><id>5</id><name>Svek</name><data_quality>Correct</data_quality>",
"<parentLabel id=\"4711\">Goldhead Music</parentLabel>",
"<sublabels><label id=\"2437\">Birdy</label></sublabels></label></labels>"
);
let labels = labels_of(xml);
assert_eq!(labels.len(), 1);
assert_eq!(labels[0].id, Some(5));
assert_eq!(labels[0].name.as_deref(), Some("Svek"));
assert_eq!(labels[0].parent_id, Some(4711));
}
#[test]
fn does_not_mistake_a_sublabel_for_the_labels_own_name() {
let xml = "<labels><label><id>1</id><name>Planet E</name><sublabels><label id=\"86537\">Antidote</label></sublabels></label></labels>";
let labels = labels_of(xml);
assert_eq!(labels[0].name.as_deref(), Some("Planet E"));
assert_eq!(labels[0].parent_id, None);
}
#[test]
fn leaves_contact_information_out() {
let xml = concat!(
"<labels><label><id>1</id><name>Planet E</name>",
"<contactinfo>P.O. Box 27218, Detroit</contactinfo>",
"<profile>Carl Craig's techno label.</profile></label></labels>"
);
let labels = labels_of(xml);
assert_eq!(labels[0].profile.as_deref(), Some("Carl Craig's techno label."));
}
}